@topgunbuild/server 0.2.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +1551 -10
- package/dist/index.d.ts +1551 -10
- package/dist/index.js +7460 -3171
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +7452 -3153
- package/dist/index.mjs.map +1 -1
- package/package.json +12 -11
- package/LICENSE +0 -97
package/dist/index.d.mts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import
|
|
1
|
+
import * as _topgunbuild_core from '@topgunbuild/core';
|
|
2
|
+
import { Timestamp, LWWRecord, ORMapRecord, Principal, EventJournalImpl, EventJournalConfig, JournalEvent, PermissionPolicy, ConsistencyLevel, ReplicationConfig, LWWMap, ORMap, PermissionType, MigrationConfig, MigrationStatus, MigrationMetrics, PartitionMap, PartitionInfo, PartitionChange, ReplicationLag, ReplicationHealth, ReplicationResult, EntryProcessorDef, EntryProcessorResult, HLC, MergeRejection, ConflictResolverDef, MergeContext, MergeResult } from '@topgunbuild/core';
|
|
2
3
|
import { WebSocket } from 'ws';
|
|
3
|
-
import {
|
|
4
|
+
import { Pool, PoolConfig } from 'pg';
|
|
4
5
|
import pino from 'pino';
|
|
6
|
+
import { EventEmitter } from 'events';
|
|
5
7
|
|
|
6
8
|
/**
|
|
7
9
|
* TaskletScheduler — Cooperative multitasking for long-running operations.
|
|
@@ -1601,16 +1603,30 @@ interface CoalescingWriterMetrics {
|
|
|
1601
1603
|
* - Larger batch size = higher throughput, higher latency
|
|
1602
1604
|
* - Longer delay = more messages per batch, higher latency
|
|
1603
1605
|
* - Larger maxBatchBytes = handles larger payloads, more memory
|
|
1606
|
+
*
|
|
1607
|
+
* NOTE: A/B testing (Dec 2024) showed maxDelayMs is the primary bottleneck:
|
|
1608
|
+
* - 10ms delay: ~10K ops/sec, p50=11ms
|
|
1609
|
+
* - 1ms delay: ~18K ops/sec, p50=8ms (+80% throughput)
|
|
1610
|
+
* - 0ms (disabled): ~18K ops/sec, p50=2ms (best latency)
|
|
1604
1611
|
*/
|
|
1605
1612
|
declare const coalescingPresets: {
|
|
1606
1613
|
/**
|
|
1607
|
-
*
|
|
1608
|
-
*
|
|
1609
|
-
*
|
|
1614
|
+
* Low latency - optimized for minimal response time.
|
|
1615
|
+
* Best for: gaming, real-time chat, interactive applications.
|
|
1616
|
+
* Benchmark: p50=2ms, ~18K ops/sec
|
|
1617
|
+
*/
|
|
1618
|
+
readonly lowLatency: {
|
|
1619
|
+
readonly maxBatchSize: 100;
|
|
1620
|
+
readonly maxDelayMs: 1;
|
|
1621
|
+
readonly maxBatchBytes: 65536;
|
|
1622
|
+
};
|
|
1623
|
+
/**
|
|
1624
|
+
* Conservative - good balance of latency and batching.
|
|
1625
|
+
* Use for: general purpose with latency sensitivity.
|
|
1610
1626
|
*/
|
|
1611
1627
|
readonly conservative: {
|
|
1612
1628
|
readonly maxBatchSize: 100;
|
|
1613
|
-
readonly maxDelayMs:
|
|
1629
|
+
readonly maxDelayMs: 2;
|
|
1614
1630
|
readonly maxBatchBytes: 65536;
|
|
1615
1631
|
};
|
|
1616
1632
|
/**
|
|
@@ -1620,17 +1636,18 @@ declare const coalescingPresets: {
|
|
|
1620
1636
|
*/
|
|
1621
1637
|
readonly balanced: {
|
|
1622
1638
|
readonly maxBatchSize: 300;
|
|
1623
|
-
readonly maxDelayMs:
|
|
1639
|
+
readonly maxDelayMs: 2;
|
|
1624
1640
|
readonly maxBatchBytes: 131072;
|
|
1625
1641
|
};
|
|
1626
1642
|
/**
|
|
1627
1643
|
* High throughput - optimized for write-heavy workloads.
|
|
1628
1644
|
* Higher batching for better network utilization.
|
|
1629
1645
|
* Use for: data ingestion, logging, IoT data streams.
|
|
1646
|
+
* Benchmark: p50=7ms, ~18K ops/sec
|
|
1630
1647
|
*/
|
|
1631
1648
|
readonly highThroughput: {
|
|
1632
1649
|
readonly maxBatchSize: 500;
|
|
1633
|
-
readonly maxDelayMs:
|
|
1650
|
+
readonly maxDelayMs: 2;
|
|
1634
1651
|
readonly maxBatchBytes: 262144;
|
|
1635
1652
|
};
|
|
1636
1653
|
/**
|
|
@@ -1640,7 +1657,7 @@ declare const coalescingPresets: {
|
|
|
1640
1657
|
*/
|
|
1641
1658
|
readonly aggressive: {
|
|
1642
1659
|
readonly maxBatchSize: 1000;
|
|
1643
|
-
readonly maxDelayMs:
|
|
1660
|
+
readonly maxDelayMs: 5;
|
|
1644
1661
|
readonly maxBatchBytes: 524288;
|
|
1645
1662
|
};
|
|
1646
1663
|
};
|
|
@@ -1655,6 +1672,116 @@ type CoalescingPreset = keyof typeof coalescingPresets;
|
|
|
1655
1672
|
*/
|
|
1656
1673
|
declare function getCoalescingPreset(preset: CoalescingPreset): CoalescingWriterOptions;
|
|
1657
1674
|
|
|
1675
|
+
/**
|
|
1676
|
+
* Export options for streaming journal events.
|
|
1677
|
+
*/
|
|
1678
|
+
interface ExportOptions {
|
|
1679
|
+
/** Start from this sequence (inclusive) */
|
|
1680
|
+
fromSequence?: bigint;
|
|
1681
|
+
/** End at this sequence (inclusive) */
|
|
1682
|
+
toSequence?: bigint;
|
|
1683
|
+
/** Filter by map name */
|
|
1684
|
+
mapName?: string;
|
|
1685
|
+
/** Filter by event types */
|
|
1686
|
+
types?: ('PUT' | 'UPDATE' | 'DELETE')[];
|
|
1687
|
+
}
|
|
1688
|
+
/**
|
|
1689
|
+
* Configuration for EventJournalService.
|
|
1690
|
+
*/
|
|
1691
|
+
interface EventJournalServiceConfig extends EventJournalConfig {
|
|
1692
|
+
/** PostgreSQL connection pool */
|
|
1693
|
+
pool: Pool;
|
|
1694
|
+
/** Table name for journal storage */
|
|
1695
|
+
tableName?: string;
|
|
1696
|
+
/** Batch size for persistence */
|
|
1697
|
+
persistBatchSize?: number;
|
|
1698
|
+
/** Interval for periodic persistence (ms) */
|
|
1699
|
+
persistIntervalMs?: number;
|
|
1700
|
+
}
|
|
1701
|
+
/**
|
|
1702
|
+
* Default configuration for EventJournalService.
|
|
1703
|
+
*/
|
|
1704
|
+
declare const DEFAULT_JOURNAL_SERVICE_CONFIG: Omit<EventJournalServiceConfig, 'pool'>;
|
|
1705
|
+
/**
|
|
1706
|
+
* Server-side Event Journal Service with PostgreSQL persistence.
|
|
1707
|
+
* Extends EventJournalImpl to add durable storage.
|
|
1708
|
+
*/
|
|
1709
|
+
declare class EventJournalService extends EventJournalImpl {
|
|
1710
|
+
private readonly pool;
|
|
1711
|
+
private readonly tableName;
|
|
1712
|
+
private readonly persistBatchSize;
|
|
1713
|
+
private readonly persistIntervalMs;
|
|
1714
|
+
private pendingPersist;
|
|
1715
|
+
private persistTimer?;
|
|
1716
|
+
private isPersisting;
|
|
1717
|
+
private isInitialized;
|
|
1718
|
+
private isLoadingFromStorage;
|
|
1719
|
+
constructor(config: EventJournalServiceConfig);
|
|
1720
|
+
/**
|
|
1721
|
+
* Initialize the journal service, creating table if needed.
|
|
1722
|
+
*/
|
|
1723
|
+
initialize(): Promise<void>;
|
|
1724
|
+
/**
|
|
1725
|
+
* Persist pending events to PostgreSQL.
|
|
1726
|
+
*/
|
|
1727
|
+
persistToStorage(): Promise<void>;
|
|
1728
|
+
/**
|
|
1729
|
+
* Load journal events from PostgreSQL on startup.
|
|
1730
|
+
*/
|
|
1731
|
+
loadFromStorage(): Promise<void>;
|
|
1732
|
+
/**
|
|
1733
|
+
* Export events as NDJSON stream.
|
|
1734
|
+
*/
|
|
1735
|
+
exportStream(options?: ExportOptions): ReadableStream<string>;
|
|
1736
|
+
/**
|
|
1737
|
+
* Get events for a specific map.
|
|
1738
|
+
*/
|
|
1739
|
+
getMapEvents(mapName: string, fromSeq?: bigint): JournalEvent[];
|
|
1740
|
+
/**
|
|
1741
|
+
* Query events from PostgreSQL with filters.
|
|
1742
|
+
*/
|
|
1743
|
+
queryFromStorage(options?: {
|
|
1744
|
+
mapName?: string;
|
|
1745
|
+
key?: string;
|
|
1746
|
+
types?: ('PUT' | 'UPDATE' | 'DELETE')[];
|
|
1747
|
+
fromSequence?: bigint;
|
|
1748
|
+
toSequence?: bigint;
|
|
1749
|
+
fromDate?: Date;
|
|
1750
|
+
toDate?: Date;
|
|
1751
|
+
limit?: number;
|
|
1752
|
+
offset?: number;
|
|
1753
|
+
}): Promise<JournalEvent[]>;
|
|
1754
|
+
/**
|
|
1755
|
+
* Count events matching filters.
|
|
1756
|
+
*/
|
|
1757
|
+
countFromStorage(options?: {
|
|
1758
|
+
mapName?: string;
|
|
1759
|
+
types?: ('PUT' | 'UPDATE' | 'DELETE')[];
|
|
1760
|
+
fromDate?: Date;
|
|
1761
|
+
toDate?: Date;
|
|
1762
|
+
}): Promise<number>;
|
|
1763
|
+
/**
|
|
1764
|
+
* Cleanup old events based on retention policy.
|
|
1765
|
+
*/
|
|
1766
|
+
cleanupOldEvents(retentionDays: number): Promise<number>;
|
|
1767
|
+
/**
|
|
1768
|
+
* Start the periodic persistence timer.
|
|
1769
|
+
*/
|
|
1770
|
+
private startPersistTimer;
|
|
1771
|
+
/**
|
|
1772
|
+
* Stop the periodic persistence timer.
|
|
1773
|
+
*/
|
|
1774
|
+
private stopPersistTimer;
|
|
1775
|
+
/**
|
|
1776
|
+
* Dispose resources and persist remaining events.
|
|
1777
|
+
*/
|
|
1778
|
+
dispose(): void;
|
|
1779
|
+
/**
|
|
1780
|
+
* Get pending persist count (for monitoring).
|
|
1781
|
+
*/
|
|
1782
|
+
getPendingPersistCount(): number;
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1658
1785
|
interface ServerCoordinatorConfig {
|
|
1659
1786
|
port: number;
|
|
1660
1787
|
nodeId: string;
|
|
@@ -1721,6 +1848,16 @@ interface ServerCoordinatorConfig {
|
|
|
1721
1848
|
workerPoolConfig?: Partial<WorkerPoolConfig>;
|
|
1722
1849
|
/** Default timeout for Write Concern acknowledgments in ms (default: 5000) */
|
|
1723
1850
|
writeAckTimeout?: number;
|
|
1851
|
+
/** Enable replication to backup nodes (default: true when cluster has peers) */
|
|
1852
|
+
replicationEnabled?: boolean;
|
|
1853
|
+
/** Default consistency level for replication (default: EVENTUAL) */
|
|
1854
|
+
defaultConsistency?: ConsistencyLevel;
|
|
1855
|
+
/** Replication configuration */
|
|
1856
|
+
replicationConfig?: Partial<ReplicationConfig>;
|
|
1857
|
+
/** Enable event journal for audit/CDC (default: false) */
|
|
1858
|
+
eventJournalEnabled?: boolean;
|
|
1859
|
+
/** Event journal configuration */
|
|
1860
|
+
eventJournalConfig?: Partial<Omit<EventJournalServiceConfig, 'pool'>>;
|
|
1724
1861
|
}
|
|
1725
1862
|
declare class ServerCoordinator {
|
|
1726
1863
|
private httpServer;
|
|
@@ -1736,6 +1873,7 @@ declare class ServerCoordinator {
|
|
|
1736
1873
|
private queryRegistry;
|
|
1737
1874
|
private cluster;
|
|
1738
1875
|
private partitionService;
|
|
1876
|
+
private replicationPipeline?;
|
|
1739
1877
|
private lockManager;
|
|
1740
1878
|
private topicManager;
|
|
1741
1879
|
private securityManager;
|
|
@@ -1759,6 +1897,12 @@ declare class ServerCoordinator {
|
|
|
1759
1897
|
private eventPayloadPool;
|
|
1760
1898
|
private taskletScheduler;
|
|
1761
1899
|
private writeAckManager;
|
|
1900
|
+
private counterHandler;
|
|
1901
|
+
private entryProcessorHandler;
|
|
1902
|
+
private conflictResolverHandler;
|
|
1903
|
+
private eventJournalService?;
|
|
1904
|
+
private journalSubscriptions;
|
|
1905
|
+
private readonly _nodeId;
|
|
1762
1906
|
private _actualPort;
|
|
1763
1907
|
private _actualClusterPort;
|
|
1764
1908
|
private _readyPromise;
|
|
@@ -1803,6 +1947,16 @@ declare class ServerCoordinator {
|
|
|
1803
1947
|
private handleConnection;
|
|
1804
1948
|
private handleMessage;
|
|
1805
1949
|
private updateClientHlc;
|
|
1950
|
+
/**
|
|
1951
|
+
* Broadcast partition map to all connected and authenticated clients.
|
|
1952
|
+
* Called when partition topology changes (node join/leave/failover).
|
|
1953
|
+
*/
|
|
1954
|
+
private broadcastPartitionMap;
|
|
1955
|
+
/**
|
|
1956
|
+
* Notify a client about a merge rejection (Phase 5.05).
|
|
1957
|
+
* Finds the client by node ID and sends MERGE_REJECTED message.
|
|
1958
|
+
*/
|
|
1959
|
+
private notifyMergeRejection;
|
|
1806
1960
|
private broadcast;
|
|
1807
1961
|
/**
|
|
1808
1962
|
* === OPTIMIZATION 2 & 3: Batched Broadcast with Serialization Caching ===
|
|
@@ -1836,6 +1990,11 @@ declare class ServerCoordinator {
|
|
|
1836
1990
|
* Broadcast event to cluster members (excluding self).
|
|
1837
1991
|
*/
|
|
1838
1992
|
private broadcastToCluster;
|
|
1993
|
+
/**
|
|
1994
|
+
* Apply replicated operation from another node (callback for ReplicationPipeline)
|
|
1995
|
+
* This is called when we receive a replicated operation as a backup node
|
|
1996
|
+
*/
|
|
1997
|
+
private applyReplicatedOperation;
|
|
1839
1998
|
/**
|
|
1840
1999
|
* Build OpContext for interceptors.
|
|
1841
2000
|
*/
|
|
@@ -2147,4 +2306,1386 @@ declare function getNativeStats(sharedMemoryManager?: SharedMemoryManager): Nati
|
|
|
2147
2306
|
*/
|
|
2148
2307
|
declare function logNativeStatus(): void;
|
|
2149
2308
|
|
|
2150
|
-
|
|
2309
|
+
/**
|
|
2310
|
+
* FailureDetector - Phi Accrual Failure Detector
|
|
2311
|
+
*
|
|
2312
|
+
* Implements the Phi Accrual Failure Detection algorithm for distributed systems.
|
|
2313
|
+
* Based on the paper: "The φ Accrual Failure Detector" by Hayashibara et al.
|
|
2314
|
+
*
|
|
2315
|
+
* The detector provides a suspicion level (phi) rather than binary alive/dead status,
|
|
2316
|
+
* allowing the application to make decisions based on configurable thresholds.
|
|
2317
|
+
*
|
|
2318
|
+
* Hazelcast equivalent: com.hazelcast.internal.cluster.fd.PhiAccrualFailureDetector
|
|
2319
|
+
*/
|
|
2320
|
+
|
|
2321
|
+
interface FailureDetectorConfig {
|
|
2322
|
+
/** Interval between heartbeat checks (ms). Default: 1000 */
|
|
2323
|
+
heartbeatIntervalMs: number;
|
|
2324
|
+
/** Time after which a node is suspected if no heartbeat received (ms). Default: 5000 */
|
|
2325
|
+
suspicionTimeoutMs: number;
|
|
2326
|
+
/** Time after suspicion before confirming failure (ms). Default: 10000 */
|
|
2327
|
+
confirmationTimeoutMs: number;
|
|
2328
|
+
/** Phi threshold above which a node is considered suspected. Default: 8 */
|
|
2329
|
+
phiThreshold: number;
|
|
2330
|
+
/** Minimum samples required for accurate phi calculation. Default: 10 */
|
|
2331
|
+
minSamples: number;
|
|
2332
|
+
/** Maximum samples to keep in history. Default: 100 */
|
|
2333
|
+
maxSamples: number;
|
|
2334
|
+
/** Initial heartbeat interval estimate (ms). Default: 1000 */
|
|
2335
|
+
initialHeartbeatIntervalMs: number;
|
|
2336
|
+
}
|
|
2337
|
+
declare class FailureDetector extends EventEmitter {
|
|
2338
|
+
private config;
|
|
2339
|
+
private nodeStates;
|
|
2340
|
+
private monitoringNodes;
|
|
2341
|
+
private checkTimer?;
|
|
2342
|
+
private confirmationTimers;
|
|
2343
|
+
private started;
|
|
2344
|
+
constructor(config?: Partial<FailureDetectorConfig>);
|
|
2345
|
+
/**
|
|
2346
|
+
* Start the failure detector monitoring loop.
|
|
2347
|
+
*/
|
|
2348
|
+
start(): void;
|
|
2349
|
+
/**
|
|
2350
|
+
* Stop the failure detector and clean up.
|
|
2351
|
+
*/
|
|
2352
|
+
stop(): void;
|
|
2353
|
+
/**
|
|
2354
|
+
* Start monitoring a node.
|
|
2355
|
+
*/
|
|
2356
|
+
startMonitoring(nodeId: string): void;
|
|
2357
|
+
/**
|
|
2358
|
+
* Stop monitoring a node.
|
|
2359
|
+
*/
|
|
2360
|
+
stopMonitoring(nodeId: string): void;
|
|
2361
|
+
/**
|
|
2362
|
+
* Record a heartbeat from a node.
|
|
2363
|
+
* This updates the node's state and clears any suspicion.
|
|
2364
|
+
*/
|
|
2365
|
+
recordHeartbeat(nodeId: string): void;
|
|
2366
|
+
/**
|
|
2367
|
+
* Check all monitored nodes for failure.
|
|
2368
|
+
*/
|
|
2369
|
+
private checkAllNodes;
|
|
2370
|
+
/**
|
|
2371
|
+
* Schedule failure confirmation after suspicion timeout.
|
|
2372
|
+
*/
|
|
2373
|
+
private scheduleConfirmation;
|
|
2374
|
+
/**
|
|
2375
|
+
* Confirm node failure after confirmation timeout.
|
|
2376
|
+
*/
|
|
2377
|
+
private confirmFailure;
|
|
2378
|
+
/**
|
|
2379
|
+
* Calculate the phi value for a node using the Phi Accrual algorithm.
|
|
2380
|
+
*
|
|
2381
|
+
* Phi = -log10(P_later(t_now - t_last))
|
|
2382
|
+
*
|
|
2383
|
+
* where P_later is the probability that a heartbeat will arrive later than expected.
|
|
2384
|
+
*/
|
|
2385
|
+
calculatePhi(nodeId: string): number;
|
|
2386
|
+
/**
|
|
2387
|
+
* Calculate mean of an array of numbers.
|
|
2388
|
+
*/
|
|
2389
|
+
private calculateMean;
|
|
2390
|
+
/**
|
|
2391
|
+
* Calculate variance of an array of numbers.
|
|
2392
|
+
*/
|
|
2393
|
+
private calculateVariance;
|
|
2394
|
+
/**
|
|
2395
|
+
* Get list of currently suspected nodes.
|
|
2396
|
+
*/
|
|
2397
|
+
getSuspectedNodes(): string[];
|
|
2398
|
+
/**
|
|
2399
|
+
* Get list of confirmed failed nodes.
|
|
2400
|
+
*/
|
|
2401
|
+
getConfirmedFailedNodes(): string[];
|
|
2402
|
+
/**
|
|
2403
|
+
* Check if a specific node is suspected.
|
|
2404
|
+
*/
|
|
2405
|
+
isSuspected(nodeId: string): boolean;
|
|
2406
|
+
/**
|
|
2407
|
+
* Check if a specific node's failure is confirmed.
|
|
2408
|
+
*/
|
|
2409
|
+
isConfirmedFailed(nodeId: string): boolean;
|
|
2410
|
+
/**
|
|
2411
|
+
* Get the current phi value for a node.
|
|
2412
|
+
*/
|
|
2413
|
+
getPhi(nodeId: string): number;
|
|
2414
|
+
/**
|
|
2415
|
+
* Get all monitored nodes.
|
|
2416
|
+
*/
|
|
2417
|
+
getMonitoredNodes(): string[];
|
|
2418
|
+
/**
|
|
2419
|
+
* Get metrics for monitoring.
|
|
2420
|
+
*/
|
|
2421
|
+
getMetrics(): {
|
|
2422
|
+
monitoredNodes: number;
|
|
2423
|
+
suspectedNodes: number;
|
|
2424
|
+
confirmedFailedNodes: number;
|
|
2425
|
+
};
|
|
2426
|
+
}
|
|
2427
|
+
|
|
2428
|
+
interface ClusterConfig {
|
|
2429
|
+
nodeId: string;
|
|
2430
|
+
host: string;
|
|
2431
|
+
port: number;
|
|
2432
|
+
peers: string[];
|
|
2433
|
+
discovery?: 'manual' | 'kubernetes';
|
|
2434
|
+
serviceName?: string;
|
|
2435
|
+
discoveryInterval?: number;
|
|
2436
|
+
tls?: ClusterTLSConfig;
|
|
2437
|
+
/** Heartbeat interval in milliseconds. Default: 1000 */
|
|
2438
|
+
heartbeatIntervalMs?: number;
|
|
2439
|
+
/** Failure detection configuration */
|
|
2440
|
+
failureDetection?: Partial<FailureDetectorConfig>;
|
|
2441
|
+
}
|
|
2442
|
+
interface ClusterMember {
|
|
2443
|
+
nodeId: string;
|
|
2444
|
+
host: string;
|
|
2445
|
+
port: number;
|
|
2446
|
+
socket: WebSocket;
|
|
2447
|
+
isSelf: boolean;
|
|
2448
|
+
}
|
|
2449
|
+
interface ClusterMessage {
|
|
2450
|
+
type: 'HELLO' | 'OP_FORWARD' | 'PARTITION_UPDATE' | 'HEARTBEAT' | 'CLUSTER_EVENT' | 'CLUSTER_QUERY_EXEC' | 'CLUSTER_QUERY_RESP' | 'CLUSTER_GC_REPORT' | 'CLUSTER_GC_COMMIT' | 'CLUSTER_LOCK_REQ' | 'CLUSTER_LOCK_RELEASE' | 'CLUSTER_LOCK_GRANTED' | 'CLUSTER_LOCK_RELEASED' | 'CLUSTER_CLIENT_DISCONNECTED' | 'CLUSTER_TOPIC_PUB';
|
|
2451
|
+
senderId: string;
|
|
2452
|
+
payload: any;
|
|
2453
|
+
}
|
|
2454
|
+
declare class ClusterManager extends EventEmitter {
|
|
2455
|
+
readonly config: ClusterConfig;
|
|
2456
|
+
private server?;
|
|
2457
|
+
private members;
|
|
2458
|
+
private pendingConnections;
|
|
2459
|
+
private reconnectIntervals;
|
|
2460
|
+
private discoveryTimer?;
|
|
2461
|
+
private heartbeatTimer?;
|
|
2462
|
+
private failureDetector;
|
|
2463
|
+
constructor(config: ClusterConfig);
|
|
2464
|
+
/**
|
|
2465
|
+
* Get the failure detector instance.
|
|
2466
|
+
*/
|
|
2467
|
+
getFailureDetector(): FailureDetector;
|
|
2468
|
+
private _actualPort;
|
|
2469
|
+
/** Get the actual port the cluster is listening on */
|
|
2470
|
+
get port(): number;
|
|
2471
|
+
start(): Promise<number>;
|
|
2472
|
+
/** Called when server is ready - registers self and initiates peer connections */
|
|
2473
|
+
private onServerReady;
|
|
2474
|
+
stop(): void;
|
|
2475
|
+
/**
|
|
2476
|
+
* Start sending heartbeats to all peers.
|
|
2477
|
+
*/
|
|
2478
|
+
private startHeartbeat;
|
|
2479
|
+
/**
|
|
2480
|
+
* Stop sending heartbeats.
|
|
2481
|
+
*/
|
|
2482
|
+
private stopHeartbeat;
|
|
2483
|
+
/**
|
|
2484
|
+
* Send heartbeat to all connected peers.
|
|
2485
|
+
*/
|
|
2486
|
+
private sendHeartbeatToAll;
|
|
2487
|
+
/**
|
|
2488
|
+
* Handle incoming heartbeat from a peer.
|
|
2489
|
+
*/
|
|
2490
|
+
private handleHeartbeat;
|
|
2491
|
+
/**
|
|
2492
|
+
* Handle confirmed node failure.
|
|
2493
|
+
*/
|
|
2494
|
+
private handleNodeFailure;
|
|
2495
|
+
private connectToPeers;
|
|
2496
|
+
private startDiscovery;
|
|
2497
|
+
private scheduleReconnect;
|
|
2498
|
+
private connectToPeerWithBackoff;
|
|
2499
|
+
private connectToPeer;
|
|
2500
|
+
private _connectToPeerInternal;
|
|
2501
|
+
private handleSocket;
|
|
2502
|
+
send(nodeId: string, type: ClusterMessage['type'], payload: any): void;
|
|
2503
|
+
sendToNode(nodeId: string, message: any): void;
|
|
2504
|
+
getMembers(): string[];
|
|
2505
|
+
isLocal(nodeId: string): boolean;
|
|
2506
|
+
private buildClusterTLSOptions;
|
|
2507
|
+
}
|
|
2508
|
+
|
|
2509
|
+
/**
|
|
2510
|
+
* MigrationManager - Manages gradual partition rebalancing
|
|
2511
|
+
*
|
|
2512
|
+
* Phase 4 Task 03: Parallel Partition Sync
|
|
2513
|
+
*
|
|
2514
|
+
* Features:
|
|
2515
|
+
* - Gradual rebalancing with configurable batch size
|
|
2516
|
+
* - State machine for migration lifecycle
|
|
2517
|
+
* - Backpressure via chunk acknowledgments
|
|
2518
|
+
* - Retry logic for failed migrations
|
|
2519
|
+
* - Metrics and observability
|
|
2520
|
+
*/
|
|
2521
|
+
|
|
2522
|
+
declare class MigrationManager extends EventEmitter {
|
|
2523
|
+
private readonly config;
|
|
2524
|
+
private readonly clusterManager;
|
|
2525
|
+
private readonly partitionService;
|
|
2526
|
+
private activeMigrations;
|
|
2527
|
+
private migrationQueue;
|
|
2528
|
+
private incomingMigrations;
|
|
2529
|
+
private pendingChunkAcks;
|
|
2530
|
+
private pendingVerifications;
|
|
2531
|
+
private metrics;
|
|
2532
|
+
private batchTimer;
|
|
2533
|
+
private dataCollector;
|
|
2534
|
+
private dataStorer;
|
|
2535
|
+
constructor(clusterManager: ClusterManager, partitionService: PartitionService, config?: Partial<MigrationConfig>);
|
|
2536
|
+
/**
|
|
2537
|
+
* Set the data collector callback
|
|
2538
|
+
* Called to collect all records for a partition before migration
|
|
2539
|
+
*/
|
|
2540
|
+
setDataCollector(collector: (partitionId: number) => Promise<Uint8Array[]>): void;
|
|
2541
|
+
/**
|
|
2542
|
+
* Set the data storer callback
|
|
2543
|
+
* Called to store received records after successful migration
|
|
2544
|
+
*/
|
|
2545
|
+
setDataStorer(storer: (partitionId: number, data: Uint8Array[]) => Promise<void>): void;
|
|
2546
|
+
/**
|
|
2547
|
+
* Plan migration for topology change
|
|
2548
|
+
*/
|
|
2549
|
+
planMigration(oldDistribution: Map<number, PartitionDistribution>, newDistribution: Map<number, PartitionDistribution>): void;
|
|
2550
|
+
/**
|
|
2551
|
+
* Start batch processing timer
|
|
2552
|
+
*/
|
|
2553
|
+
private startBatchProcessing;
|
|
2554
|
+
/**
|
|
2555
|
+
* Stop batch processing
|
|
2556
|
+
*/
|
|
2557
|
+
private stopBatchProcessing;
|
|
2558
|
+
/**
|
|
2559
|
+
* Start next batch of migrations
|
|
2560
|
+
*/
|
|
2561
|
+
startNextBatch(): Promise<void>;
|
|
2562
|
+
/**
|
|
2563
|
+
* Start migration for a single partition
|
|
2564
|
+
*/
|
|
2565
|
+
private startPartitionMigration;
|
|
2566
|
+
/**
|
|
2567
|
+
* Split records into chunks
|
|
2568
|
+
*/
|
|
2569
|
+
private chunkify;
|
|
2570
|
+
/**
|
|
2571
|
+
* Calculate checksum for a chunk using native xxhash
|
|
2572
|
+
*/
|
|
2573
|
+
private calculateChecksum;
|
|
2574
|
+
/**
|
|
2575
|
+
* Calculate checksum for all partition records using streaming xxhash
|
|
2576
|
+
*/
|
|
2577
|
+
private calculatePartitionChecksum;
|
|
2578
|
+
/**
|
|
2579
|
+
* Wait for chunk acknowledgment
|
|
2580
|
+
*/
|
|
2581
|
+
private waitForChunkAck;
|
|
2582
|
+
/**
|
|
2583
|
+
* Wait for migration verification
|
|
2584
|
+
*/
|
|
2585
|
+
private waitForVerification;
|
|
2586
|
+
/**
|
|
2587
|
+
* Handle successful migration completion
|
|
2588
|
+
*/
|
|
2589
|
+
private onMigrationComplete;
|
|
2590
|
+
/**
|
|
2591
|
+
* Handle migration failure
|
|
2592
|
+
*/
|
|
2593
|
+
private onMigrationFailed;
|
|
2594
|
+
/**
|
|
2595
|
+
* Handle MIGRATION_START message
|
|
2596
|
+
*/
|
|
2597
|
+
private handleMigrationStart;
|
|
2598
|
+
/**
|
|
2599
|
+
* Handle MIGRATION_CHUNK message
|
|
2600
|
+
*/
|
|
2601
|
+
private handleMigrationChunk;
|
|
2602
|
+
/**
|
|
2603
|
+
* Handle MIGRATION_COMPLETE message
|
|
2604
|
+
*/
|
|
2605
|
+
private handleMigrationComplete;
|
|
2606
|
+
/**
|
|
2607
|
+
* Handle MIGRATION_CHUNK_ACK message
|
|
2608
|
+
*/
|
|
2609
|
+
private handleMigrationChunkAck;
|
|
2610
|
+
/**
|
|
2611
|
+
* Handle MIGRATION_VERIFY message
|
|
2612
|
+
*/
|
|
2613
|
+
private handleMigrationVerify;
|
|
2614
|
+
/**
|
|
2615
|
+
* Reassemble chunks into continuous data
|
|
2616
|
+
*/
|
|
2617
|
+
private reassemble;
|
|
2618
|
+
/**
|
|
2619
|
+
* Deserialize records from chunk data
|
|
2620
|
+
*/
|
|
2621
|
+
private deserializeRecords;
|
|
2622
|
+
/**
|
|
2623
|
+
* Setup cluster message handlers
|
|
2624
|
+
*/
|
|
2625
|
+
private setupMessageHandlers;
|
|
2626
|
+
/**
|
|
2627
|
+
* Check if a partition is currently migrating
|
|
2628
|
+
*/
|
|
2629
|
+
isActive(partitionId: number): boolean;
|
|
2630
|
+
/**
|
|
2631
|
+
* Get migration status
|
|
2632
|
+
*/
|
|
2633
|
+
getStatus(): MigrationStatus;
|
|
2634
|
+
/**
|
|
2635
|
+
* Get migration metrics
|
|
2636
|
+
*/
|
|
2637
|
+
getMetrics(): MigrationMetrics;
|
|
2638
|
+
/**
|
|
2639
|
+
* Cancel all active and queued migrations
|
|
2640
|
+
*/
|
|
2641
|
+
cancelAll(): Promise<void>;
|
|
2642
|
+
/**
|
|
2643
|
+
* Cleanup resources (sync version for backwards compatibility)
|
|
2644
|
+
*/
|
|
2645
|
+
close(): void;
|
|
2646
|
+
/**
|
|
2647
|
+
* Async cleanup - waits for cancellation to complete
|
|
2648
|
+
*/
|
|
2649
|
+
closeAsync(): Promise<void>;
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
interface PartitionDistribution {
|
|
2653
|
+
owner: string;
|
|
2654
|
+
backups: string[];
|
|
2655
|
+
}
|
|
2656
|
+
interface PartitionServiceEvents {
|
|
2657
|
+
'rebalanced': (map: PartitionMap, changes: PartitionChange[]) => void;
|
|
2658
|
+
'partitionMoved': (info: {
|
|
2659
|
+
partitionId: number;
|
|
2660
|
+
previousOwner: string;
|
|
2661
|
+
newOwner: string;
|
|
2662
|
+
version: number;
|
|
2663
|
+
}) => void;
|
|
2664
|
+
}
|
|
2665
|
+
interface PartitionServiceConfig {
|
|
2666
|
+
/** Enable gradual rebalancing (default: false for backward compatibility) */
|
|
2667
|
+
gradualRebalancing: boolean;
|
|
2668
|
+
/** Migration configuration */
|
|
2669
|
+
migration: Partial<MigrationConfig>;
|
|
2670
|
+
}
|
|
2671
|
+
declare class PartitionService extends EventEmitter {
|
|
2672
|
+
private cluster;
|
|
2673
|
+
private partitions;
|
|
2674
|
+
private readonly PARTITION_COUNT;
|
|
2675
|
+
private readonly BACKUP_COUNT;
|
|
2676
|
+
private mapVersion;
|
|
2677
|
+
private lastRebalanceTime;
|
|
2678
|
+
private config;
|
|
2679
|
+
private migrationManager;
|
|
2680
|
+
constructor(cluster: ClusterManager, config?: Partial<PartitionServiceConfig>);
|
|
2681
|
+
/**
|
|
2682
|
+
* Handle membership change
|
|
2683
|
+
*/
|
|
2684
|
+
private onMembershipChange;
|
|
2685
|
+
getPartitionId(key: string): number;
|
|
2686
|
+
getDistribution(key: string): PartitionDistribution;
|
|
2687
|
+
getOwner(key: string): string;
|
|
2688
|
+
isLocalOwner(key: string): boolean;
|
|
2689
|
+
isLocalBackup(key: string): boolean;
|
|
2690
|
+
isRelated(key: string): boolean;
|
|
2691
|
+
/**
|
|
2692
|
+
* Get current partition map version
|
|
2693
|
+
*/
|
|
2694
|
+
getMapVersion(): number;
|
|
2695
|
+
/**
|
|
2696
|
+
* Generate full PartitionMap for client consumption
|
|
2697
|
+
*/
|
|
2698
|
+
getPartitionMap(): PartitionMap;
|
|
2699
|
+
/**
|
|
2700
|
+
* Get partition info by ID
|
|
2701
|
+
*/
|
|
2702
|
+
getPartitionInfo(partitionId: number): PartitionInfo | null;
|
|
2703
|
+
/**
|
|
2704
|
+
* Get owner node for a partition ID
|
|
2705
|
+
*/
|
|
2706
|
+
getPartitionOwner(partitionId: number): string | null;
|
|
2707
|
+
private rebalance;
|
|
2708
|
+
/**
|
|
2709
|
+
* Perform gradual rebalancing using MigrationManager
|
|
2710
|
+
*/
|
|
2711
|
+
private rebalanceGradual;
|
|
2712
|
+
/**
|
|
2713
|
+
* Set partition owner (called after migration completes)
|
|
2714
|
+
*/
|
|
2715
|
+
setOwner(partitionId: number, nodeId: string): void;
|
|
2716
|
+
/**
|
|
2717
|
+
* Get backups for a partition
|
|
2718
|
+
*/
|
|
2719
|
+
getBackups(partitionId: number): string[];
|
|
2720
|
+
/**
|
|
2721
|
+
* Get migration status
|
|
2722
|
+
*/
|
|
2723
|
+
getMigrationStatus(): MigrationStatus | null;
|
|
2724
|
+
/**
|
|
2725
|
+
* Check if partition is currently migrating
|
|
2726
|
+
*/
|
|
2727
|
+
isMigrating(partitionId: number): boolean;
|
|
2728
|
+
/**
|
|
2729
|
+
* Check if any partition is currently migrating
|
|
2730
|
+
*/
|
|
2731
|
+
isRebalancing(): boolean;
|
|
2732
|
+
/**
|
|
2733
|
+
* Get MigrationManager for configuration
|
|
2734
|
+
*/
|
|
2735
|
+
getMigrationManager(): MigrationManager | null;
|
|
2736
|
+
/**
|
|
2737
|
+
* Cancel all migrations
|
|
2738
|
+
*/
|
|
2739
|
+
cancelMigrations(): Promise<void>;
|
|
2740
|
+
}
|
|
2741
|
+
|
|
2742
|
+
/**
|
|
2743
|
+
* LagTracker - Monitors replication lag across cluster nodes
|
|
2744
|
+
*
|
|
2745
|
+
* Phase 4 Task 04: Async Replication Pipeline
|
|
2746
|
+
*
|
|
2747
|
+
* Features:
|
|
2748
|
+
* - Tracks replication lag per node
|
|
2749
|
+
* - Maintains historical lag data for percentile calculations
|
|
2750
|
+
* - Identifies unhealthy and laggy nodes
|
|
2751
|
+
* - Provides health metrics for monitoring
|
|
2752
|
+
*/
|
|
2753
|
+
|
|
2754
|
+
interface LagInfo {
|
|
2755
|
+
current: number;
|
|
2756
|
+
history: number[];
|
|
2757
|
+
lastUpdate: number;
|
|
2758
|
+
pendingOps: number;
|
|
2759
|
+
}
|
|
2760
|
+
interface LagTrackerConfig {
|
|
2761
|
+
/** Number of lag samples to keep in history (default: 100) */
|
|
2762
|
+
historySize: number;
|
|
2763
|
+
/** Threshold in ms for considering a node laggy (default: 5000) */
|
|
2764
|
+
laggyThresholdMs: number;
|
|
2765
|
+
/** Threshold in ms for considering a node unhealthy (default: 30000) */
|
|
2766
|
+
unhealthyThresholdMs: number;
|
|
2767
|
+
}
|
|
2768
|
+
declare const DEFAULT_LAG_TRACKER_CONFIG: LagTrackerConfig;
|
|
2769
|
+
declare class LagTracker {
|
|
2770
|
+
private readonly config;
|
|
2771
|
+
private lagByNode;
|
|
2772
|
+
constructor(config?: Partial<LagTrackerConfig>);
|
|
2773
|
+
/**
|
|
2774
|
+
* Update lag measurement for a node
|
|
2775
|
+
*/
|
|
2776
|
+
update(nodeId: string, lagMs: number): void;
|
|
2777
|
+
/**
|
|
2778
|
+
* Record acknowledgment from a node (lag effectively becomes 0)
|
|
2779
|
+
*/
|
|
2780
|
+
recordAck(nodeId: string): void;
|
|
2781
|
+
/**
|
|
2782
|
+
* Increment pending operations counter for a node
|
|
2783
|
+
*/
|
|
2784
|
+
incrementPending(nodeId: string): void;
|
|
2785
|
+
/**
|
|
2786
|
+
* Get lag statistics for a specific node
|
|
2787
|
+
*/
|
|
2788
|
+
getLag(nodeId: string): ReplicationLag;
|
|
2789
|
+
/**
|
|
2790
|
+
* Get pending operations count for a node
|
|
2791
|
+
*/
|
|
2792
|
+
getPendingOps(nodeId: string): number;
|
|
2793
|
+
/**
|
|
2794
|
+
* Get overall replication health status
|
|
2795
|
+
*/
|
|
2796
|
+
getHealth(): ReplicationHealth;
|
|
2797
|
+
/**
|
|
2798
|
+
* Get average lag across all tracked nodes
|
|
2799
|
+
*/
|
|
2800
|
+
getAverageLag(): number;
|
|
2801
|
+
/**
|
|
2802
|
+
* Check if a specific node is considered healthy
|
|
2803
|
+
*/
|
|
2804
|
+
isNodeHealthy(nodeId: string): boolean;
|
|
2805
|
+
/**
|
|
2806
|
+
* Check if a specific node is considered laggy
|
|
2807
|
+
*/
|
|
2808
|
+
isNodeLaggy(nodeId: string): boolean;
|
|
2809
|
+
/**
|
|
2810
|
+
* Remove a node from tracking
|
|
2811
|
+
*/
|
|
2812
|
+
removeNode(nodeId: string): void;
|
|
2813
|
+
/**
|
|
2814
|
+
* Get all tracked node IDs
|
|
2815
|
+
*/
|
|
2816
|
+
getTrackedNodes(): string[];
|
|
2817
|
+
/**
|
|
2818
|
+
* Get raw lag info for a node (for advanced monitoring)
|
|
2819
|
+
*/
|
|
2820
|
+
getRawLagInfo(nodeId: string): LagInfo | undefined;
|
|
2821
|
+
/**
|
|
2822
|
+
* Clear all tracking data
|
|
2823
|
+
*/
|
|
2824
|
+
clear(): void;
|
|
2825
|
+
/**
|
|
2826
|
+
* Export metrics in Prometheus format
|
|
2827
|
+
*/
|
|
2828
|
+
toPrometheusMetrics(): string;
|
|
2829
|
+
}
|
|
2830
|
+
|
|
2831
|
+
/**
|
|
2832
|
+
* ReplicationPipeline - Manages async replication with configurable consistency levels
|
|
2833
|
+
*
|
|
2834
|
+
* Phase 4 Task 04: Async Replication Pipeline
|
|
2835
|
+
*
|
|
2836
|
+
* Features:
|
|
2837
|
+
* - Three consistency levels: STRONG, QUORUM, EVENTUAL
|
|
2838
|
+
* - Async replication queue for high throughput
|
|
2839
|
+
* - Backpressure handling with queue limits
|
|
2840
|
+
* - Retry logic for failed replications
|
|
2841
|
+
* - Integration with LagTracker for monitoring
|
|
2842
|
+
* - Pluggable operation applier for storage integration
|
|
2843
|
+
*/
|
|
2844
|
+
|
|
2845
|
+
/**
|
|
2846
|
+
* Callback to apply replicated operation to local storage
|
|
2847
|
+
* @param operation - The operation to apply
|
|
2848
|
+
* @param opId - Unique operation ID
|
|
2849
|
+
* @param sourceNode - Node that originated the operation
|
|
2850
|
+
* @returns Promise<boolean> - true if applied successfully
|
|
2851
|
+
*/
|
|
2852
|
+
type OperationApplier = (operation: unknown, opId: string, sourceNode: string) => Promise<boolean>;
|
|
2853
|
+
declare class ReplicationPipeline extends EventEmitter {
|
|
2854
|
+
private readonly config;
|
|
2855
|
+
private readonly clusterManager;
|
|
2856
|
+
private readonly partitionService;
|
|
2857
|
+
private readonly lagTracker;
|
|
2858
|
+
private readonly nodeId;
|
|
2859
|
+
private replicationQueue;
|
|
2860
|
+
private pendingAcks;
|
|
2861
|
+
private queueProcessorTimer;
|
|
2862
|
+
private operationApplier;
|
|
2863
|
+
constructor(clusterManager: ClusterManager, partitionService: PartitionService, config?: Partial<ReplicationConfig>);
|
|
2864
|
+
/**
|
|
2865
|
+
* Set the operation applier callback
|
|
2866
|
+
* This is called when replicated operations are received from other nodes
|
|
2867
|
+
*/
|
|
2868
|
+
setOperationApplier(applier: OperationApplier): void;
|
|
2869
|
+
/**
|
|
2870
|
+
* Replicate operation to backup nodes
|
|
2871
|
+
*/
|
|
2872
|
+
replicate(operation: unknown, opId: string, key: string, options?: {
|
|
2873
|
+
consistency?: ConsistencyLevel;
|
|
2874
|
+
timeout?: number;
|
|
2875
|
+
}): Promise<ReplicationResult>;
|
|
2876
|
+
/**
|
|
2877
|
+
* STRONG: Wait for all replicas to acknowledge
|
|
2878
|
+
*/
|
|
2879
|
+
private replicateStrong;
|
|
2880
|
+
/**
|
|
2881
|
+
* QUORUM: Wait for majority of replicas
|
|
2882
|
+
*/
|
|
2883
|
+
private replicateQuorum;
|
|
2884
|
+
/**
|
|
2885
|
+
* EVENTUAL: Fire-and-forget with queue
|
|
2886
|
+
*/
|
|
2887
|
+
private replicateEventual;
|
|
2888
|
+
/**
|
|
2889
|
+
* Add task to replication queue
|
|
2890
|
+
*/
|
|
2891
|
+
private enqueue;
|
|
2892
|
+
/**
|
|
2893
|
+
* Start queue processor
|
|
2894
|
+
*/
|
|
2895
|
+
private startQueueProcessor;
|
|
2896
|
+
/**
|
|
2897
|
+
* Stop queue processor
|
|
2898
|
+
*/
|
|
2899
|
+
private stopQueueProcessor;
|
|
2900
|
+
/**
|
|
2901
|
+
* Process replication queue for a node
|
|
2902
|
+
*/
|
|
2903
|
+
private processQueue;
|
|
2904
|
+
/**
|
|
2905
|
+
* Send replication message to a node
|
|
2906
|
+
*/
|
|
2907
|
+
private sendReplication;
|
|
2908
|
+
/**
|
|
2909
|
+
* Setup cluster message handlers
|
|
2910
|
+
*/
|
|
2911
|
+
private setupMessageHandlers;
|
|
2912
|
+
/**
|
|
2913
|
+
* Handle incoming replication request (on backup node)
|
|
2914
|
+
*/
|
|
2915
|
+
private handleReplication;
|
|
2916
|
+
/**
|
|
2917
|
+
* Handle incoming batch replication (on backup node)
|
|
2918
|
+
*/
|
|
2919
|
+
private handleReplicationBatch;
|
|
2920
|
+
/**
|
|
2921
|
+
* Handle replication acknowledgment (on owner node)
|
|
2922
|
+
*/
|
|
2923
|
+
private handleReplicationAck;
|
|
2924
|
+
/**
|
|
2925
|
+
* Handle batch acknowledgment (on owner node)
|
|
2926
|
+
*/
|
|
2927
|
+
private handleReplicationBatchAck;
|
|
2928
|
+
/**
|
|
2929
|
+
* Get replication lag for a specific node
|
|
2930
|
+
*/
|
|
2931
|
+
getLag(nodeId: string): ReplicationLag;
|
|
2932
|
+
/**
|
|
2933
|
+
* Get overall replication health
|
|
2934
|
+
*/
|
|
2935
|
+
getHealth(): ReplicationHealth;
|
|
2936
|
+
/**
|
|
2937
|
+
* Get queue size for a specific node
|
|
2938
|
+
*/
|
|
2939
|
+
getQueueSize(nodeId: string): number;
|
|
2940
|
+
/**
|
|
2941
|
+
* Get total pending operations across all nodes
|
|
2942
|
+
*/
|
|
2943
|
+
getTotalPending(): number;
|
|
2944
|
+
/**
|
|
2945
|
+
* Check if a node is considered synced (low lag)
|
|
2946
|
+
*/
|
|
2947
|
+
isSynced(nodeId: string, maxLagMs?: number): boolean;
|
|
2948
|
+
/**
|
|
2949
|
+
* Get LagTracker for advanced monitoring
|
|
2950
|
+
*/
|
|
2951
|
+
getLagTracker(): LagTracker;
|
|
2952
|
+
/**
|
|
2953
|
+
* Export metrics in Prometheus format
|
|
2954
|
+
*/
|
|
2955
|
+
toPrometheusMetrics(): string;
|
|
2956
|
+
/**
|
|
2957
|
+
* Cleanup resources
|
|
2958
|
+
*/
|
|
2959
|
+
close(): void;
|
|
2960
|
+
}
|
|
2961
|
+
|
|
2962
|
+
declare class LockManager extends EventEmitter {
|
|
2963
|
+
private locks;
|
|
2964
|
+
private checkInterval;
|
|
2965
|
+
private static readonly MIN_TTL;
|
|
2966
|
+
private static readonly MAX_TTL;
|
|
2967
|
+
constructor();
|
|
2968
|
+
stop(): void;
|
|
2969
|
+
acquire(name: string, clientId: string, requestId: string, ttl: number): {
|
|
2970
|
+
granted: boolean;
|
|
2971
|
+
fencingToken?: number;
|
|
2972
|
+
error?: string;
|
|
2973
|
+
};
|
|
2974
|
+
release(name: string, clientId: string, fencingToken: number): boolean;
|
|
2975
|
+
handleClientDisconnect(clientId: string): void;
|
|
2976
|
+
private grantLock;
|
|
2977
|
+
private processNext;
|
|
2978
|
+
private cleanupExpiredLocks;
|
|
2979
|
+
}
|
|
2980
|
+
|
|
2981
|
+
/**
|
|
2982
|
+
* ClusterCoordinator - Unified cluster integration layer
|
|
2983
|
+
*
|
|
2984
|
+
* Phase 4 Task 06: System Integration
|
|
2985
|
+
*
|
|
2986
|
+
* Coordinates all cluster components:
|
|
2987
|
+
* - ClusterManager: P2P WebSocket mesh
|
|
2988
|
+
* - PartitionService: Consistent hashing & routing
|
|
2989
|
+
* - MigrationManager: Gradual rebalancing
|
|
2990
|
+
* - ReplicationPipeline: Async replication with consistency levels
|
|
2991
|
+
* - LagTracker: Replication health monitoring
|
|
2992
|
+
*/
|
|
2993
|
+
|
|
2994
|
+
interface ClusterCoordinatorConfig {
|
|
2995
|
+
/** Cluster node configuration */
|
|
2996
|
+
cluster: ClusterConfig;
|
|
2997
|
+
/** Enable gradual partition rebalancing (default: true) */
|
|
2998
|
+
gradualRebalancing: boolean;
|
|
2999
|
+
/** Migration configuration for gradual rebalancing */
|
|
3000
|
+
migration: Partial<MigrationConfig>;
|
|
3001
|
+
/** Replication configuration */
|
|
3002
|
+
replication: Partial<ReplicationConfig>;
|
|
3003
|
+
/** Enable async replication pipeline (default: true) */
|
|
3004
|
+
replicationEnabled: boolean;
|
|
3005
|
+
/** Data collector callback for migrations */
|
|
3006
|
+
dataCollector?: (partitionId: number) => Promise<Uint8Array[]>;
|
|
3007
|
+
/** Data storer callback for incoming migrations */
|
|
3008
|
+
dataStorer?: (partitionId: number, data: Uint8Array[]) => Promise<void>;
|
|
3009
|
+
}
|
|
3010
|
+
declare const DEFAULT_CLUSTER_COORDINATOR_CONFIG: Omit<ClusterCoordinatorConfig, 'cluster'>;
|
|
3011
|
+
interface ClusterCoordinatorEvents {
|
|
3012
|
+
'started': () => void;
|
|
3013
|
+
'stopped': () => void;
|
|
3014
|
+
'member:joined': (nodeId: string) => void;
|
|
3015
|
+
'member:left': (nodeId: string) => void;
|
|
3016
|
+
'partition:rebalanced': (map: PartitionMap, changes: PartitionChange[]) => void;
|
|
3017
|
+
'partition:moved': (info: {
|
|
3018
|
+
partitionId: number;
|
|
3019
|
+
previousOwner: string;
|
|
3020
|
+
newOwner: string;
|
|
3021
|
+
version: number;
|
|
3022
|
+
}) => void;
|
|
3023
|
+
'migration:started': (partitionId: number, targetNode: string) => void;
|
|
3024
|
+
'migration:completed': (partitionId: number) => void;
|
|
3025
|
+
'migration:failed': (partitionId: number, error: Error) => void;
|
|
3026
|
+
'replication:unhealthy': (nodeId: string) => void;
|
|
3027
|
+
'replication:healthy': (nodeId: string) => void;
|
|
3028
|
+
'error': (error: Error) => void;
|
|
3029
|
+
}
|
|
3030
|
+
declare class ClusterCoordinator extends EventEmitter {
|
|
3031
|
+
private readonly config;
|
|
3032
|
+
private clusterManager;
|
|
3033
|
+
private partitionService;
|
|
3034
|
+
private replicationPipeline;
|
|
3035
|
+
private lagTracker;
|
|
3036
|
+
private started;
|
|
3037
|
+
private actualPort;
|
|
3038
|
+
constructor(config: ClusterCoordinatorConfig);
|
|
3039
|
+
/**
|
|
3040
|
+
* Start the cluster coordinator
|
|
3041
|
+
*/
|
|
3042
|
+
start(): Promise<number>;
|
|
3043
|
+
/**
|
|
3044
|
+
* Stop the cluster coordinator
|
|
3045
|
+
*/
|
|
3046
|
+
stop(): Promise<void>;
|
|
3047
|
+
/**
|
|
3048
|
+
* Get local node ID
|
|
3049
|
+
*/
|
|
3050
|
+
getNodeId(): string;
|
|
3051
|
+
/**
|
|
3052
|
+
* Get cluster port
|
|
3053
|
+
*/
|
|
3054
|
+
getPort(): number;
|
|
3055
|
+
/**
|
|
3056
|
+
* Get all cluster members
|
|
3057
|
+
*/
|
|
3058
|
+
getMembers(): string[];
|
|
3059
|
+
/**
|
|
3060
|
+
* Check if this is the local node
|
|
3061
|
+
*/
|
|
3062
|
+
isLocal(nodeId: string): boolean;
|
|
3063
|
+
/**
|
|
3064
|
+
* Check if coordinator is started
|
|
3065
|
+
*/
|
|
3066
|
+
isStarted(): boolean;
|
|
3067
|
+
/**
|
|
3068
|
+
* Get current partition map
|
|
3069
|
+
*/
|
|
3070
|
+
getPartitionMap(): PartitionMap;
|
|
3071
|
+
/**
|
|
3072
|
+
* Get partition map version
|
|
3073
|
+
*/
|
|
3074
|
+
getPartitionMapVersion(): number;
|
|
3075
|
+
/**
|
|
3076
|
+
* Get partition ID for a key
|
|
3077
|
+
*/
|
|
3078
|
+
getPartitionId(key: string): number;
|
|
3079
|
+
/**
|
|
3080
|
+
* Get owner node for a key
|
|
3081
|
+
*/
|
|
3082
|
+
getOwner(key: string): string;
|
|
3083
|
+
/**
|
|
3084
|
+
* Check if this node owns the key
|
|
3085
|
+
*/
|
|
3086
|
+
isLocalOwner(key: string): boolean;
|
|
3087
|
+
/**
|
|
3088
|
+
* Check if this node is a backup for the key
|
|
3089
|
+
*/
|
|
3090
|
+
isLocalBackup(key: string): boolean;
|
|
3091
|
+
/**
|
|
3092
|
+
* Get backup nodes for a partition
|
|
3093
|
+
*/
|
|
3094
|
+
getBackups(partitionId: number): string[];
|
|
3095
|
+
/**
|
|
3096
|
+
* Check if partition is currently migrating
|
|
3097
|
+
*/
|
|
3098
|
+
isMigrating(partitionId: number): boolean;
|
|
3099
|
+
/**
|
|
3100
|
+
* Check if any rebalancing is in progress
|
|
3101
|
+
*/
|
|
3102
|
+
isRebalancing(): boolean;
|
|
3103
|
+
/**
|
|
3104
|
+
* Get migration status
|
|
3105
|
+
*/
|
|
3106
|
+
getMigrationStatus(): MigrationStatus | null;
|
|
3107
|
+
/**
|
|
3108
|
+
* Get migration metrics
|
|
3109
|
+
*/
|
|
3110
|
+
getMigrationMetrics(): MigrationMetrics | null;
|
|
3111
|
+
/**
|
|
3112
|
+
* Cancel all active migrations
|
|
3113
|
+
*/
|
|
3114
|
+
cancelMigrations(): Promise<void>;
|
|
3115
|
+
/**
|
|
3116
|
+
* Set data collector for migrations
|
|
3117
|
+
*/
|
|
3118
|
+
setDataCollector(collector: (partitionId: number) => Promise<Uint8Array[]>): void;
|
|
3119
|
+
/**
|
|
3120
|
+
* Set data storer for incoming migrations
|
|
3121
|
+
*/
|
|
3122
|
+
setDataStorer(storer: (partitionId: number, data: Uint8Array[]) => Promise<void>): void;
|
|
3123
|
+
/**
|
|
3124
|
+
* Replicate an operation to backup nodes
|
|
3125
|
+
*/
|
|
3126
|
+
replicate(operation: unknown, opId: string, key: string, options?: {
|
|
3127
|
+
consistency?: ConsistencyLevel;
|
|
3128
|
+
timeout?: number;
|
|
3129
|
+
}): Promise<ReplicationResult>;
|
|
3130
|
+
/**
|
|
3131
|
+
* Get replication health status
|
|
3132
|
+
*/
|
|
3133
|
+
getReplicationHealth(): ReplicationHealth;
|
|
3134
|
+
/**
|
|
3135
|
+
* Get replication lag for a specific node
|
|
3136
|
+
*/
|
|
3137
|
+
getReplicationLag(nodeId: string): ReplicationLag;
|
|
3138
|
+
/**
|
|
3139
|
+
* Check if a node is healthy for replication
|
|
3140
|
+
*/
|
|
3141
|
+
isNodeHealthy(nodeId: string): boolean;
|
|
3142
|
+
/**
|
|
3143
|
+
* Check if a node is laggy
|
|
3144
|
+
*/
|
|
3145
|
+
isNodeLaggy(nodeId: string): boolean;
|
|
3146
|
+
/**
|
|
3147
|
+
* Send message to a specific node
|
|
3148
|
+
*/
|
|
3149
|
+
send(nodeId: string, message: unknown): void;
|
|
3150
|
+
/**
|
|
3151
|
+
* Broadcast message to all nodes
|
|
3152
|
+
*/
|
|
3153
|
+
broadcast(message: unknown): void;
|
|
3154
|
+
/**
|
|
3155
|
+
* Get underlying ClusterManager
|
|
3156
|
+
*/
|
|
3157
|
+
getClusterManager(): ClusterManager;
|
|
3158
|
+
/**
|
|
3159
|
+
* Get underlying PartitionService
|
|
3160
|
+
*/
|
|
3161
|
+
getPartitionService(): PartitionService;
|
|
3162
|
+
/**
|
|
3163
|
+
* Get underlying ReplicationPipeline
|
|
3164
|
+
*/
|
|
3165
|
+
getReplicationPipeline(): ReplicationPipeline | null;
|
|
3166
|
+
/**
|
|
3167
|
+
* Get underlying LagTracker
|
|
3168
|
+
*/
|
|
3169
|
+
getLagTracker(): LagTracker;
|
|
3170
|
+
/**
|
|
3171
|
+
* Get all metrics in Prometheus format
|
|
3172
|
+
*/
|
|
3173
|
+
getPrometheusMetrics(): string;
|
|
3174
|
+
private setupEventHandlers;
|
|
3175
|
+
}
|
|
3176
|
+
|
|
3177
|
+
/**
|
|
3178
|
+
* Configuration for the processor sandbox.
|
|
3179
|
+
*/
|
|
3180
|
+
interface ProcessorSandboxConfig {
|
|
3181
|
+
/** Memory limit in MB per isolate */
|
|
3182
|
+
memoryLimitMb: number;
|
|
3183
|
+
/** Execution timeout in milliseconds */
|
|
3184
|
+
timeoutMs: number;
|
|
3185
|
+
/** Maximum number of cached isolates */
|
|
3186
|
+
maxCachedIsolates: number;
|
|
3187
|
+
/** Enable strict code validation */
|
|
3188
|
+
strictValidation: boolean;
|
|
3189
|
+
}
|
|
3190
|
+
/**
|
|
3191
|
+
* Default sandbox configuration.
|
|
3192
|
+
*/
|
|
3193
|
+
declare const DEFAULT_SANDBOX_CONFIG: ProcessorSandboxConfig;
|
|
3194
|
+
/**
|
|
3195
|
+
* Sandbox for executing entry processor code securely.
|
|
3196
|
+
*
|
|
3197
|
+
* Uses isolated-vm for production environments with:
|
|
3198
|
+
* - Memory limits to prevent memory bombs
|
|
3199
|
+
* - CPU limits via timeout to prevent infinite loops
|
|
3200
|
+
* - No I/O access (no require, fs, net, etc.)
|
|
3201
|
+
* - Minimal exposed globals (only value, key, args)
|
|
3202
|
+
*
|
|
3203
|
+
* Falls back to Node.js vm module for development/testing
|
|
3204
|
+
* when isolated-vm is not available.
|
|
3205
|
+
*/
|
|
3206
|
+
declare class ProcessorSandbox {
|
|
3207
|
+
private config;
|
|
3208
|
+
private isolateCache;
|
|
3209
|
+
private scriptCache;
|
|
3210
|
+
private fallbackScriptCache;
|
|
3211
|
+
private disposed;
|
|
3212
|
+
constructor(config?: Partial<ProcessorSandboxConfig>);
|
|
3213
|
+
/**
|
|
3214
|
+
* Execute an entry processor in the sandbox.
|
|
3215
|
+
*
|
|
3216
|
+
* @param processor The processor definition (name, code, args)
|
|
3217
|
+
* @param value The current value for the key (or undefined)
|
|
3218
|
+
* @param key The key being processed
|
|
3219
|
+
* @returns Result containing success status, result, and new value
|
|
3220
|
+
*/
|
|
3221
|
+
execute<V, R>(processor: EntryProcessorDef<V, R>, value: V | undefined, key: string): Promise<EntryProcessorResult<R>>;
|
|
3222
|
+
/**
|
|
3223
|
+
* Execute processor in isolated-vm (secure production mode).
|
|
3224
|
+
*/
|
|
3225
|
+
private executeInIsolate;
|
|
3226
|
+
/**
|
|
3227
|
+
* Execute processor in fallback VM (less secure, for development).
|
|
3228
|
+
*/
|
|
3229
|
+
private executeInFallback;
|
|
3230
|
+
/**
|
|
3231
|
+
* Get or create an isolate for a processor.
|
|
3232
|
+
*/
|
|
3233
|
+
private getOrCreateIsolate;
|
|
3234
|
+
/**
|
|
3235
|
+
* Get or compile a script for a processor.
|
|
3236
|
+
*/
|
|
3237
|
+
private getOrCompileScript;
|
|
3238
|
+
/**
|
|
3239
|
+
* Clear script cache for a specific processor (e.g., when code changes).
|
|
3240
|
+
*/
|
|
3241
|
+
clearCache(processorName?: string): void;
|
|
3242
|
+
/**
|
|
3243
|
+
* Check if using secure isolated-vm mode.
|
|
3244
|
+
*/
|
|
3245
|
+
isSecureMode(): boolean;
|
|
3246
|
+
/**
|
|
3247
|
+
* Get current cache sizes.
|
|
3248
|
+
*/
|
|
3249
|
+
getCacheStats(): {
|
|
3250
|
+
isolates: number;
|
|
3251
|
+
scripts: number;
|
|
3252
|
+
fallbackScripts: number;
|
|
3253
|
+
};
|
|
3254
|
+
/**
|
|
3255
|
+
* Dispose of all isolates and clear caches.
|
|
3256
|
+
*/
|
|
3257
|
+
dispose(): void;
|
|
3258
|
+
}
|
|
3259
|
+
|
|
3260
|
+
/**
|
|
3261
|
+
* Configuration for the EntryProcessorHandler.
|
|
3262
|
+
*/
|
|
3263
|
+
interface EntryProcessorHandlerConfig {
|
|
3264
|
+
/** HLC instance for timestamp generation */
|
|
3265
|
+
hlc: HLC;
|
|
3266
|
+
/** Optional sandbox configuration override */
|
|
3267
|
+
sandboxConfig?: Partial<ProcessorSandboxConfig>;
|
|
3268
|
+
}
|
|
3269
|
+
/**
|
|
3270
|
+
* Server-side handler for Entry Processor execution.
|
|
3271
|
+
*
|
|
3272
|
+
* Responsibilities:
|
|
3273
|
+
* - Validate incoming processor definitions
|
|
3274
|
+
* - Execute processors in sandboxed environment
|
|
3275
|
+
* - Update map state atomically
|
|
3276
|
+
* - Return results with new values for client cache sync
|
|
3277
|
+
*/
|
|
3278
|
+
declare class EntryProcessorHandler {
|
|
3279
|
+
private sandbox;
|
|
3280
|
+
private hlc;
|
|
3281
|
+
constructor(config: EntryProcessorHandlerConfig);
|
|
3282
|
+
/**
|
|
3283
|
+
* Execute a processor on a single key atomically.
|
|
3284
|
+
*
|
|
3285
|
+
* @param map The LWWMap to operate on
|
|
3286
|
+
* @param key The key to process
|
|
3287
|
+
* @param processorDef The processor definition (will be validated)
|
|
3288
|
+
* @returns Result with success status, processor result, and new value
|
|
3289
|
+
*/
|
|
3290
|
+
executeOnKey<V, R>(map: LWWMap<string, V>, key: string, processorDef: unknown): Promise<{
|
|
3291
|
+
result: EntryProcessorResult<R>;
|
|
3292
|
+
timestamp?: Timestamp;
|
|
3293
|
+
}>;
|
|
3294
|
+
/**
|
|
3295
|
+
* Execute a processor on multiple keys.
|
|
3296
|
+
*
|
|
3297
|
+
* Each key is processed sequentially to ensure atomicity per-key.
|
|
3298
|
+
* For parallel execution across keys, use multiple calls.
|
|
3299
|
+
*
|
|
3300
|
+
* @param map The LWWMap to operate on
|
|
3301
|
+
* @param keys The keys to process
|
|
3302
|
+
* @param processorDef The processor definition
|
|
3303
|
+
* @returns Map of key -> result
|
|
3304
|
+
*/
|
|
3305
|
+
executeOnKeys<V, R>(map: LWWMap<string, V>, keys: string[], processorDef: unknown): Promise<{
|
|
3306
|
+
results: Map<string, EntryProcessorResult<R>>;
|
|
3307
|
+
timestamps: Map<string, Timestamp>;
|
|
3308
|
+
}>;
|
|
3309
|
+
/**
|
|
3310
|
+
* Execute a processor on all entries matching a predicate.
|
|
3311
|
+
*
|
|
3312
|
+
* WARNING: This can be expensive for large maps.
|
|
3313
|
+
*
|
|
3314
|
+
* @param map The LWWMap to operate on
|
|
3315
|
+
* @param processorDef The processor definition
|
|
3316
|
+
* @param predicateCode Optional predicate code to filter entries
|
|
3317
|
+
* @returns Map of key -> result for processed entries
|
|
3318
|
+
*/
|
|
3319
|
+
executeOnEntries<V, R>(map: LWWMap<string, V>, processorDef: unknown, predicateCode?: string): Promise<{
|
|
3320
|
+
results: Map<string, EntryProcessorResult<R>>;
|
|
3321
|
+
timestamps: Map<string, Timestamp>;
|
|
3322
|
+
}>;
|
|
3323
|
+
/**
|
|
3324
|
+
* Check if sandbox is in secure mode (using isolated-vm).
|
|
3325
|
+
*/
|
|
3326
|
+
isSecureMode(): boolean;
|
|
3327
|
+
/**
|
|
3328
|
+
* Get sandbox cache statistics.
|
|
3329
|
+
*/
|
|
3330
|
+
getCacheStats(): {
|
|
3331
|
+
isolates: number;
|
|
3332
|
+
scripts: number;
|
|
3333
|
+
fallbackScripts: number;
|
|
3334
|
+
};
|
|
3335
|
+
/**
|
|
3336
|
+
* Clear sandbox cache.
|
|
3337
|
+
*/
|
|
3338
|
+
clearCache(processorName?: string): void;
|
|
3339
|
+
/**
|
|
3340
|
+
* Dispose of the handler and its sandbox.
|
|
3341
|
+
*/
|
|
3342
|
+
dispose(): void;
|
|
3343
|
+
}
|
|
3344
|
+
|
|
3345
|
+
/**
|
|
3346
|
+
* Configuration for ConflictResolverService.
|
|
3347
|
+
*/
|
|
3348
|
+
interface ConflictResolverServiceConfig {
|
|
3349
|
+
/** Maximum resolvers per map */
|
|
3350
|
+
maxResolversPerMap: number;
|
|
3351
|
+
/** Enable sandboxed code execution (requires isolated-vm) */
|
|
3352
|
+
enableSandboxedResolvers: boolean;
|
|
3353
|
+
/** Default timeout for resolver execution in milliseconds */
|
|
3354
|
+
resolverTimeoutMs: number;
|
|
3355
|
+
}
|
|
3356
|
+
/**
|
|
3357
|
+
* Default service configuration.
|
|
3358
|
+
*/
|
|
3359
|
+
declare const DEFAULT_CONFLICT_RESOLVER_CONFIG: ConflictResolverServiceConfig;
|
|
3360
|
+
/**
|
|
3361
|
+
* Service for managing and executing conflict resolvers.
|
|
3362
|
+
*
|
|
3363
|
+
* Resolvers are executed in priority order (highest first).
|
|
3364
|
+
* The first resolver that returns a non-'local' action wins.
|
|
3365
|
+
* If all resolvers return 'local', LWW is used as fallback.
|
|
3366
|
+
*
|
|
3367
|
+
* ## Design Decisions
|
|
3368
|
+
*
|
|
3369
|
+
* ### In-Memory Storage
|
|
3370
|
+
* Resolvers are stored in memory only (not persisted to database).
|
|
3371
|
+
* This is intentional - resolvers represent application logic that should
|
|
3372
|
+
* be registered by clients on connection. Benefits:
|
|
3373
|
+
* - Simpler architecture without resolver schema migrations
|
|
3374
|
+
* - Clients control their own conflict resolution logic
|
|
3375
|
+
* - Natural cleanup when client disconnects
|
|
3376
|
+
*
|
|
3377
|
+
* ### Permission Model
|
|
3378
|
+
* Resolver registration requires PUT permission on the target map.
|
|
3379
|
+
* This aligns with the principle that if you can write to a map,
|
|
3380
|
+
* you can define how your writes are resolved. For stricter control,
|
|
3381
|
+
* implement custom permission checks in ServerCoordinator.
|
|
3382
|
+
*
|
|
3383
|
+
* ### Deletion Handling
|
|
3384
|
+
* Deletions (tombstones with null value) are passed through resolvers
|
|
3385
|
+
* with `remoteValue: null`. This allows resolvers like IMMUTABLE or
|
|
3386
|
+
* OWNER_ONLY to protect against unauthorized deletions.
|
|
3387
|
+
*/
|
|
3388
|
+
declare class ConflictResolverService {
|
|
3389
|
+
private resolvers;
|
|
3390
|
+
private sandbox;
|
|
3391
|
+
private config;
|
|
3392
|
+
private onRejectionCallback?;
|
|
3393
|
+
private disposed;
|
|
3394
|
+
constructor(sandbox: ProcessorSandbox, config?: Partial<ConflictResolverServiceConfig>);
|
|
3395
|
+
/**
|
|
3396
|
+
* Set callback for merge rejections.
|
|
3397
|
+
*/
|
|
3398
|
+
onRejection(callback: (rejection: MergeRejection) => void): void;
|
|
3399
|
+
/**
|
|
3400
|
+
* Register a resolver for a map.
|
|
3401
|
+
*
|
|
3402
|
+
* @param mapName The map this resolver applies to
|
|
3403
|
+
* @param resolver The resolver definition
|
|
3404
|
+
* @param registeredBy Optional client ID that registered this resolver
|
|
3405
|
+
*/
|
|
3406
|
+
register<V>(mapName: string, resolver: ConflictResolverDef<V>, registeredBy?: string): void;
|
|
3407
|
+
/**
|
|
3408
|
+
* Unregister a resolver.
|
|
3409
|
+
*
|
|
3410
|
+
* @param mapName The map name
|
|
3411
|
+
* @param resolverName The resolver name to unregister
|
|
3412
|
+
* @param clientId Optional - only unregister if registered by this client
|
|
3413
|
+
*/
|
|
3414
|
+
unregister(mapName: string, resolverName: string, clientId?: string): boolean;
|
|
3415
|
+
/**
|
|
3416
|
+
* Resolve a merge conflict using registered resolvers.
|
|
3417
|
+
*
|
|
3418
|
+
* @param context The merge context
|
|
3419
|
+
* @returns The merge result
|
|
3420
|
+
*/
|
|
3421
|
+
resolve<V>(context: MergeContext<V>): Promise<MergeResult<V>>;
|
|
3422
|
+
/**
|
|
3423
|
+
* List registered resolvers.
|
|
3424
|
+
*
|
|
3425
|
+
* @param mapName Optional - filter by map name
|
|
3426
|
+
*/
|
|
3427
|
+
list(mapName?: string): Array<{
|
|
3428
|
+
mapName: string;
|
|
3429
|
+
name: string;
|
|
3430
|
+
priority?: number;
|
|
3431
|
+
keyPattern?: string;
|
|
3432
|
+
registeredBy?: string;
|
|
3433
|
+
}>;
|
|
3434
|
+
/**
|
|
3435
|
+
* Check if a map has any registered resolvers.
|
|
3436
|
+
*/
|
|
3437
|
+
hasResolvers(mapName: string): boolean;
|
|
3438
|
+
/**
|
|
3439
|
+
* Get the number of registered resolvers.
|
|
3440
|
+
*/
|
|
3441
|
+
get size(): number;
|
|
3442
|
+
/**
|
|
3443
|
+
* Clear all registered resolvers.
|
|
3444
|
+
*
|
|
3445
|
+
* @param mapName Optional - only clear resolvers for specific map
|
|
3446
|
+
*/
|
|
3447
|
+
clear(mapName?: string): void;
|
|
3448
|
+
/**
|
|
3449
|
+
* Clear resolvers registered by a specific client.
|
|
3450
|
+
*/
|
|
3451
|
+
clearByClient(clientId: string): number;
|
|
3452
|
+
/**
|
|
3453
|
+
* Dispose the service.
|
|
3454
|
+
*/
|
|
3455
|
+
dispose(): void;
|
|
3456
|
+
/**
|
|
3457
|
+
* Match a key against a glob-like pattern.
|
|
3458
|
+
* Supports * (any chars) and ? (single char).
|
|
3459
|
+
*/
|
|
3460
|
+
private matchKeyPattern;
|
|
3461
|
+
/**
|
|
3462
|
+
* Compile sandboxed resolver code.
|
|
3463
|
+
*/
|
|
3464
|
+
private compileSandboxed;
|
|
3465
|
+
}
|
|
3466
|
+
|
|
3467
|
+
/**
|
|
3468
|
+
* Configuration for MapWithResolver.
|
|
3469
|
+
*/
|
|
3470
|
+
interface MapWithResolverConfig {
|
|
3471
|
+
/** Map name */
|
|
3472
|
+
name: string;
|
|
3473
|
+
/** Node ID for HLC */
|
|
3474
|
+
nodeId: string;
|
|
3475
|
+
/** Conflict resolver service */
|
|
3476
|
+
resolverService: ConflictResolverService;
|
|
3477
|
+
/** Callback for merge rejections */
|
|
3478
|
+
onRejection?: (rejection: MergeRejection) => void;
|
|
3479
|
+
}
|
|
3480
|
+
/**
|
|
3481
|
+
* Result of setWithResolver operation.
|
|
3482
|
+
*/
|
|
3483
|
+
interface SetWithResolverResult<V> {
|
|
3484
|
+
/** Whether the value was applied */
|
|
3485
|
+
applied: boolean;
|
|
3486
|
+
/** The merge result */
|
|
3487
|
+
result: MergeResult<V>;
|
|
3488
|
+
/** The final record if applied */
|
|
3489
|
+
record?: LWWRecord<V>;
|
|
3490
|
+
}
|
|
3491
|
+
/**
|
|
3492
|
+
* Extended LWWMap that supports custom conflict resolvers.
|
|
3493
|
+
*
|
|
3494
|
+
* This wrapper delegates merge operations to ConflictResolverService,
|
|
3495
|
+
* allowing custom business logic to intercept and modify merge behavior.
|
|
3496
|
+
*/
|
|
3497
|
+
declare class MapWithResolver<K extends string, V> {
|
|
3498
|
+
private map;
|
|
3499
|
+
private resolverService;
|
|
3500
|
+
private mapName;
|
|
3501
|
+
private hlc;
|
|
3502
|
+
private onRejection?;
|
|
3503
|
+
constructor(config: MapWithResolverConfig);
|
|
3504
|
+
/**
|
|
3505
|
+
* Get the map name.
|
|
3506
|
+
*/
|
|
3507
|
+
get name(): string;
|
|
3508
|
+
/**
|
|
3509
|
+
* Get the underlying LWWMap.
|
|
3510
|
+
*/
|
|
3511
|
+
get rawMap(): LWWMap<K, V>;
|
|
3512
|
+
/**
|
|
3513
|
+
* Get a value by key.
|
|
3514
|
+
*/
|
|
3515
|
+
get(key: K): V | undefined;
|
|
3516
|
+
/**
|
|
3517
|
+
* Get the full record for a key.
|
|
3518
|
+
*/
|
|
3519
|
+
getRecord(key: K): LWWRecord<V> | undefined;
|
|
3520
|
+
/**
|
|
3521
|
+
* Get the timestamp for a key.
|
|
3522
|
+
*/
|
|
3523
|
+
getTimestamp(key: K): Timestamp | undefined;
|
|
3524
|
+
/**
|
|
3525
|
+
* Set a value locally (no resolver).
|
|
3526
|
+
* Use for server-initiated writes.
|
|
3527
|
+
*/
|
|
3528
|
+
set(key: K, value: V, ttlMs?: number): LWWRecord<V>;
|
|
3529
|
+
/**
|
|
3530
|
+
* Set a value with conflict resolution.
|
|
3531
|
+
* Use for client-initiated writes.
|
|
3532
|
+
*
|
|
3533
|
+
* @param key The key to set
|
|
3534
|
+
* @param value The new value
|
|
3535
|
+
* @param timestamp The client's timestamp
|
|
3536
|
+
* @param remoteNodeId The client's node ID
|
|
3537
|
+
* @param auth Optional authentication context
|
|
3538
|
+
* @returns Result containing applied status and merge result
|
|
3539
|
+
*/
|
|
3540
|
+
setWithResolver(key: K, value: V, timestamp: Timestamp, remoteNodeId: string, auth?: MergeContext['auth']): Promise<SetWithResolverResult<V>>;
|
|
3541
|
+
/**
|
|
3542
|
+
* Remove a key.
|
|
3543
|
+
*/
|
|
3544
|
+
remove(key: K): LWWRecord<V>;
|
|
3545
|
+
/**
|
|
3546
|
+
* Standard merge without resolver (for sync operations).
|
|
3547
|
+
*/
|
|
3548
|
+
merge(key: K, record: LWWRecord<V>): boolean;
|
|
3549
|
+
/**
|
|
3550
|
+
* Merge with resolver support.
|
|
3551
|
+
* Equivalent to setWithResolver but takes a full record.
|
|
3552
|
+
*/
|
|
3553
|
+
mergeWithResolver(key: K, record: LWWRecord<V>, remoteNodeId: string, auth?: MergeContext['auth']): Promise<SetWithResolverResult<V>>;
|
|
3554
|
+
/**
|
|
3555
|
+
* Clear all data.
|
|
3556
|
+
*/
|
|
3557
|
+
clear(): void;
|
|
3558
|
+
/**
|
|
3559
|
+
* Get map size.
|
|
3560
|
+
*/
|
|
3561
|
+
get size(): number;
|
|
3562
|
+
/**
|
|
3563
|
+
* Iterate over entries.
|
|
3564
|
+
*/
|
|
3565
|
+
entries(): IterableIterator<[K, V]>;
|
|
3566
|
+
/**
|
|
3567
|
+
* Get all keys.
|
|
3568
|
+
*/
|
|
3569
|
+
allKeys(): IterableIterator<K>;
|
|
3570
|
+
/**
|
|
3571
|
+
* Subscribe to changes.
|
|
3572
|
+
*/
|
|
3573
|
+
onChange(callback: () => void): () => void;
|
|
3574
|
+
/**
|
|
3575
|
+
* Get MerkleTree for sync.
|
|
3576
|
+
*/
|
|
3577
|
+
getMerkleTree(): _topgunbuild_core.MerkleTree;
|
|
3578
|
+
/**
|
|
3579
|
+
* Prune old tombstones.
|
|
3580
|
+
*/
|
|
3581
|
+
prune(olderThan: Timestamp): K[];
|
|
3582
|
+
}
|
|
3583
|
+
|
|
3584
|
+
/**
|
|
3585
|
+
* Configuration for ConflictResolverHandler.
|
|
3586
|
+
*/
|
|
3587
|
+
interface ConflictResolverHandlerConfig {
|
|
3588
|
+
/** Node ID for identifying server-side resolvers */
|
|
3589
|
+
nodeId: string;
|
|
3590
|
+
/** Optional sandbox configuration override */
|
|
3591
|
+
sandboxConfig?: Partial<ProcessorSandboxConfig>;
|
|
3592
|
+
/** Optional resolver service configuration */
|
|
3593
|
+
resolverConfig?: Partial<ConflictResolverServiceConfig>;
|
|
3594
|
+
}
|
|
3595
|
+
/**
|
|
3596
|
+
* Result of merge operation with resolver.
|
|
3597
|
+
*/
|
|
3598
|
+
interface MergeWithResolverResult<V> {
|
|
3599
|
+
/** Whether the merge was applied */
|
|
3600
|
+
applied: boolean;
|
|
3601
|
+
/** The merge result details */
|
|
3602
|
+
result: MergeResult<V>;
|
|
3603
|
+
/** The final record if applied */
|
|
3604
|
+
record?: LWWRecord<V>;
|
|
3605
|
+
/** Rejection details if rejected */
|
|
3606
|
+
rejection?: MergeRejection;
|
|
3607
|
+
}
|
|
3608
|
+
/**
|
|
3609
|
+
* Server-side handler for Conflict Resolver operations.
|
|
3610
|
+
*
|
|
3611
|
+
* Responsibilities:
|
|
3612
|
+
* - Manage conflict resolver registrations
|
|
3613
|
+
* - Execute resolvers during merge operations
|
|
3614
|
+
* - Provide merge rejection notifications
|
|
3615
|
+
*/
|
|
3616
|
+
declare class ConflictResolverHandler {
|
|
3617
|
+
private sandbox;
|
|
3618
|
+
private resolverService;
|
|
3619
|
+
/** Reserved for future use (server-side resolver identification) */
|
|
3620
|
+
private readonly nodeId;
|
|
3621
|
+
private rejectionListeners;
|
|
3622
|
+
constructor(config: ConflictResolverHandlerConfig);
|
|
3623
|
+
/**
|
|
3624
|
+
* Register a conflict resolver for a map.
|
|
3625
|
+
*
|
|
3626
|
+
* @param mapName The map name
|
|
3627
|
+
* @param resolver The resolver definition
|
|
3628
|
+
* @param clientId Optional client ID that registered this resolver
|
|
3629
|
+
*/
|
|
3630
|
+
registerResolver<V>(mapName: string, resolver: ConflictResolverDef<V>, clientId?: string): void;
|
|
3631
|
+
/**
|
|
3632
|
+
* Unregister a conflict resolver.
|
|
3633
|
+
*
|
|
3634
|
+
* @param mapName The map name
|
|
3635
|
+
* @param resolverName The resolver name
|
|
3636
|
+
* @param clientId Optional - only unregister if registered by this client
|
|
3637
|
+
*/
|
|
3638
|
+
unregisterResolver(mapName: string, resolverName: string, clientId?: string): boolean;
|
|
3639
|
+
/**
|
|
3640
|
+
* List registered resolvers.
|
|
3641
|
+
*
|
|
3642
|
+
* @param mapName Optional - filter by map name
|
|
3643
|
+
*/
|
|
3644
|
+
listResolvers(mapName?: string): Array<{
|
|
3645
|
+
mapName: string;
|
|
3646
|
+
name: string;
|
|
3647
|
+
priority?: number;
|
|
3648
|
+
keyPattern?: string;
|
|
3649
|
+
}>;
|
|
3650
|
+
/**
|
|
3651
|
+
* Apply a merge with conflict resolution.
|
|
3652
|
+
*
|
|
3653
|
+
* Deletions (tombstones) are also passed through resolvers to allow
|
|
3654
|
+
* protection via IMMUTABLE, OWNER_ONLY, or similar resolvers.
|
|
3655
|
+
* If no custom resolvers are registered, deletions use standard LWW.
|
|
3656
|
+
*
|
|
3657
|
+
* @param map The LWWMap to merge into
|
|
3658
|
+
* @param mapName The map name (for resolver lookup)
|
|
3659
|
+
* @param key The key being merged
|
|
3660
|
+
* @param record The incoming record
|
|
3661
|
+
* @param remoteNodeId The source node ID
|
|
3662
|
+
* @param auth Optional authentication context
|
|
3663
|
+
*/
|
|
3664
|
+
mergeWithResolver<V>(map: LWWMap<string, V>, mapName: string, key: string, record: LWWRecord<V>, remoteNodeId: string, auth?: MergeContext['auth']): Promise<MergeWithResolverResult<V>>;
|
|
3665
|
+
/**
|
|
3666
|
+
* Check if a map has custom resolvers registered.
|
|
3667
|
+
*/
|
|
3668
|
+
hasResolvers(mapName: string): boolean;
|
|
3669
|
+
/**
|
|
3670
|
+
* Add a listener for merge rejections.
|
|
3671
|
+
*/
|
|
3672
|
+
onRejection(listener: (rejection: MergeRejection) => void): () => void;
|
|
3673
|
+
/**
|
|
3674
|
+
* Clear resolvers registered by a specific client.
|
|
3675
|
+
*/
|
|
3676
|
+
clearByClient(clientId: string): number;
|
|
3677
|
+
/**
|
|
3678
|
+
* Get the number of registered resolvers.
|
|
3679
|
+
*/
|
|
3680
|
+
get resolverCount(): number;
|
|
3681
|
+
/**
|
|
3682
|
+
* Check if sandbox is in secure mode.
|
|
3683
|
+
*/
|
|
3684
|
+
isSecureMode(): boolean;
|
|
3685
|
+
/**
|
|
3686
|
+
* Dispose of the handler.
|
|
3687
|
+
*/
|
|
3688
|
+
dispose(): void;
|
|
3689
|
+
}
|
|
3690
|
+
|
|
3691
|
+
export { BufferPool, type BufferPoolConfig, type BufferPoolStats, type ClusterConfig, ClusterCoordinator, type ClusterCoordinatorConfig, type ClusterCoordinatorEvents, ClusterManager, type ClusterMember, type ClusterMessage, type CoalescingPreset, type CoalescingWriterMetrics, type CoalescingWriterOptions, ConflictResolverHandler, type ConflictResolverHandlerConfig, ConflictResolverService, type ConflictResolverServiceConfig, type ConnectionContext, ConnectionRateLimiter, DEFAULT_CLUSTER_COORDINATOR_CONFIG, DEFAULT_CONFLICT_RESOLVER_CONFIG, DEFAULT_JOURNAL_SERVICE_CONFIG, DEFAULT_LAG_TRACKER_CONFIG, DEFAULT_SANDBOX_CONFIG, EntryProcessorHandler, type EntryProcessorHandlerConfig, EventJournalService, type EventJournalServiceConfig, type ExportOptions, FilterTasklet, ForEachTasklet, type IInterceptor, type IServerStorage, IteratorTasklet, type IteratorTaskletConfig, type LagInfo, LagTracker, type LagTrackerConfig, LockManager, type Logger, MapTasklet, MapWithResolver, type MapWithResolverConfig, MemoryServerAdapter, type MergeWithResolverResult, MigrationManager, type NativeModuleStatus, type NativeStats, type ORMapTombstones, type ORMapValue, ObjectPool, type ObjectPoolConfig, type ObjectPoolStats, type OpContext, type PartitionDistribution, PartitionService, type PartitionServiceConfig, type PartitionServiceEvents, type PooledEventPayload, type PooledMessage, type PooledRecord, type PooledTimestamp, PostgresAdapter, type PostgresAdapterOptions, ProcessorSandbox, type ProcessorSandboxConfig, type ProgressState, RateLimitInterceptor, type RateLimiterConfig, type RateLimiterStats, ReduceTasklet, ReplicationPipeline, SecurityManager, ServerCoordinator, type ServerCoordinatorConfig, type ServerOp, type SetWithResolverResult, type StorageValue, type Tasklet, TaskletScheduler, type TaskletSchedulerConfig, type TaskletSchedulerStats, TimestampInterceptor, coalescingPresets, createEventPayloadPool, createMessagePool, createRecordPool, createTimestampPool, getCoalescingPreset, getGlobalBufferPool, getGlobalEventPayloadPool, getGlobalMessagePool, getGlobalRecordPool, getGlobalTimestampPool, getNativeModuleStatus, getNativeStats, logNativeStatus, logger, setGlobalBufferPool, setGlobalEventPayloadPool, setGlobalMessagePool, setGlobalRecordPool, setGlobalTimestampPool };
|