@topgunbuild/server 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -1,8 +1,1387 @@
1
- import { LWWRecord, ORMapRecord, Principal, PermissionPolicy, LWWMap, ORMap, PermissionType } from '@topgunbuild/core';
1
+ import { Timestamp, LWWRecord, ORMapRecord, Principal, PermissionPolicy, LWWMap, ORMap, PermissionType } from '@topgunbuild/core';
2
2
  import { WebSocket } from 'ws';
3
3
  import { PoolConfig, Pool } from 'pg';
4
4
  import pino from 'pino';
5
5
 
6
+ /**
7
+ * TaskletScheduler — Cooperative multitasking for long-running operations.
8
+ *
9
+ * Inspired by Hazelcast's Tasklet pattern, this scheduler allows long operations
10
+ * to yield control back to the event loop periodically, preventing event loop
11
+ * blocking and maintaining responsiveness.
12
+ *
13
+ * Key concepts:
14
+ * - Tasklet: A unit of work that can be paused and resumed
15
+ * - Time budget: Maximum time a tasklet can run before yielding
16
+ * - Cooperative scheduling: Tasklets voluntarily yield when time budget is exhausted
17
+ */
18
+ /**
19
+ * Progress state returned by a tasklet after each execution step.
20
+ */
21
+ type ProgressState = 'DONE' | 'MADE_PROGRESS' | 'NO_PROGRESS';
22
+ /**
23
+ * Interface for a tasklet that can be scheduled.
24
+ */
25
+ interface Tasklet<T = void> {
26
+ /** Unique name for logging/metrics */
27
+ readonly name: string;
28
+ /**
29
+ * Execute one chunk of work.
30
+ * Should check time budget and return appropriate state.
31
+ */
32
+ call(): ProgressState;
33
+ /**
34
+ * Get the result after tasklet is DONE.
35
+ * Only valid when call() returns 'DONE'.
36
+ */
37
+ getResult(): T;
38
+ /**
39
+ * Called when tasklet is cancelled before completion.
40
+ */
41
+ onCancel?(): void;
42
+ }
43
+ /**
44
+ * Configuration for TaskletScheduler.
45
+ */
46
+ interface TaskletSchedulerConfig {
47
+ /** Default time budget per tasklet execution in ms (default: 5) */
48
+ defaultTimeBudgetMs?: number;
49
+ /** Maximum concurrent tasklets (default: 10) */
50
+ maxConcurrent?: number;
51
+ /** Interval between scheduler ticks in ms (default: 1) */
52
+ tickIntervalMs?: number;
53
+ /** Enable metrics collection (default: true) */
54
+ metricsEnabled?: boolean;
55
+ }
56
+ /**
57
+ * Metrics for monitoring scheduler performance.
58
+ */
59
+ interface TaskletSchedulerStats {
60
+ /** Total tasklets scheduled */
61
+ totalScheduled: number;
62
+ /** Tasklets currently running */
63
+ activeTasklets: number;
64
+ /** Tasklets completed successfully */
65
+ completedTasklets: number;
66
+ /** Tasklets cancelled */
67
+ cancelledTasklets: number;
68
+ /** Total iterations across all tasklets */
69
+ totalIterations: number;
70
+ /** Average iterations per tasklet */
71
+ avgIterationsPerTasklet: number;
72
+ /** Tasklets that completed in a single iteration */
73
+ singleIterationCompletions: number;
74
+ /** Time spent in tasklet execution (ms) */
75
+ totalExecutionTimeMs: number;
76
+ }
77
+ /**
78
+ * TaskletScheduler manages cooperative multitasking for long-running operations.
79
+ *
80
+ * Usage:
81
+ * ```typescript
82
+ * const scheduler = new TaskletScheduler();
83
+ *
84
+ * // Schedule a tasklet
85
+ * const result = await scheduler.schedule(new QueryExecutionTasklet(records, query));
86
+ *
87
+ * // Cancel all running tasklets
88
+ * scheduler.cancelAll();
89
+ *
90
+ * // Shutdown scheduler
91
+ * scheduler.shutdown();
92
+ * ```
93
+ */
94
+ declare class TaskletScheduler {
95
+ private readonly config;
96
+ private readonly activeTasklets;
97
+ private tickTimer;
98
+ private isRunning;
99
+ private isShuttingDown;
100
+ private totalScheduled;
101
+ private completedTasklets;
102
+ private cancelledTasklets;
103
+ private totalIterations;
104
+ private singleIterationCompletions;
105
+ private totalExecutionTimeMs;
106
+ constructor(config?: TaskletSchedulerConfig);
107
+ /**
108
+ * Schedule a tasklet for execution.
109
+ * Returns a promise that resolves when the tasklet completes.
110
+ */
111
+ schedule<T>(tasklet: Tasklet<T>): Promise<T>;
112
+ /**
113
+ * Run a tasklet synchronously (blocking).
114
+ * Useful for small operations or when cooperative scheduling isn't needed.
115
+ */
116
+ runSync<T>(tasklet: Tasklet<T>): T;
117
+ /**
118
+ * Cancel a specific tasklet by name pattern.
119
+ * Returns the number of tasklets cancelled.
120
+ */
121
+ cancel(namePattern: string | RegExp): number;
122
+ /**
123
+ * Cancel all running tasklets.
124
+ */
125
+ cancelAll(): number;
126
+ /**
127
+ * Get scheduler statistics.
128
+ */
129
+ getStats(): TaskletSchedulerStats;
130
+ /**
131
+ * Reset statistics.
132
+ */
133
+ resetStats(): void;
134
+ /**
135
+ * Shutdown the scheduler.
136
+ * Cancels all running tasklets and stops the tick timer.
137
+ */
138
+ shutdown(): void;
139
+ /**
140
+ * Check if scheduler is running.
141
+ */
142
+ get running(): boolean;
143
+ /**
144
+ * Get number of active tasklets.
145
+ */
146
+ get activeCount(): number;
147
+ private startScheduler;
148
+ private stopScheduler;
149
+ private scheduleTick;
150
+ private tick;
151
+ private completeTasklet;
152
+ private failTasklet;
153
+ private cancelTasklet;
154
+ }
155
+
156
+ /**
157
+ * IteratorTasklet — Base class for tasklets that iterate over collections.
158
+ *
159
+ * Provides cooperative iteration with time-budgeted chunks.
160
+ * Subclasses implement processItem() to handle each item.
161
+ */
162
+
163
+ /**
164
+ * Configuration for iterator tasklets.
165
+ */
166
+ interface IteratorTaskletConfig {
167
+ /** Time budget per iteration in ms (default: 5) */
168
+ timeBudgetMs?: number;
169
+ /** Maximum items to process per iteration (default: 1000) */
170
+ maxItemsPerIteration?: number;
171
+ }
172
+ /**
173
+ * Abstract base class for iterator-based tasklets.
174
+ *
175
+ * Usage:
176
+ * ```typescript
177
+ * class MyTasklet extends IteratorTasklet<[string, Record], Record[]> {
178
+ * constructor(map: Map<string, Record>) {
179
+ * super('my-tasklet', map.entries());
180
+ * }
181
+ *
182
+ * protected processItem([key, record]: [string, Record]): void {
183
+ * if (matchesCriteria(record)) {
184
+ * this.results.push(record);
185
+ * }
186
+ * }
187
+ *
188
+ * getResult(): Record[] {
189
+ * return this.results;
190
+ * }
191
+ * }
192
+ * ```
193
+ */
194
+ declare abstract class IteratorTasklet<TItem, TResult> implements Tasklet<TResult> {
195
+ abstract readonly name: string;
196
+ protected readonly config: Required<IteratorTaskletConfig>;
197
+ protected readonly iterator: Iterator<TItem>;
198
+ protected itemsProcessed: number;
199
+ protected isDone: boolean;
200
+ constructor(iterator: Iterator<TItem>, config?: IteratorTaskletConfig);
201
+ /**
202
+ * Process a single item from the iterator.
203
+ * Override this in subclasses.
204
+ */
205
+ protected abstract processItem(item: TItem): void;
206
+ /**
207
+ * Get the final result after iteration completes.
208
+ */
209
+ abstract getResult(): TResult;
210
+ /**
211
+ * Execute one chunk of iteration.
212
+ */
213
+ call(): ProgressState;
214
+ /**
215
+ * Called when tasklet is cancelled.
216
+ */
217
+ onCancel(): void;
218
+ /**
219
+ * Get number of items processed so far.
220
+ */
221
+ get processed(): number;
222
+ }
223
+ /**
224
+ * Simple iterator tasklet that collects items matching a predicate.
225
+ */
226
+ declare class FilterTasklet<T> extends IteratorTasklet<T, T[]> {
227
+ readonly name: string;
228
+ protected readonly results: T[];
229
+ private readonly predicate;
230
+ constructor(name: string, iterator: Iterator<T>, predicate: (item: T) => boolean, config?: IteratorTaskletConfig);
231
+ protected processItem(item: T): void;
232
+ getResult(): T[];
233
+ }
234
+ /**
235
+ * Iterator tasklet that transforms items.
236
+ */
237
+ declare class MapTasklet<TIn, TOut> extends IteratorTasklet<TIn, TOut[]> {
238
+ readonly name: string;
239
+ protected readonly results: TOut[];
240
+ private readonly mapper;
241
+ constructor(name: string, iterator: Iterator<TIn>, mapper: (item: TIn) => TOut, config?: IteratorTaskletConfig);
242
+ protected processItem(item: TIn): void;
243
+ getResult(): TOut[];
244
+ }
245
+ /**
246
+ * Iterator tasklet that applies a function to each item (side effects).
247
+ */
248
+ declare class ForEachTasklet<T> extends IteratorTasklet<T, number> {
249
+ readonly name: string;
250
+ private readonly action;
251
+ constructor(name: string, iterator: Iterator<T>, action: (item: T) => void, config?: IteratorTaskletConfig);
252
+ protected processItem(item: T): void;
253
+ getResult(): number;
254
+ }
255
+ /**
256
+ * Iterator tasklet that reduces items to a single value.
257
+ */
258
+ declare class ReduceTasklet<T, TAccum> extends IteratorTasklet<T, TAccum> {
259
+ readonly name: string;
260
+ private accumulator;
261
+ private readonly reducer;
262
+ constructor(name: string, iterator: Iterator<T>, initialValue: TAccum, reducer: (acc: TAccum, item: T) => TAccum, config?: IteratorTaskletConfig);
263
+ protected processItem(item: T): void;
264
+ getResult(): TAccum;
265
+ }
266
+
267
+ /**
268
+ * BufferPool - High-performance buffer reuse for serialization operations.
269
+ * Phase 2.03: Memory Pooling
270
+ *
271
+ * Reduces GC pressure by reusing pre-allocated Uint8Array buffers instead of
272
+ * creating new ones for each serialization operation.
273
+ *
274
+ * Reference: Hazelcast MemoryAllocator pattern
275
+ */
276
+ interface BufferPoolConfig {
277
+ /**
278
+ * Size of each buffer chunk in bytes.
279
+ * Larger chunks = fewer allocations, but more memory waste.
280
+ * Default: 64KB (65536)
281
+ */
282
+ chunkSize?: number;
283
+ /**
284
+ * Initial number of buffers to pre-allocate.
285
+ * Default: 16
286
+ */
287
+ initialSize?: number;
288
+ /**
289
+ * Maximum buffers to keep in pool.
290
+ * Excess buffers are released to GC.
291
+ * Default: 256
292
+ */
293
+ maxSize?: number;
294
+ /**
295
+ * Auto-shrink pool when idle buffers exceed this ratio.
296
+ * Default: true
297
+ */
298
+ autoShrink?: boolean;
299
+ /**
300
+ * Shrink threshold - shrink if (available / maxSize) > threshold.
301
+ * Default: 0.75 (75% idle)
302
+ */
303
+ shrinkThreshold?: number;
304
+ /**
305
+ * Enable metrics collection.
306
+ * Default: true
307
+ */
308
+ metricsEnabled?: boolean;
309
+ }
310
+ interface BufferPoolStats {
311
+ /** Buffers currently available in pool */
312
+ available: number;
313
+ /** Buffers currently in use */
314
+ inUse: number;
315
+ /** Total buffers ever created */
316
+ created: number;
317
+ /** Total times a buffer was reused */
318
+ reused: number;
319
+ /** Reuse ratio: reused / (created + reused) */
320
+ reuseRatio: number;
321
+ /** Total bytes in pool (available buffers only) */
322
+ poolSizeBytes: number;
323
+ /** Peak concurrent usage */
324
+ peakUsage: number;
325
+ /** Times pool was exhausted (had to create new buffer) */
326
+ misses: number;
327
+ /** Chunk size configuration */
328
+ chunkSize: number;
329
+ /** Max pool size configuration */
330
+ maxSize: number;
331
+ }
332
+ /**
333
+ * High-performance buffer pool for serialization operations.
334
+ *
335
+ * Usage:
336
+ * ```typescript
337
+ * const pool = new BufferPool({ chunkSize: 64 * 1024 });
338
+ * const buffer = pool.acquire();
339
+ * // ... use buffer for serialization ...
340
+ * pool.release(buffer);
341
+ * ```
342
+ *
343
+ * Thread Safety:
344
+ * This pool is designed for single-threaded use (Node.js main thread).
345
+ * For worker threads, create separate pools or copy buffers.
346
+ */
347
+ declare class BufferPool {
348
+ private readonly pool;
349
+ private readonly chunkSize;
350
+ private readonly maxSize;
351
+ private readonly autoShrink;
352
+ private readonly shrinkThreshold;
353
+ private readonly metricsEnabled;
354
+ private inUseCount;
355
+ private createdCount;
356
+ private reusedCount;
357
+ private peakUsage;
358
+ private missCount;
359
+ private readonly pooledBuffers;
360
+ private readonly releasedBuffers;
361
+ constructor(config?: BufferPoolConfig);
362
+ /**
363
+ * Acquire a buffer from the pool.
364
+ * Creates new buffer if pool is empty.
365
+ *
366
+ * @returns A Uint8Array of exactly chunkSize bytes
367
+ */
368
+ acquire(): Uint8Array;
369
+ /**
370
+ * Release a buffer back to the pool.
371
+ * Buffer contents are cleared before returning to pool.
372
+ *
373
+ * @param buffer - The buffer to release
374
+ */
375
+ release(buffer: Uint8Array): void;
376
+ /**
377
+ * Acquire a buffer of specific minimum size.
378
+ * May return a buffer larger than requested.
379
+ * For sizes > chunkSize, creates new buffer (not pooled).
380
+ *
381
+ * @param minSize - Minimum required size in bytes
382
+ * @returns A Uint8Array of at least minSize bytes
383
+ */
384
+ acquireSize(minSize: number): Uint8Array;
385
+ /**
386
+ * Get current pool statistics.
387
+ */
388
+ getStats(): BufferPoolStats;
389
+ /**
390
+ * Clear all buffers from pool.
391
+ * Use for shutdown or memory pressure situations.
392
+ */
393
+ clear(): void;
394
+ /**
395
+ * Manually trigger shrink operation.
396
+ * Removes excess idle buffers to reduce memory footprint.
397
+ */
398
+ shrink(): void;
399
+ /**
400
+ * Pre-warm pool by creating buffers up to specified count.
401
+ * Called automatically in constructor.
402
+ *
403
+ * @param count - Number of buffers to create
404
+ */
405
+ prewarm(count?: number): void;
406
+ /**
407
+ * Reset all metrics to zero.
408
+ * Useful for testing or periodic metric collection.
409
+ */
410
+ resetStats(): void;
411
+ /**
412
+ * Get configuration values.
413
+ */
414
+ getConfig(): Readonly<Required<BufferPoolConfig>>;
415
+ private createBuffer;
416
+ private shouldShrink;
417
+ }
418
+ /**
419
+ * Get or create the global buffer pool.
420
+ * Uses default configuration (64KB chunks, 256 max).
421
+ */
422
+ declare function getGlobalBufferPool(): BufferPool;
423
+ /**
424
+ * Replace the global buffer pool with a custom instance.
425
+ * Useful for testing or custom configurations.
426
+ *
427
+ * @param pool - New pool instance, or null to reset to default
428
+ */
429
+ declare function setGlobalBufferPool(pool: BufferPool | null): void;
430
+
431
+ /**
432
+ * ObjectPool - Generic object pooling for reducing GC pressure.
433
+ * Phase 2.04: Object Pool Implementation
434
+ *
435
+ * Reuses objects instead of creating new ones in hot paths.
436
+ * Works with any plain data structure type.
437
+ */
438
+ interface ObjectPoolConfig<T> {
439
+ /**
440
+ * Factory function to create new objects.
441
+ * Called when pool is empty.
442
+ */
443
+ factory: () => T;
444
+ /**
445
+ * Reset function to clean object before reuse.
446
+ * Should clear all fields to initial state.
447
+ */
448
+ reset: (obj: T) => void;
449
+ /**
450
+ * Optional validator to check if object is reusable.
451
+ * Return false to discard corrupted objects.
452
+ */
453
+ validate?: (obj: T) => boolean;
454
+ /**
455
+ * Initial pool size.
456
+ * Default: 32
457
+ */
458
+ initialSize?: number;
459
+ /**
460
+ * Maximum pool size.
461
+ * Excess objects are discarded.
462
+ * Default: 512
463
+ */
464
+ maxSize?: number;
465
+ /**
466
+ * Name for debugging/metrics.
467
+ */
468
+ name?: string;
469
+ }
470
+ interface ObjectPoolStats {
471
+ /** Pool name */
472
+ name: string;
473
+ /** Objects currently available */
474
+ available: number;
475
+ /** Objects currently in use */
476
+ inUse: number;
477
+ /** Total objects created */
478
+ created: number;
479
+ /** Total times an object was reused */
480
+ reused: number;
481
+ /** Reuse ratio */
482
+ reuseRatio: number;
483
+ /** Objects discarded (failed validation) */
484
+ discarded: number;
485
+ /** Max pool size */
486
+ maxSize: number;
487
+ /** Peak concurrent usage */
488
+ peakUsage: number;
489
+ }
490
+ /**
491
+ * Generic object pool for reusing plain data structures.
492
+ *
493
+ * Usage:
494
+ * ```typescript
495
+ * const pool = new ObjectPool({
496
+ * factory: () => ({ name: '', value: 0 }),
497
+ * reset: (obj) => { obj.name = ''; obj.value = 0; },
498
+ * });
499
+ *
500
+ * const obj = pool.acquire();
501
+ * obj.name = 'test';
502
+ * obj.value = 42;
503
+ * // ... use obj ...
504
+ * pool.release(obj);
505
+ * ```
506
+ *
507
+ * Important:
508
+ * - Only use for plain data objects
509
+ * - Don't pool objects with closures or event listeners
510
+ * - Don't keep references after release
511
+ */
512
+ declare class ObjectPool<T> {
513
+ private readonly pool;
514
+ private readonly factory;
515
+ private readonly resetFn;
516
+ private readonly validate?;
517
+ private readonly maxSize;
518
+ private readonly name;
519
+ private inUseCount;
520
+ private createdCount;
521
+ private reusedCount;
522
+ private discardedCount;
523
+ private peakUsage;
524
+ private readonly releasedObjects;
525
+ constructor(config: ObjectPoolConfig<T>);
526
+ /**
527
+ * Acquire an object from the pool.
528
+ * Creates new object via factory if pool is empty.
529
+ *
530
+ * @returns A clean, ready-to-use object
531
+ */
532
+ acquire(): T;
533
+ /**
534
+ * Release an object back to the pool.
535
+ * Object is reset before returning to pool.
536
+ *
537
+ * @param obj - The object to release
538
+ */
539
+ release(obj: T): void;
540
+ /**
541
+ * Acquire multiple objects at once.
542
+ * More efficient than multiple acquire() calls.
543
+ *
544
+ * @param count - Number of objects to acquire
545
+ * @returns Array of objects
546
+ */
547
+ acquireBatch(count: number): T[];
548
+ /**
549
+ * Release multiple objects at once.
550
+ *
551
+ * @param objects - Objects to release
552
+ */
553
+ releaseBatch(objects: T[]): void;
554
+ /**
555
+ * Get pool statistics.
556
+ */
557
+ getStats(): ObjectPoolStats;
558
+ /**
559
+ * Clear all objects from pool.
560
+ */
561
+ clear(): void;
562
+ /**
563
+ * Pre-warm pool with objects.
564
+ *
565
+ * @param count - Number of objects to create
566
+ */
567
+ prewarm(count?: number): void;
568
+ /**
569
+ * Reset statistics.
570
+ */
571
+ resetStats(): void;
572
+ private createObject;
573
+ }
574
+
575
+ /**
576
+ * MessagePool - Pre-configured ObjectPool for message objects.
577
+ * Phase 2.04: Object Pool Implementation
578
+ *
579
+ * Reduces GC pressure when processing incoming messages.
580
+ */
581
+
582
+ /**
583
+ * Pooled message structure matching common message format.
584
+ */
585
+ interface PooledMessage {
586
+ type: string;
587
+ payload: unknown;
588
+ timestamp: number | null;
589
+ clientId: string | null;
590
+ mapName: string | null;
591
+ key: string | null;
592
+ }
593
+ /**
594
+ * Create a new MessagePool instance.
595
+ *
596
+ * @param config - Pool configuration
597
+ * @returns Configured ObjectPool for messages
598
+ */
599
+ declare function createMessagePool(config?: {
600
+ maxSize?: number;
601
+ initialSize?: number;
602
+ }): ObjectPool<PooledMessage>;
603
+ /**
604
+ * Get or create the global message pool.
605
+ */
606
+ declare function getGlobalMessagePool(): ObjectPool<PooledMessage>;
607
+ /**
608
+ * Replace the global message pool (for testing).
609
+ */
610
+ declare function setGlobalMessagePool(pool: ObjectPool<PooledMessage> | null): void;
611
+
612
+ /**
613
+ * TimestampPool - Pre-configured ObjectPool for HLC timestamp objects.
614
+ * Phase 2.04: Object Pool Implementation
615
+ *
616
+ * Reduces GC pressure for frequent timestamp operations.
617
+ * Timestamps are created on every operation (set, merge, etc.).
618
+ */
619
+
620
+ /**
621
+ * Pooled timestamp structure matching HLC format.
622
+ */
623
+ interface PooledTimestamp {
624
+ millis: number;
625
+ counter: number;
626
+ nodeId: string;
627
+ }
628
+ /**
629
+ * Create a new TimestampPool instance.
630
+ *
631
+ * @param config - Pool configuration
632
+ * @returns Configured ObjectPool for timestamps
633
+ */
634
+ declare function createTimestampPool(config?: {
635
+ maxSize?: number;
636
+ initialSize?: number;
637
+ }): ObjectPool<PooledTimestamp>;
638
+ /**
639
+ * Get or create the global timestamp pool.
640
+ */
641
+ declare function getGlobalTimestampPool(): ObjectPool<PooledTimestamp>;
642
+ /**
643
+ * Replace the global timestamp pool (for testing).
644
+ */
645
+ declare function setGlobalTimestampPool(pool: ObjectPool<PooledTimestamp> | null): void;
646
+
647
+ /**
648
+ * RecordPool - Pre-configured ObjectPool for LWW/OR record objects.
649
+ * Phase 2.04: Object Pool Implementation
650
+ *
651
+ * Reduces GC pressure when processing CRDT operations.
652
+ */
653
+
654
+ /**
655
+ * Pooled record structure matching LWWRecord format.
656
+ */
657
+ interface PooledRecord<T = unknown> {
658
+ value: T | null;
659
+ timestamp: PooledTimestamp | null;
660
+ ttlMs?: number;
661
+ }
662
+ /**
663
+ * Pooled event payload structure.
664
+ */
665
+ interface PooledEventPayload {
666
+ mapName: string;
667
+ key: string;
668
+ eventType: string;
669
+ record: PooledRecord | null;
670
+ orRecord: PooledRecord | null;
671
+ orTag: string | null;
672
+ }
673
+ /**
674
+ * Create a new RecordPool instance.
675
+ *
676
+ * @param config - Pool configuration
677
+ * @returns Configured ObjectPool for records
678
+ */
679
+ declare function createRecordPool<T = unknown>(config?: {
680
+ maxSize?: number;
681
+ initialSize?: number;
682
+ }): ObjectPool<PooledRecord<T>>;
683
+ /**
684
+ * Create a new EventPayloadPool instance.
685
+ *
686
+ * @param config - Pool configuration
687
+ * @returns Configured ObjectPool for event payloads
688
+ */
689
+ declare function createEventPayloadPool(config?: {
690
+ maxSize?: number;
691
+ initialSize?: number;
692
+ }): ObjectPool<PooledEventPayload>;
693
+ /**
694
+ * Get or create the global record pool.
695
+ */
696
+ declare function getGlobalRecordPool(): ObjectPool<PooledRecord>;
697
+ /**
698
+ * Replace the global record pool (for testing).
699
+ */
700
+ declare function setGlobalRecordPool(pool: ObjectPool<PooledRecord> | null): void;
701
+ /**
702
+ * Get or create the global event payload pool.
703
+ */
704
+ declare function getGlobalEventPayloadPool(): ObjectPool<PooledEventPayload>;
705
+ /**
706
+ * Replace the global event payload pool (for testing).
707
+ */
708
+ declare function setGlobalEventPayloadPool(pool: ObjectPool<PooledEventPayload> | null): void;
709
+
710
+ /**
711
+ * WorkerPool Types
712
+ * Phase 1.02: Worker Threads Implementation
713
+ */
714
+ /**
715
+ * Configuration for WorkerPool
716
+ */
717
+ interface WorkerPoolConfig {
718
+ /** Minimum number of workers (default: 2) */
719
+ minWorkers?: number;
720
+ /** Maximum number of workers (default: os.cpus().length - 1) */
721
+ maxWorkers?: number;
722
+ /** Task execution timeout in ms (default: 5000) */
723
+ taskTimeout?: number;
724
+ /** Worker idle timeout before termination in ms (default: 30000) */
725
+ idleTimeout?: number;
726
+ /** Enable worker restart on crash (default: true) */
727
+ autoRestart?: boolean;
728
+ /** Custom worker script path (for testing or custom workers) */
729
+ workerScript?: string;
730
+ }
731
+ /**
732
+ * Task types supported by workers
733
+ */
734
+ type WorkerTaskType = 'merkle-hash' | 'merkle-hash-ormap' | 'merkle-diff' | 'merkle-rebuild' | 'merkle-rebuild-ormap' | 'lww-merge' | 'ormap-merge' | 'serialize' | 'deserialize';
735
+ /**
736
+ * Task priority levels
737
+ */
738
+ type TaskPriority = 'high' | 'normal' | 'low';
739
+ /**
740
+ * Task to be executed by a worker
741
+ */
742
+ interface WorkerTask<TPayload = unknown, TResult = unknown> {
743
+ /** Unique task ID */
744
+ id: string;
745
+ /** Task type for routing to correct handler */
746
+ type: WorkerTaskType;
747
+ /** Task payload (must be serializable) */
748
+ payload: TPayload;
749
+ /** Priority (default: 'normal') */
750
+ priority?: TaskPriority;
751
+ /** Internal: Expected result type marker */
752
+ _resultType?: TResult;
753
+ }
754
+ /**
755
+ * Pool statistics
756
+ */
757
+ interface WorkerPoolStats {
758
+ /** Number of workers currently executing tasks */
759
+ activeWorkers: number;
760
+ /** Number of workers waiting for tasks */
761
+ idleWorkers: number;
762
+ /** Number of tasks in queue */
763
+ pendingTasks: number;
764
+ /** Total tasks completed successfully */
765
+ completedTasks: number;
766
+ /** Total tasks that failed */
767
+ failedTasks: number;
768
+ /** Average task execution time in ms */
769
+ avgTaskDuration: number;
770
+ }
771
+
772
+ /**
773
+ * WorkerPool Implementation
774
+ * Phase 1.02: Worker Threads Implementation
775
+ *
776
+ * Manages a pool of worker threads for CPU-bound operations.
777
+ * Features:
778
+ * - Auto-scaling (minWorkers to maxWorkers)
779
+ * - Priority queue (high > normal > low)
780
+ * - Task timeouts
781
+ * - Worker crash recovery
782
+ * - Graceful shutdown
783
+ */
784
+
785
+ declare class WorkerPool {
786
+ private readonly config;
787
+ private readonly workers;
788
+ private readonly taskQueue;
789
+ private readonly pendingTasks;
790
+ private workerIdCounter;
791
+ private isShuttingDown;
792
+ private isShutdown;
793
+ private completedTaskCount;
794
+ private failedTaskCount;
795
+ private totalTaskDuration;
796
+ private idleCheckInterval?;
797
+ constructor(config?: WorkerPoolConfig);
798
+ /**
799
+ * Resolve the worker script path based on environment
800
+ */
801
+ private resolveWorkerScript;
802
+ /**
803
+ * Submit a task to the pool
804
+ */
805
+ submit<TPayload, TResult>(task: WorkerTask<TPayload, TResult>): Promise<TResult>;
806
+ /**
807
+ * Get current pool statistics
808
+ */
809
+ getStats(): WorkerPoolStats;
810
+ /**
811
+ * Gracefully shutdown all workers
812
+ */
813
+ shutdown(timeout?: number): Promise<void>;
814
+ /**
815
+ * Check if pool is accepting tasks
816
+ */
817
+ isRunning(): boolean;
818
+ private initializeWorkers;
819
+ private createWorker;
820
+ private findIdleWorker;
821
+ private assignTaskToWorker;
822
+ private enqueueTask;
823
+ private tryScaleUp;
824
+ private handleWorkerResponse;
825
+ private handleTaskTimeout;
826
+ private handleWorkerError;
827
+ private handleWorkerExit;
828
+ private startIdleCheck;
829
+ private checkIdleWorkers;
830
+ }
831
+
832
+ /**
833
+ * Merkle Worker Types
834
+ * Phase 1.03: MerkleWorker Implementation
835
+ */
836
+
837
+ /**
838
+ * Entry for Merkle hash computation (LWWMap style)
839
+ */
840
+ interface MerkleHashEntry {
841
+ key: string;
842
+ timestamp: Timestamp;
843
+ }
844
+ /**
845
+ * Entry for ORMap Merkle hash computation
846
+ */
847
+ interface ORMapMerkleHashEntry {
848
+ key: string;
849
+ records: Array<{
850
+ tag: string;
851
+ timestamp: Timestamp;
852
+ }>;
853
+ }
854
+ /**
855
+ * Payload for merkle-hash task (LWWMap)
856
+ */
857
+ interface MerkleHashPayload {
858
+ entries: MerkleHashEntry[];
859
+ depth?: number;
860
+ }
861
+ /**
862
+ * Result of merkle-hash task
863
+ */
864
+ interface MerkleHashResult {
865
+ /** Map of key -> hash for each entry */
866
+ hashes: Array<[string, number]>;
867
+ /** Root hash of all entries */
868
+ rootHash: number;
869
+ /** Bucket structure: path -> { hash, keys } */
870
+ buckets: Array<[string, {
871
+ hash: number;
872
+ keys: string[];
873
+ }]>;
874
+ }
875
+ /**
876
+ * Payload for merkle-hash-ormap task
877
+ */
878
+ interface ORMapMerkleHashPayload {
879
+ entries: ORMapMerkleHashEntry[];
880
+ depth?: number;
881
+ }
882
+ /**
883
+ * Result of merkle-hash-ormap task
884
+ */
885
+ interface ORMapMerkleHashResult {
886
+ /** Map of key -> hash for each entry */
887
+ hashes: Array<[string, number]>;
888
+ /** Root hash of all entries */
889
+ rootHash: number;
890
+ /** Bucket structure: path -> { hash, keys } */
891
+ buckets: Array<[string, {
892
+ hash: number;
893
+ keys: string[];
894
+ }]>;
895
+ }
896
+ /**
897
+ * Bucket info for diff comparison
898
+ */
899
+ interface BucketInfo {
900
+ hash: number;
901
+ keys: string[];
902
+ }
903
+ /**
904
+ * Payload for merkle-diff task
905
+ */
906
+ interface MerkleDiffPayload {
907
+ /** Local buckets: path -> bucket info */
908
+ localBuckets: Array<[string, BucketInfo]>;
909
+ /** Remote buckets: path -> bucket info */
910
+ remoteBuckets: Array<[string, BucketInfo]>;
911
+ }
912
+ /**
913
+ * Result of merkle-diff task
914
+ */
915
+ interface MerkleDiffResult {
916
+ /** Keys that exist on remote but not locally */
917
+ missingLocal: string[];
918
+ /** Keys that exist locally but not on remote */
919
+ missingRemote: string[];
920
+ /** Paths where buckets differ (need deeper comparison) */
921
+ differingPaths: string[];
922
+ }
923
+ /**
924
+ * Payload for merkle-rebuild task (LWWMap)
925
+ */
926
+ interface MerkleRebuildPayload {
927
+ records: Array<{
928
+ key: string;
929
+ timestamp: Timestamp;
930
+ }>;
931
+ depth?: number;
932
+ }
933
+ /**
934
+ * Payload for merkle-rebuild-ormap task
935
+ */
936
+ interface ORMapMerkleRebuildPayload {
937
+ records: Array<{
938
+ key: string;
939
+ tags: Array<{
940
+ tag: string;
941
+ timestamp: Timestamp;
942
+ }>;
943
+ }>;
944
+ depth?: number;
945
+ }
946
+ /**
947
+ * Result of merkle-rebuild task
948
+ */
949
+ interface MerkleRebuildResult {
950
+ /** Root hash of rebuilt tree */
951
+ rootHash: number;
952
+ /** All buckets in the tree */
953
+ buckets: Array<[string, {
954
+ hash: number;
955
+ keys: string[];
956
+ }]>;
957
+ }
958
+
959
+ /**
960
+ * MerkleWorker - High-level API for Merkle tree operations in worker threads
961
+ * Phase 1.03: MerkleWorker Implementation
962
+ *
963
+ * Provides a clean interface for CPU-intensive Merkle tree operations.
964
+ * Delegates actual work to worker threads via WorkerPool.
965
+ */
966
+
967
+ /**
968
+ * MerkleWorker provides methods for Merkle tree operations.
969
+ * Automatically decides whether to use worker threads based on workload size.
970
+ */
971
+ declare class MerkleWorker {
972
+ private readonly pool;
973
+ private readonly workerScript;
974
+ constructor(pool: WorkerPool);
975
+ private resolveMerkleWorkerScript;
976
+ /**
977
+ * Compute hashes for a batch of LWWMap entries.
978
+ * Uses worker thread if entries count exceeds threshold.
979
+ */
980
+ computeHashes(payload: MerkleHashPayload): Promise<MerkleHashResult>;
981
+ /**
982
+ * Compute hashes for a batch of ORMap entries.
983
+ * Uses worker thread if entries count exceeds threshold.
984
+ */
985
+ computeORMapHashes(payload: ORMapMerkleHashPayload): Promise<ORMapMerkleHashResult>;
986
+ /**
987
+ * Find differences between local and remote Merkle trees.
988
+ */
989
+ diff(payload: MerkleDiffPayload): Promise<MerkleDiffResult>;
990
+ /**
991
+ * Rebuild Merkle tree from LWWMap records.
992
+ * Always uses worker thread as this is typically a heavy operation.
993
+ */
994
+ rebuild(payload: MerkleRebuildPayload): Promise<MerkleRebuildResult>;
995
+ /**
996
+ * Rebuild Merkle tree from ORMap records.
997
+ */
998
+ rebuildORMap(payload: ORMapMerkleRebuildPayload): Promise<MerkleRebuildResult>;
999
+ private computeHashesInline;
1000
+ private computeORMapHashesInline;
1001
+ private diffInline;
1002
+ private rebuildInline;
1003
+ private rebuildORMapInline;
1004
+ private hashString;
1005
+ private buildTree;
1006
+ }
1007
+
1008
+ /**
1009
+ * CRDT Worker Types
1010
+ * Phase 1.04: CRDTMergeWorker Implementation
1011
+ */
1012
+
1013
+ /**
1014
+ * Single LWW record for merge
1015
+ */
1016
+ interface LWWMergeRecord {
1017
+ key: string;
1018
+ value: unknown;
1019
+ timestamp: Timestamp;
1020
+ ttlMs?: number;
1021
+ }
1022
+ /**
1023
+ * Existing state for a key (for merge comparison)
1024
+ * Note: value is optional since we only need timestamp for comparison
1025
+ */
1026
+ interface LWWExistingRecord {
1027
+ key: string;
1028
+ timestamp: Timestamp;
1029
+ value?: unknown;
1030
+ ttlMs?: number;
1031
+ }
1032
+ /**
1033
+ * Payload for lww-merge task
1034
+ */
1035
+ interface LWWMergePayload {
1036
+ mapName: string;
1037
+ /** Records to merge */
1038
+ records: LWWMergeRecord[];
1039
+ /** Current state of affected keys (for comparison) */
1040
+ existingState: LWWExistingRecord[];
1041
+ }
1042
+ /**
1043
+ * Result of lww-merge task
1044
+ */
1045
+ interface LWWMergeResult {
1046
+ /** Records that should be applied (newer than existing) */
1047
+ toApply: Array<{
1048
+ key: string;
1049
+ value: unknown;
1050
+ timestamp: Timestamp;
1051
+ ttlMs?: number;
1052
+ }>;
1053
+ /** Number of records skipped (older timestamp) */
1054
+ skipped: number;
1055
+ /** Keys with concurrent updates (same timestamp, different nodeId) */
1056
+ conflicts: string[];
1057
+ }
1058
+ /**
1059
+ * Single ORMap item for merge
1060
+ */
1061
+ interface ORMapMergeItem {
1062
+ key: string;
1063
+ value: unknown;
1064
+ timestamp: Timestamp;
1065
+ tag: string;
1066
+ ttlMs?: number;
1067
+ }
1068
+ /**
1069
+ * Single ORMap tombstone for merge
1070
+ */
1071
+ interface ORMapMergeTombstone {
1072
+ tag: string;
1073
+ timestamp: Timestamp;
1074
+ }
1075
+ /**
1076
+ * Payload for ormap-merge task
1077
+ */
1078
+ interface ORMapMergePayload {
1079
+ mapName: string;
1080
+ /** Items to merge */
1081
+ items: ORMapMergeItem[];
1082
+ /** Tombstones to merge */
1083
+ tombstones: ORMapMergeTombstone[];
1084
+ /** Existing tags in the map (for tombstone check) */
1085
+ existingTags: string[];
1086
+ /** Existing tombstones (to avoid re-adding deleted items) */
1087
+ existingTombstones: string[];
1088
+ }
1089
+ /**
1090
+ * Result of ormap-merge task
1091
+ */
1092
+ interface ORMapMergeResult {
1093
+ /** Items that should be applied */
1094
+ itemsToApply: Array<{
1095
+ key: string;
1096
+ value: unknown;
1097
+ timestamp: Timestamp;
1098
+ tag: string;
1099
+ ttlMs?: number;
1100
+ }>;
1101
+ /** Tombstones that should be applied */
1102
+ tombstonesToApply: string[];
1103
+ /** Tags that need to be removed due to new tombstones */
1104
+ tagsToRemove: string[];
1105
+ /** Number of items skipped */
1106
+ itemsSkipped: number;
1107
+ /** Number of tombstones skipped */
1108
+ tombstonesSkipped: number;
1109
+ }
1110
+
1111
+ /**
1112
+ * CRDTMergeWorker - High-level API for CRDT merge operations in worker threads
1113
+ * Phase 1.04: CRDTMergeWorker Implementation
1114
+ *
1115
+ * Provides a clean interface for CPU-intensive CRDT merge operations.
1116
+ * Delegates actual work to worker threads via WorkerPool for large batches.
1117
+ */
1118
+
1119
+ /**
1120
+ * CRDTMergeWorker provides methods for CRDT merge operations.
1121
+ * Automatically decides whether to use worker threads based on batch size.
1122
+ */
1123
+ declare class CRDTMergeWorker {
1124
+ private readonly pool;
1125
+ /** Threshold for using worker (operations below this go to main thread) */
1126
+ static readonly BATCH_THRESHOLD = 10;
1127
+ constructor(pool: WorkerPool);
1128
+ /**
1129
+ * Decide if batch should go to worker
1130
+ */
1131
+ shouldUseWorker(batchSize: number): boolean;
1132
+ /**
1133
+ * Merge LWW records
1134
+ * @param payload - Records to merge and existing state
1135
+ * @returns Which records should be applied
1136
+ */
1137
+ mergeLWW(payload: LWWMergePayload): Promise<LWWMergeResult>;
1138
+ /**
1139
+ * Merge ORMap items and tombstones
1140
+ * @param payload - Items, tombstones, and existing state
1141
+ * @returns Which items/tombstones should be applied
1142
+ */
1143
+ mergeORMap(payload: ORMapMergePayload): Promise<ORMapMergeResult>;
1144
+ private mergeLWWInline;
1145
+ private mergeORMapInline;
1146
+ }
1147
+
1148
+ /**
1149
+ * SerializationWorker - High-level API for serialization operations in worker threads
1150
+ * Phase 1.07: SerializationWorker Implementation
1151
+ *
1152
+ * Provides a clean interface for CPU-intensive serialization/deserialization.
1153
+ * Delegates actual work to worker threads via WorkerPool for large payloads.
1154
+ *
1155
+ * Uses base64 encoding to transfer binary data through postMessage.
1156
+ * This adds ~33% overhead but is necessary since Uint8Array cannot be
1157
+ * directly transferred through structured clone algorithm for our use case.
1158
+ */
1159
+
1160
+ /**
1161
+ * SerializationWorker provides methods for serialization operations.
1162
+ * Automatically decides whether to use worker threads based on payload size.
1163
+ */
1164
+ declare class SerializationWorker {
1165
+ private readonly pool;
1166
+ private readonly workerScript;
1167
+ /** Threshold for using worker (items below this go to main thread) */
1168
+ static readonly BATCH_THRESHOLD = 10;
1169
+ /** Size threshold for using worker (bytes) */
1170
+ static readonly SIZE_THRESHOLD: number;
1171
+ constructor(pool: WorkerPool);
1172
+ private resolveWorkerScript;
1173
+ /**
1174
+ * Decide if batch should go to worker based on count or size
1175
+ */
1176
+ shouldUseWorker(items: unknown[]): boolean;
1177
+ /**
1178
+ * Serialize multiple objects to MessagePack binary format.
1179
+ * Uses worker thread for large batches.
1180
+ *
1181
+ * @param items - Objects to serialize
1182
+ * @returns Array of Uint8Array containing serialized data
1183
+ */
1184
+ serializeBatch(items: unknown[]): Promise<Uint8Array[]>;
1185
+ /**
1186
+ * Deserialize multiple MessagePack binary payloads to objects.
1187
+ * Uses worker thread for large batches.
1188
+ *
1189
+ * @param items - Binary data to deserialize
1190
+ * @returns Array of deserialized objects
1191
+ */
1192
+ deserializeBatch<T = unknown>(items: Uint8Array[]): Promise<T[]>;
1193
+ /**
1194
+ * Serialize a single object (always inline, too small for worker)
1195
+ */
1196
+ serialize(data: unknown): Uint8Array;
1197
+ /**
1198
+ * Deserialize a single payload (always inline, too small for worker)
1199
+ */
1200
+ deserialize<T = unknown>(data: Uint8Array | ArrayBuffer): T;
1201
+ private serializeBatchInline;
1202
+ private deserializeBatchInline;
1203
+ }
1204
+
1205
+ /**
1206
+ * SharedMemoryManager - Zero-copy data transfer between main thread and workers
1207
+ *
1208
+ * Uses SharedArrayBuffer with Atomics for synchronization.
1209
+ * Provides slot-based allocation for concurrent operations.
1210
+ *
1211
+ * Phase 3.04: SharedArrayBuffer Integration
1212
+ */
1213
+ interface SharedMemoryConfig {
1214
+ /**
1215
+ * Size of shared buffer in bytes.
1216
+ * Default: 16MB
1217
+ */
1218
+ bufferSize?: number;
1219
+ /**
1220
+ * Number of slots for concurrent operations.
1221
+ * Each slot can hold one transfer.
1222
+ * Default: 256
1223
+ */
1224
+ slotCount?: number;
1225
+ /**
1226
+ * Reserved bytes per slot for metadata (length, status, etc).
1227
+ * Default: 16 (must be multiple of 8 for alignment)
1228
+ */
1229
+ metadataSize?: number;
1230
+ }
1231
+ interface SharedMemoryStats {
1232
+ /** Total buffer size in bytes */
1233
+ totalSize: number;
1234
+ /** Number of slots */
1235
+ slotCount: number;
1236
+ /** Size of each slot in bytes */
1237
+ slotSize: number;
1238
+ /** Currently allocated slots */
1239
+ allocatedSlots: number;
1240
+ /** Available slots */
1241
+ availableSlots: number;
1242
+ /** Peak concurrent allocations */
1243
+ peakUsage: number;
1244
+ /** Total allocations since creation */
1245
+ totalAllocations: number;
1246
+ /** Total releases since creation */
1247
+ totalReleases: number;
1248
+ }
1249
+ interface SharedSlot {
1250
+ /** Slot index */
1251
+ index: number;
1252
+ /** View into shared buffer for this slot's data area */
1253
+ dataView: Uint8Array;
1254
+ /** Maximum data size (excluding metadata) */
1255
+ maxDataSize: number;
1256
+ }
1257
+ /**
1258
+ * Slot status values (stored in first 4 bytes of slot metadata)
1259
+ */
1260
+ declare enum SlotStatus {
1261
+ FREE = 0,// Slot is available
1262
+ ALLOCATED = 1,// Slot is allocated, no data yet
1263
+ DATA_READY = 2,// Data written, ready for reading
1264
+ PROCESSING = 3,// Worker is processing
1265
+ RESULT_READY = 4,// Worker has written result
1266
+ ERROR = 255
1267
+ }
1268
+ /**
1269
+ * Manages shared memory for zero-copy data transfer between threads.
1270
+ *
1271
+ * Usage:
1272
+ * 1. Main thread allocates a slot
1273
+ * 2. Main thread writes data to slot
1274
+ * 3. Main thread sends slot index to worker via postMessage
1275
+ * 4. Worker reads data from shared memory (zero-copy)
1276
+ * 5. Worker writes result to slot
1277
+ * 6. Main thread reads result (zero-copy)
1278
+ * 7. Main thread releases slot
1279
+ */
1280
+ declare class SharedMemoryManager {
1281
+ private readonly buffer;
1282
+ private readonly statusArray;
1283
+ private readonly slotSize;
1284
+ private readonly slotCount;
1285
+ private readonly metadataSize;
1286
+ private readonly freeSlots;
1287
+ private allocatedCount;
1288
+ private peakUsage;
1289
+ private totalAllocations;
1290
+ private totalReleases;
1291
+ constructor(config?: SharedMemoryConfig);
1292
+ /**
1293
+ * Get offset in Int32Array for slot status.
1294
+ * Status is at the beginning of each slot's metadata.
1295
+ */
1296
+ private getStatusOffset;
1297
+ /**
1298
+ * Get byte offset for slot length field.
1299
+ */
1300
+ private getLengthOffset;
1301
+ /**
1302
+ * Get byte offset for slot data (after metadata).
1303
+ */
1304
+ private getDataOffset;
1305
+ /**
1306
+ * Allocate a slot for data transfer.
1307
+ * Returns null if no slots available.
1308
+ */
1309
+ allocate(): SharedSlot | null;
1310
+ /**
1311
+ * Release a slot back to the pool.
1312
+ */
1313
+ release(slot: SharedSlot): void;
1314
+ /**
1315
+ * Write data to a slot and signal it's ready.
1316
+ * Returns false if data is too large.
1317
+ */
1318
+ writeData(slot: SharedSlot, data: Uint8Array): boolean;
1319
+ /**
1320
+ * Read data length from a slot.
1321
+ */
1322
+ getDataLength(slotIndex: number): number;
1323
+ /**
1324
+ * Get data view for a slot (for reading).
1325
+ */
1326
+ getDataView(slotIndex: number): Uint8Array;
1327
+ /**
1328
+ * Get slot status.
1329
+ */
1330
+ getStatus(slotIndex: number): SlotStatus;
1331
+ /**
1332
+ * Wait for a specific status with timeout.
1333
+ * Returns the actual status (may differ if timeout occurred).
1334
+ */
1335
+ waitForStatus(slotIndex: number, expectedStatus: SlotStatus, timeoutMs?: number): SlotStatus;
1336
+ /**
1337
+ * Wait for result and read it.
1338
+ * Returns null on timeout or error.
1339
+ */
1340
+ waitForResult(slot: SharedSlot, timeoutMs?: number): Uint8Array | null;
1341
+ /**
1342
+ * Get the SharedArrayBuffer for passing to workers.
1343
+ */
1344
+ getBuffer(): SharedArrayBuffer;
1345
+ /**
1346
+ * Get configuration needed by workers.
1347
+ */
1348
+ getWorkerConfig(): SharedWorkerConfig;
1349
+ /**
1350
+ * Get statistics.
1351
+ */
1352
+ getStats(): SharedMemoryStats;
1353
+ /**
1354
+ * Check if SharedArrayBuffer is available in current environment.
1355
+ */
1356
+ static isAvailable(): boolean;
1357
+ /**
1358
+ * Shutdown and release resources.
1359
+ * Resets all slots to FREE status.
1360
+ */
1361
+ shutdown(): void;
1362
+ }
1363
+ /**
1364
+ * Configuration passed to workers for shared memory access.
1365
+ */
1366
+ interface SharedWorkerConfig {
1367
+ sharedBuffer: SharedArrayBuffer;
1368
+ slotSize: number;
1369
+ slotCount: number;
1370
+ metadataSize: number;
1371
+ }
1372
+
1373
+ interface QueueMetrics {
1374
+ enqueued: number;
1375
+ dequeued: number;
1376
+ rejected: number;
1377
+ currentSize: number;
1378
+ }
1379
+
1380
+ interface StripeMetrics {
1381
+ stripe: number;
1382
+ metrics: QueueMetrics;
1383
+ }
1384
+
6
1385
  type ORMapValue<V> = {
7
1386
  type: 'OR';
8
1387
  records: ORMapRecord<V>[];
@@ -160,6 +1539,122 @@ interface ClusterTLSConfig extends TLSConfig {
160
1539
  rejectUnauthorized?: boolean;
161
1540
  }
162
1541
 
1542
+ interface CoalescingWriterOptions {
1543
+ /**
1544
+ * Maximum messages to batch before forcing flush.
1545
+ * Default: 100
1546
+ */
1547
+ maxBatchSize: number;
1548
+ /**
1549
+ * Maximum time to wait before flushing (ms).
1550
+ * Default: 5 (similar to Nagle's algorithm)
1551
+ */
1552
+ maxDelayMs: number;
1553
+ /**
1554
+ * Maximum batch size in bytes.
1555
+ * Default: 65536 (64KB)
1556
+ */
1557
+ maxBatchBytes: number;
1558
+ /**
1559
+ * Optional BufferPool for batch buffer reuse.
1560
+ * If not provided, uses the global buffer pool.
1561
+ */
1562
+ bufferPool?: BufferPool;
1563
+ }
1564
+ /**
1565
+ * Extended metrics for CoalescingWriter performance analysis.
1566
+ */
1567
+ interface CoalescingWriterMetrics {
1568
+ /** Total messages sent */
1569
+ messagesSent: number;
1570
+ /** Total batches sent */
1571
+ batchesSent: number;
1572
+ /** Total bytes sent */
1573
+ bytesSent: number;
1574
+ /** Average messages per batch */
1575
+ avgMessagesPerBatch: number;
1576
+ /** Average bytes per batch */
1577
+ avgBytesPerBatch: number;
1578
+ /** Messages currently in queue */
1579
+ pendingMessages: number;
1580
+ /** Bytes currently pending */
1581
+ pendingBytes: number;
1582
+ /** Count of flushes triggered by size limits (batch full or bytes exceeded) */
1583
+ immediateFlushes: number;
1584
+ /** Count of flushes triggered by timer expiration */
1585
+ timedFlushes: number;
1586
+ /** Ratio of actual batch size to maxBatchSize (0-1, higher = better utilization) */
1587
+ batchUtilization: number;
1588
+ /** Ratio of immediate flushes to total flushes (high = batches filling up quickly) */
1589
+ immediateFlushRatio: number;
1590
+ /** Count of pooled buffer acquisitions (for monitoring buffer pool usage) */
1591
+ pooledBuffersUsed: number;
1592
+ /** Count of oversized (non-pooled) buffers that were allocated directly */
1593
+ oversizedBuffers: number;
1594
+ }
1595
+
1596
+ /**
1597
+ * Preset configurations for CoalescingWriter.
1598
+ * Based on Hazelcast OutboxImpl (batch size 2048) and real-world benchmarking.
1599
+ *
1600
+ * Trade-offs:
1601
+ * - Larger batch size = higher throughput, higher latency
1602
+ * - Longer delay = more messages per batch, higher latency
1603
+ * - Larger maxBatchBytes = handles larger payloads, more memory
1604
+ */
1605
+ declare const coalescingPresets: {
1606
+ /**
1607
+ * Conservative defaults - good for low-latency workloads.
1608
+ * Minimizes batching delay at the cost of more network calls.
1609
+ * Use for: gaming, real-time chat, interactive applications.
1610
+ */
1611
+ readonly conservative: {
1612
+ readonly maxBatchSize: 100;
1613
+ readonly maxDelayMs: 5;
1614
+ readonly maxBatchBytes: 65536;
1615
+ };
1616
+ /**
1617
+ * Balanced - good for most workloads.
1618
+ * Reasonable trade-off between throughput and latency.
1619
+ * Use for: mixed read/write applications, general purpose.
1620
+ */
1621
+ readonly balanced: {
1622
+ readonly maxBatchSize: 300;
1623
+ readonly maxDelayMs: 8;
1624
+ readonly maxBatchBytes: 131072;
1625
+ };
1626
+ /**
1627
+ * High throughput - optimized for write-heavy workloads.
1628
+ * Higher batching for better network utilization.
1629
+ * Use for: data ingestion, logging, IoT data streams.
1630
+ */
1631
+ readonly highThroughput: {
1632
+ readonly maxBatchSize: 500;
1633
+ readonly maxDelayMs: 10;
1634
+ readonly maxBatchBytes: 262144;
1635
+ };
1636
+ /**
1637
+ * Aggressive - maximum batching for batch processing.
1638
+ * Closest to Hazelcast's OutboxImpl (batch size 2048).
1639
+ * Use for: batch imports, bulk operations, offline sync.
1640
+ */
1641
+ readonly aggressive: {
1642
+ readonly maxBatchSize: 1000;
1643
+ readonly maxDelayMs: 15;
1644
+ readonly maxBatchBytes: 524288;
1645
+ };
1646
+ };
1647
+ /**
1648
+ * Available preset names for type safety.
1649
+ */
1650
+ type CoalescingPreset = keyof typeof coalescingPresets;
1651
+ /**
1652
+ * Get preset configuration by name.
1653
+ * @param preset - Preset name
1654
+ * @returns CoalescingWriterOptions
1655
+ */
1656
+ declare function getCoalescingPreset(preset: CoalescingPreset): CoalescingWriterOptions;
1657
+
163
1658
  interface ServerCoordinatorConfig {
164
1659
  port: number;
165
1660
  nodeId: string;
@@ -178,6 +1673,54 @@ interface ServerCoordinatorConfig {
178
1673
  discoveryInterval?: number;
179
1674
  tls?: TLSConfig;
180
1675
  clusterTls?: ClusterTLSConfig;
1676
+ /** Total event queue capacity for bounded queue (default: 10000) */
1677
+ eventQueueCapacity?: number;
1678
+ /** Number of event queue stripes for parallel processing (default: 4) */
1679
+ eventStripeCount?: number;
1680
+ /** Enable/disable backpressure (default: true) */
1681
+ backpressureEnabled?: boolean;
1682
+ /** How often to force sync processing (default: 100 operations) */
1683
+ backpressureSyncFrequency?: number;
1684
+ /** Maximum pending async operations before blocking (default: 1000) */
1685
+ backpressureMaxPending?: number;
1686
+ /** Backoff timeout in ms when at capacity (default: 5000) */
1687
+ backpressureBackoffMs?: number;
1688
+ /** Enable/disable write coalescing (default: true) */
1689
+ writeCoalescingEnabled?: boolean;
1690
+ /** Coalescing preset: 'conservative', 'balanced', 'highThroughput', 'aggressive' (default: 'highThroughput') */
1691
+ writeCoalescingPreset?: CoalescingPreset;
1692
+ /** Maximum messages to batch before forcing flush (default: 500 for highThroughput) */
1693
+ writeCoalescingMaxBatch?: number;
1694
+ /** Maximum delay before flushing in ms (default: 10 for highThroughput) */
1695
+ writeCoalescingMaxDelayMs?: number;
1696
+ /** Maximum batch size in bytes (default: 262144/256KB for highThroughput) */
1697
+ writeCoalescingMaxBytes?: number;
1698
+ /** WebSocket backlog for pending connections (default: 511) */
1699
+ wsBacklog?: number;
1700
+ /** Enable WebSocket per-message compression (default: false for CPU savings) */
1701
+ wsCompression?: boolean;
1702
+ /** Maximum WebSocket payload size in bytes (default: 64MB) */
1703
+ wsMaxPayload?: number;
1704
+ /** Maximum server connections (default: 10000) */
1705
+ maxConnections?: number;
1706
+ /** Server timeout in ms (default: 120000 = 2 min) */
1707
+ serverTimeout?: number;
1708
+ /** Keep-alive timeout in ms (default: 5000) */
1709
+ keepAliveTimeout?: number;
1710
+ /** Headers timeout in ms (default: 60000) */
1711
+ headersTimeout?: number;
1712
+ /** Enable connection rate limiting (default: true) */
1713
+ rateLimitingEnabled?: boolean;
1714
+ /** Maximum new connections per second (default: 100) */
1715
+ maxConnectionsPerSecond?: number;
1716
+ /** Maximum pending connections (default: 1000) */
1717
+ maxPendingConnections?: number;
1718
+ /** Enable worker pool for CPU-bound operations (default: false) */
1719
+ workerPoolEnabled?: boolean;
1720
+ /** Worker pool configuration */
1721
+ workerPoolConfig?: Partial<WorkerPoolConfig>;
1722
+ /** Default timeout for Write Concern acknowledgments in ms (default: 5000) */
1723
+ writeAckTimeout?: number;
181
1724
  }
182
1725
  declare class ServerCoordinator {
183
1726
  private httpServer;
@@ -202,6 +1745,20 @@ declare class ServerCoordinator {
202
1745
  private heartbeatCheckInterval?;
203
1746
  private gcReports;
204
1747
  private mapLoadingPromises;
1748
+ private pendingBatchOperations;
1749
+ private eventExecutor;
1750
+ private backpressure;
1751
+ private writeCoalescingEnabled;
1752
+ private writeCoalescingOptions;
1753
+ private rateLimiter;
1754
+ private rateLimitingEnabled;
1755
+ private workerPool?;
1756
+ private merkleWorker?;
1757
+ private crdtMergeWorker?;
1758
+ private serializationWorker?;
1759
+ private eventPayloadPool;
1760
+ private taskletScheduler;
1761
+ private writeAckManager;
205
1762
  private _actualPort;
206
1763
  private _actualClusterPort;
207
1764
  private _readyPromise;
@@ -209,20 +1766,113 @@ declare class ServerCoordinator {
209
1766
  constructor(config: ServerCoordinatorConfig);
210
1767
  /** Wait for server to be fully ready (ports assigned) */
211
1768
  ready(): Promise<void>;
1769
+ /**
1770
+ * Wait for all pending batch operations to complete.
1771
+ * Useful for tests that need to verify state after OP_BATCH.
1772
+ */
1773
+ waitForPendingBatches(): Promise<void>;
212
1774
  /** Get the actual port the server is listening on */
213
1775
  get port(): number;
214
1776
  /** Get the actual cluster port */
215
1777
  get clusterPort(): number;
1778
+ /** Get event executor metrics for monitoring */
1779
+ getEventExecutorMetrics(): StripeMetrics[];
1780
+ /** Get total event executor metrics across all stripes */
1781
+ getEventExecutorTotalMetrics(): QueueMetrics;
1782
+ /** Get connection rate limiter stats for monitoring */
1783
+ getRateLimiterStats(): RateLimiterStats;
1784
+ /** Get worker pool stats for monitoring */
1785
+ getWorkerPoolStats(): WorkerPoolStats | null;
1786
+ /** Check if worker pool is enabled */
1787
+ get workerPoolEnabled(): boolean;
1788
+ /** Get MerkleWorker for external use (null if worker pool disabled) */
1789
+ getMerkleWorker(): MerkleWorker | null;
1790
+ /** Get CRDTMergeWorker for external use (null if worker pool disabled) */
1791
+ getCRDTMergeWorker(): CRDTMergeWorker | null;
1792
+ /** Get SerializationWorker for external use (null if worker pool disabled) */
1793
+ getSerializationWorker(): SerializationWorker | null;
1794
+ /** Get memory pool stats for monitoring GC pressure reduction */
1795
+ getMemoryPoolStats(): {
1796
+ eventPayloadPool: ObjectPoolStats;
1797
+ };
1798
+ /** Get tasklet scheduler stats for monitoring cooperative multitasking */
1799
+ getTaskletSchedulerStats(): TaskletSchedulerStats;
1800
+ /** Get tasklet scheduler for scheduling long-running operations */
1801
+ getTaskletScheduler(): TaskletScheduler;
216
1802
  shutdown(): Promise<void>;
217
1803
  private handleConnection;
218
1804
  private handleMessage;
219
1805
  private updateClientHlc;
220
1806
  private broadcast;
1807
+ /**
1808
+ * === OPTIMIZATION 2 & 3: Batched Broadcast with Serialization Caching ===
1809
+ * Groups clients by their permission roles and serializes once per group.
1810
+ * Also batches multiple events into a single SERVER_BATCH_EVENT message.
1811
+ * === OPTIMIZATION 4: Subscription-based Routing ===
1812
+ * Only sends events to clients with active subscriptions for affected maps.
1813
+ */
1814
+ private broadcastBatch;
1815
+ /**
1816
+ * Helper method to get role signature for a client (for caching key)
1817
+ */
1818
+ private getClientRoleSignature;
1819
+ /**
1820
+ * === BACKPRESSURE: Synchronous Broadcast ===
1821
+ * Same as broadcastBatch but waits for all sends to complete.
1822
+ * Used when backpressure forces sync processing to drain the pipeline.
1823
+ */
1824
+ private broadcastBatchSync;
221
1825
  private setupClusterListeners;
222
1826
  private executeLocalQuery;
223
1827
  private finalizeClusterQuery;
1828
+ /**
1829
+ * Core operation application logic shared between processLocalOp and processLocalOpForBatch.
1830
+ * Handles map merge, storage persistence, query evaluation, and event generation.
1831
+ *
1832
+ * @returns Event payload for broadcasting (or null if operation failed)
1833
+ */
1834
+ private applyOpToMap;
1835
+ /**
1836
+ * Broadcast event to cluster members (excluding self).
1837
+ */
1838
+ private broadcastToCluster;
1839
+ /**
1840
+ * Build OpContext for interceptors.
1841
+ */
1842
+ private buildOpContext;
1843
+ /**
1844
+ * Run onBeforeOp interceptors. Returns modified op or null if dropped.
1845
+ */
1846
+ private runBeforeInterceptors;
1847
+ /**
1848
+ * Run onAfterOp interceptors (fire-and-forget).
1849
+ */
1850
+ private runAfterInterceptors;
224
1851
  private handleLockGranted;
225
1852
  private processLocalOp;
1853
+ /**
1854
+ * === OPTIMIZATION 1: Async Batch Processing with Backpressure ===
1855
+ * Processes validated operations asynchronously after ACK has been sent.
1856
+ * Uses BackpressureRegulator to periodically force sync processing and
1857
+ * prevent unbounded accumulation of async work.
1858
+ */
1859
+ private processBatchAsync;
1860
+ /**
1861
+ * === BACKPRESSURE: Synchronous Batch Processing ===
1862
+ * Processes operations synchronously, waiting for broadcast completion.
1863
+ * Used when backpressure forces sync to drain the pipeline.
1864
+ */
1865
+ private processBatchSync;
1866
+ /**
1867
+ * Forward operation to owner node and wait for completion.
1868
+ * Used in sync processing mode.
1869
+ */
1870
+ private forwardOpAndWait;
1871
+ /**
1872
+ * Process a single operation for batch processing.
1873
+ * Uses shared applyOpToMap but collects events instead of broadcasting immediately.
1874
+ */
1875
+ private processLocalOpForBatch;
226
1876
  private handleClusterEvent;
227
1877
  getMap(name: string, typeHint?: 'LWW' | 'OR'): LWWMap<string, any> | ORMap<string, any>;
228
1878
  /**
@@ -257,6 +1907,38 @@ declare class ServerCoordinator {
257
1907
  private handleGcReport;
258
1908
  private performGarbageCollection;
259
1909
  private buildTLSOptions;
1910
+ /**
1911
+ * Get effective Write Concern level for an operation.
1912
+ * Per-op writeConcern overrides batch-level.
1913
+ */
1914
+ private getEffectiveWriteConcern;
1915
+ /**
1916
+ * Convert string WriteConcern value to enum.
1917
+ */
1918
+ private stringToWriteConcern;
1919
+ /**
1920
+ * Process batch with Write Concern tracking.
1921
+ * Notifies WriteAckManager at each stage of processing.
1922
+ */
1923
+ private processBatchAsyncWithWriteConcern;
1924
+ /**
1925
+ * Synchronous batch processing with Write Concern.
1926
+ */
1927
+ private processBatchSyncWithWriteConcern;
1928
+ /**
1929
+ * Process a single operation with Write Concern level notifications.
1930
+ */
1931
+ private processLocalOpWithWriteConcern;
1932
+ /**
1933
+ * Persist operation synchronously (blocking).
1934
+ * Used for PERSISTED Write Concern.
1935
+ */
1936
+ private persistOpSync;
1937
+ /**
1938
+ * Persist operation asynchronously (fire-and-forget).
1939
+ * Used for non-PERSISTED Write Concern levels.
1940
+ */
1941
+ private persistOpAsync;
260
1942
  }
261
1943
 
262
1944
  interface PostgresAdapterOptions {
@@ -312,6 +1994,98 @@ declare class SecurityManager {
312
1994
  declare const logger: pino.Logger<never, boolean>;
313
1995
  type Logger = typeof logger;
314
1996
 
1997
+ /**
1998
+ * ConnectionRateLimiter - Rate limiter for incoming WebSocket connections.
1999
+ *
2000
+ * Implements connection rate limiting to prevent connection storms and
2001
+ * protect the server from being overwhelmed during high load scenarios.
2002
+ *
2003
+ * Features:
2004
+ * - Limits connections per second to prevent TCP backlog exhaustion
2005
+ * - Tracks pending connections (in-progress handshakes)
2006
+ * - Provides graceful rejection when limits are exceeded
2007
+ * - Auto-resets counters after cooldown period
2008
+ */
2009
+ interface RateLimiterConfig {
2010
+ /** Maximum new connections allowed per second (default: 100) */
2011
+ maxConnectionsPerSecond: number;
2012
+ /** Maximum pending connections waiting for handshake (default: 1000) */
2013
+ maxPendingConnections: number;
2014
+ /** Cooldown period in ms after which counters reset (default: 1000) */
2015
+ cooldownMs: number;
2016
+ }
2017
+ interface RateLimiterStats {
2018
+ /** Current connections per second rate */
2019
+ connectionsPerSecond: number;
2020
+ /** Number of connections currently pending (handshake in progress) */
2021
+ pendingConnections: number;
2022
+ /** Total connections established since start */
2023
+ totalConnections: number;
2024
+ /** Total connections rejected due to rate limiting */
2025
+ totalRejected: number;
2026
+ }
2027
+ declare class ConnectionRateLimiter {
2028
+ private config;
2029
+ /** Connection attempts in current window */
2030
+ private connectionCount;
2031
+ /** Timestamp when current window started */
2032
+ private windowStart;
2033
+ /** Currently pending connections (handshake in progress) */
2034
+ private pendingCount;
2035
+ /** Total connections established since start */
2036
+ private totalConnections;
2037
+ /** Total connections rejected since start */
2038
+ private totalRejected;
2039
+ constructor(config?: Partial<RateLimiterConfig>);
2040
+ /**
2041
+ * Check if a new connection should be accepted.
2042
+ * @returns true if the connection should be accepted, false if it should be rejected
2043
+ */
2044
+ shouldAccept(): boolean;
2045
+ /**
2046
+ * Register a connection attempt.
2047
+ * Call this when a new connection is initiated (before handshake completes).
2048
+ */
2049
+ onConnectionAttempt(): void;
2050
+ /**
2051
+ * Register that a connection has been established (handshake complete).
2052
+ * Call this when the connection is fully established and authenticated.
2053
+ */
2054
+ onConnectionEstablished(): void;
2055
+ /**
2056
+ * Register that a connection has been closed.
2057
+ * Call this when a pending connection is closed before completing handshake.
2058
+ */
2059
+ onConnectionClosed(): void;
2060
+ /**
2061
+ * Register that a connection was rejected.
2062
+ * Call this when shouldAccept() returns false and the connection is rejected.
2063
+ */
2064
+ onConnectionRejected(): void;
2065
+ /**
2066
+ * Decrease pending count when a connection times out or fails.
2067
+ * Call this when a pending connection fails to complete handshake.
2068
+ */
2069
+ onPendingConnectionFailed(): void;
2070
+ /**
2071
+ * Get current rate limiter statistics.
2072
+ */
2073
+ getStats(): RateLimiterStats;
2074
+ /**
2075
+ * Reset the rate limiter state.
2076
+ * Useful for testing or when recovering from errors.
2077
+ */
2078
+ reset(): void;
2079
+ /**
2080
+ * Update configuration at runtime.
2081
+ */
2082
+ updateConfig(config: Partial<RateLimiterConfig>): void;
2083
+ /**
2084
+ * Check if window has expired and reset if needed.
2085
+ */
2086
+ private maybeResetWindow;
2087
+ }
2088
+
315
2089
  declare class TimestampInterceptor implements IInterceptor {
316
2090
  name: string;
317
2091
  onBeforeOp(op: ServerOp, context: OpContext): Promise<ServerOp>;
@@ -330,4 +2104,47 @@ declare class RateLimitInterceptor implements IInterceptor {
330
2104
  onDisconnect(context: any): Promise<void>;
331
2105
  }
332
2106
 
333
- export { type ConnectionContext, type IInterceptor, type IServerStorage, type Logger, MemoryServerAdapter, type ORMapTombstones, type ORMapValue, type OpContext, PostgresAdapter, type PostgresAdapterOptions, RateLimitInterceptor, SecurityManager, ServerCoordinator, type ServerCoordinatorConfig, type ServerOp, type StorageValue, TimestampInterceptor, logger };
2107
+ /**
2108
+ * Native Module Statistics
2109
+ *
2110
+ * Provides information about available native optimizations.
2111
+ *
2112
+ * Phase 3.05: Integration
2113
+ */
2114
+
2115
+ /**
2116
+ * Native module availability status
2117
+ */
2118
+ interface NativeModuleStatus {
2119
+ /** Native xxHash64 is available and being used */
2120
+ nativeHash: boolean;
2121
+ /** SharedArrayBuffer is available */
2122
+ sharedArrayBuffer: boolean;
2123
+ }
2124
+ /**
2125
+ * Comprehensive native statistics
2126
+ */
2127
+ interface NativeStats {
2128
+ /** Module availability status */
2129
+ modules: NativeModuleStatus;
2130
+ /** Shared memory statistics (if enabled) */
2131
+ sharedMemory: SharedMemoryStats | null;
2132
+ /** Summary of what's being used */
2133
+ summary: string;
2134
+ }
2135
+ /**
2136
+ * Check which native modules are available.
2137
+ */
2138
+ declare function getNativeModuleStatus(): NativeModuleStatus;
2139
+ /**
2140
+ * Get native statistics including shared memory.
2141
+ *
2142
+ * @param sharedMemoryManager - Optional SharedMemoryManager instance
2143
+ */
2144
+ declare function getNativeStats(sharedMemoryManager?: SharedMemoryManager): NativeStats;
2145
+ /**
2146
+ * Log native module status to console.
2147
+ */
2148
+ declare function logNativeStatus(): void;
2149
+
2150
+ export { BufferPool, type BufferPoolConfig, type BufferPoolStats, type CoalescingPreset, type CoalescingWriterMetrics, type CoalescingWriterOptions, type ConnectionContext, ConnectionRateLimiter, FilterTasklet, ForEachTasklet, type IInterceptor, type IServerStorage, IteratorTasklet, type IteratorTaskletConfig, type Logger, MapTasklet, MemoryServerAdapter, type NativeModuleStatus, type NativeStats, type ORMapTombstones, type ORMapValue, ObjectPool, type ObjectPoolConfig, type ObjectPoolStats, type OpContext, type PooledEventPayload, type PooledMessage, type PooledRecord, type PooledTimestamp, PostgresAdapter, type PostgresAdapterOptions, type ProgressState, RateLimitInterceptor, type RateLimiterConfig, type RateLimiterStats, ReduceTasklet, SecurityManager, ServerCoordinator, type ServerCoordinatorConfig, type ServerOp, type StorageValue, type Tasklet, TaskletScheduler, type TaskletSchedulerConfig, type TaskletSchedulerStats, TimestampInterceptor, coalescingPresets, createEventPayloadPool, createMessagePool, createRecordPool, createTimestampPool, getCoalescingPreset, getGlobalBufferPool, getGlobalEventPayloadPool, getGlobalMessagePool, getGlobalRecordPool, getGlobalTimestampPool, getNativeModuleStatus, getNativeStats, logNativeStatus, logger, setGlobalBufferPool, setGlobalEventPayloadPool, setGlobalMessagePool, setGlobalRecordPool, setGlobalTimestampPool };