@topgunbuild/server 0.8.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -9,13 +9,13 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
9
9
  import { createServer as createHttpServer } from "http";
10
10
  import { createServer as createHttpsServer } from "https";
11
11
  import { readFileSync as readFileSync2 } from "fs";
12
- import { WebSocketServer as WebSocketServer2, WebSocket as WebSocket3 } from "ws";
13
- import { HLC as HLC2, LWWMap as LWWMap3, ORMap as ORMap2, serialize as serialize4, deserialize, MessageSchema, WriteConcern as WriteConcern2, ConsistencyLevel as ConsistencyLevel3, DEFAULT_REPLICATION_CONFIG as DEFAULT_REPLICATION_CONFIG2, IndexedLWWMap as IndexedLWWMap2, IndexedORMap as IndexedORMap2 } from "@topgunbuild/core";
12
+ import { WebSocketServer as WebSocketServer2, WebSocket as WebSocket5 } from "ws";
13
+ import { HLC as HLC2, LWWMap as LWWMap3, ORMap as ORMap2, serialize as serialize4, deserialize, MessageSchema, WriteConcern as WriteConcern2, ConsistencyLevel as ConsistencyLevel3, DEFAULT_REPLICATION_CONFIG as DEFAULT_REPLICATION_CONFIG2, IndexedLWWMap as IndexedLWWMap2, IndexedORMap as IndexedORMap2, QueryCursor as QueryCursor2, DEFAULT_QUERY_CURSOR_MAX_AGE_MS as DEFAULT_QUERY_CURSOR_MAX_AGE_MS2 } from "@topgunbuild/core";
14
14
  import * as jwt from "jsonwebtoken";
15
15
  import * as crypto from "crypto";
16
16
 
17
17
  // src/query/Matcher.ts
18
- import { evaluatePredicate } from "@topgunbuild/core";
18
+ import { evaluatePredicate, QueryCursor, DEFAULT_QUERY_CURSOR_MAX_AGE_MS } from "@topgunbuild/core";
19
19
  function matchesQuery(record, query) {
20
20
  const data = record.value;
21
21
  if (!data) return false;
@@ -64,6 +64,10 @@ function matchesQuery(record, query) {
64
64
  return true;
65
65
  }
66
66
  function executeQuery(records, query) {
67
+ const result = executeQueryWithCursor(records, query);
68
+ return result.results;
69
+ }
70
+ function executeQueryWithCursor(records, query) {
67
71
  if (!query) {
68
72
  query = {};
69
73
  }
@@ -81,9 +85,13 @@ function executeQuery(records, query) {
81
85
  }
82
86
  }
83
87
  }
84
- if (query.sort) {
88
+ const sort = query.sort || {};
89
+ const sortEntries = Object.entries(sort);
90
+ const sortField = sortEntries.length > 0 ? sortEntries[0][0] : "_key";
91
+ const sortDirection = sortEntries.length > 0 ? sortEntries[0][1] : "asc";
92
+ if (sortEntries.length > 0) {
85
93
  results.sort((a, b) => {
86
- for (const [field, direction] of Object.entries(query.sort)) {
94
+ for (const [field, direction] of sortEntries) {
87
95
  const valA = a.record.value[field];
88
96
  const valB = b.record.value[field];
89
97
  if (valA < valB) return direction === "asc" ? -1 : 1;
@@ -92,16 +100,55 @@ function executeQuery(records, query) {
92
100
  return 0;
93
101
  });
94
102
  }
95
- if (query.offset || query.limit) {
96
- const offset = query.offset || 0;
97
- const limit = query.limit || results.length;
98
- results = results.slice(offset, offset + limit);
103
+ let cursorStatus = "none";
104
+ if (query.cursor) {
105
+ const cursorData = QueryCursor.decode(query.cursor);
106
+ if (!cursorData) {
107
+ cursorStatus = "invalid";
108
+ } else if (!QueryCursor.isValid(cursorData, query.predicate ?? query.where, sort)) {
109
+ if (Date.now() - cursorData.timestamp > DEFAULT_QUERY_CURSOR_MAX_AGE_MS) {
110
+ cursorStatus = "expired";
111
+ } else {
112
+ cursorStatus = "invalid";
113
+ }
114
+ } else {
115
+ cursorStatus = "valid";
116
+ results = results.filter((r) => {
117
+ const sortValue = r.record.value[sortField];
118
+ return QueryCursor.isAfterCursor(
119
+ { key: r.key, sortValue },
120
+ cursorData
121
+ );
122
+ });
123
+ }
124
+ }
125
+ const hasLimit = query.limit !== void 0 && query.limit > 0;
126
+ const totalBeforeLimit = results.length;
127
+ if (hasLimit) {
128
+ results = results.slice(0, query.limit);
129
+ }
130
+ const hasMore = hasLimit && totalBeforeLimit > query.limit;
131
+ let nextCursor;
132
+ if (hasMore && results.length > 0) {
133
+ const lastResult = results[results.length - 1];
134
+ const sortValue = lastResult.record.value[sortField];
135
+ nextCursor = QueryCursor.fromLastResult(
136
+ { key: lastResult.key, sortValue },
137
+ sort,
138
+ query.predicate ?? query.where
139
+ );
99
140
  }
100
- return results.map((r) => ({ key: r.key, value: r.record.value }));
141
+ return {
142
+ results: results.map((r) => ({ key: r.key, value: r.record.value })),
143
+ nextCursor,
144
+ hasMore,
145
+ cursorStatus
146
+ };
101
147
  }
102
148
 
103
149
  // src/query/QueryRegistry.ts
104
150
  import { serialize, IndexedLWWMap } from "@topgunbuild/core";
151
+ import { WebSocket } from "ws";
105
152
 
106
153
  // src/utils/logger.ts
107
154
  import pino from "pino";
@@ -290,6 +337,106 @@ var QueryRegistry = class {
290
337
  }
291
338
  }
292
339
  }
340
+ /**
341
+ * Set the ClusterManager for distributed subscriptions.
342
+ */
343
+ setClusterManager(clusterManager, nodeId) {
344
+ this.clusterManager = clusterManager;
345
+ this.nodeId = nodeId;
346
+ }
347
+ /**
348
+ * Set the callback for getting maps by name.
349
+ * Required for distributed subscriptions to return initial results.
350
+ */
351
+ setMapGetter(getter) {
352
+ this.getMap = getter;
353
+ }
354
+ /**
355
+ * Register a distributed subscription from a remote coordinator.
356
+ * Called when receiving CLUSTER_SUB_REGISTER message.
357
+ *
358
+ * @param subscriptionId - Unique subscription ID
359
+ * @param mapName - Map name to query
360
+ * @param query - Query predicate
361
+ * @param coordinatorNodeId - Node ID of the coordinator (receives updates)
362
+ * @returns Initial query results from this node
363
+ */
364
+ registerDistributed(subscriptionId, mapName, query, coordinatorNodeId) {
365
+ const dummySocket = {
366
+ readyState: 1,
367
+ send: () => {
368
+ }
369
+ // Updates go via cluster messages, not socket
370
+ };
371
+ let initialResults = [];
372
+ const previousResultKeys = /* @__PURE__ */ new Set();
373
+ if (this.getMap) {
374
+ const map = this.getMap(mapName);
375
+ if (map) {
376
+ const records = this.getMapRecords(map);
377
+ const queryResults = executeQuery(records, query);
378
+ initialResults = queryResults.map((r) => {
379
+ previousResultKeys.add(r.key);
380
+ return { key: r.key, value: r.value };
381
+ });
382
+ }
383
+ }
384
+ const sub = {
385
+ id: subscriptionId,
386
+ clientId: `cluster:${coordinatorNodeId}`,
387
+ mapName,
388
+ query,
389
+ socket: dummySocket,
390
+ previousResultKeys,
391
+ coordinatorNodeId,
392
+ isDistributed: true
393
+ };
394
+ this.register(sub);
395
+ logger.debug(
396
+ { subscriptionId, mapName, coordinatorNodeId, resultCount: initialResults.length },
397
+ "Distributed query subscription registered"
398
+ );
399
+ return initialResults;
400
+ }
401
+ /**
402
+ * Get a distributed subscription by ID.
403
+ * Returns undefined if not found or not distributed.
404
+ */
405
+ getDistributedSubscription(subscriptionId) {
406
+ for (const subs of this.subscriptions.values()) {
407
+ for (const sub of subs) {
408
+ if (sub.id === subscriptionId && sub.isDistributed) {
409
+ return sub;
410
+ }
411
+ }
412
+ }
413
+ return void 0;
414
+ }
415
+ /**
416
+ * Unregister all distributed subscriptions where the given node was the coordinator.
417
+ * Called when a cluster node disconnects.
418
+ *
419
+ * @param coordinatorNodeId - Node ID of the disconnected coordinator
420
+ */
421
+ unregisterByCoordinator(coordinatorNodeId) {
422
+ const subscriptionsToRemove = [];
423
+ for (const subs of this.subscriptions.values()) {
424
+ for (const sub of subs) {
425
+ if (sub.isDistributed && sub.coordinatorNodeId === coordinatorNodeId) {
426
+ subscriptionsToRemove.push(sub.id);
427
+ }
428
+ }
429
+ }
430
+ for (const subId of subscriptionsToRemove) {
431
+ this.unregister(subId);
432
+ }
433
+ if (subscriptionsToRemove.length > 0) {
434
+ logger.debug(
435
+ { coordinatorNodeId, count: subscriptionsToRemove.length },
436
+ "Cleaned up distributed query subscriptions for disconnected coordinator"
437
+ );
438
+ }
439
+ }
293
440
  /**
294
441
  * Returns all active subscriptions for a specific map.
295
442
  * Used for subscription-based event routing to avoid broadcasting to all clients.
@@ -572,7 +719,9 @@ var QueryRegistry = class {
572
719
  return record.value;
573
720
  }
574
721
  sendUpdate(sub, key, value, type) {
575
- if (sub.socket.readyState === 1) {
722
+ if (sub.isDistributed && sub.coordinatorNodeId && this.clusterManager) {
723
+ this.sendDistributedUpdate(sub, key, value, type);
724
+ } else if (sub.socket.readyState === WebSocket.OPEN) {
576
725
  sub.socket.send(serialize({
577
726
  type: "QUERY_UPDATE",
578
727
  payload: {
@@ -584,6 +733,26 @@ var QueryRegistry = class {
584
733
  }));
585
734
  }
586
735
  }
736
+ /**
737
+ * Send update to remote coordinator node for a distributed subscription.
738
+ */
739
+ sendDistributedUpdate(sub, key, value, type) {
740
+ if (!this.clusterManager || !sub.coordinatorNodeId) return;
741
+ const changeType = type === "UPDATE" ? sub.previousResultKeys.has(key) ? "UPDATE" : "ENTER" : "LEAVE";
742
+ const payload = {
743
+ subscriptionId: sub.id,
744
+ sourceNodeId: this.nodeId || "unknown",
745
+ key,
746
+ value,
747
+ changeType,
748
+ timestamp: Date.now()
749
+ };
750
+ this.clusterManager.send(sub.coordinatorNodeId, "CLUSTER_SUB_UPDATE", payload);
751
+ logger.debug(
752
+ { subscriptionId: sub.id, key, changeType, coordinator: sub.coordinatorNodeId },
753
+ "Sent distributed query update"
754
+ );
755
+ }
587
756
  analyzeQueryFields(query) {
588
757
  const fields = /* @__PURE__ */ new Set();
589
758
  try {
@@ -722,7 +891,7 @@ var TopicManager = class {
722
891
  };
723
892
 
724
893
  // src/cluster/ClusterManager.ts
725
- import { WebSocket, WebSocketServer } from "ws";
894
+ import { WebSocket as WebSocket2, WebSocketServer } from "ws";
726
895
  import { EventEmitter as EventEmitter2 } from "events";
727
896
  import * as dns from "dns";
728
897
  import { readFileSync } from "fs";
@@ -1115,7 +1284,7 @@ var ClusterManager = class extends EventEmitter2 {
1115
1284
  sendHeartbeatToAll() {
1116
1285
  for (const [nodeId, member] of this.members) {
1117
1286
  if (member.isSelf) continue;
1118
- if (member.socket && member.socket.readyState === WebSocket.OPEN) {
1287
+ if (member.socket && member.socket.readyState === WebSocket2.OPEN) {
1119
1288
  this.send(nodeId, "HEARTBEAT", { timestamp: Date.now() });
1120
1289
  }
1121
1290
  }
@@ -1149,7 +1318,7 @@ var ClusterManager = class extends EventEmitter2 {
1149
1318
  broadcastMemberList() {
1150
1319
  for (const [nodeId, member] of this.members) {
1151
1320
  if (member.isSelf) continue;
1152
- if (member.socket && member.socket.readyState === WebSocket.OPEN) {
1321
+ if (member.socket && member.socket.readyState === WebSocket2.OPEN) {
1153
1322
  this.sendMemberList(nodeId);
1154
1323
  }
1155
1324
  }
@@ -1174,7 +1343,7 @@ var ClusterManager = class extends EventEmitter2 {
1174
1343
  const member = this.members.get(nodeId);
1175
1344
  if (!member) return;
1176
1345
  logger.warn({ nodeId }, "Removing failed node from cluster");
1177
- if (member.socket && member.socket.readyState !== WebSocket.CLOSED) {
1346
+ if (member.socket && member.socket.readyState !== WebSocket2.CLOSED) {
1178
1347
  try {
1179
1348
  member.socket.terminate();
1180
1349
  } catch (e) {
@@ -1248,9 +1417,9 @@ var ClusterManager = class extends EventEmitter2 {
1248
1417
  if (this.config.tls.caCertPath) {
1249
1418
  wsOptions.ca = readFileSync(this.config.tls.caCertPath);
1250
1419
  }
1251
- ws = new WebSocket(`${protocol}${peerAddress}`, wsOptions);
1420
+ ws = new WebSocket2(`${protocol}${peerAddress}`, wsOptions);
1252
1421
  } else {
1253
- ws = new WebSocket(`ws://${peerAddress}`);
1422
+ ws = new WebSocket2(`ws://${peerAddress}`);
1254
1423
  }
1255
1424
  ws.on("open", () => {
1256
1425
  this.pendingConnections.delete(peerAddress);
@@ -1337,7 +1506,7 @@ var ClusterManager = class extends EventEmitter2 {
1337
1506
  }
1338
1507
  send(nodeId, type, payload) {
1339
1508
  const member = this.members.get(nodeId);
1340
- if (member && member.socket && member.socket.readyState === WebSocket.OPEN) {
1509
+ if (member && member.socket && member.socket.readyState === WebSocket2.OPEN) {
1341
1510
  const msg = {
1342
1511
  type,
1343
1512
  senderId: this.config.nodeId,
@@ -2452,7 +2621,7 @@ var SecurityManager = class {
2452
2621
  };
2453
2622
 
2454
2623
  // src/monitoring/MetricsService.ts
2455
- import { Registry, Gauge, Counter, Summary, collectDefaultMetrics } from "prom-client";
2624
+ import { Registry, Gauge, Counter, Summary, Histogram, collectDefaultMetrics } from "prom-client";
2456
2625
  var MetricsService = class {
2457
2626
  constructor() {
2458
2627
  this.registry = new Registry();
@@ -2564,6 +2733,90 @@ var MetricsService = class {
2564
2733
  help: "Current connection rate per second",
2565
2734
  registers: [this.registry]
2566
2735
  });
2736
+ this.distributedSearchTotal = new Counter({
2737
+ name: "topgun_distributed_search_total",
2738
+ help: "Total number of distributed search requests",
2739
+ labelNames: ["map", "status"],
2740
+ registers: [this.registry]
2741
+ });
2742
+ this.distributedSearchDuration = new Summary({
2743
+ name: "topgun_distributed_search_duration_ms",
2744
+ help: "Distribution of distributed search execution times in milliseconds",
2745
+ labelNames: ["map"],
2746
+ percentiles: [0.5, 0.9, 0.95, 0.99],
2747
+ registers: [this.registry]
2748
+ });
2749
+ this.distributedSearchFailedNodes = new Counter({
2750
+ name: "topgun_distributed_search_failed_nodes_total",
2751
+ help: "Total number of node failures during distributed search",
2752
+ registers: [this.registry]
2753
+ });
2754
+ this.distributedSearchPartialResults = new Counter({
2755
+ name: "topgun_distributed_search_partial_results_total",
2756
+ help: "Total number of searches that returned partial results due to node failures",
2757
+ registers: [this.registry]
2758
+ });
2759
+ this.distributedSubTotal = new Counter({
2760
+ name: "topgun_distributed_sub_total",
2761
+ help: "Total distributed subscriptions created",
2762
+ labelNames: ["type", "status"],
2763
+ registers: [this.registry]
2764
+ });
2765
+ this.distributedSubUnsubscribeTotal = new Counter({
2766
+ name: "topgun_distributed_sub_unsubscribe_total",
2767
+ help: "Total unsubscriptions from distributed subscriptions",
2768
+ labelNames: ["type"],
2769
+ registers: [this.registry]
2770
+ });
2771
+ this.distributedSubActive = new Gauge({
2772
+ name: "topgun_distributed_sub_active",
2773
+ help: "Currently active distributed subscriptions",
2774
+ labelNames: ["type"],
2775
+ registers: [this.registry]
2776
+ });
2777
+ this.distributedSubPendingAcks = new Gauge({
2778
+ name: "topgun_distributed_sub_pending_acks",
2779
+ help: "Subscriptions waiting for ACKs from cluster nodes",
2780
+ registers: [this.registry]
2781
+ });
2782
+ this.distributedSubUpdates = new Counter({
2783
+ name: "topgun_distributed_sub_updates_total",
2784
+ help: "Delta updates processed for distributed subscriptions",
2785
+ labelNames: ["direction", "change_type"],
2786
+ registers: [this.registry]
2787
+ });
2788
+ this.distributedSubAckTotal = new Counter({
2789
+ name: "topgun_distributed_sub_ack_total",
2790
+ help: "Node ACK responses for distributed subscriptions",
2791
+ labelNames: ["status"],
2792
+ registers: [this.registry]
2793
+ });
2794
+ this.distributedSubNodeDisconnect = new Counter({
2795
+ name: "topgun_distributed_sub_node_disconnect_total",
2796
+ help: "Node disconnects affecting distributed subscriptions",
2797
+ registers: [this.registry]
2798
+ });
2799
+ this.distributedSubRegistrationDuration = new Histogram({
2800
+ name: "topgun_distributed_sub_registration_duration_ms",
2801
+ help: "Time to register subscription on all nodes",
2802
+ labelNames: ["type"],
2803
+ buckets: [10, 50, 100, 250, 500, 1e3, 2500],
2804
+ registers: [this.registry]
2805
+ });
2806
+ this.distributedSubUpdateLatency = new Histogram({
2807
+ name: "topgun_distributed_sub_update_latency_ms",
2808
+ help: "Latency from data change to client notification",
2809
+ labelNames: ["type"],
2810
+ buckets: [1, 5, 10, 25, 50, 100, 250],
2811
+ registers: [this.registry]
2812
+ });
2813
+ this.distributedSubInitialResultsCount = new Histogram({
2814
+ name: "topgun_distributed_sub_initial_results_count",
2815
+ help: "Initial result set size for distributed subscriptions",
2816
+ labelNames: ["type"],
2817
+ buckets: [0, 1, 5, 10, 25, 50, 100],
2818
+ registers: [this.registry]
2819
+ });
2567
2820
  }
2568
2821
  destroy() {
2569
2822
  this.registry.clear();
@@ -2674,6 +2927,115 @@ var MetricsService = class {
2674
2927
  setConnectionRatePerSecond(rate) {
2675
2928
  this.connectionRatePerSecond.set(rate);
2676
2929
  }
2930
+ // === Distributed search metric methods (Phase 14) ===
2931
+ /**
2932
+ * Record a distributed search request.
2933
+ * @param mapName - Name of the map being searched
2934
+ * @param status - 'success', 'partial', or 'error'
2935
+ */
2936
+ incDistributedSearch(mapName, status) {
2937
+ this.distributedSearchTotal.inc({ map: mapName, status });
2938
+ }
2939
+ /**
2940
+ * Record the duration of a distributed search.
2941
+ * @param mapName - Name of the map being searched
2942
+ * @param durationMs - Duration in milliseconds
2943
+ */
2944
+ recordDistributedSearchDuration(mapName, durationMs) {
2945
+ this.distributedSearchDuration.observe({ map: mapName }, durationMs);
2946
+ }
2947
+ /**
2948
+ * Increment counter for failed nodes during distributed search.
2949
+ * @param count - Number of nodes that failed (default 1)
2950
+ */
2951
+ incDistributedSearchFailedNodes(count = 1) {
2952
+ this.distributedSearchFailedNodes.inc(count);
2953
+ }
2954
+ /**
2955
+ * Increment counter for searches returning partial results.
2956
+ */
2957
+ incDistributedSearchPartialResults() {
2958
+ this.distributedSearchPartialResults.inc();
2959
+ }
2960
+ // === Distributed subscription metric methods (Phase 14.2) ===
2961
+ /**
2962
+ * Record a distributed subscription creation.
2963
+ * @param type - Subscription type (SEARCH or QUERY)
2964
+ * @param status - Result status (success, failed, timeout)
2965
+ */
2966
+ incDistributedSub(type, status) {
2967
+ this.distributedSubTotal.inc({ type, status });
2968
+ if (status === "success") {
2969
+ this.distributedSubActive.inc({ type });
2970
+ }
2971
+ }
2972
+ /**
2973
+ * Record a distributed subscription unsubscription.
2974
+ * @param type - Subscription type (SEARCH or QUERY)
2975
+ */
2976
+ incDistributedSubUnsubscribe(type) {
2977
+ this.distributedSubUnsubscribeTotal.inc({ type });
2978
+ }
2979
+ /**
2980
+ * Decrement the active distributed subscriptions gauge.
2981
+ * @param type - Subscription type (SEARCH or QUERY)
2982
+ */
2983
+ decDistributedSubActive(type) {
2984
+ this.distributedSubActive.dec({ type });
2985
+ }
2986
+ /**
2987
+ * Set the number of subscriptions waiting for ACKs.
2988
+ * @param count - Number of pending ACKs
2989
+ */
2990
+ setDistributedSubPendingAcks(count) {
2991
+ this.distributedSubPendingAcks.set(count);
2992
+ }
2993
+ /**
2994
+ * Record a delta update for distributed subscriptions.
2995
+ * @param direction - Direction of update (sent or received)
2996
+ * @param changeType - Type of change (ENTER, UPDATE, LEAVE)
2997
+ */
2998
+ incDistributedSubUpdates(direction, changeType) {
2999
+ this.distributedSubUpdates.inc({ direction, change_type: changeType });
3000
+ }
3001
+ /**
3002
+ * Record node ACK responses.
3003
+ * @param status - ACK status (success, failed, timeout)
3004
+ * @param count - Number of ACKs to record (default 1)
3005
+ */
3006
+ incDistributedSubAck(status, count = 1) {
3007
+ this.distributedSubAckTotal.inc({ status }, count);
3008
+ }
3009
+ /**
3010
+ * Record a node disconnect affecting distributed subscriptions.
3011
+ */
3012
+ incDistributedSubNodeDisconnect() {
3013
+ this.distributedSubNodeDisconnect.inc();
3014
+ }
3015
+ /**
3016
+ * Record the time to register a subscription on all nodes.
3017
+ * @param type - Subscription type (SEARCH or QUERY)
3018
+ * @param durationMs - Duration in milliseconds
3019
+ */
3020
+ recordDistributedSubRegistration(type, durationMs) {
3021
+ this.distributedSubRegistrationDuration.observe({ type }, durationMs);
3022
+ }
3023
+ /**
3024
+ * Record the latency from data change to client notification.
3025
+ * @param type - Subscription type (SEARCH or QUERY)
3026
+ * @param latencyMs - Latency in milliseconds
3027
+ */
3028
+ recordDistributedSubUpdateLatency(type, latencyMs) {
3029
+ this.distributedSubUpdateLatency.observe({ type }, latencyMs);
3030
+ }
3031
+ /**
3032
+ * Record the initial result set size for a subscription.
3033
+ * @param type - Subscription type (SEARCH or QUERY)
3034
+ * @param count - Number of initial results
3035
+ */
3036
+ recordDistributedSubInitialResultsCount(type, count) {
3037
+ this.distributedSubInitialResultsCount.observe({ type }, count);
3038
+ }
2677
3039
  async getMetrics() {
2678
3040
  return this.registry.metrics();
2679
3041
  }
@@ -3204,7 +3566,7 @@ var BackpressureRegulator = class {
3204
3566
  };
3205
3567
 
3206
3568
  // src/utils/CoalescingWriter.ts
3207
- import { WebSocket as WebSocket2 } from "ws";
3569
+ import { WebSocket as WebSocket3 } from "ws";
3208
3570
  import { serialize as serialize3 } from "@topgunbuild/core";
3209
3571
 
3210
3572
  // src/memory/BufferPool.ts
@@ -3774,7 +4136,7 @@ var CoalescingWriter = class {
3774
4136
  }
3775
4137
  this.state = 2 /* FLUSHING */;
3776
4138
  try {
3777
- if (this.socket.readyState !== WebSocket2.OPEN) {
4139
+ if (this.socket.readyState !== WebSocket3.OPEN) {
3778
4140
  this.queue = [];
3779
4141
  this.pendingBytes = 0;
3780
4142
  this.state = 0 /* IDLE */;
@@ -3847,7 +4209,7 @@ var CoalescingWriter = class {
3847
4209
  * Send a message immediately without batching.
3848
4210
  */
3849
4211
  sendImmediate(data) {
3850
- if (this.socket.readyState !== WebSocket2.OPEN) {
4212
+ if (this.socket.readyState !== WebSocket3.OPEN) {
3851
4213
  return;
3852
4214
  }
3853
4215
  try {
@@ -7437,7 +7799,7 @@ var RepairScheduler = class extends EventEmitter12 {
7437
7799
  this.processTimer = setInterval(() => {
7438
7800
  this.processRepairQueue();
7439
7801
  }, 1e3);
7440
- setTimeout(() => {
7802
+ this.initialScanTimer = setTimeout(() => {
7441
7803
  this.scheduleFullScan();
7442
7804
  }, 6e4);
7443
7805
  }
@@ -7455,9 +7817,13 @@ var RepairScheduler = class extends EventEmitter12 {
7455
7817
  clearInterval(this.processTimer);
7456
7818
  this.processTimer = void 0;
7457
7819
  }
7820
+ if (this.initialScanTimer) {
7821
+ clearTimeout(this.initialScanTimer);
7822
+ this.initialScanTimer = void 0;
7823
+ }
7458
7824
  this.repairQueue = [];
7459
7825
  this.activeRepairs.clear();
7460
- for (const [id, req] of this.pendingRequests) {
7826
+ for (const [, req] of this.pendingRequests) {
7461
7827
  clearTimeout(req.timer);
7462
7828
  req.reject(new Error("Scheduler stopped"));
7463
7829
  }
@@ -9188,11 +9554,13 @@ var EventJournalService = class extends EventJournalImpl {
9188
9554
  };
9189
9555
 
9190
9556
  // src/search/SearchCoordinator.ts
9557
+ import { EventEmitter as EventEmitter13 } from "events";
9191
9558
  import {
9192
9559
  FullTextIndex
9193
9560
  } from "@topgunbuild/core";
9194
- var SearchCoordinator = class {
9561
+ var SearchCoordinator = class extends EventEmitter13 {
9195
9562
  constructor() {
9563
+ super();
9196
9564
  /** Map name → FullTextIndex */
9197
9565
  this.indexes = /* @__PURE__ */ new Map();
9198
9566
  /** Map name → FullTextIndexConfig (for reference) */
@@ -9217,6 +9585,13 @@ var SearchCoordinator = class {
9217
9585
  this.BATCH_INTERVAL = 16;
9218
9586
  logger.debug("SearchCoordinator initialized");
9219
9587
  }
9588
+ /**
9589
+ * Set the node ID for this server.
9590
+ * Required for distributed subscriptions.
9591
+ */
9592
+ setNodeId(nodeId) {
9593
+ this.nodeId = nodeId;
9594
+ }
9220
9595
  /**
9221
9596
  * Set the callback for sending updates to clients.
9222
9597
  * Called by ServerCoordinator during initialization.
@@ -9518,9 +9893,112 @@ var SearchCoordinator = class {
9518
9893
  getSubscriptionCount() {
9519
9894
  return this.subscriptions.size;
9520
9895
  }
9896
+ // ============================================
9897
+ // Phase 14.2: Distributed Subscription Methods
9898
+ // ============================================
9899
+ /**
9900
+ * Register a distributed subscription from a remote coordinator.
9901
+ * Called when receiving CLUSTER_SUB_REGISTER message.
9902
+ *
9903
+ * @param subscriptionId - Unique subscription ID
9904
+ * @param mapName - Map name to search
9905
+ * @param query - Search query string
9906
+ * @param options - Search options
9907
+ * @param coordinatorNodeId - Node ID of the coordinator (receives updates)
9908
+ * @returns Initial search results from this node
9909
+ */
9910
+ registerDistributedSubscription(subscriptionId, mapName, query, options, coordinatorNodeId) {
9911
+ const index = this.indexes.get(mapName);
9912
+ if (!index) {
9913
+ logger.warn({ mapName }, "Distributed subscription for map without FTS enabled");
9914
+ return { results: [], totalHits: 0 };
9915
+ }
9916
+ const queryTerms = index.tokenizeQuery(query);
9917
+ const searchResults = index.search(query, options);
9918
+ const currentResults = /* @__PURE__ */ new Map();
9919
+ const results = [];
9920
+ for (const result of searchResults) {
9921
+ const value = this.getDocumentValue ? this.getDocumentValue(mapName, result.docId) : void 0;
9922
+ currentResults.set(result.docId, {
9923
+ score: result.score,
9924
+ matchedTerms: result.matchedTerms || []
9925
+ });
9926
+ results.push({
9927
+ key: result.docId,
9928
+ value,
9929
+ score: result.score,
9930
+ matchedTerms: result.matchedTerms || []
9931
+ });
9932
+ }
9933
+ const subscription = {
9934
+ id: subscriptionId,
9935
+ clientId: `cluster:${coordinatorNodeId}`,
9936
+ // Virtual client ID
9937
+ mapName,
9938
+ query,
9939
+ queryTerms,
9940
+ options: options || {},
9941
+ currentResults,
9942
+ // Distributed fields
9943
+ coordinatorNodeId,
9944
+ isDistributed: true
9945
+ };
9946
+ this.subscriptions.set(subscriptionId, subscription);
9947
+ if (!this.subscriptionsByMap.has(mapName)) {
9948
+ this.subscriptionsByMap.set(mapName, /* @__PURE__ */ new Set());
9949
+ }
9950
+ this.subscriptionsByMap.get(mapName).add(subscriptionId);
9951
+ if (!this.subscriptionsByClient.has(subscription.clientId)) {
9952
+ this.subscriptionsByClient.set(subscription.clientId, /* @__PURE__ */ new Set());
9953
+ }
9954
+ this.subscriptionsByClient.get(subscription.clientId).add(subscriptionId);
9955
+ logger.debug(
9956
+ { subscriptionId, mapName, query, coordinatorNodeId, resultCount: results.length },
9957
+ "Distributed search subscription registered"
9958
+ );
9959
+ return {
9960
+ results,
9961
+ totalHits: results.length
9962
+ };
9963
+ }
9964
+ /**
9965
+ * Get a distributed subscription by ID.
9966
+ * Returns undefined if not found or not distributed.
9967
+ */
9968
+ getDistributedSubscription(subscriptionId) {
9969
+ const sub = this.subscriptions.get(subscriptionId);
9970
+ if (sub?.isDistributed) {
9971
+ return sub;
9972
+ }
9973
+ return void 0;
9974
+ }
9975
+ /**
9976
+ * Unsubscribe all distributed subscriptions where the given node was the coordinator.
9977
+ * Called when a cluster node disconnects.
9978
+ *
9979
+ * @param coordinatorNodeId - Node ID of the disconnected coordinator
9980
+ */
9981
+ unsubscribeByCoordinator(coordinatorNodeId) {
9982
+ const subscriptionsToRemove = [];
9983
+ for (const [subId, sub] of this.subscriptions) {
9984
+ if (sub.isDistributed && sub.coordinatorNodeId === coordinatorNodeId) {
9985
+ subscriptionsToRemove.push(subId);
9986
+ }
9987
+ }
9988
+ for (const subId of subscriptionsToRemove) {
9989
+ this.unsubscribe(subId);
9990
+ }
9991
+ if (subscriptionsToRemove.length > 0) {
9992
+ logger.debug(
9993
+ { coordinatorNodeId, count: subscriptionsToRemove.length },
9994
+ "Cleaned up distributed subscriptions for disconnected coordinator"
9995
+ );
9996
+ }
9997
+ }
9521
9998
  /**
9522
9999
  * Notify subscribers about a document change.
9523
10000
  * Computes delta (ENTER/UPDATE/LEAVE) for each affected subscription.
10001
+ * For distributed subscriptions, emits 'distributedUpdate' event instead of sending to client.
9524
10002
  *
9525
10003
  * @param mapName - Name of the map that changed
9526
10004
  * @param key - Document key that changed
@@ -9528,9 +10006,6 @@ var SearchCoordinator = class {
9528
10006
  * @param changeType - Type of change
9529
10007
  */
9530
10008
  notifySubscribers(mapName, key, value, changeType) {
9531
- if (!this.sendUpdate) {
9532
- return;
9533
- }
9534
10009
  const subscriptionIds = this.subscriptionsByMap.get(mapName);
9535
10010
  if (!subscriptionIds || subscriptionIds.size === 0) {
9536
10011
  return;
@@ -9571,18 +10046,50 @@ var SearchCoordinator = class {
9571
10046
  }
9572
10047
  logger.debug({ subId, key, wasInResults, isInResults, updateType, newScore }, "Update decision");
9573
10048
  if (updateType) {
9574
- this.sendUpdate(
9575
- sub.clientId,
9576
- subId,
9577
- key,
9578
- value,
9579
- newScore,
9580
- matchedTerms,
9581
- updateType
9582
- );
10049
+ if (sub.isDistributed && sub.coordinatorNodeId) {
10050
+ this.sendDistributedUpdate(sub, key, value, newScore, matchedTerms, updateType);
10051
+ } else if (this.sendUpdate) {
10052
+ this.sendUpdate(
10053
+ sub.clientId,
10054
+ subId,
10055
+ key,
10056
+ value,
10057
+ newScore,
10058
+ matchedTerms,
10059
+ updateType
10060
+ );
10061
+ }
9583
10062
  }
9584
10063
  }
9585
10064
  }
10065
+ /**
10066
+ * Send update to remote coordinator node for a distributed subscription.
10067
+ * Emits 'distributedUpdate' event with ClusterSubUpdatePayload.
10068
+ *
10069
+ * @param sub - The distributed subscription
10070
+ * @param key - Document key
10071
+ * @param value - Document value
10072
+ * @param score - Search score
10073
+ * @param matchedTerms - Matched search terms
10074
+ * @param changeType - Change type (ENTER/UPDATE/LEAVE)
10075
+ */
10076
+ sendDistributedUpdate(sub, key, value, score, matchedTerms, changeType) {
10077
+ const payload = {
10078
+ subscriptionId: sub.id,
10079
+ sourceNodeId: this.nodeId || "unknown",
10080
+ key,
10081
+ value,
10082
+ score,
10083
+ matchedTerms,
10084
+ changeType,
10085
+ timestamp: Date.now()
10086
+ };
10087
+ this.emit("distributedUpdate", payload);
10088
+ logger.debug(
10089
+ { subscriptionId: sub.id, key, changeType, coordinator: sub.coordinatorNodeId },
10090
+ "Emitted distributed update"
10091
+ );
10092
+ }
9586
10093
  /**
9587
10094
  * Score a single document against a subscription's query.
9588
10095
  *
@@ -9737,44 +10244,1223 @@ var SearchCoordinator = class {
9737
10244
  }
9738
10245
  };
9739
10246
 
9740
- // src/ServerCoordinator.ts
9741
- var GC_INTERVAL_MS = 60 * 60 * 1e3;
9742
- var GC_AGE_MS = 30 * 24 * 60 * 60 * 1e3;
9743
- var CLIENT_HEARTBEAT_TIMEOUT_MS = 2e4;
9744
- var CLIENT_HEARTBEAT_CHECK_INTERVAL_MS = 5e3;
9745
- var ServerCoordinator = class {
9746
- constructor(config) {
9747
- this.clients = /* @__PURE__ */ new Map();
9748
- // Interceptors
9749
- this.interceptors = [];
9750
- // In-memory storage (partitioned later)
9751
- this.maps = /* @__PURE__ */ new Map();
9752
- this.pendingClusterQueries = /* @__PURE__ */ new Map();
9753
- // GC Consensus State
9754
- this.gcReports = /* @__PURE__ */ new Map();
9755
- // Track map loading state to avoid returning empty results during async load
9756
- this.mapLoadingPromises = /* @__PURE__ */ new Map();
9757
- // Track pending batch operations for testing purposes
9758
- this.pendingBatchOperations = /* @__PURE__ */ new Set();
9759
- this.journalSubscriptions = /* @__PURE__ */ new Map();
9760
- this._actualPort = 0;
9761
- this._actualClusterPort = 0;
9762
- this._readyPromise = new Promise((resolve) => {
9763
- this._readyResolve = resolve;
9764
- });
9765
- this._nodeId = config.nodeId;
9766
- this.hlc = new HLC2(config.nodeId);
9767
- this.storage = config.storage;
9768
- const rawSecret = config.jwtSecret || process.env.JWT_SECRET || "topgun-secret-dev";
9769
- this.jwtSecret = rawSecret.replace(/\\n/g, "\n");
9770
- this.queryRegistry = new QueryRegistry();
9771
- this.securityManager = new SecurityManager(config.securityPolicies || []);
9772
- this.interceptors = config.interceptors || [];
9773
- this.metricsService = new MetricsService();
9774
- this.eventExecutor = new StripedEventExecutor({
9775
- stripeCount: config.eventStripeCount ?? 4,
9776
- queueCapacity: config.eventQueueCapacity ?? 1e4,
9777
- name: `${config.nodeId}-event-executor`,
10247
+ // src/search/ClusterSearchCoordinator.ts
10248
+ import { EventEmitter as EventEmitter14 } from "events";
10249
+ import {
10250
+ ReciprocalRankFusion,
10251
+ SearchCursor,
10252
+ ClusterSearchReqPayloadSchema,
10253
+ ClusterSearchRespPayloadSchema
10254
+ } from "@topgunbuild/core";
10255
+ var DEFAULT_CONFIG6 = {
10256
+ rrfK: 60,
10257
+ defaultTimeoutMs: 5e3,
10258
+ defaultMinResponses: 0
10259
+ };
10260
+ var ClusterSearchCoordinator = class extends EventEmitter14 {
10261
+ constructor(clusterManager, partitionService, localSearchCoordinator, config, metricsService) {
10262
+ super();
10263
+ /** Pending requests awaiting responses */
10264
+ this.pendingRequests = /* @__PURE__ */ new Map();
10265
+ this.clusterManager = clusterManager;
10266
+ this.partitionService = partitionService;
10267
+ this.localSearchCoordinator = localSearchCoordinator;
10268
+ this.config = { ...DEFAULT_CONFIG6, ...config };
10269
+ this.rrf = new ReciprocalRankFusion({ k: this.config.rrfK });
10270
+ this.metricsService = metricsService;
10271
+ this.clusterManager.on("message", this.handleClusterMessage.bind(this));
10272
+ }
10273
+ /**
10274
+ * Execute a distributed search across the cluster.
10275
+ *
10276
+ * @param mapName - Name of the map to search
10277
+ * @param query - Search query text
10278
+ * @param options - Search options
10279
+ * @returns Promise resolving to merged search results
10280
+ */
10281
+ async search(mapName, query, options = { limit: 10 }) {
10282
+ const startTime = performance.now();
10283
+ const requestId = this.generateRequestId();
10284
+ const timeoutMs = options.timeoutMs ?? this.config.defaultTimeoutMs;
10285
+ const allNodes = new Set(this.clusterManager.getMembers());
10286
+ const myNodeId = this.clusterManager.config.nodeId;
10287
+ if (allNodes.size === 1 && allNodes.has(myNodeId)) {
10288
+ return this.executeLocalSearch(mapName, query, options, startTime);
10289
+ }
10290
+ const perNodeLimit = this.calculatePerNodeLimit(options.limit, options.cursor);
10291
+ let cursorData = null;
10292
+ if (options.cursor) {
10293
+ cursorData = SearchCursor.decode(options.cursor);
10294
+ if (cursorData && !SearchCursor.isValid(cursorData, query)) {
10295
+ cursorData = null;
10296
+ logger.warn({ requestId }, "Invalid or expired cursor, ignoring");
10297
+ }
10298
+ }
10299
+ const promise = new Promise((resolve, reject) => {
10300
+ const timeoutHandle = setTimeout(() => {
10301
+ this.resolvePartialResults(requestId);
10302
+ }, timeoutMs);
10303
+ this.pendingRequests.set(requestId, {
10304
+ resolve,
10305
+ reject,
10306
+ responses: /* @__PURE__ */ new Map(),
10307
+ expectedNodes: allNodes,
10308
+ startTime,
10309
+ timeoutHandle,
10310
+ options,
10311
+ mapName,
10312
+ query
10313
+ });
10314
+ });
10315
+ const payload = {
10316
+ requestId,
10317
+ mapName,
10318
+ query,
10319
+ options: {
10320
+ limit: perNodeLimit,
10321
+ minScore: options.minScore,
10322
+ boost: options.boost,
10323
+ includeMatchedTerms: true,
10324
+ // Add cursor position if available
10325
+ ...cursorData ? {
10326
+ afterScore: cursorData.nodeScores[myNodeId],
10327
+ afterKey: cursorData.nodeKeys[myNodeId]
10328
+ } : {}
10329
+ },
10330
+ timeoutMs
10331
+ };
10332
+ for (const nodeId of allNodes) {
10333
+ if (nodeId === myNodeId) {
10334
+ this.executeLocalAndRespond(requestId, mapName, query, perNodeLimit, cursorData);
10335
+ } else {
10336
+ this.clusterManager.send(nodeId, "CLUSTER_SEARCH_REQ", payload);
10337
+ }
10338
+ }
10339
+ return promise;
10340
+ }
10341
+ /**
10342
+ * Handle incoming cluster messages.
10343
+ */
10344
+ handleClusterMessage(msg) {
10345
+ switch (msg.type) {
10346
+ case "CLUSTER_SEARCH_REQ":
10347
+ this.handleSearchRequest(msg.senderId, msg.payload);
10348
+ break;
10349
+ case "CLUSTER_SEARCH_RESP":
10350
+ this.handleSearchResponse(msg.senderId, msg.payload);
10351
+ break;
10352
+ }
10353
+ }
10354
+ /**
10355
+ * Handle incoming search request from another node.
10356
+ */
10357
+ async handleSearchRequest(senderId, rawPayload) {
10358
+ const startTime = performance.now();
10359
+ const myNodeId = this.clusterManager.config.nodeId;
10360
+ const parsed = ClusterSearchReqPayloadSchema.safeParse(rawPayload);
10361
+ if (!parsed.success) {
10362
+ logger.warn(
10363
+ { senderId, error: parsed.error.message },
10364
+ "Invalid cluster search request payload"
10365
+ );
10366
+ this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", {
10367
+ requestId: rawPayload?.requestId ?? "unknown",
10368
+ nodeId: myNodeId,
10369
+ results: [],
10370
+ totalHits: 0,
10371
+ executionTimeMs: performance.now() - startTime,
10372
+ error: "Invalid request payload"
10373
+ });
10374
+ return;
10375
+ }
10376
+ const payload = parsed.data;
10377
+ try {
10378
+ const localResult = this.localSearchCoordinator.search(
10379
+ payload.mapName,
10380
+ payload.query,
10381
+ {
10382
+ limit: payload.options.limit,
10383
+ minScore: payload.options.minScore,
10384
+ boost: payload.options.boost
10385
+ }
10386
+ );
10387
+ let results = localResult.results;
10388
+ if (payload.options.afterScore !== void 0) {
10389
+ results = results.filter((r) => {
10390
+ if (r.score < payload.options.afterScore) {
10391
+ return true;
10392
+ }
10393
+ if (r.score === payload.options.afterScore && payload.options.afterKey) {
10394
+ return r.key > payload.options.afterKey;
10395
+ }
10396
+ return false;
10397
+ });
10398
+ }
10399
+ const response = {
10400
+ requestId: payload.requestId,
10401
+ nodeId: myNodeId,
10402
+ results: results.map((r) => ({
10403
+ key: r.key,
10404
+ value: r.value,
10405
+ score: r.score,
10406
+ matchedTerms: r.matchedTerms
10407
+ })),
10408
+ totalHits: localResult.totalCount ?? results.length,
10409
+ executionTimeMs: performance.now() - startTime
10410
+ };
10411
+ this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", response);
10412
+ } catch (error) {
10413
+ this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", {
10414
+ requestId: payload.requestId,
10415
+ nodeId: myNodeId,
10416
+ results: [],
10417
+ totalHits: 0,
10418
+ executionTimeMs: performance.now() - startTime,
10419
+ error: error instanceof Error ? error.message : "Unknown error"
10420
+ });
10421
+ }
10422
+ }
10423
+ /**
10424
+ * Handle search response from a node.
10425
+ */
10426
+ handleSearchResponse(_senderId, rawPayload) {
10427
+ const parsed = ClusterSearchRespPayloadSchema.safeParse(rawPayload);
10428
+ if (!parsed.success) {
10429
+ logger.warn(
10430
+ { error: parsed.error.message },
10431
+ "Invalid cluster search response payload"
10432
+ );
10433
+ return;
10434
+ }
10435
+ const payload = parsed.data;
10436
+ const pending = this.pendingRequests.get(payload.requestId);
10437
+ if (!pending) {
10438
+ logger.warn({ requestId: payload.requestId }, "Received response for unknown request");
10439
+ return;
10440
+ }
10441
+ pending.responses.set(payload.nodeId, payload);
10442
+ const minResponses = pending.options.minResponses ?? this.config.defaultMinResponses;
10443
+ const requiredResponses = minResponses > 0 ? minResponses : pending.expectedNodes.size;
10444
+ if (pending.responses.size >= requiredResponses) {
10445
+ clearTimeout(pending.timeoutHandle);
10446
+ this.mergeAndResolve(payload.requestId);
10447
+ }
10448
+ }
10449
+ /**
10450
+ * Merge results from all nodes using RRF and resolve the promise.
10451
+ */
10452
+ mergeAndResolve(requestId) {
10453
+ const pending = this.pendingRequests.get(requestId);
10454
+ if (!pending) return;
10455
+ const resultSets = [];
10456
+ const respondedNodes = [];
10457
+ const failedNodes = [];
10458
+ let totalHits = 0;
10459
+ const resultsWithNodes = [];
10460
+ for (const [nodeId, response] of pending.responses) {
10461
+ if (response.error) {
10462
+ failedNodes.push(nodeId);
10463
+ logger.warn({ nodeId, error: response.error }, "Node returned error for search");
10464
+ } else {
10465
+ respondedNodes.push(nodeId);
10466
+ totalHits += response.totalHits;
10467
+ const rankedResults = response.results.map((r) => ({
10468
+ docId: r.key,
10469
+ score: r.score,
10470
+ source: nodeId
10471
+ }));
10472
+ resultSets.push(rankedResults);
10473
+ for (const r of response.results) {
10474
+ resultsWithNodes.push({
10475
+ key: r.key,
10476
+ score: r.score,
10477
+ nodeId,
10478
+ value: r.value,
10479
+ matchedTerms: r.matchedTerms || []
10480
+ });
10481
+ }
10482
+ }
10483
+ }
10484
+ for (const nodeId of pending.expectedNodes) {
10485
+ if (!pending.responses.has(nodeId)) {
10486
+ failedNodes.push(nodeId);
10487
+ }
10488
+ }
10489
+ const merged = this.rrf.merge(resultSets);
10490
+ const limit = pending.options.limit;
10491
+ const results = [];
10492
+ const cursorResults = [];
10493
+ for (const mergedResult of merged.slice(0, limit)) {
10494
+ const original = resultsWithNodes.find((r) => r.key === mergedResult.docId);
10495
+ if (original) {
10496
+ results.push({
10497
+ key: original.key,
10498
+ value: original.value,
10499
+ score: mergedResult.score,
10500
+ // Use RRF score
10501
+ matchedTerms: original.matchedTerms
10502
+ });
10503
+ cursorResults.push({
10504
+ key: original.key,
10505
+ score: original.score,
10506
+ // Use original score for cursor
10507
+ nodeId: original.nodeId
10508
+ });
10509
+ }
10510
+ }
10511
+ let nextCursor;
10512
+ if (merged.length > limit && cursorResults.length > 0) {
10513
+ nextCursor = SearchCursor.fromResults(cursorResults, pending.query);
10514
+ }
10515
+ const executionTimeMs = performance.now() - pending.startTime;
10516
+ if (this.metricsService) {
10517
+ const status = failedNodes.length === 0 ? "success" : respondedNodes.length === 0 ? "error" : "partial";
10518
+ this.metricsService.incDistributedSearch(pending.mapName, status);
10519
+ this.metricsService.recordDistributedSearchDuration(pending.mapName, executionTimeMs);
10520
+ if (failedNodes.length > 0) {
10521
+ this.metricsService.incDistributedSearchFailedNodes(failedNodes.length);
10522
+ if (respondedNodes.length > 0) {
10523
+ this.metricsService.incDistributedSearchPartialResults();
10524
+ }
10525
+ }
10526
+ }
10527
+ pending.resolve({
10528
+ results,
10529
+ totalHits,
10530
+ nextCursor,
10531
+ respondedNodes,
10532
+ failedNodes,
10533
+ executionTimeMs
10534
+ });
10535
+ this.pendingRequests.delete(requestId);
10536
+ logger.debug({
10537
+ requestId,
10538
+ mapName: pending.mapName,
10539
+ query: pending.query,
10540
+ resultCount: results.length,
10541
+ totalHits,
10542
+ respondedNodes: respondedNodes.length,
10543
+ failedNodes: failedNodes.length,
10544
+ executionTimeMs
10545
+ }, "Distributed search completed");
10546
+ }
10547
+ /**
10548
+ * Resolve with partial results when timeout occurs.
10549
+ */
10550
+ resolvePartialResults(requestId) {
10551
+ const pending = this.pendingRequests.get(requestId);
10552
+ if (!pending) return;
10553
+ logger.warn(
10554
+ {
10555
+ requestId,
10556
+ received: pending.responses.size,
10557
+ expected: pending.expectedNodes.size
10558
+ },
10559
+ "Search request timed out, returning partial results"
10560
+ );
10561
+ this.mergeAndResolve(requestId);
10562
+ }
10563
+ /**
10564
+ * Execute local search and add response to pending request.
10565
+ */
10566
+ async executeLocalAndRespond(requestId, mapName, query, limit, cursorData) {
10567
+ const startTime = performance.now();
10568
+ const myNodeId = this.clusterManager.config.nodeId;
10569
+ const pending = this.pendingRequests.get(requestId);
10570
+ if (!pending) return;
10571
+ try {
10572
+ const localResult = this.localSearchCoordinator.search(mapName, query, {
10573
+ limit,
10574
+ minScore: pending.options.minScore,
10575
+ boost: pending.options.boost
10576
+ });
10577
+ let results = localResult.results;
10578
+ if (cursorData) {
10579
+ const position = SearchCursor.getNodePosition(cursorData, myNodeId);
10580
+ if (position) {
10581
+ results = results.filter((r) => {
10582
+ if (r.score < position.afterScore) {
10583
+ return true;
10584
+ }
10585
+ if (r.score === position.afterScore) {
10586
+ return r.key > position.afterKey;
10587
+ }
10588
+ return false;
10589
+ });
10590
+ }
10591
+ }
10592
+ const response = {
10593
+ requestId,
10594
+ nodeId: myNodeId,
10595
+ results: results.map((r) => ({
10596
+ key: r.key,
10597
+ value: r.value,
10598
+ score: r.score,
10599
+ matchedTerms: r.matchedTerms
10600
+ })),
10601
+ totalHits: localResult.totalCount ?? results.length,
10602
+ executionTimeMs: performance.now() - startTime
10603
+ };
10604
+ this.handleSearchResponse(myNodeId, response);
10605
+ } catch (error) {
10606
+ this.handleSearchResponse(myNodeId, {
10607
+ requestId,
10608
+ nodeId: myNodeId,
10609
+ results: [],
10610
+ totalHits: 0,
10611
+ executionTimeMs: performance.now() - startTime,
10612
+ error: error instanceof Error ? error.message : "Unknown error"
10613
+ });
10614
+ }
10615
+ }
10616
+ /**
10617
+ * Execute search locally only (single-node optimization).
10618
+ */
10619
+ async executeLocalSearch(mapName, query, options, startTime) {
10620
+ const myNodeId = this.clusterManager.config.nodeId;
10621
+ const localResult = this.localSearchCoordinator.search(mapName, query, {
10622
+ limit: options.limit,
10623
+ minScore: options.minScore,
10624
+ boost: options.boost
10625
+ });
10626
+ let results = localResult.results;
10627
+ if (options.cursor) {
10628
+ const cursorData = SearchCursor.decode(options.cursor);
10629
+ if (cursorData && SearchCursor.isValid(cursorData, query)) {
10630
+ const position = SearchCursor.getNodePosition(cursorData, myNodeId);
10631
+ if (position) {
10632
+ results = results.filter((r) => {
10633
+ if (r.score < position.afterScore) {
10634
+ return true;
10635
+ }
10636
+ if (r.score === position.afterScore) {
10637
+ return r.key > position.afterKey;
10638
+ }
10639
+ return false;
10640
+ });
10641
+ }
10642
+ }
10643
+ }
10644
+ results = results.slice(0, options.limit);
10645
+ const totalCount = localResult.totalCount ?? localResult.results.length;
10646
+ let nextCursor;
10647
+ if (totalCount > options.limit && results.length > 0) {
10648
+ const lastResult = results[results.length - 1];
10649
+ nextCursor = SearchCursor.fromResults(
10650
+ [{ key: lastResult.key, score: lastResult.score, nodeId: myNodeId }],
10651
+ query
10652
+ );
10653
+ }
10654
+ const executionTimeMs = performance.now() - startTime;
10655
+ if (this.metricsService) {
10656
+ this.metricsService.incDistributedSearch(mapName, "success");
10657
+ this.metricsService.recordDistributedSearchDuration(mapName, executionTimeMs);
10658
+ }
10659
+ return {
10660
+ results,
10661
+ totalHits: localResult.totalCount ?? results.length,
10662
+ nextCursor,
10663
+ respondedNodes: [myNodeId],
10664
+ failedNodes: [],
10665
+ executionTimeMs
10666
+ };
10667
+ }
10668
+ /**
10669
+ * Calculate per-node limit for distributed query.
10670
+ *
10671
+ * For quality RRF merge, each node should return more than the final limit.
10672
+ * Rule of thumb: 2x limit for good merge quality.
10673
+ */
10674
+ calculatePerNodeLimit(limit, cursor) {
10675
+ if (cursor) {
10676
+ return limit;
10677
+ }
10678
+ return Math.min(limit * 2, 1e3);
10679
+ }
10680
+ /**
10681
+ * Generate unique request ID.
10682
+ */
10683
+ generateRequestId() {
10684
+ return `search-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;
10685
+ }
10686
+ /**
10687
+ * Get RRF constant k.
10688
+ */
10689
+ getRrfK() {
10690
+ return this.config.rrfK;
10691
+ }
10692
+ /**
10693
+ * Clean up resources.
10694
+ */
10695
+ destroy() {
10696
+ for (const [requestId, pending] of this.pendingRequests) {
10697
+ clearTimeout(pending.timeoutHandle);
10698
+ pending.reject(new Error("ClusterSearchCoordinator destroyed"));
10699
+ }
10700
+ this.pendingRequests.clear();
10701
+ this.removeAllListeners();
10702
+ }
10703
+ };
10704
+
10705
+ // src/subscriptions/DistributedSubscriptionCoordinator.ts
10706
+ import { EventEmitter as EventEmitter15 } from "events";
10707
+ import {
10708
+ ReciprocalRankFusion as ReciprocalRankFusion2,
10709
+ ClusterSubRegisterPayloadSchema,
10710
+ ClusterSubAckPayloadSchema,
10711
+ ClusterSubUpdatePayloadSchema,
10712
+ ClusterSubUnregisterPayloadSchema
10713
+ } from "@topgunbuild/core";
10714
+ import { WebSocket as WebSocket4 } from "ws";
10715
+ var DEFAULT_CONFIG7 = {
10716
+ ackTimeoutMs: 5e3,
10717
+ rrfK: 60
10718
+ };
10719
+ var DistributedSubscriptionCoordinator = class extends EventEmitter15 {
10720
+ constructor(clusterManager, queryRegistry, searchCoordinator, config, metricsService) {
10721
+ super();
10722
+ /**
10723
+ * Active subscriptions where this node is coordinator.
10724
+ * subscriptionId → SubscriptionState
10725
+ */
10726
+ this.subscriptions = /* @__PURE__ */ new Map();
10727
+ /**
10728
+ * Track which nodes have acknowledged subscription registration.
10729
+ * subscriptionId → Set<nodeId>
10730
+ */
10731
+ this.nodeAcks = /* @__PURE__ */ new Map();
10732
+ /**
10733
+ * Pending ACK promises for subscription registration.
10734
+ * subscriptionId → { resolve, reject, timeout, startTime }
10735
+ */
10736
+ this.pendingAcks = /* @__PURE__ */ new Map();
10737
+ this.clusterManager = clusterManager;
10738
+ this.localQueryRegistry = queryRegistry;
10739
+ this.localSearchCoordinator = searchCoordinator;
10740
+ this.config = { ...DEFAULT_CONFIG7, ...config };
10741
+ this.rrf = new ReciprocalRankFusion2({ k: this.config.rrfK });
10742
+ this.metricsService = metricsService;
10743
+ this.clusterManager.on("message", this.handleClusterMessage.bind(this));
10744
+ this.localSearchCoordinator.on("distributedUpdate", this.handleLocalSearchUpdate.bind(this));
10745
+ this.clusterManager.on("memberLeft", this.handleMemberLeft.bind(this));
10746
+ logger.debug("DistributedSubscriptionCoordinator initialized");
10747
+ }
10748
+ /**
10749
+ * Create a new distributed search subscription.
10750
+ *
10751
+ * @param subscriptionId - Unique subscription ID
10752
+ * @param clientSocket - Client WebSocket for sending updates
10753
+ * @param mapName - Map name to search
10754
+ * @param query - Search query string
10755
+ * @param options - Search options
10756
+ * @returns Promise resolving to initial results
10757
+ */
10758
+ async subscribeSearch(subscriptionId, clientSocket, mapName, query, options = {}) {
10759
+ const myNodeId = this.clusterManager.config.nodeId;
10760
+ const allNodes = new Set(this.clusterManager.getMembers());
10761
+ logger.debug(
10762
+ { subscriptionId, mapName, query, nodes: Array.from(allNodes) },
10763
+ "Creating distributed search subscription"
10764
+ );
10765
+ const subscription = {
10766
+ id: subscriptionId,
10767
+ type: "SEARCH",
10768
+ coordinatorNodeId: myNodeId,
10769
+ clientSocket,
10770
+ mapName,
10771
+ searchQuery: query,
10772
+ searchOptions: options,
10773
+ registeredNodes: /* @__PURE__ */ new Set(),
10774
+ pendingResults: /* @__PURE__ */ new Map(),
10775
+ createdAt: Date.now(),
10776
+ currentResults: /* @__PURE__ */ new Map()
10777
+ };
10778
+ this.subscriptions.set(subscriptionId, subscription);
10779
+ this.nodeAcks.set(subscriptionId, /* @__PURE__ */ new Set());
10780
+ const registerPayload = {
10781
+ subscriptionId,
10782
+ coordinatorNodeId: myNodeId,
10783
+ mapName,
10784
+ type: "SEARCH",
10785
+ searchQuery: query,
10786
+ searchOptions: {
10787
+ limit: options.limit,
10788
+ minScore: options.minScore,
10789
+ boost: options.boost
10790
+ }
10791
+ };
10792
+ const localResult = this.registerLocalSearchSubscription(subscription);
10793
+ this.handleLocalAck(subscriptionId, myNodeId, localResult);
10794
+ for (const nodeId of allNodes) {
10795
+ if (nodeId !== myNodeId) {
10796
+ this.clusterManager.send(nodeId, "CLUSTER_SUB_REGISTER", registerPayload);
10797
+ }
10798
+ }
10799
+ return this.waitForAcks(subscriptionId, allNodes);
10800
+ }
10801
+ /**
10802
+ * Create a new distributed query subscription.
10803
+ *
10804
+ * @param subscriptionId - Unique subscription ID
10805
+ * @param clientSocket - Client WebSocket for sending updates
10806
+ * @param mapName - Map name to query
10807
+ * @param query - Query predicate
10808
+ * @returns Promise resolving to initial results
10809
+ */
10810
+ async subscribeQuery(subscriptionId, clientSocket, mapName, query) {
10811
+ const myNodeId = this.clusterManager.config.nodeId;
10812
+ const allNodes = new Set(this.clusterManager.getMembers());
10813
+ logger.debug(
10814
+ { subscriptionId, mapName, nodes: Array.from(allNodes) },
10815
+ "Creating distributed query subscription"
10816
+ );
10817
+ const subscription = {
10818
+ id: subscriptionId,
10819
+ type: "QUERY",
10820
+ coordinatorNodeId: myNodeId,
10821
+ clientSocket,
10822
+ mapName,
10823
+ queryPredicate: query,
10824
+ registeredNodes: /* @__PURE__ */ new Set(),
10825
+ pendingResults: /* @__PURE__ */ new Map(),
10826
+ createdAt: Date.now(),
10827
+ currentResults: /* @__PURE__ */ new Map()
10828
+ };
10829
+ this.subscriptions.set(subscriptionId, subscription);
10830
+ this.nodeAcks.set(subscriptionId, /* @__PURE__ */ new Set());
10831
+ const registerPayload = {
10832
+ subscriptionId,
10833
+ coordinatorNodeId: myNodeId,
10834
+ mapName,
10835
+ type: "QUERY",
10836
+ queryPredicate: query
10837
+ };
10838
+ const localResults = this.registerLocalQuerySubscription(subscription);
10839
+ this.handleLocalAck(subscriptionId, myNodeId, { results: localResults, totalHits: localResults.length });
10840
+ for (const nodeId of allNodes) {
10841
+ if (nodeId !== myNodeId) {
10842
+ this.clusterManager.send(nodeId, "CLUSTER_SUB_REGISTER", registerPayload);
10843
+ }
10844
+ }
10845
+ return this.waitForAcks(subscriptionId, allNodes);
10846
+ }
10847
+ /**
10848
+ * Unsubscribe from a distributed subscription.
10849
+ *
10850
+ * @param subscriptionId - Subscription ID to unsubscribe
10851
+ */
10852
+ async unsubscribe(subscriptionId) {
10853
+ const subscription = this.subscriptions.get(subscriptionId);
10854
+ if (!subscription) {
10855
+ logger.warn({ subscriptionId }, "Attempt to unsubscribe from unknown subscription");
10856
+ return;
10857
+ }
10858
+ logger.debug({ subscriptionId }, "Unsubscribing from distributed subscription");
10859
+ const myNodeId = this.clusterManager.config.nodeId;
10860
+ const payload = { subscriptionId };
10861
+ for (const nodeId of subscription.registeredNodes) {
10862
+ if (nodeId !== myNodeId) {
10863
+ this.clusterManager.send(nodeId, "CLUSTER_SUB_UNREGISTER", payload);
10864
+ }
10865
+ }
10866
+ this.unregisterLocalSubscription(subscription);
10867
+ this.subscriptions.delete(subscriptionId);
10868
+ this.nodeAcks.delete(subscriptionId);
10869
+ const pendingAck = this.pendingAcks.get(subscriptionId);
10870
+ if (pendingAck) {
10871
+ clearTimeout(pendingAck.timeoutHandle);
10872
+ this.pendingAcks.delete(subscriptionId);
10873
+ }
10874
+ this.metricsService?.incDistributedSubUnsubscribe(subscription.type);
10875
+ this.metricsService?.decDistributedSubActive(subscription.type);
10876
+ this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
10877
+ }
10878
+ /**
10879
+ * Handle client disconnect - unsubscribe all their subscriptions.
10880
+ */
10881
+ unsubscribeClient(clientSocket) {
10882
+ const subscriptionsToRemove = [];
10883
+ for (const [subId, sub] of this.subscriptions) {
10884
+ if (sub.clientSocket === clientSocket) {
10885
+ subscriptionsToRemove.push(subId);
10886
+ }
10887
+ }
10888
+ for (const subId of subscriptionsToRemove) {
10889
+ this.unsubscribe(subId);
10890
+ }
10891
+ }
10892
+ /**
10893
+ * Get active subscription count.
10894
+ */
10895
+ getActiveSubscriptionCount() {
10896
+ return this.subscriptions.size;
10897
+ }
10898
+ /**
10899
+ * Handle incoming cluster messages with Zod validation.
10900
+ */
10901
+ handleClusterMessage(msg) {
10902
+ switch (msg.type) {
10903
+ case "CLUSTER_SUB_REGISTER": {
10904
+ const parsed = ClusterSubRegisterPayloadSchema.safeParse(msg.payload);
10905
+ if (!parsed.success) {
10906
+ logger.warn(
10907
+ { senderId: msg.senderId, error: parsed.error.message },
10908
+ "Invalid CLUSTER_SUB_REGISTER payload"
10909
+ );
10910
+ return;
10911
+ }
10912
+ this.handleSubRegister(msg.senderId, parsed.data);
10913
+ break;
10914
+ }
10915
+ case "CLUSTER_SUB_ACK": {
10916
+ const parsed = ClusterSubAckPayloadSchema.safeParse(msg.payload);
10917
+ if (!parsed.success) {
10918
+ logger.warn(
10919
+ { senderId: msg.senderId, error: parsed.error.message },
10920
+ "Invalid CLUSTER_SUB_ACK payload"
10921
+ );
10922
+ return;
10923
+ }
10924
+ this.handleSubAck(msg.senderId, parsed.data);
10925
+ break;
10926
+ }
10927
+ case "CLUSTER_SUB_UPDATE": {
10928
+ const parsed = ClusterSubUpdatePayloadSchema.safeParse(msg.payload);
10929
+ if (!parsed.success) {
10930
+ logger.warn(
10931
+ { senderId: msg.senderId, error: parsed.error.message },
10932
+ "Invalid CLUSTER_SUB_UPDATE payload"
10933
+ );
10934
+ return;
10935
+ }
10936
+ this.handleSubUpdate(msg.senderId, parsed.data);
10937
+ break;
10938
+ }
10939
+ case "CLUSTER_SUB_UNREGISTER": {
10940
+ const parsed = ClusterSubUnregisterPayloadSchema.safeParse(msg.payload);
10941
+ if (!parsed.success) {
10942
+ logger.warn(
10943
+ { senderId: msg.senderId, error: parsed.error.message },
10944
+ "Invalid CLUSTER_SUB_UNREGISTER payload"
10945
+ );
10946
+ return;
10947
+ }
10948
+ this.handleSubUnregister(msg.senderId, parsed.data);
10949
+ break;
10950
+ }
10951
+ }
10952
+ }
10953
+ /**
10954
+ * Handle cluster node disconnect - cleanup subscriptions involving this node.
10955
+ */
10956
+ handleMemberLeft(nodeId) {
10957
+ logger.debug({ nodeId }, "Handling member left for distributed subscriptions");
10958
+ const subscriptionsToRemove = [];
10959
+ const myNodeId = this.clusterManager.config.nodeId;
10960
+ for (const [subId, subscription] of this.subscriptions) {
10961
+ if (subscription.registeredNodes.has(nodeId)) {
10962
+ subscription.registeredNodes.delete(nodeId);
10963
+ for (const [key, result] of subscription.currentResults) {
10964
+ if (result.sourceNode === nodeId) {
10965
+ subscription.currentResults.delete(key);
10966
+ }
10967
+ }
10968
+ logger.debug(
10969
+ { subscriptionId: subId, nodeId, remainingNodes: subscription.registeredNodes.size },
10970
+ "Removed disconnected node from subscription"
10971
+ );
10972
+ }
10973
+ }
10974
+ for (const [subId, pending] of this.pendingAcks) {
10975
+ const acks = this.nodeAcks.get(subId);
10976
+ if (acks && !acks.has(nodeId)) {
10977
+ acks.add(nodeId);
10978
+ this.checkAcksComplete(subId);
10979
+ }
10980
+ }
10981
+ this.localSearchCoordinator.unsubscribeByCoordinator(nodeId);
10982
+ this.localQueryRegistry.unregisterByCoordinator(nodeId);
10983
+ this.metricsService?.incDistributedSubNodeDisconnect();
10984
+ }
10985
+ /**
10986
+ * Handle CLUSTER_SUB_REGISTER from coordinator (we are a data node).
10987
+ */
10988
+ handleSubRegister(senderId, payload) {
10989
+ const myNodeId = this.clusterManager.config.nodeId;
10990
+ logger.debug(
10991
+ { subscriptionId: payload.subscriptionId, coordinator: payload.coordinatorNodeId },
10992
+ "Received distributed subscription registration"
10993
+ );
10994
+ let ackPayload;
10995
+ try {
10996
+ if (payload.type === "SEARCH") {
10997
+ const result = this.localSearchCoordinator.registerDistributedSubscription(
10998
+ payload.subscriptionId,
10999
+ payload.mapName,
11000
+ payload.searchQuery,
11001
+ payload.searchOptions || {},
11002
+ payload.coordinatorNodeId
11003
+ );
11004
+ ackPayload = {
11005
+ subscriptionId: payload.subscriptionId,
11006
+ nodeId: myNodeId,
11007
+ success: true,
11008
+ initialResults: result.results.map((r) => ({
11009
+ key: r.key,
11010
+ value: r.value,
11011
+ score: r.score,
11012
+ matchedTerms: r.matchedTerms
11013
+ })),
11014
+ totalHits: result.totalHits
11015
+ };
11016
+ } else {
11017
+ const results = this.localQueryRegistry.registerDistributed(
11018
+ payload.subscriptionId,
11019
+ payload.mapName,
11020
+ payload.queryPredicate,
11021
+ payload.coordinatorNodeId
11022
+ );
11023
+ ackPayload = {
11024
+ subscriptionId: payload.subscriptionId,
11025
+ nodeId: myNodeId,
11026
+ success: true,
11027
+ initialResults: results.map((r) => ({
11028
+ key: r.key,
11029
+ value: r.value
11030
+ })),
11031
+ totalHits: results.length
11032
+ };
11033
+ }
11034
+ } catch (error) {
11035
+ logger.error(
11036
+ { subscriptionId: payload.subscriptionId, error },
11037
+ "Failed to register distributed subscription locally"
11038
+ );
11039
+ ackPayload = {
11040
+ subscriptionId: payload.subscriptionId,
11041
+ nodeId: myNodeId,
11042
+ success: false,
11043
+ error: error instanceof Error ? error.message : "Unknown error"
11044
+ };
11045
+ }
11046
+ this.clusterManager.send(payload.coordinatorNodeId, "CLUSTER_SUB_ACK", ackPayload);
11047
+ }
11048
+ /**
11049
+ * Handle CLUSTER_SUB_ACK from a data node.
11050
+ */
11051
+ handleSubAck(senderId, payload) {
11052
+ const subscription = this.subscriptions.get(payload.subscriptionId);
11053
+ if (!subscription) {
11054
+ logger.warn(
11055
+ { subscriptionId: payload.subscriptionId, nodeId: payload.nodeId },
11056
+ "Received ACK for unknown subscription"
11057
+ );
11058
+ return;
11059
+ }
11060
+ const acks = this.nodeAcks.get(payload.subscriptionId);
11061
+ if (!acks) return;
11062
+ logger.debug(
11063
+ { subscriptionId: payload.subscriptionId, nodeId: payload.nodeId, success: payload.success },
11064
+ "Received subscription ACK"
11065
+ );
11066
+ if (payload.success) {
11067
+ subscription.registeredNodes.add(payload.nodeId);
11068
+ subscription.pendingResults.set(payload.nodeId, payload);
11069
+ }
11070
+ acks.add(payload.nodeId);
11071
+ this.checkAcksComplete(payload.subscriptionId);
11072
+ }
11073
+ /**
11074
+ * Handle CLUSTER_SUB_UPDATE from a data node.
11075
+ */
11076
+ handleSubUpdate(senderId, payload) {
11077
+ const subscription = this.subscriptions.get(payload.subscriptionId);
11078
+ if (!subscription) {
11079
+ logger.warn(
11080
+ { subscriptionId: payload.subscriptionId },
11081
+ "Update for unknown subscription"
11082
+ );
11083
+ return;
11084
+ }
11085
+ logger.debug(
11086
+ { subscriptionId: payload.subscriptionId, key: payload.key, changeType: payload.changeType },
11087
+ "Received subscription update"
11088
+ );
11089
+ if (payload.changeType === "LEAVE") {
11090
+ subscription.currentResults.delete(payload.key);
11091
+ } else {
11092
+ subscription.currentResults.set(payload.key, {
11093
+ value: payload.value,
11094
+ score: payload.score,
11095
+ sourceNode: payload.sourceNodeId
11096
+ });
11097
+ }
11098
+ this.forwardUpdateToClient(subscription, payload);
11099
+ this.metricsService?.incDistributedSubUpdates("received", payload.changeType);
11100
+ if (payload.timestamp) {
11101
+ const latencyMs = Date.now() - payload.timestamp;
11102
+ this.metricsService?.recordDistributedSubUpdateLatency(subscription.type, latencyMs);
11103
+ }
11104
+ }
11105
+ /**
11106
+ * Handle CLUSTER_SUB_UNREGISTER from coordinator.
11107
+ */
11108
+ handleSubUnregister(senderId, payload) {
11109
+ logger.debug(
11110
+ { subscriptionId: payload.subscriptionId },
11111
+ "Received subscription unregister request"
11112
+ );
11113
+ this.localSearchCoordinator.unsubscribe(payload.subscriptionId);
11114
+ this.localQueryRegistry.unregister(payload.subscriptionId);
11115
+ }
11116
+ /**
11117
+ * Handle local search update (emitted by SearchCoordinator for distributed subscriptions).
11118
+ */
11119
+ handleLocalSearchUpdate(payload) {
11120
+ const coordinatorNodeId = this.getCoordinatorForSubscription(payload.subscriptionId);
11121
+ if (!coordinatorNodeId) return;
11122
+ const myNodeId = this.clusterManager.config.nodeId;
11123
+ if (coordinatorNodeId === myNodeId) {
11124
+ this.handleSubUpdate(myNodeId, payload);
11125
+ } else {
11126
+ this.clusterManager.send(coordinatorNodeId, "CLUSTER_SUB_UPDATE", payload);
11127
+ }
11128
+ this.metricsService?.incDistributedSubUpdates("sent", payload.changeType);
11129
+ }
11130
+ /**
11131
+ * Register a local search subscription for a distributed coordinator.
11132
+ */
11133
+ registerLocalSearchSubscription(subscription) {
11134
+ return this.localSearchCoordinator.registerDistributedSubscription(
11135
+ subscription.id,
11136
+ subscription.mapName,
11137
+ subscription.searchQuery,
11138
+ subscription.searchOptions || {},
11139
+ subscription.coordinatorNodeId
11140
+ );
11141
+ }
11142
+ /**
11143
+ * Register a local query subscription for a distributed coordinator.
11144
+ */
11145
+ registerLocalQuerySubscription(subscription) {
11146
+ return this.localQueryRegistry.registerDistributed(
11147
+ subscription.id,
11148
+ subscription.mapName,
11149
+ subscription.queryPredicate,
11150
+ subscription.coordinatorNodeId
11151
+ );
11152
+ }
11153
+ /**
11154
+ * Unregister local subscription.
11155
+ */
11156
+ unregisterLocalSubscription(subscription) {
11157
+ if (subscription.type === "SEARCH") {
11158
+ this.localSearchCoordinator.unsubscribe(subscription.id);
11159
+ } else {
11160
+ this.localQueryRegistry.unregister(subscription.id);
11161
+ }
11162
+ }
11163
+ /**
11164
+ * Handle local ACK (from this node's registration).
11165
+ */
11166
+ handleLocalAck(subscriptionId, nodeId, result) {
11167
+ const subscription = this.subscriptions.get(subscriptionId);
11168
+ if (!subscription) return;
11169
+ const acks = this.nodeAcks.get(subscriptionId);
11170
+ if (!acks) return;
11171
+ subscription.registeredNodes.add(nodeId);
11172
+ subscription.pendingResults.set(nodeId, {
11173
+ subscriptionId,
11174
+ nodeId,
11175
+ success: true,
11176
+ initialResults: (result.results || []).map((r) => ({
11177
+ key: r.key,
11178
+ value: r.value,
11179
+ score: r.score,
11180
+ matchedTerms: r.matchedTerms
11181
+ })),
11182
+ totalHits: result.totalHits || (result.results?.length ?? 0)
11183
+ });
11184
+ acks.add(nodeId);
11185
+ this.checkAcksComplete(subscriptionId);
11186
+ }
11187
+ /**
11188
+ * Wait for all node ACKs with timeout.
11189
+ */
11190
+ waitForAcks(subscriptionId, expectedNodes) {
11191
+ return new Promise((resolve, reject) => {
11192
+ const startTime = performance.now();
11193
+ const timeoutHandle = setTimeout(() => {
11194
+ this.resolveWithPartialAcks(subscriptionId);
11195
+ }, this.config.ackTimeoutMs);
11196
+ this.pendingAcks.set(subscriptionId, { resolve, reject, timeoutHandle, startTime });
11197
+ this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
11198
+ this.checkAcksComplete(subscriptionId);
11199
+ });
11200
+ }
11201
+ /**
11202
+ * Check if all ACKs have been received.
11203
+ */
11204
+ checkAcksComplete(subscriptionId) {
11205
+ const subscription = this.subscriptions.get(subscriptionId);
11206
+ const acks = this.nodeAcks.get(subscriptionId);
11207
+ const pendingAck = this.pendingAcks.get(subscriptionId);
11208
+ if (!subscription || !acks || !pendingAck) return;
11209
+ const allNodes = new Set(this.clusterManager.getMembers());
11210
+ if (acks.size >= allNodes.size) {
11211
+ clearTimeout(pendingAck.timeoutHandle);
11212
+ this.pendingAcks.delete(subscriptionId);
11213
+ const duration = performance.now() - pendingAck.startTime;
11214
+ const result = this.mergeInitialResults(subscription);
11215
+ const hasFailures = result.failedNodes.length > 0;
11216
+ this.recordCompletionMetrics(
11217
+ subscription,
11218
+ result,
11219
+ duration,
11220
+ hasFailures ? "timeout" : "success"
11221
+ );
11222
+ pendingAck.resolve(result);
11223
+ }
11224
+ }
11225
+ /**
11226
+ * Resolve with partial ACKs (on timeout).
11227
+ */
11228
+ resolveWithPartialAcks(subscriptionId) {
11229
+ const subscription = this.subscriptions.get(subscriptionId);
11230
+ const pendingAck = this.pendingAcks.get(subscriptionId);
11231
+ if (!subscription || !pendingAck) return;
11232
+ this.pendingAcks.delete(subscriptionId);
11233
+ logger.warn(
11234
+ { subscriptionId, registeredNodes: Array.from(subscription.registeredNodes) },
11235
+ "Subscription ACK timeout, resolving with partial results"
11236
+ );
11237
+ const duration = performance.now() - pendingAck.startTime;
11238
+ const result = this.mergeInitialResults(subscription);
11239
+ this.recordCompletionMetrics(subscription, result, duration, "timeout");
11240
+ pendingAck.resolve(result);
11241
+ }
11242
+ /**
11243
+ * Record metrics when subscription registration completes.
11244
+ */
11245
+ recordCompletionMetrics(subscription, result, durationMs, status) {
11246
+ this.metricsService?.incDistributedSub(subscription.type, status);
11247
+ this.metricsService?.recordDistributedSubRegistration(subscription.type, durationMs);
11248
+ this.metricsService?.recordDistributedSubInitialResultsCount(subscription.type, result.results.length);
11249
+ this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
11250
+ this.metricsService?.incDistributedSubAck("success", subscription.registeredNodes.size);
11251
+ this.metricsService?.incDistributedSubAck("timeout", result.failedNodes.length);
11252
+ }
11253
+ /**
11254
+ * Merge initial results from all nodes.
11255
+ */
11256
+ mergeInitialResults(subscription) {
11257
+ const allNodes = new Set(this.clusterManager.getMembers());
11258
+ const failedNodes = Array.from(allNodes).filter((n) => !subscription.registeredNodes.has(n));
11259
+ if (subscription.type === "SEARCH") {
11260
+ return this.mergeSearchResults(subscription, failedNodes);
11261
+ } else {
11262
+ return this.mergeQueryResults(subscription, failedNodes);
11263
+ }
11264
+ }
11265
+ /**
11266
+ * Merge search results using RRF.
11267
+ */
11268
+ mergeSearchResults(subscription, failedNodes) {
11269
+ const resultSets = [];
11270
+ const resultDataMap = /* @__PURE__ */ new Map();
11271
+ for (const [nodeId, ack] of subscription.pendingResults) {
11272
+ if (ack.success && ack.initialResults) {
11273
+ const rankedResults = [];
11274
+ for (const r of ack.initialResults) {
11275
+ rankedResults.push({
11276
+ docId: r.key,
11277
+ score: r.score ?? 0,
11278
+ source: nodeId
11279
+ });
11280
+ if (!resultDataMap.has(r.key)) {
11281
+ resultDataMap.set(r.key, {
11282
+ key: r.key,
11283
+ value: r.value,
11284
+ score: r.score ?? 0,
11285
+ matchedTerms: r.matchedTerms,
11286
+ nodeId
11287
+ });
11288
+ }
11289
+ }
11290
+ if (rankedResults.length > 0) {
11291
+ resultSets.push(rankedResults);
11292
+ }
11293
+ }
11294
+ }
11295
+ const mergedResults = this.rrf.merge(resultSets);
11296
+ const limit = subscription.searchOptions?.limit ?? 10;
11297
+ const results = [];
11298
+ for (const merged of mergedResults.slice(0, limit)) {
11299
+ const original = resultDataMap.get(merged.docId);
11300
+ if (original) {
11301
+ results.push({
11302
+ key: original.key,
11303
+ value: original.value,
11304
+ score: merged.score,
11305
+ // Use RRF score
11306
+ matchedTerms: original.matchedTerms
11307
+ });
11308
+ subscription.currentResults.set(original.key, {
11309
+ value: original.value,
11310
+ score: merged.score,
11311
+ sourceNode: original.nodeId
11312
+ });
11313
+ }
11314
+ }
11315
+ let totalHits = 0;
11316
+ for (const ack of subscription.pendingResults.values()) {
11317
+ totalHits += ack.totalHits ?? 0;
11318
+ }
11319
+ return {
11320
+ subscriptionId: subscription.id,
11321
+ results,
11322
+ totalHits,
11323
+ registeredNodes: Array.from(subscription.registeredNodes),
11324
+ failedNodes
11325
+ };
11326
+ }
11327
+ /**
11328
+ * Merge query results (simple dedupe by key).
11329
+ */
11330
+ mergeQueryResults(subscription, failedNodes) {
11331
+ const resultMap = /* @__PURE__ */ new Map();
11332
+ for (const [nodeId, ack] of subscription.pendingResults) {
11333
+ if (ack.success && ack.initialResults) {
11334
+ for (const result of ack.initialResults) {
11335
+ if (!resultMap.has(result.key)) {
11336
+ resultMap.set(result.key, { key: result.key, value: result.value });
11337
+ subscription.currentResults.set(result.key, {
11338
+ value: result.value,
11339
+ sourceNode: nodeId
11340
+ });
11341
+ }
11342
+ }
11343
+ }
11344
+ }
11345
+ const results = Array.from(resultMap.values());
11346
+ return {
11347
+ subscriptionId: subscription.id,
11348
+ results,
11349
+ totalHits: results.length,
11350
+ registeredNodes: Array.from(subscription.registeredNodes),
11351
+ failedNodes
11352
+ };
11353
+ }
11354
+ /**
11355
+ * Forward update to client WebSocket.
11356
+ */
11357
+ forwardUpdateToClient(subscription, payload) {
11358
+ if (subscription.clientSocket.readyState !== WebSocket4.OPEN) {
11359
+ logger.warn(
11360
+ { subscriptionId: subscription.id },
11361
+ "Cannot forward update, client socket not open"
11362
+ );
11363
+ return;
11364
+ }
11365
+ const message = subscription.type === "SEARCH" ? {
11366
+ type: "SEARCH_UPDATE",
11367
+ payload: {
11368
+ subscriptionId: payload.subscriptionId,
11369
+ key: payload.key,
11370
+ value: payload.value,
11371
+ score: payload.score,
11372
+ matchedTerms: payload.matchedTerms,
11373
+ type: payload.changeType
11374
+ }
11375
+ } : {
11376
+ type: "QUERY_UPDATE",
11377
+ payload: {
11378
+ queryId: payload.subscriptionId,
11379
+ key: payload.key,
11380
+ value: payload.value,
11381
+ type: payload.changeType
11382
+ }
11383
+ };
11384
+ try {
11385
+ subscription.clientSocket.send(JSON.stringify(message));
11386
+ } catch (error) {
11387
+ logger.error(
11388
+ { subscriptionId: subscription.id, error },
11389
+ "Failed to send update to client"
11390
+ );
11391
+ }
11392
+ }
11393
+ /**
11394
+ * Get coordinator node for a subscription.
11395
+ * For remote subscriptions (where we are a data node), this returns the coordinator.
11396
+ */
11397
+ getCoordinatorForSubscription(subscriptionId) {
11398
+ if (this.subscriptions.has(subscriptionId)) {
11399
+ return this.clusterManager.config.nodeId;
11400
+ }
11401
+ const searchSub = this.localSearchCoordinator.getDistributedSubscription(subscriptionId);
11402
+ if (searchSub?.coordinatorNodeId) {
11403
+ return searchSub.coordinatorNodeId;
11404
+ }
11405
+ const querySub = this.localQueryRegistry.getDistributedSubscription(subscriptionId);
11406
+ if (querySub?.coordinatorNodeId) {
11407
+ return querySub.coordinatorNodeId;
11408
+ }
11409
+ return null;
11410
+ }
11411
+ /**
11412
+ * Cleanup on destroy.
11413
+ */
11414
+ destroy() {
11415
+ for (const subscriptionId of this.subscriptions.keys()) {
11416
+ this.unsubscribe(subscriptionId);
11417
+ }
11418
+ for (const pending of this.pendingAcks.values()) {
11419
+ clearTimeout(pending.timeoutHandle);
11420
+ }
11421
+ this.pendingAcks.clear();
11422
+ this.removeAllListeners();
11423
+ }
11424
+ };
11425
+
11426
+ // src/ServerCoordinator.ts
11427
+ var GC_INTERVAL_MS = 60 * 60 * 1e3;
11428
+ var GC_AGE_MS = 30 * 24 * 60 * 60 * 1e3;
11429
+ var CLIENT_HEARTBEAT_TIMEOUT_MS = 2e4;
11430
+ var CLIENT_HEARTBEAT_CHECK_INTERVAL_MS = 5e3;
11431
+ var ServerCoordinator = class {
11432
+ constructor(config) {
11433
+ this.clients = /* @__PURE__ */ new Map();
11434
+ // Interceptors
11435
+ this.interceptors = [];
11436
+ // In-memory storage (partitioned later)
11437
+ this.maps = /* @__PURE__ */ new Map();
11438
+ this.pendingClusterQueries = /* @__PURE__ */ new Map();
11439
+ // GC Consensus State
11440
+ this.gcReports = /* @__PURE__ */ new Map();
11441
+ // Track map loading state to avoid returning empty results during async load
11442
+ this.mapLoadingPromises = /* @__PURE__ */ new Map();
11443
+ // Track pending batch operations for testing purposes
11444
+ this.pendingBatchOperations = /* @__PURE__ */ new Set();
11445
+ this.journalSubscriptions = /* @__PURE__ */ new Map();
11446
+ this._actualPort = 0;
11447
+ this._actualClusterPort = 0;
11448
+ this._readyPromise = new Promise((resolve) => {
11449
+ this._readyResolve = resolve;
11450
+ });
11451
+ this._nodeId = config.nodeId;
11452
+ this.hlc = new HLC2(config.nodeId);
11453
+ this.storage = config.storage;
11454
+ const rawSecret = config.jwtSecret || process.env.JWT_SECRET || "topgun-secret-dev";
11455
+ this.jwtSecret = rawSecret.replace(/\\n/g, "\n");
11456
+ this.queryRegistry = new QueryRegistry();
11457
+ this.securityManager = new SecurityManager(config.securityPolicies || []);
11458
+ this.interceptors = config.interceptors || [];
11459
+ this.metricsService = new MetricsService();
11460
+ this.eventExecutor = new StripedEventExecutor({
11461
+ stripeCount: config.eventStripeCount ?? 4,
11462
+ queueCapacity: config.eventQueueCapacity ?? 1e4,
11463
+ name: `${config.nodeId}-event-executor`,
9778
11464
  onReject: (task) => {
9779
11465
  logger.warn({ nodeId: config.nodeId, key: task.key }, "Event task rejected due to queue capacity");
9780
11466
  this.metricsService.incEventQueueRejected();
@@ -9922,7 +11608,7 @@ var ServerCoordinator = class {
9922
11608
  cluster: this.cluster,
9923
11609
  sendToClient: (clientId, message) => {
9924
11610
  const client = this.clients.get(clientId);
9925
- if (client && client.socket.readyState === WebSocket3.OPEN) {
11611
+ if (client && client.socket.readyState === WebSocket5.OPEN) {
9926
11612
  client.writer.write(message);
9927
11613
  }
9928
11614
  }
@@ -10022,6 +11708,26 @@ var ServerCoordinator = class {
10022
11708
  logger.info({ mapName, fields: ftsConfig.fields }, "FTS enabled for map");
10023
11709
  }
10024
11710
  }
11711
+ this.clusterSearchCoordinator = new ClusterSearchCoordinator(
11712
+ this.cluster,
11713
+ this.partitionService,
11714
+ this.searchCoordinator,
11715
+ config.distributedSearch,
11716
+ this.metricsService
11717
+ );
11718
+ logger.info("ClusterSearchCoordinator initialized for distributed search");
11719
+ this.distributedSubCoordinator = new DistributedSubscriptionCoordinator(
11720
+ this.cluster,
11721
+ this.queryRegistry,
11722
+ this.searchCoordinator,
11723
+ void 0,
11724
+ // Use default config
11725
+ this.metricsService
11726
+ );
11727
+ logger.info("DistributedSubscriptionCoordinator initialized for distributed live subscriptions");
11728
+ this.searchCoordinator.setNodeId(config.nodeId);
11729
+ this.queryRegistry.setClusterManager(this.cluster, config.nodeId);
11730
+ this.queryRegistry.setMapGetter((name) => this.getMap(name));
10025
11731
  this.systemManager = new SystemManager(
10026
11732
  this.cluster,
10027
11733
  this.metricsService,
@@ -10290,7 +11996,7 @@ var ServerCoordinator = class {
10290
11996
  const shutdownMsg = serialize4({ type: "SHUTDOWN_PENDING", retryAfter: 5e3 });
10291
11997
  for (const client of this.clients.values()) {
10292
11998
  try {
10293
- if (client.socket.readyState === WebSocket3.OPEN) {
11999
+ if (client.socket.readyState === WebSocket5.OPEN) {
10294
12000
  client.socket.send(shutdownMsg);
10295
12001
  if (client.writer) {
10296
12002
  client.writer.close();
@@ -10353,6 +12059,12 @@ var ServerCoordinator = class {
10353
12059
  if (this.eventJournalService) {
10354
12060
  this.eventJournalService.dispose();
10355
12061
  }
12062
+ if (this.clusterSearchCoordinator) {
12063
+ this.clusterSearchCoordinator.destroy();
12064
+ }
12065
+ if (this.distributedSubCoordinator) {
12066
+ this.distributedSubCoordinator.destroy();
12067
+ }
10356
12068
  logger.info("Server Coordinator shutdown complete.");
10357
12069
  }
10358
12070
  async handleConnection(ws) {
@@ -10461,6 +12173,9 @@ var ServerCoordinator = class {
10461
12173
  this.topicManager.unsubscribeAll(clientId);
10462
12174
  this.counterHandler.unsubscribeAll(clientId);
10463
12175
  this.searchCoordinator.unsubscribeClient(clientId);
12176
+ if (this.distributedSubCoordinator && connection) {
12177
+ this.distributedSubCoordinator.unsubscribeClient(connection.socket);
12178
+ }
10464
12179
  const members = this.cluster.getMembers();
10465
12180
  for (const memberId of members) {
10466
12181
  if (!this.cluster.isLocal(memberId)) {
@@ -10535,70 +12250,115 @@ var ServerCoordinator = class {
10535
12250
  }
10536
12251
  logger.info({ clientId: client.id, mapName, query }, "Client subscribed");
10537
12252
  this.metricsService.incOp("SUBSCRIBE", mapName);
10538
- const allMembers = this.cluster.getMembers();
10539
- let remoteMembers = allMembers.filter((id) => !this.cluster.isLocal(id));
10540
- const queryKey = query._id || query.where?._id;
10541
- if (queryKey && typeof queryKey === "string" && this.readReplicaHandler) {
10542
- try {
10543
- const targetNode = this.readReplicaHandler.selectReadNode({
12253
+ if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
12254
+ this.distributedSubCoordinator.subscribeQuery(
12255
+ queryId,
12256
+ client.socket,
12257
+ mapName,
12258
+ query
12259
+ ).then((result) => {
12260
+ const filteredResults = result.results.map((res) => {
12261
+ const filteredValue = this.securityManager.filterObject(res.value, client.principal, mapName);
12262
+ return { ...res, value: filteredValue };
12263
+ });
12264
+ client.writer.write({
12265
+ type: "QUERY_RESP",
12266
+ payload: {
12267
+ queryId,
12268
+ results: filteredResults
12269
+ }
12270
+ });
12271
+ client.subscriptions.add(queryId);
12272
+ logger.debug({
12273
+ clientId: client.id,
12274
+ queryId,
10544
12275
  mapName,
10545
- key: queryKey,
10546
- options: {
10547
- // Default to EVENTUAL for read scaling unless specified otherwise
10548
- // In future, we could extract consistency from query options if available
10549
- consistency: ConsistencyLevel3.EVENTUAL
12276
+ resultCount: result.results.length,
12277
+ totalHits: result.totalHits,
12278
+ nodes: result.registeredNodes
12279
+ }, "Distributed query subscription created");
12280
+ }).catch((err) => {
12281
+ logger.error({ err, queryId }, "Distributed query subscription failed");
12282
+ client.writer.write({
12283
+ type: "QUERY_RESP",
12284
+ payload: {
12285
+ queryId,
12286
+ results: [],
12287
+ error: "Failed to create distributed subscription"
10550
12288
  }
10551
12289
  });
10552
- if (targetNode) {
10553
- if (this.cluster.isLocal(targetNode)) {
10554
- remoteMembers = [];
10555
- logger.debug({ clientId: client.id, mapName, key: queryKey }, "Read optimization: Serving locally");
10556
- } else if (remoteMembers.includes(targetNode)) {
10557
- remoteMembers = [targetNode];
10558
- logger.debug({ clientId: client.id, mapName, key: queryKey, targetNode }, "Read optimization: Routing to replica");
12290
+ });
12291
+ } else {
12292
+ const allMembers = this.cluster.getMembers();
12293
+ let remoteMembers = allMembers.filter((id) => !this.cluster.isLocal(id));
12294
+ const queryKey = query._id || query.where?._id;
12295
+ if (queryKey && typeof queryKey === "string" && this.readReplicaHandler) {
12296
+ try {
12297
+ const targetNode = this.readReplicaHandler.selectReadNode({
12298
+ mapName,
12299
+ key: queryKey,
12300
+ options: {
12301
+ // Default to EVENTUAL for read scaling unless specified otherwise
12302
+ // In future, we could extract consistency from query options if available
12303
+ consistency: ConsistencyLevel3.EVENTUAL
12304
+ }
12305
+ });
12306
+ if (targetNode) {
12307
+ if (this.cluster.isLocal(targetNode)) {
12308
+ remoteMembers = [];
12309
+ logger.debug({ clientId: client.id, mapName, key: queryKey }, "Read optimization: Serving locally");
12310
+ } else if (remoteMembers.includes(targetNode)) {
12311
+ remoteMembers = [targetNode];
12312
+ logger.debug({ clientId: client.id, mapName, key: queryKey, targetNode }, "Read optimization: Routing to replica");
12313
+ }
10559
12314
  }
12315
+ } catch (e) {
12316
+ logger.warn({ err: e }, "Error in ReadReplicaHandler selection");
10560
12317
  }
10561
- } catch (e) {
10562
- logger.warn({ err: e }, "Error in ReadReplicaHandler selection");
10563
12318
  }
10564
- }
10565
- const requestId = crypto.randomUUID();
10566
- const pending = {
10567
- requestId,
10568
- client,
10569
- queryId,
10570
- mapName,
10571
- query,
10572
- results: [],
10573
- // Will populate with local results first
10574
- expectedNodes: new Set(remoteMembers),
10575
- respondedNodes: /* @__PURE__ */ new Set(),
10576
- timer: setTimeout(() => this.finalizeClusterQuery(requestId, true), 5e3)
10577
- // 5s timeout
10578
- };
10579
- this.pendingClusterQueries.set(requestId, pending);
10580
- try {
10581
- const localResults = await this.executeLocalQuery(mapName, query);
10582
- pending.results.push(...localResults);
10583
- if (remoteMembers.length > 0) {
10584
- for (const nodeId of remoteMembers) {
10585
- this.cluster.send(nodeId, "CLUSTER_QUERY_EXEC", {
10586
- requestId,
10587
- mapName,
10588
- query
10589
- });
12319
+ const requestId = crypto.randomUUID();
12320
+ const pending = {
12321
+ requestId,
12322
+ client,
12323
+ queryId,
12324
+ mapName,
12325
+ query,
12326
+ results: [],
12327
+ // Will populate with local results first
12328
+ expectedNodes: new Set(remoteMembers),
12329
+ respondedNodes: /* @__PURE__ */ new Set(),
12330
+ timer: setTimeout(() => this.finalizeClusterQuery(requestId, true), 5e3)
12331
+ // 5s timeout
12332
+ };
12333
+ this.pendingClusterQueries.set(requestId, pending);
12334
+ try {
12335
+ const localResults = await this.executeLocalQuery(mapName, query);
12336
+ pending.results.push(...localResults);
12337
+ if (remoteMembers.length > 0) {
12338
+ for (const nodeId of remoteMembers) {
12339
+ this.cluster.send(nodeId, "CLUSTER_QUERY_EXEC", {
12340
+ requestId,
12341
+ mapName,
12342
+ query
12343
+ });
12344
+ }
12345
+ } else {
12346
+ this.finalizeClusterQuery(requestId);
10590
12347
  }
10591
- } else {
12348
+ } catch (err) {
12349
+ logger.error({ err, mapName }, "Failed to execute local query");
10592
12350
  this.finalizeClusterQuery(requestId);
10593
12351
  }
10594
- } catch (err) {
10595
- logger.error({ err, mapName }, "Failed to execute local query");
10596
- this.finalizeClusterQuery(requestId);
10597
12352
  }
10598
12353
  break;
10599
12354
  }
10600
12355
  case "QUERY_UNSUB": {
10601
12356
  const { queryId: unsubId } = message.payload;
12357
+ if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
12358
+ this.distributedSubCoordinator.unsubscribe(unsubId).catch((err) => {
12359
+ logger.warn({ err, queryId: unsubId }, "Failed to unsubscribe from distributed coordinator");
12360
+ });
12361
+ }
10602
12362
  this.queryRegistry.unregister(unsubId);
10603
12363
  client.subscriptions.delete(unsubId);
10604
12364
  break;
@@ -10922,7 +12682,7 @@ var ServerCoordinator = class {
10922
12682
  client.writer.write(result.response);
10923
12683
  for (const targetClientId of result.broadcastTo) {
10924
12684
  const targetClient = this.clients.get(targetClientId);
10925
- if (targetClient && targetClient.socket.readyState === WebSocket3.OPEN) {
12685
+ if (targetClient && targetClient.socket.readyState === WebSocket5.OPEN) {
10926
12686
  targetClient.writer.write(result.broadcastMessage);
10927
12687
  }
10928
12688
  }
@@ -11402,7 +13162,7 @@ var ServerCoordinator = class {
11402
13162
  });
11403
13163
  break;
11404
13164
  }
11405
- // Phase 11.1: Full-Text Search
13165
+ // Phase 11.1: Full-Text Search (Phase 14: Distributed Search)
11406
13166
  case "SEARCH": {
11407
13167
  const { requestId: searchReqId, mapName: searchMapName, query: searchQuery, options: searchOptions } = message.payload;
11408
13168
  if (!this.securityManager.checkPermission(client.principal, searchMapName, "READ")) {
@@ -11430,18 +13190,58 @@ var ServerCoordinator = class {
11430
13190
  });
11431
13191
  break;
11432
13192
  }
11433
- const searchResult = this.searchCoordinator.search(searchMapName, searchQuery, searchOptions);
11434
- searchResult.requestId = searchReqId;
11435
- logger.debug({
11436
- clientId: client.id,
11437
- mapName: searchMapName,
11438
- query: searchQuery,
11439
- resultCount: searchResult.results.length
11440
- }, "Search executed");
11441
- client.writer.write({
11442
- type: "SEARCH_RESP",
11443
- payload: searchResult
11444
- });
13193
+ if (this.clusterSearchCoordinator && this.cluster.getMembers().length > 1) {
13194
+ this.clusterSearchCoordinator.search(searchMapName, searchQuery, {
13195
+ limit: searchOptions?.limit ?? 10,
13196
+ minScore: searchOptions?.minScore,
13197
+ boost: searchOptions?.boost
13198
+ }).then((distributedResult) => {
13199
+ logger.debug({
13200
+ clientId: client.id,
13201
+ mapName: searchMapName,
13202
+ query: searchQuery,
13203
+ resultCount: distributedResult.results.length,
13204
+ totalHits: distributedResult.totalHits,
13205
+ respondedNodes: distributedResult.respondedNodes.length,
13206
+ failedNodes: distributedResult.failedNodes.length,
13207
+ executionTimeMs: distributedResult.executionTimeMs
13208
+ }, "Distributed search executed");
13209
+ client.writer.write({
13210
+ type: "SEARCH_RESP",
13211
+ payload: {
13212
+ requestId: searchReqId,
13213
+ results: distributedResult.results,
13214
+ totalCount: distributedResult.totalHits,
13215
+ // Include cursor for pagination if available
13216
+ nextCursor: distributedResult.nextCursor
13217
+ }
13218
+ });
13219
+ }).catch((err) => {
13220
+ logger.error({ err, mapName: searchMapName, query: searchQuery }, "Distributed search failed");
13221
+ client.writer.write({
13222
+ type: "SEARCH_RESP",
13223
+ payload: {
13224
+ requestId: searchReqId,
13225
+ results: [],
13226
+ totalCount: 0,
13227
+ error: `Distributed search failed: ${err.message}`
13228
+ }
13229
+ });
13230
+ });
13231
+ } else {
13232
+ const searchResult = this.searchCoordinator.search(searchMapName, searchQuery, searchOptions);
13233
+ searchResult.requestId = searchReqId;
13234
+ logger.debug({
13235
+ clientId: client.id,
13236
+ mapName: searchMapName,
13237
+ query: searchQuery,
13238
+ resultCount: searchResult.results.length
13239
+ }, "Local search executed");
13240
+ client.writer.write({
13241
+ type: "SEARCH_RESP",
13242
+ payload: searchResult
13243
+ });
13244
+ }
11445
13245
  break;
11446
13246
  }
11447
13247
  // Phase 11.1b: Live Search Subscriptions
@@ -11472,33 +13272,75 @@ var ServerCoordinator = class {
11472
13272
  });
11473
13273
  break;
11474
13274
  }
11475
- const initialResults = this.searchCoordinator.subscribe(
11476
- client.id,
11477
- subscriptionId,
11478
- subMapName,
11479
- subQuery,
11480
- subOptions
11481
- );
11482
- logger.debug({
11483
- clientId: client.id,
11484
- subscriptionId,
11485
- mapName: subMapName,
11486
- query: subQuery,
11487
- resultCount: initialResults.length
11488
- }, "Search subscription created");
11489
- client.writer.write({
11490
- type: "SEARCH_RESP",
11491
- payload: {
11492
- requestId: subscriptionId,
11493
- results: initialResults,
11494
- totalCount: initialResults.length
11495
- }
11496
- });
13275
+ if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
13276
+ this.distributedSubCoordinator.subscribeSearch(
13277
+ subscriptionId,
13278
+ client.socket,
13279
+ subMapName,
13280
+ subQuery,
13281
+ subOptions || {}
13282
+ ).then((result) => {
13283
+ client.writer.write({
13284
+ type: "SEARCH_RESP",
13285
+ payload: {
13286
+ requestId: subscriptionId,
13287
+ results: result.results,
13288
+ totalCount: result.totalHits
13289
+ }
13290
+ });
13291
+ logger.debug({
13292
+ clientId: client.id,
13293
+ subscriptionId,
13294
+ mapName: subMapName,
13295
+ query: subQuery,
13296
+ resultCount: result.results.length,
13297
+ totalHits: result.totalHits,
13298
+ nodes: result.registeredNodes
13299
+ }, "Distributed search subscription created");
13300
+ }).catch((err) => {
13301
+ logger.error({ err, subscriptionId }, "Distributed search subscription failed");
13302
+ client.writer.write({
13303
+ type: "SEARCH_RESP",
13304
+ payload: {
13305
+ requestId: subscriptionId,
13306
+ results: [],
13307
+ totalCount: 0,
13308
+ error: "Failed to create distributed subscription"
13309
+ }
13310
+ });
13311
+ });
13312
+ } else {
13313
+ const initialResults = this.searchCoordinator.subscribe(
13314
+ client.id,
13315
+ subscriptionId,
13316
+ subMapName,
13317
+ subQuery,
13318
+ subOptions
13319
+ );
13320
+ logger.debug({
13321
+ clientId: client.id,
13322
+ subscriptionId,
13323
+ mapName: subMapName,
13324
+ query: subQuery,
13325
+ resultCount: initialResults.length
13326
+ }, "Search subscription created (local)");
13327
+ client.writer.write({
13328
+ type: "SEARCH_RESP",
13329
+ payload: {
13330
+ requestId: subscriptionId,
13331
+ results: initialResults,
13332
+ totalCount: initialResults.length
13333
+ }
13334
+ });
13335
+ }
11497
13336
  break;
11498
13337
  }
11499
13338
  case "SEARCH_UNSUB": {
11500
13339
  const { subscriptionId: unsubId } = message.payload;
11501
13340
  this.searchCoordinator.unsubscribe(unsubId);
13341
+ if (this.distributedSubCoordinator) {
13342
+ this.distributedSubCoordinator.unsubscribe(unsubId);
13343
+ }
11502
13344
  logger.debug({ clientId: client.id, subscriptionId: unsubId }, "Search unsubscription");
11503
13345
  break;
11504
13346
  }
@@ -11539,7 +13381,7 @@ var ServerCoordinator = class {
11539
13381
  };
11540
13382
  let broadcastCount = 0;
11541
13383
  for (const client of this.clients.values()) {
11542
- if (client.isAuthenticated && client.socket.readyState === WebSocket3.OPEN && client.writer) {
13384
+ if (client.isAuthenticated && client.socket.readyState === WebSocket5.OPEN && client.writer) {
11543
13385
  client.writer.write(message);
11544
13386
  broadcastCount++;
11545
13387
  }
@@ -11971,7 +13813,7 @@ var ServerCoordinator = class {
11971
13813
  async executeLocalQuery(mapName, query) {
11972
13814
  const map = await this.getMapAsync(mapName);
11973
13815
  const localQuery = { ...query };
11974
- delete localQuery.offset;
13816
+ delete localQuery.cursor;
11975
13817
  delete localQuery.limit;
11976
13818
  if (map instanceof IndexedLWWMap2) {
11977
13819
  const coreQuery = this.convertToCoreQuery(localQuery);
@@ -12090,7 +13932,7 @@ var ServerCoordinator = class {
12090
13932
  };
12091
13933
  return mapping[op] || null;
12092
13934
  }
12093
- finalizeClusterQuery(requestId, timeout = false) {
13935
+ async finalizeClusterQuery(requestId, timeout = false) {
12094
13936
  const pending = this.pendingClusterQueries.get(requestId);
12095
13937
  if (!pending) return;
12096
13938
  if (timeout) {
@@ -12115,7 +13957,49 @@ var ServerCoordinator = class {
12115
13957
  return 0;
12116
13958
  });
12117
13959
  }
12118
- const slicedResults = query.offset || query.limit ? finalResults.slice(query.offset || 0, (query.offset || 0) + (query.limit || finalResults.length)) : finalResults;
13960
+ let slicedResults = finalResults;
13961
+ let nextCursor;
13962
+ let hasMore = false;
13963
+ let cursorStatus = "none";
13964
+ if (query.cursor || query.limit) {
13965
+ const sort = query.sort || {};
13966
+ const sortEntries = Object.entries(sort);
13967
+ const sortField = sortEntries.length > 0 ? sortEntries[0][0] : "_key";
13968
+ if (query.cursor) {
13969
+ const cursorData = QueryCursor2.decode(query.cursor);
13970
+ if (!cursorData) {
13971
+ cursorStatus = "invalid";
13972
+ } else if (!QueryCursor2.isValid(cursorData, query.predicate ?? query.where, sort)) {
13973
+ if (Date.now() - cursorData.timestamp > DEFAULT_QUERY_CURSOR_MAX_AGE_MS2) {
13974
+ cursorStatus = "expired";
13975
+ } else {
13976
+ cursorStatus = "invalid";
13977
+ }
13978
+ } else {
13979
+ cursorStatus = "valid";
13980
+ slicedResults = finalResults.filter((r) => {
13981
+ const sortValue = r.value[sortField];
13982
+ return QueryCursor2.isAfterCursor(
13983
+ { key: r.key, sortValue },
13984
+ cursorData
13985
+ );
13986
+ });
13987
+ }
13988
+ }
13989
+ if (query.limit) {
13990
+ hasMore = slicedResults.length > query.limit;
13991
+ slicedResults = slicedResults.slice(0, query.limit);
13992
+ if (hasMore && slicedResults.length > 0) {
13993
+ const lastResult = slicedResults[slicedResults.length - 1];
13994
+ const sortValue = lastResult.value[sortField];
13995
+ nextCursor = QueryCursor2.fromLastResult(
13996
+ { key: lastResult.key, sortValue },
13997
+ sort,
13998
+ query.predicate ?? query.where
13999
+ );
14000
+ }
14001
+ }
14002
+ }
12119
14003
  const resultKeys = new Set(slicedResults.map((r) => r.key));
12120
14004
  const sub = {
12121
14005
  id: queryId,
@@ -12134,7 +14018,7 @@ var ServerCoordinator = class {
12134
14018
  });
12135
14019
  client.writer.write({
12136
14020
  type: "QUERY_RESP",
12137
- payload: { queryId, results: filteredResults }
14021
+ payload: { queryId, results: filteredResults, nextCursor, hasMore, cursorStatus }
12138
14022
  });
12139
14023
  }
12140
14024
  /**
@@ -12246,17 +14130,6 @@ var ServerCoordinator = class {
12246
14130
  }
12247
14131
  return { eventPayload, oldRecord };
12248
14132
  }
12249
- /**
12250
- * Broadcast event to cluster members (excluding self).
12251
- */
12252
- broadcastToCluster(eventPayload) {
12253
- const members = this.cluster.getMembers();
12254
- for (const memberId of members) {
12255
- if (!this.cluster.isLocal(memberId)) {
12256
- this.cluster.send(memberId, "CLUSTER_EVENT", eventPayload);
12257
- }
12258
- }
12259
- }
12260
14133
  /**
12261
14134
  * Apply replicated operation from another node (callback for ReplicationPipeline)
12262
14135
  * This is called when we receive a replicated operation as a backup node
@@ -12382,7 +14255,6 @@ var ServerCoordinator = class {
12382
14255
  payload: eventPayload,
12383
14256
  timestamp: this.hlc.now()
12384
14257
  }, originalSenderId);
12385
- this.broadcastToCluster(eventPayload);
12386
14258
  this.runAfterInterceptors(op, context);
12387
14259
  }
12388
14260
  /**
@@ -12509,7 +14381,6 @@ var ServerCoordinator = class {
12509
14381
  });
12510
14382
  }
12511
14383
  batchedEvents.push(eventPayload);
12512
- this.broadcastToCluster(eventPayload);
12513
14384
  this.runAfterInterceptors(op, context);
12514
14385
  }
12515
14386
  handleClusterEvent(payload) {
@@ -12754,7 +14625,7 @@ var ServerCoordinator = class {
12754
14625
  idleTime: now - client.lastPingReceived,
12755
14626
  timeoutMs: CLIENT_HEARTBEAT_TIMEOUT_MS
12756
14627
  }, "Evicting dead client (heartbeat timeout)");
12757
- if (client.socket.readyState === WebSocket3.OPEN) {
14628
+ if (client.socket.readyState === WebSocket5.OPEN) {
12758
14629
  client.socket.close(4002, "Heartbeat timeout");
12759
14630
  }
12760
14631
  }
@@ -12852,11 +14723,18 @@ var ServerCoordinator = class {
12852
14723
  payload: eventPayload,
12853
14724
  timestamp: this.hlc.now()
12854
14725
  });
12855
- const members = this.cluster.getMembers();
12856
- for (const memberId of members) {
12857
- if (!this.cluster.isLocal(memberId)) {
12858
- this.cluster.send(memberId, "CLUSTER_EVENT", eventPayload);
12859
- }
14726
+ this.queryRegistry.processChange(name, map, key, tombstone, record);
14727
+ if (this.replicationPipeline) {
14728
+ const op = {
14729
+ opType: "set",
14730
+ mapName: name,
14731
+ key,
14732
+ record: tombstone
14733
+ };
14734
+ const opId = `ttl:${name}:${key}:${Date.now()}`;
14735
+ this.replicationPipeline.replicate(op, opId, key).catch((err) => {
14736
+ logger.warn({ opId, key, err }, "TTL expiration replication failed (non-fatal)");
14737
+ });
12860
14738
  }
12861
14739
  }
12862
14740
  }
@@ -12889,6 +14767,7 @@ var ServerCoordinator = class {
12889
14767
  }
12890
14768
  for (const { key, tag } of tagsToExpire) {
12891
14769
  logger.info({ mapName: name, key, tag }, "ORMap Record expired (TTL). Removing.");
14770
+ const oldRecords = map.getRecords(key);
12892
14771
  map.applyTombstone(tag);
12893
14772
  if (this.storage) {
12894
14773
  const records = map.getRecords(key);
@@ -12914,11 +14793,19 @@ var ServerCoordinator = class {
12914
14793
  payload: eventPayload,
12915
14794
  timestamp: this.hlc.now()
12916
14795
  });
12917
- const members = this.cluster.getMembers();
12918
- for (const memberId of members) {
12919
- if (!this.cluster.isLocal(memberId)) {
12920
- this.cluster.send(memberId, "CLUSTER_EVENT", eventPayload);
12921
- }
14796
+ const newRecords = map.getRecords(key);
14797
+ this.queryRegistry.processChange(name, map, key, newRecords, oldRecords);
14798
+ if (this.replicationPipeline) {
14799
+ const op = {
14800
+ opType: "OR_REMOVE",
14801
+ mapName: name,
14802
+ key,
14803
+ orTag: tag
14804
+ };
14805
+ const opId = `ttl:${name}:${key}:${tag}:${Date.now()}`;
14806
+ this.replicationPipeline.replicate(op, opId, key).catch((err) => {
14807
+ logger.warn({ opId, key, err }, "ORMap TTL expiration replication failed (non-fatal)");
14808
+ });
12922
14809
  }
12923
14810
  }
12924
14811
  const removedTags = map.prune(olderThan);
@@ -13512,7 +15399,7 @@ function logNativeStatus() {
13512
15399
  }
13513
15400
 
13514
15401
  // src/cluster/ClusterCoordinator.ts
13515
- import { EventEmitter as EventEmitter13 } from "events";
15402
+ import { EventEmitter as EventEmitter16 } from "events";
13516
15403
  import {
13517
15404
  DEFAULT_MIGRATION_CONFIG as DEFAULT_MIGRATION_CONFIG3,
13518
15405
  DEFAULT_REPLICATION_CONFIG as DEFAULT_REPLICATION_CONFIG3
@@ -13523,7 +15410,7 @@ var DEFAULT_CLUSTER_COORDINATOR_CONFIG = {
13523
15410
  replication: DEFAULT_REPLICATION_CONFIG3,
13524
15411
  replicationEnabled: true
13525
15412
  };
13526
- var ClusterCoordinator = class extends EventEmitter13 {
15413
+ var ClusterCoordinator = class extends EventEmitter16 {
13527
15414
  constructor(config) {
13528
15415
  super();
13529
15416
  this.replicationPipeline = null;