@topgunbuild/server 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -105,8 +105,8 @@ module.exports = __toCommonJS(index_exports);
105
105
  var import_http = require("http");
106
106
  var import_https = require("https");
107
107
  var import_fs2 = require("fs");
108
- var import_ws3 = require("ws");
109
- var import_core20 = require("@topgunbuild/core");
108
+ var import_ws5 = require("ws");
109
+ var import_core22 = require("@topgunbuild/core");
110
110
  var jwt = __toESM(require("jsonwebtoken"));
111
111
  var crypto = __toESM(require("crypto"));
112
112
 
@@ -160,6 +160,10 @@ function matchesQuery(record, query) {
160
160
  return true;
161
161
  }
162
162
  function executeQuery(records, query) {
163
+ const result = executeQueryWithCursor(records, query);
164
+ return result.results;
165
+ }
166
+ function executeQueryWithCursor(records, query) {
163
167
  if (!query) {
164
168
  query = {};
165
169
  }
@@ -177,9 +181,13 @@ function executeQuery(records, query) {
177
181
  }
178
182
  }
179
183
  }
180
- if (query.sort) {
184
+ const sort = query.sort || {};
185
+ const sortEntries = Object.entries(sort);
186
+ const sortField = sortEntries.length > 0 ? sortEntries[0][0] : "_key";
187
+ const sortDirection = sortEntries.length > 0 ? sortEntries[0][1] : "asc";
188
+ if (sortEntries.length > 0) {
181
189
  results.sort((a, b) => {
182
- for (const [field, direction] of Object.entries(query.sort)) {
190
+ for (const [field, direction] of sortEntries) {
183
191
  const valA = a.record.value[field];
184
192
  const valB = b.record.value[field];
185
193
  if (valA < valB) return direction === "asc" ? -1 : 1;
@@ -188,16 +196,55 @@ function executeQuery(records, query) {
188
196
  return 0;
189
197
  });
190
198
  }
191
- if (query.offset || query.limit) {
192
- const offset = query.offset || 0;
193
- const limit = query.limit || results.length;
194
- results = results.slice(offset, offset + limit);
199
+ let cursorStatus = "none";
200
+ if (query.cursor) {
201
+ const cursorData = import_core.QueryCursor.decode(query.cursor);
202
+ if (!cursorData) {
203
+ cursorStatus = "invalid";
204
+ } else if (!import_core.QueryCursor.isValid(cursorData, query.predicate ?? query.where, sort)) {
205
+ if (Date.now() - cursorData.timestamp > import_core.DEFAULT_QUERY_CURSOR_MAX_AGE_MS) {
206
+ cursorStatus = "expired";
207
+ } else {
208
+ cursorStatus = "invalid";
209
+ }
210
+ } else {
211
+ cursorStatus = "valid";
212
+ results = results.filter((r) => {
213
+ const sortValue = r.record.value[sortField];
214
+ return import_core.QueryCursor.isAfterCursor(
215
+ { key: r.key, sortValue },
216
+ cursorData
217
+ );
218
+ });
219
+ }
220
+ }
221
+ const hasLimit = query.limit !== void 0 && query.limit > 0;
222
+ const totalBeforeLimit = results.length;
223
+ if (hasLimit) {
224
+ results = results.slice(0, query.limit);
225
+ }
226
+ const hasMore = hasLimit && totalBeforeLimit > query.limit;
227
+ let nextCursor;
228
+ if (hasMore && results.length > 0) {
229
+ const lastResult = results[results.length - 1];
230
+ const sortValue = lastResult.record.value[sortField];
231
+ nextCursor = import_core.QueryCursor.fromLastResult(
232
+ { key: lastResult.key, sortValue },
233
+ sort,
234
+ query.predicate ?? query.where
235
+ );
195
236
  }
196
- return results.map((r) => ({ key: r.key, value: r.record.value }));
237
+ return {
238
+ results: results.map((r) => ({ key: r.key, value: r.record.value })),
239
+ nextCursor,
240
+ hasMore,
241
+ cursorStatus
242
+ };
197
243
  }
198
244
 
199
245
  // src/query/QueryRegistry.ts
200
246
  var import_core2 = require("@topgunbuild/core");
247
+ var import_ws = require("ws");
201
248
 
202
249
  // src/utils/logger.ts
203
250
  var import_pino = __toESM(require("pino"));
@@ -386,6 +433,106 @@ var QueryRegistry = class {
386
433
  }
387
434
  }
388
435
  }
436
+ /**
437
+ * Set the ClusterManager for distributed subscriptions.
438
+ */
439
+ setClusterManager(clusterManager, nodeId) {
440
+ this.clusterManager = clusterManager;
441
+ this.nodeId = nodeId;
442
+ }
443
+ /**
444
+ * Set the callback for getting maps by name.
445
+ * Required for distributed subscriptions to return initial results.
446
+ */
447
+ setMapGetter(getter) {
448
+ this.getMap = getter;
449
+ }
450
+ /**
451
+ * Register a distributed subscription from a remote coordinator.
452
+ * Called when receiving CLUSTER_SUB_REGISTER message.
453
+ *
454
+ * @param subscriptionId - Unique subscription ID
455
+ * @param mapName - Map name to query
456
+ * @param query - Query predicate
457
+ * @param coordinatorNodeId - Node ID of the coordinator (receives updates)
458
+ * @returns Initial query results from this node
459
+ */
460
+ registerDistributed(subscriptionId, mapName, query, coordinatorNodeId) {
461
+ const dummySocket = {
462
+ readyState: 1,
463
+ send: () => {
464
+ }
465
+ // Updates go via cluster messages, not socket
466
+ };
467
+ let initialResults = [];
468
+ const previousResultKeys = /* @__PURE__ */ new Set();
469
+ if (this.getMap) {
470
+ const map = this.getMap(mapName);
471
+ if (map) {
472
+ const records = this.getMapRecords(map);
473
+ const queryResults = executeQuery(records, query);
474
+ initialResults = queryResults.map((r) => {
475
+ previousResultKeys.add(r.key);
476
+ return { key: r.key, value: r.value };
477
+ });
478
+ }
479
+ }
480
+ const sub = {
481
+ id: subscriptionId,
482
+ clientId: `cluster:${coordinatorNodeId}`,
483
+ mapName,
484
+ query,
485
+ socket: dummySocket,
486
+ previousResultKeys,
487
+ coordinatorNodeId,
488
+ isDistributed: true
489
+ };
490
+ this.register(sub);
491
+ logger.debug(
492
+ { subscriptionId, mapName, coordinatorNodeId, resultCount: initialResults.length },
493
+ "Distributed query subscription registered"
494
+ );
495
+ return initialResults;
496
+ }
497
+ /**
498
+ * Get a distributed subscription by ID.
499
+ * Returns undefined if not found or not distributed.
500
+ */
501
+ getDistributedSubscription(subscriptionId) {
502
+ for (const subs of this.subscriptions.values()) {
503
+ for (const sub of subs) {
504
+ if (sub.id === subscriptionId && sub.isDistributed) {
505
+ return sub;
506
+ }
507
+ }
508
+ }
509
+ return void 0;
510
+ }
511
+ /**
512
+ * Unregister all distributed subscriptions where the given node was the coordinator.
513
+ * Called when a cluster node disconnects.
514
+ *
515
+ * @param coordinatorNodeId - Node ID of the disconnected coordinator
516
+ */
517
+ unregisterByCoordinator(coordinatorNodeId) {
518
+ const subscriptionsToRemove = [];
519
+ for (const subs of this.subscriptions.values()) {
520
+ for (const sub of subs) {
521
+ if (sub.isDistributed && sub.coordinatorNodeId === coordinatorNodeId) {
522
+ subscriptionsToRemove.push(sub.id);
523
+ }
524
+ }
525
+ }
526
+ for (const subId of subscriptionsToRemove) {
527
+ this.unregister(subId);
528
+ }
529
+ if (subscriptionsToRemove.length > 0) {
530
+ logger.debug(
531
+ { coordinatorNodeId, count: subscriptionsToRemove.length },
532
+ "Cleaned up distributed query subscriptions for disconnected coordinator"
533
+ );
534
+ }
535
+ }
389
536
  /**
390
537
  * Returns all active subscriptions for a specific map.
391
538
  * Used for subscription-based event routing to avoid broadcasting to all clients.
@@ -668,7 +815,9 @@ var QueryRegistry = class {
668
815
  return record.value;
669
816
  }
670
817
  sendUpdate(sub, key, value, type) {
671
- if (sub.socket.readyState === 1) {
818
+ if (sub.isDistributed && sub.coordinatorNodeId && this.clusterManager) {
819
+ this.sendDistributedUpdate(sub, key, value, type);
820
+ } else if (sub.socket.readyState === import_ws.WebSocket.OPEN) {
672
821
  sub.socket.send((0, import_core2.serialize)({
673
822
  type: "QUERY_UPDATE",
674
823
  payload: {
@@ -680,6 +829,26 @@ var QueryRegistry = class {
680
829
  }));
681
830
  }
682
831
  }
832
+ /**
833
+ * Send update to remote coordinator node for a distributed subscription.
834
+ */
835
+ sendDistributedUpdate(sub, key, value, type) {
836
+ if (!this.clusterManager || !sub.coordinatorNodeId) return;
837
+ const changeType = type === "UPDATE" ? sub.previousResultKeys.has(key) ? "UPDATE" : "ENTER" : "LEAVE";
838
+ const payload = {
839
+ subscriptionId: sub.id,
840
+ sourceNodeId: this.nodeId || "unknown",
841
+ key,
842
+ value,
843
+ changeType,
844
+ timestamp: Date.now()
845
+ };
846
+ this.clusterManager.send(sub.coordinatorNodeId, "CLUSTER_SUB_UPDATE", payload);
847
+ logger.debug(
848
+ { subscriptionId: sub.id, key, changeType, coordinator: sub.coordinatorNodeId },
849
+ "Sent distributed query update"
850
+ );
851
+ }
683
852
  analyzeQueryFields(query) {
684
853
  const fields = /* @__PURE__ */ new Set();
685
854
  try {
@@ -818,7 +987,7 @@ var TopicManager = class {
818
987
  };
819
988
 
820
989
  // src/cluster/ClusterManager.ts
821
- var import_ws = require("ws");
990
+ var import_ws2 = require("ws");
822
991
  var import_events2 = require("events");
823
992
  var dns = __toESM(require("dns"));
824
993
  var import_fs = require("fs");
@@ -1123,7 +1292,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
1123
1292
  if (this.config.tls?.enabled) {
1124
1293
  const tlsOptions = this.buildClusterTLSOptions();
1125
1294
  const httpsServer = https.createServer(tlsOptions);
1126
- this.server = new import_ws.WebSocketServer({ server: httpsServer });
1295
+ this.server = new import_ws2.WebSocketServer({ server: httpsServer });
1127
1296
  httpsServer.listen(this.config.port, () => {
1128
1297
  const addr = httpsServer.address();
1129
1298
  this._actualPort = typeof addr === "object" && addr ? addr.port : this.config.port;
@@ -1131,7 +1300,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
1131
1300
  this.onServerReady(resolve);
1132
1301
  });
1133
1302
  } else {
1134
- this.server = new import_ws.WebSocketServer({ port: this.config.port });
1303
+ this.server = new import_ws2.WebSocketServer({ port: this.config.port });
1135
1304
  this.server.on("listening", () => {
1136
1305
  const addr = this.server.address();
1137
1306
  this._actualPort = typeof addr === "object" && addr ? addr.port : this.config.port;
@@ -1211,7 +1380,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
1211
1380
  sendHeartbeatToAll() {
1212
1381
  for (const [nodeId, member] of this.members) {
1213
1382
  if (member.isSelf) continue;
1214
- if (member.socket && member.socket.readyState === import_ws.WebSocket.OPEN) {
1383
+ if (member.socket && member.socket.readyState === import_ws2.WebSocket.OPEN) {
1215
1384
  this.send(nodeId, "HEARTBEAT", { timestamp: Date.now() });
1216
1385
  }
1217
1386
  }
@@ -1245,7 +1414,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
1245
1414
  broadcastMemberList() {
1246
1415
  for (const [nodeId, member] of this.members) {
1247
1416
  if (member.isSelf) continue;
1248
- if (member.socket && member.socket.readyState === import_ws.WebSocket.OPEN) {
1417
+ if (member.socket && member.socket.readyState === import_ws2.WebSocket.OPEN) {
1249
1418
  this.sendMemberList(nodeId);
1250
1419
  }
1251
1420
  }
@@ -1270,7 +1439,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
1270
1439
  const member = this.members.get(nodeId);
1271
1440
  if (!member) return;
1272
1441
  logger.warn({ nodeId }, "Removing failed node from cluster");
1273
- if (member.socket && member.socket.readyState !== import_ws.WebSocket.CLOSED) {
1442
+ if (member.socket && member.socket.readyState !== import_ws2.WebSocket.CLOSED) {
1274
1443
  try {
1275
1444
  member.socket.terminate();
1276
1445
  } catch (e) {
@@ -1344,9 +1513,9 @@ var ClusterManager = class extends import_events2.EventEmitter {
1344
1513
  if (this.config.tls.caCertPath) {
1345
1514
  wsOptions.ca = (0, import_fs.readFileSync)(this.config.tls.caCertPath);
1346
1515
  }
1347
- ws = new import_ws.WebSocket(`${protocol}${peerAddress}`, wsOptions);
1516
+ ws = new import_ws2.WebSocket(`${protocol}${peerAddress}`, wsOptions);
1348
1517
  } else {
1349
- ws = new import_ws.WebSocket(`ws://${peerAddress}`);
1518
+ ws = new import_ws2.WebSocket(`ws://${peerAddress}`);
1350
1519
  }
1351
1520
  ws.on("open", () => {
1352
1521
  this.pendingConnections.delete(peerAddress);
@@ -1433,7 +1602,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
1433
1602
  }
1434
1603
  send(nodeId, type, payload) {
1435
1604
  const member = this.members.get(nodeId);
1436
- if (member && member.socket && member.socket.readyState === import_ws.WebSocket.OPEN) {
1605
+ if (member && member.socket && member.socket.readyState === import_ws2.WebSocket.OPEN) {
1437
1606
  const msg = {
1438
1607
  type,
1439
1608
  senderId: this.config.nodeId,
@@ -2652,6 +2821,90 @@ var MetricsService = class {
2652
2821
  help: "Current connection rate per second",
2653
2822
  registers: [this.registry]
2654
2823
  });
2824
+ this.distributedSearchTotal = new import_prom_client.Counter({
2825
+ name: "topgun_distributed_search_total",
2826
+ help: "Total number of distributed search requests",
2827
+ labelNames: ["map", "status"],
2828
+ registers: [this.registry]
2829
+ });
2830
+ this.distributedSearchDuration = new import_prom_client.Summary({
2831
+ name: "topgun_distributed_search_duration_ms",
2832
+ help: "Distribution of distributed search execution times in milliseconds",
2833
+ labelNames: ["map"],
2834
+ percentiles: [0.5, 0.9, 0.95, 0.99],
2835
+ registers: [this.registry]
2836
+ });
2837
+ this.distributedSearchFailedNodes = new import_prom_client.Counter({
2838
+ name: "topgun_distributed_search_failed_nodes_total",
2839
+ help: "Total number of node failures during distributed search",
2840
+ registers: [this.registry]
2841
+ });
2842
+ this.distributedSearchPartialResults = new import_prom_client.Counter({
2843
+ name: "topgun_distributed_search_partial_results_total",
2844
+ help: "Total number of searches that returned partial results due to node failures",
2845
+ registers: [this.registry]
2846
+ });
2847
+ this.distributedSubTotal = new import_prom_client.Counter({
2848
+ name: "topgun_distributed_sub_total",
2849
+ help: "Total distributed subscriptions created",
2850
+ labelNames: ["type", "status"],
2851
+ registers: [this.registry]
2852
+ });
2853
+ this.distributedSubUnsubscribeTotal = new import_prom_client.Counter({
2854
+ name: "topgun_distributed_sub_unsubscribe_total",
2855
+ help: "Total unsubscriptions from distributed subscriptions",
2856
+ labelNames: ["type"],
2857
+ registers: [this.registry]
2858
+ });
2859
+ this.distributedSubActive = new import_prom_client.Gauge({
2860
+ name: "topgun_distributed_sub_active",
2861
+ help: "Currently active distributed subscriptions",
2862
+ labelNames: ["type"],
2863
+ registers: [this.registry]
2864
+ });
2865
+ this.distributedSubPendingAcks = new import_prom_client.Gauge({
2866
+ name: "topgun_distributed_sub_pending_acks",
2867
+ help: "Subscriptions waiting for ACKs from cluster nodes",
2868
+ registers: [this.registry]
2869
+ });
2870
+ this.distributedSubUpdates = new import_prom_client.Counter({
2871
+ name: "topgun_distributed_sub_updates_total",
2872
+ help: "Delta updates processed for distributed subscriptions",
2873
+ labelNames: ["direction", "change_type"],
2874
+ registers: [this.registry]
2875
+ });
2876
+ this.distributedSubAckTotal = new import_prom_client.Counter({
2877
+ name: "topgun_distributed_sub_ack_total",
2878
+ help: "Node ACK responses for distributed subscriptions",
2879
+ labelNames: ["status"],
2880
+ registers: [this.registry]
2881
+ });
2882
+ this.distributedSubNodeDisconnect = new import_prom_client.Counter({
2883
+ name: "topgun_distributed_sub_node_disconnect_total",
2884
+ help: "Node disconnects affecting distributed subscriptions",
2885
+ registers: [this.registry]
2886
+ });
2887
+ this.distributedSubRegistrationDuration = new import_prom_client.Histogram({
2888
+ name: "topgun_distributed_sub_registration_duration_ms",
2889
+ help: "Time to register subscription on all nodes",
2890
+ labelNames: ["type"],
2891
+ buckets: [10, 50, 100, 250, 500, 1e3, 2500],
2892
+ registers: [this.registry]
2893
+ });
2894
+ this.distributedSubUpdateLatency = new import_prom_client.Histogram({
2895
+ name: "topgun_distributed_sub_update_latency_ms",
2896
+ help: "Latency from data change to client notification",
2897
+ labelNames: ["type"],
2898
+ buckets: [1, 5, 10, 25, 50, 100, 250],
2899
+ registers: [this.registry]
2900
+ });
2901
+ this.distributedSubInitialResultsCount = new import_prom_client.Histogram({
2902
+ name: "topgun_distributed_sub_initial_results_count",
2903
+ help: "Initial result set size for distributed subscriptions",
2904
+ labelNames: ["type"],
2905
+ buckets: [0, 1, 5, 10, 25, 50, 100],
2906
+ registers: [this.registry]
2907
+ });
2655
2908
  }
2656
2909
  destroy() {
2657
2910
  this.registry.clear();
@@ -2762,6 +3015,115 @@ var MetricsService = class {
2762
3015
  setConnectionRatePerSecond(rate) {
2763
3016
  this.connectionRatePerSecond.set(rate);
2764
3017
  }
3018
+ // === Distributed search metric methods (Phase 14) ===
3019
+ /**
3020
+ * Record a distributed search request.
3021
+ * @param mapName - Name of the map being searched
3022
+ * @param status - 'success', 'partial', or 'error'
3023
+ */
3024
+ incDistributedSearch(mapName, status) {
3025
+ this.distributedSearchTotal.inc({ map: mapName, status });
3026
+ }
3027
+ /**
3028
+ * Record the duration of a distributed search.
3029
+ * @param mapName - Name of the map being searched
3030
+ * @param durationMs - Duration in milliseconds
3031
+ */
3032
+ recordDistributedSearchDuration(mapName, durationMs) {
3033
+ this.distributedSearchDuration.observe({ map: mapName }, durationMs);
3034
+ }
3035
+ /**
3036
+ * Increment counter for failed nodes during distributed search.
3037
+ * @param count - Number of nodes that failed (default 1)
3038
+ */
3039
+ incDistributedSearchFailedNodes(count = 1) {
3040
+ this.distributedSearchFailedNodes.inc(count);
3041
+ }
3042
+ /**
3043
+ * Increment counter for searches returning partial results.
3044
+ */
3045
+ incDistributedSearchPartialResults() {
3046
+ this.distributedSearchPartialResults.inc();
3047
+ }
3048
+ // === Distributed subscription metric methods (Phase 14.2) ===
3049
+ /**
3050
+ * Record a distributed subscription creation.
3051
+ * @param type - Subscription type (SEARCH or QUERY)
3052
+ * @param status - Result status (success, failed, timeout)
3053
+ */
3054
+ incDistributedSub(type, status) {
3055
+ this.distributedSubTotal.inc({ type, status });
3056
+ if (status === "success") {
3057
+ this.distributedSubActive.inc({ type });
3058
+ }
3059
+ }
3060
+ /**
3061
+ * Record a distributed subscription unsubscription.
3062
+ * @param type - Subscription type (SEARCH or QUERY)
3063
+ */
3064
+ incDistributedSubUnsubscribe(type) {
3065
+ this.distributedSubUnsubscribeTotal.inc({ type });
3066
+ }
3067
+ /**
3068
+ * Decrement the active distributed subscriptions gauge.
3069
+ * @param type - Subscription type (SEARCH or QUERY)
3070
+ */
3071
+ decDistributedSubActive(type) {
3072
+ this.distributedSubActive.dec({ type });
3073
+ }
3074
+ /**
3075
+ * Set the number of subscriptions waiting for ACKs.
3076
+ * @param count - Number of pending ACKs
3077
+ */
3078
+ setDistributedSubPendingAcks(count) {
3079
+ this.distributedSubPendingAcks.set(count);
3080
+ }
3081
+ /**
3082
+ * Record a delta update for distributed subscriptions.
3083
+ * @param direction - Direction of update (sent or received)
3084
+ * @param changeType - Type of change (ENTER, UPDATE, LEAVE)
3085
+ */
3086
+ incDistributedSubUpdates(direction, changeType) {
3087
+ this.distributedSubUpdates.inc({ direction, change_type: changeType });
3088
+ }
3089
+ /**
3090
+ * Record node ACK responses.
3091
+ * @param status - ACK status (success, failed, timeout)
3092
+ * @param count - Number of ACKs to record (default 1)
3093
+ */
3094
+ incDistributedSubAck(status, count = 1) {
3095
+ this.distributedSubAckTotal.inc({ status }, count);
3096
+ }
3097
+ /**
3098
+ * Record a node disconnect affecting distributed subscriptions.
3099
+ */
3100
+ incDistributedSubNodeDisconnect() {
3101
+ this.distributedSubNodeDisconnect.inc();
3102
+ }
3103
+ /**
3104
+ * Record the time to register a subscription on all nodes.
3105
+ * @param type - Subscription type (SEARCH or QUERY)
3106
+ * @param durationMs - Duration in milliseconds
3107
+ */
3108
+ recordDistributedSubRegistration(type, durationMs) {
3109
+ this.distributedSubRegistrationDuration.observe({ type }, durationMs);
3110
+ }
3111
+ /**
3112
+ * Record the latency from data change to client notification.
3113
+ * @param type - Subscription type (SEARCH or QUERY)
3114
+ * @param latencyMs - Latency in milliseconds
3115
+ */
3116
+ recordDistributedSubUpdateLatency(type, latencyMs) {
3117
+ this.distributedSubUpdateLatency.observe({ type }, latencyMs);
3118
+ }
3119
+ /**
3120
+ * Record the initial result set size for a subscription.
3121
+ * @param type - Subscription type (SEARCH or QUERY)
3122
+ * @param count - Number of initial results
3123
+ */
3124
+ recordDistributedSubInitialResultsCount(type, count) {
3125
+ this.distributedSubInitialResultsCount.observe({ type }, count);
3126
+ }
2765
3127
  async getMetrics() {
2766
3128
  return this.registry.metrics();
2767
3129
  }
@@ -3292,7 +3654,7 @@ var BackpressureRegulator = class {
3292
3654
  };
3293
3655
 
3294
3656
  // src/utils/CoalescingWriter.ts
3295
- var import_ws2 = require("ws");
3657
+ var import_ws3 = require("ws");
3296
3658
  var import_core5 = require("@topgunbuild/core");
3297
3659
 
3298
3660
  // src/memory/BufferPool.ts
@@ -3862,7 +4224,7 @@ var CoalescingWriter = class {
3862
4224
  }
3863
4225
  this.state = 2 /* FLUSHING */;
3864
4226
  try {
3865
- if (this.socket.readyState !== import_ws2.WebSocket.OPEN) {
4227
+ if (this.socket.readyState !== import_ws3.WebSocket.OPEN) {
3866
4228
  this.queue = [];
3867
4229
  this.pendingBytes = 0;
3868
4230
  this.state = 0 /* IDLE */;
@@ -3935,7 +4297,7 @@ var CoalescingWriter = class {
3935
4297
  * Send a message immediately without batching.
3936
4298
  */
3937
4299
  sendImmediate(data) {
3938
- if (this.socket.readyState !== import_ws2.WebSocket.OPEN) {
4300
+ if (this.socket.readyState !== import_ws3.WebSocket.OPEN) {
3939
4301
  return;
3940
4302
  }
3941
4303
  try {
@@ -7517,7 +7879,7 @@ var RepairScheduler = class extends import_events12.EventEmitter {
7517
7879
  this.processTimer = setInterval(() => {
7518
7880
  this.processRepairQueue();
7519
7881
  }, 1e3);
7520
- setTimeout(() => {
7882
+ this.initialScanTimer = setTimeout(() => {
7521
7883
  this.scheduleFullScan();
7522
7884
  }, 6e4);
7523
7885
  }
@@ -7535,9 +7897,13 @@ var RepairScheduler = class extends import_events12.EventEmitter {
7535
7897
  clearInterval(this.processTimer);
7536
7898
  this.processTimer = void 0;
7537
7899
  }
7900
+ if (this.initialScanTimer) {
7901
+ clearTimeout(this.initialScanTimer);
7902
+ this.initialScanTimer = void 0;
7903
+ }
7538
7904
  this.repairQueue = [];
7539
7905
  this.activeRepairs.clear();
7540
- for (const [id, req] of this.pendingRequests) {
7906
+ for (const [, req] of this.pendingRequests) {
7541
7907
  clearTimeout(req.timer);
7542
7908
  req.reject(new Error("Scheduler stopped"));
7543
7909
  }
@@ -9257,9 +9623,11 @@ var EventJournalService = class extends import_core18.EventJournalImpl {
9257
9623
  };
9258
9624
 
9259
9625
  // src/search/SearchCoordinator.ts
9626
+ var import_events13 = require("events");
9260
9627
  var import_core19 = require("@topgunbuild/core");
9261
- var SearchCoordinator = class {
9628
+ var SearchCoordinator = class extends import_events13.EventEmitter {
9262
9629
  constructor() {
9630
+ super();
9263
9631
  /** Map name → FullTextIndex */
9264
9632
  this.indexes = /* @__PURE__ */ new Map();
9265
9633
  /** Map name → FullTextIndexConfig (for reference) */
@@ -9284,6 +9652,13 @@ var SearchCoordinator = class {
9284
9652
  this.BATCH_INTERVAL = 16;
9285
9653
  logger.debug("SearchCoordinator initialized");
9286
9654
  }
9655
+ /**
9656
+ * Set the node ID for this server.
9657
+ * Required for distributed subscriptions.
9658
+ */
9659
+ setNodeId(nodeId) {
9660
+ this.nodeId = nodeId;
9661
+ }
9287
9662
  /**
9288
9663
  * Set the callback for sending updates to clients.
9289
9664
  * Called by ServerCoordinator during initialization.
@@ -9585,9 +9960,112 @@ var SearchCoordinator = class {
9585
9960
  getSubscriptionCount() {
9586
9961
  return this.subscriptions.size;
9587
9962
  }
9963
+ // ============================================
9964
+ // Phase 14.2: Distributed Subscription Methods
9965
+ // ============================================
9966
+ /**
9967
+ * Register a distributed subscription from a remote coordinator.
9968
+ * Called when receiving CLUSTER_SUB_REGISTER message.
9969
+ *
9970
+ * @param subscriptionId - Unique subscription ID
9971
+ * @param mapName - Map name to search
9972
+ * @param query - Search query string
9973
+ * @param options - Search options
9974
+ * @param coordinatorNodeId - Node ID of the coordinator (receives updates)
9975
+ * @returns Initial search results from this node
9976
+ */
9977
+ registerDistributedSubscription(subscriptionId, mapName, query, options, coordinatorNodeId) {
9978
+ const index = this.indexes.get(mapName);
9979
+ if (!index) {
9980
+ logger.warn({ mapName }, "Distributed subscription for map without FTS enabled");
9981
+ return { results: [], totalHits: 0 };
9982
+ }
9983
+ const queryTerms = index.tokenizeQuery(query);
9984
+ const searchResults = index.search(query, options);
9985
+ const currentResults = /* @__PURE__ */ new Map();
9986
+ const results = [];
9987
+ for (const result of searchResults) {
9988
+ const value = this.getDocumentValue ? this.getDocumentValue(mapName, result.docId) : void 0;
9989
+ currentResults.set(result.docId, {
9990
+ score: result.score,
9991
+ matchedTerms: result.matchedTerms || []
9992
+ });
9993
+ results.push({
9994
+ key: result.docId,
9995
+ value,
9996
+ score: result.score,
9997
+ matchedTerms: result.matchedTerms || []
9998
+ });
9999
+ }
10000
+ const subscription = {
10001
+ id: subscriptionId,
10002
+ clientId: `cluster:${coordinatorNodeId}`,
10003
+ // Virtual client ID
10004
+ mapName,
10005
+ query,
10006
+ queryTerms,
10007
+ options: options || {},
10008
+ currentResults,
10009
+ // Distributed fields
10010
+ coordinatorNodeId,
10011
+ isDistributed: true
10012
+ };
10013
+ this.subscriptions.set(subscriptionId, subscription);
10014
+ if (!this.subscriptionsByMap.has(mapName)) {
10015
+ this.subscriptionsByMap.set(mapName, /* @__PURE__ */ new Set());
10016
+ }
10017
+ this.subscriptionsByMap.get(mapName).add(subscriptionId);
10018
+ if (!this.subscriptionsByClient.has(subscription.clientId)) {
10019
+ this.subscriptionsByClient.set(subscription.clientId, /* @__PURE__ */ new Set());
10020
+ }
10021
+ this.subscriptionsByClient.get(subscription.clientId).add(subscriptionId);
10022
+ logger.debug(
10023
+ { subscriptionId, mapName, query, coordinatorNodeId, resultCount: results.length },
10024
+ "Distributed search subscription registered"
10025
+ );
10026
+ return {
10027
+ results,
10028
+ totalHits: results.length
10029
+ };
10030
+ }
10031
+ /**
10032
+ * Get a distributed subscription by ID.
10033
+ * Returns undefined if not found or not distributed.
10034
+ */
10035
+ getDistributedSubscription(subscriptionId) {
10036
+ const sub = this.subscriptions.get(subscriptionId);
10037
+ if (sub?.isDistributed) {
10038
+ return sub;
10039
+ }
10040
+ return void 0;
10041
+ }
10042
+ /**
10043
+ * Unsubscribe all distributed subscriptions where the given node was the coordinator.
10044
+ * Called when a cluster node disconnects.
10045
+ *
10046
+ * @param coordinatorNodeId - Node ID of the disconnected coordinator
10047
+ */
10048
+ unsubscribeByCoordinator(coordinatorNodeId) {
10049
+ const subscriptionsToRemove = [];
10050
+ for (const [subId, sub] of this.subscriptions) {
10051
+ if (sub.isDistributed && sub.coordinatorNodeId === coordinatorNodeId) {
10052
+ subscriptionsToRemove.push(subId);
10053
+ }
10054
+ }
10055
+ for (const subId of subscriptionsToRemove) {
10056
+ this.unsubscribe(subId);
10057
+ }
10058
+ if (subscriptionsToRemove.length > 0) {
10059
+ logger.debug(
10060
+ { coordinatorNodeId, count: subscriptionsToRemove.length },
10061
+ "Cleaned up distributed subscriptions for disconnected coordinator"
10062
+ );
10063
+ }
10064
+ }
9588
10065
  /**
9589
10066
  * Notify subscribers about a document change.
9590
10067
  * Computes delta (ENTER/UPDATE/LEAVE) for each affected subscription.
10068
+ * For distributed subscriptions, emits 'distributedUpdate' event instead of sending to client.
9591
10069
  *
9592
10070
  * @param mapName - Name of the map that changed
9593
10071
  * @param key - Document key that changed
@@ -9595,9 +10073,6 @@ var SearchCoordinator = class {
9595
10073
  * @param changeType - Type of change
9596
10074
  */
9597
10075
  notifySubscribers(mapName, key, value, changeType) {
9598
- if (!this.sendUpdate) {
9599
- return;
9600
- }
9601
10076
  const subscriptionIds = this.subscriptionsByMap.get(mapName);
9602
10077
  if (!subscriptionIds || subscriptionIds.size === 0) {
9603
10078
  return;
@@ -9638,18 +10113,50 @@ var SearchCoordinator = class {
9638
10113
  }
9639
10114
  logger.debug({ subId, key, wasInResults, isInResults, updateType, newScore }, "Update decision");
9640
10115
  if (updateType) {
9641
- this.sendUpdate(
9642
- sub.clientId,
9643
- subId,
9644
- key,
9645
- value,
9646
- newScore,
9647
- matchedTerms,
9648
- updateType
9649
- );
10116
+ if (sub.isDistributed && sub.coordinatorNodeId) {
10117
+ this.sendDistributedUpdate(sub, key, value, newScore, matchedTerms, updateType);
10118
+ } else if (this.sendUpdate) {
10119
+ this.sendUpdate(
10120
+ sub.clientId,
10121
+ subId,
10122
+ key,
10123
+ value,
10124
+ newScore,
10125
+ matchedTerms,
10126
+ updateType
10127
+ );
10128
+ }
9650
10129
  }
9651
10130
  }
9652
10131
  }
10132
+ /**
10133
+ * Send update to remote coordinator node for a distributed subscription.
10134
+ * Emits 'distributedUpdate' event with ClusterSubUpdatePayload.
10135
+ *
10136
+ * @param sub - The distributed subscription
10137
+ * @param key - Document key
10138
+ * @param value - Document value
10139
+ * @param score - Search score
10140
+ * @param matchedTerms - Matched search terms
10141
+ * @param changeType - Change type (ENTER/UPDATE/LEAVE)
10142
+ */
10143
+ sendDistributedUpdate(sub, key, value, score, matchedTerms, changeType) {
10144
+ const payload = {
10145
+ subscriptionId: sub.id,
10146
+ sourceNodeId: this.nodeId || "unknown",
10147
+ key,
10148
+ value,
10149
+ score,
10150
+ matchedTerms,
10151
+ changeType,
10152
+ timestamp: Date.now()
10153
+ };
10154
+ this.emit("distributedUpdate", payload);
10155
+ logger.debug(
10156
+ { subscriptionId: sub.id, key, changeType, coordinator: sub.coordinatorNodeId },
10157
+ "Emitted distributed update"
10158
+ );
10159
+ }
9653
10160
  /**
9654
10161
  * Score a single document against a subscription's query.
9655
10162
  *
@@ -9804,37 +10311,1205 @@ var SearchCoordinator = class {
9804
10311
  }
9805
10312
  };
9806
10313
 
9807
- // src/ServerCoordinator.ts
9808
- var GC_INTERVAL_MS = 60 * 60 * 1e3;
9809
- var GC_AGE_MS = 30 * 24 * 60 * 60 * 1e3;
9810
- var CLIENT_HEARTBEAT_TIMEOUT_MS = 2e4;
9811
- var CLIENT_HEARTBEAT_CHECK_INTERVAL_MS = 5e3;
9812
- var ServerCoordinator = class {
9813
- constructor(config) {
9814
- this.clients = /* @__PURE__ */ new Map();
9815
- // Interceptors
9816
- this.interceptors = [];
9817
- // In-memory storage (partitioned later)
9818
- this.maps = /* @__PURE__ */ new Map();
9819
- this.pendingClusterQueries = /* @__PURE__ */ new Map();
9820
- // GC Consensus State
9821
- this.gcReports = /* @__PURE__ */ new Map();
9822
- // Track map loading state to avoid returning empty results during async load
9823
- this.mapLoadingPromises = /* @__PURE__ */ new Map();
9824
- // Track pending batch operations for testing purposes
9825
- this.pendingBatchOperations = /* @__PURE__ */ new Set();
9826
- this.journalSubscriptions = /* @__PURE__ */ new Map();
9827
- this._actualPort = 0;
9828
- this._actualClusterPort = 0;
9829
- this._readyPromise = new Promise((resolve) => {
9830
- this._readyResolve = resolve;
9831
- });
9832
- this._nodeId = config.nodeId;
9833
- this.hlc = new import_core20.HLC(config.nodeId);
9834
- this.storage = config.storage;
9835
- const rawSecret = config.jwtSecret || process.env.JWT_SECRET || "topgun-secret-dev";
9836
- this.jwtSecret = rawSecret.replace(/\\n/g, "\n");
9837
- this.queryRegistry = new QueryRegistry();
10314
+ // src/search/ClusterSearchCoordinator.ts
10315
+ var import_events14 = require("events");
10316
+ var import_core20 = require("@topgunbuild/core");
10317
+ var DEFAULT_CONFIG6 = {
10318
+ rrfK: 60,
10319
+ defaultTimeoutMs: 5e3,
10320
+ defaultMinResponses: 0
10321
+ };
10322
+ var ClusterSearchCoordinator = class extends import_events14.EventEmitter {
10323
+ constructor(clusterManager, partitionService, localSearchCoordinator, config, metricsService) {
10324
+ super();
10325
+ /** Pending requests awaiting responses */
10326
+ this.pendingRequests = /* @__PURE__ */ new Map();
10327
+ this.clusterManager = clusterManager;
10328
+ this.partitionService = partitionService;
10329
+ this.localSearchCoordinator = localSearchCoordinator;
10330
+ this.config = { ...DEFAULT_CONFIG6, ...config };
10331
+ this.rrf = new import_core20.ReciprocalRankFusion({ k: this.config.rrfK });
10332
+ this.metricsService = metricsService;
10333
+ this.clusterManager.on("message", this.handleClusterMessage.bind(this));
10334
+ }
10335
+ /**
10336
+ * Execute a distributed search across the cluster.
10337
+ *
10338
+ * @param mapName - Name of the map to search
10339
+ * @param query - Search query text
10340
+ * @param options - Search options
10341
+ * @returns Promise resolving to merged search results
10342
+ */
10343
+ async search(mapName, query, options = { limit: 10 }) {
10344
+ const startTime = performance.now();
10345
+ const requestId = this.generateRequestId();
10346
+ const timeoutMs = options.timeoutMs ?? this.config.defaultTimeoutMs;
10347
+ const allNodes = new Set(this.clusterManager.getMembers());
10348
+ const myNodeId = this.clusterManager.config.nodeId;
10349
+ if (allNodes.size === 1 && allNodes.has(myNodeId)) {
10350
+ return this.executeLocalSearch(mapName, query, options, startTime);
10351
+ }
10352
+ const perNodeLimit = this.calculatePerNodeLimit(options.limit, options.cursor);
10353
+ let cursorData = null;
10354
+ if (options.cursor) {
10355
+ cursorData = import_core20.SearchCursor.decode(options.cursor);
10356
+ if (cursorData && !import_core20.SearchCursor.isValid(cursorData, query)) {
10357
+ cursorData = null;
10358
+ logger.warn({ requestId }, "Invalid or expired cursor, ignoring");
10359
+ }
10360
+ }
10361
+ const promise = new Promise((resolve, reject) => {
10362
+ const timeoutHandle = setTimeout(() => {
10363
+ this.resolvePartialResults(requestId);
10364
+ }, timeoutMs);
10365
+ this.pendingRequests.set(requestId, {
10366
+ resolve,
10367
+ reject,
10368
+ responses: /* @__PURE__ */ new Map(),
10369
+ expectedNodes: allNodes,
10370
+ startTime,
10371
+ timeoutHandle,
10372
+ options,
10373
+ mapName,
10374
+ query
10375
+ });
10376
+ });
10377
+ const payload = {
10378
+ requestId,
10379
+ mapName,
10380
+ query,
10381
+ options: {
10382
+ limit: perNodeLimit,
10383
+ minScore: options.minScore,
10384
+ boost: options.boost,
10385
+ includeMatchedTerms: true,
10386
+ // Add cursor position if available
10387
+ ...cursorData ? {
10388
+ afterScore: cursorData.nodeScores[myNodeId],
10389
+ afterKey: cursorData.nodeKeys[myNodeId]
10390
+ } : {}
10391
+ },
10392
+ timeoutMs
10393
+ };
10394
+ for (const nodeId of allNodes) {
10395
+ if (nodeId === myNodeId) {
10396
+ this.executeLocalAndRespond(requestId, mapName, query, perNodeLimit, cursorData);
10397
+ } else {
10398
+ this.clusterManager.send(nodeId, "CLUSTER_SEARCH_REQ", payload);
10399
+ }
10400
+ }
10401
+ return promise;
10402
+ }
10403
+ /**
10404
+ * Handle incoming cluster messages.
10405
+ */
10406
+ handleClusterMessage(msg) {
10407
+ switch (msg.type) {
10408
+ case "CLUSTER_SEARCH_REQ":
10409
+ this.handleSearchRequest(msg.senderId, msg.payload);
10410
+ break;
10411
+ case "CLUSTER_SEARCH_RESP":
10412
+ this.handleSearchResponse(msg.senderId, msg.payload);
10413
+ break;
10414
+ }
10415
+ }
10416
+ /**
10417
+ * Handle incoming search request from another node.
10418
+ */
10419
+ async handleSearchRequest(senderId, rawPayload) {
10420
+ const startTime = performance.now();
10421
+ const myNodeId = this.clusterManager.config.nodeId;
10422
+ const parsed = import_core20.ClusterSearchReqPayloadSchema.safeParse(rawPayload);
10423
+ if (!parsed.success) {
10424
+ logger.warn(
10425
+ { senderId, error: parsed.error.message },
10426
+ "Invalid cluster search request payload"
10427
+ );
10428
+ this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", {
10429
+ requestId: rawPayload?.requestId ?? "unknown",
10430
+ nodeId: myNodeId,
10431
+ results: [],
10432
+ totalHits: 0,
10433
+ executionTimeMs: performance.now() - startTime,
10434
+ error: "Invalid request payload"
10435
+ });
10436
+ return;
10437
+ }
10438
+ const payload = parsed.data;
10439
+ try {
10440
+ const localResult = this.localSearchCoordinator.search(
10441
+ payload.mapName,
10442
+ payload.query,
10443
+ {
10444
+ limit: payload.options.limit,
10445
+ minScore: payload.options.minScore,
10446
+ boost: payload.options.boost
10447
+ }
10448
+ );
10449
+ let results = localResult.results;
10450
+ if (payload.options.afterScore !== void 0) {
10451
+ results = results.filter((r) => {
10452
+ if (r.score < payload.options.afterScore) {
10453
+ return true;
10454
+ }
10455
+ if (r.score === payload.options.afterScore && payload.options.afterKey) {
10456
+ return r.key > payload.options.afterKey;
10457
+ }
10458
+ return false;
10459
+ });
10460
+ }
10461
+ const response = {
10462
+ requestId: payload.requestId,
10463
+ nodeId: myNodeId,
10464
+ results: results.map((r) => ({
10465
+ key: r.key,
10466
+ value: r.value,
10467
+ score: r.score,
10468
+ matchedTerms: r.matchedTerms
10469
+ })),
10470
+ totalHits: localResult.totalCount ?? results.length,
10471
+ executionTimeMs: performance.now() - startTime
10472
+ };
10473
+ this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", response);
10474
+ } catch (error) {
10475
+ this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", {
10476
+ requestId: payload.requestId,
10477
+ nodeId: myNodeId,
10478
+ results: [],
10479
+ totalHits: 0,
10480
+ executionTimeMs: performance.now() - startTime,
10481
+ error: error instanceof Error ? error.message : "Unknown error"
10482
+ });
10483
+ }
10484
+ }
10485
+ /**
10486
+ * Handle search response from a node.
10487
+ */
10488
+ handleSearchResponse(_senderId, rawPayload) {
10489
+ const parsed = import_core20.ClusterSearchRespPayloadSchema.safeParse(rawPayload);
10490
+ if (!parsed.success) {
10491
+ logger.warn(
10492
+ { error: parsed.error.message },
10493
+ "Invalid cluster search response payload"
10494
+ );
10495
+ return;
10496
+ }
10497
+ const payload = parsed.data;
10498
+ const pending = this.pendingRequests.get(payload.requestId);
10499
+ if (!pending) {
10500
+ logger.warn({ requestId: payload.requestId }, "Received response for unknown request");
10501
+ return;
10502
+ }
10503
+ pending.responses.set(payload.nodeId, payload);
10504
+ const minResponses = pending.options.minResponses ?? this.config.defaultMinResponses;
10505
+ const requiredResponses = minResponses > 0 ? minResponses : pending.expectedNodes.size;
10506
+ if (pending.responses.size >= requiredResponses) {
10507
+ clearTimeout(pending.timeoutHandle);
10508
+ this.mergeAndResolve(payload.requestId);
10509
+ }
10510
+ }
10511
+ /**
10512
+ * Merge results from all nodes using RRF and resolve the promise.
10513
+ */
10514
+ mergeAndResolve(requestId) {
10515
+ const pending = this.pendingRequests.get(requestId);
10516
+ if (!pending) return;
10517
+ const resultSets = [];
10518
+ const respondedNodes = [];
10519
+ const failedNodes = [];
10520
+ let totalHits = 0;
10521
+ const resultsWithNodes = [];
10522
+ for (const [nodeId, response] of pending.responses) {
10523
+ if (response.error) {
10524
+ failedNodes.push(nodeId);
10525
+ logger.warn({ nodeId, error: response.error }, "Node returned error for search");
10526
+ } else {
10527
+ respondedNodes.push(nodeId);
10528
+ totalHits += response.totalHits;
10529
+ const rankedResults = response.results.map((r) => ({
10530
+ docId: r.key,
10531
+ score: r.score,
10532
+ source: nodeId
10533
+ }));
10534
+ resultSets.push(rankedResults);
10535
+ for (const r of response.results) {
10536
+ resultsWithNodes.push({
10537
+ key: r.key,
10538
+ score: r.score,
10539
+ nodeId,
10540
+ value: r.value,
10541
+ matchedTerms: r.matchedTerms || []
10542
+ });
10543
+ }
10544
+ }
10545
+ }
10546
+ for (const nodeId of pending.expectedNodes) {
10547
+ if (!pending.responses.has(nodeId)) {
10548
+ failedNodes.push(nodeId);
10549
+ }
10550
+ }
10551
+ const merged = this.rrf.merge(resultSets);
10552
+ const limit = pending.options.limit;
10553
+ const results = [];
10554
+ const cursorResults = [];
10555
+ for (const mergedResult of merged.slice(0, limit)) {
10556
+ const original = resultsWithNodes.find((r) => r.key === mergedResult.docId);
10557
+ if (original) {
10558
+ results.push({
10559
+ key: original.key,
10560
+ value: original.value,
10561
+ score: mergedResult.score,
10562
+ // Use RRF score
10563
+ matchedTerms: original.matchedTerms
10564
+ });
10565
+ cursorResults.push({
10566
+ key: original.key,
10567
+ score: original.score,
10568
+ // Use original score for cursor
10569
+ nodeId: original.nodeId
10570
+ });
10571
+ }
10572
+ }
10573
+ let nextCursor;
10574
+ if (merged.length > limit && cursorResults.length > 0) {
10575
+ nextCursor = import_core20.SearchCursor.fromResults(cursorResults, pending.query);
10576
+ }
10577
+ const executionTimeMs = performance.now() - pending.startTime;
10578
+ if (this.metricsService) {
10579
+ const status = failedNodes.length === 0 ? "success" : respondedNodes.length === 0 ? "error" : "partial";
10580
+ this.metricsService.incDistributedSearch(pending.mapName, status);
10581
+ this.metricsService.recordDistributedSearchDuration(pending.mapName, executionTimeMs);
10582
+ if (failedNodes.length > 0) {
10583
+ this.metricsService.incDistributedSearchFailedNodes(failedNodes.length);
10584
+ if (respondedNodes.length > 0) {
10585
+ this.metricsService.incDistributedSearchPartialResults();
10586
+ }
10587
+ }
10588
+ }
10589
+ pending.resolve({
10590
+ results,
10591
+ totalHits,
10592
+ nextCursor,
10593
+ respondedNodes,
10594
+ failedNodes,
10595
+ executionTimeMs
10596
+ });
10597
+ this.pendingRequests.delete(requestId);
10598
+ logger.debug({
10599
+ requestId,
10600
+ mapName: pending.mapName,
10601
+ query: pending.query,
10602
+ resultCount: results.length,
10603
+ totalHits,
10604
+ respondedNodes: respondedNodes.length,
10605
+ failedNodes: failedNodes.length,
10606
+ executionTimeMs
10607
+ }, "Distributed search completed");
10608
+ }
10609
+ /**
10610
+ * Resolve with partial results when timeout occurs.
10611
+ */
10612
+ resolvePartialResults(requestId) {
10613
+ const pending = this.pendingRequests.get(requestId);
10614
+ if (!pending) return;
10615
+ logger.warn(
10616
+ {
10617
+ requestId,
10618
+ received: pending.responses.size,
10619
+ expected: pending.expectedNodes.size
10620
+ },
10621
+ "Search request timed out, returning partial results"
10622
+ );
10623
+ this.mergeAndResolve(requestId);
10624
+ }
10625
+ /**
10626
+ * Execute local search and add response to pending request.
10627
+ */
10628
+ async executeLocalAndRespond(requestId, mapName, query, limit, cursorData) {
10629
+ const startTime = performance.now();
10630
+ const myNodeId = this.clusterManager.config.nodeId;
10631
+ const pending = this.pendingRequests.get(requestId);
10632
+ if (!pending) return;
10633
+ try {
10634
+ const localResult = this.localSearchCoordinator.search(mapName, query, {
10635
+ limit,
10636
+ minScore: pending.options.minScore,
10637
+ boost: pending.options.boost
10638
+ });
10639
+ let results = localResult.results;
10640
+ if (cursorData) {
10641
+ const position = import_core20.SearchCursor.getNodePosition(cursorData, myNodeId);
10642
+ if (position) {
10643
+ results = results.filter((r) => {
10644
+ if (r.score < position.afterScore) {
10645
+ return true;
10646
+ }
10647
+ if (r.score === position.afterScore) {
10648
+ return r.key > position.afterKey;
10649
+ }
10650
+ return false;
10651
+ });
10652
+ }
10653
+ }
10654
+ const response = {
10655
+ requestId,
10656
+ nodeId: myNodeId,
10657
+ results: results.map((r) => ({
10658
+ key: r.key,
10659
+ value: r.value,
10660
+ score: r.score,
10661
+ matchedTerms: r.matchedTerms
10662
+ })),
10663
+ totalHits: localResult.totalCount ?? results.length,
10664
+ executionTimeMs: performance.now() - startTime
10665
+ };
10666
+ this.handleSearchResponse(myNodeId, response);
10667
+ } catch (error) {
10668
+ this.handleSearchResponse(myNodeId, {
10669
+ requestId,
10670
+ nodeId: myNodeId,
10671
+ results: [],
10672
+ totalHits: 0,
10673
+ executionTimeMs: performance.now() - startTime,
10674
+ error: error instanceof Error ? error.message : "Unknown error"
10675
+ });
10676
+ }
10677
+ }
10678
+ /**
10679
+ * Execute search locally only (single-node optimization).
10680
+ */
10681
+ async executeLocalSearch(mapName, query, options, startTime) {
10682
+ const myNodeId = this.clusterManager.config.nodeId;
10683
+ const localResult = this.localSearchCoordinator.search(mapName, query, {
10684
+ limit: options.limit,
10685
+ minScore: options.minScore,
10686
+ boost: options.boost
10687
+ });
10688
+ let results = localResult.results;
10689
+ if (options.cursor) {
10690
+ const cursorData = import_core20.SearchCursor.decode(options.cursor);
10691
+ if (cursorData && import_core20.SearchCursor.isValid(cursorData, query)) {
10692
+ const position = import_core20.SearchCursor.getNodePosition(cursorData, myNodeId);
10693
+ if (position) {
10694
+ results = results.filter((r) => {
10695
+ if (r.score < position.afterScore) {
10696
+ return true;
10697
+ }
10698
+ if (r.score === position.afterScore) {
10699
+ return r.key > position.afterKey;
10700
+ }
10701
+ return false;
10702
+ });
10703
+ }
10704
+ }
10705
+ }
10706
+ results = results.slice(0, options.limit);
10707
+ const totalCount = localResult.totalCount ?? localResult.results.length;
10708
+ let nextCursor;
10709
+ if (totalCount > options.limit && results.length > 0) {
10710
+ const lastResult = results[results.length - 1];
10711
+ nextCursor = import_core20.SearchCursor.fromResults(
10712
+ [{ key: lastResult.key, score: lastResult.score, nodeId: myNodeId }],
10713
+ query
10714
+ );
10715
+ }
10716
+ const executionTimeMs = performance.now() - startTime;
10717
+ if (this.metricsService) {
10718
+ this.metricsService.incDistributedSearch(mapName, "success");
10719
+ this.metricsService.recordDistributedSearchDuration(mapName, executionTimeMs);
10720
+ }
10721
+ return {
10722
+ results,
10723
+ totalHits: localResult.totalCount ?? results.length,
10724
+ nextCursor,
10725
+ respondedNodes: [myNodeId],
10726
+ failedNodes: [],
10727
+ executionTimeMs
10728
+ };
10729
+ }
10730
+ /**
10731
+ * Calculate per-node limit for distributed query.
10732
+ *
10733
+ * For quality RRF merge, each node should return more than the final limit.
10734
+ * Rule of thumb: 2x limit for good merge quality.
10735
+ */
10736
+ calculatePerNodeLimit(limit, cursor) {
10737
+ if (cursor) {
10738
+ return limit;
10739
+ }
10740
+ return Math.min(limit * 2, 1e3);
10741
+ }
10742
+ /**
10743
+ * Generate unique request ID.
10744
+ */
10745
+ generateRequestId() {
10746
+ return `search-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;
10747
+ }
10748
+ /**
10749
+ * Get RRF constant k.
10750
+ */
10751
+ getRrfK() {
10752
+ return this.config.rrfK;
10753
+ }
10754
+ /**
10755
+ * Clean up resources.
10756
+ */
10757
+ destroy() {
10758
+ for (const [requestId, pending] of this.pendingRequests) {
10759
+ clearTimeout(pending.timeoutHandle);
10760
+ pending.reject(new Error("ClusterSearchCoordinator destroyed"));
10761
+ }
10762
+ this.pendingRequests.clear();
10763
+ this.removeAllListeners();
10764
+ }
10765
+ };
10766
+
10767
+ // src/subscriptions/DistributedSubscriptionCoordinator.ts
10768
+ var import_events15 = require("events");
10769
+ var import_core21 = require("@topgunbuild/core");
10770
+ var import_ws4 = require("ws");
10771
+ var DEFAULT_CONFIG7 = {
10772
+ ackTimeoutMs: 5e3,
10773
+ rrfK: 60
10774
+ };
10775
+ var DistributedSubscriptionCoordinator = class extends import_events15.EventEmitter {
10776
+ constructor(clusterManager, queryRegistry, searchCoordinator, config, metricsService) {
10777
+ super();
10778
+ /**
10779
+ * Active subscriptions where this node is coordinator.
10780
+ * subscriptionId → SubscriptionState
10781
+ */
10782
+ this.subscriptions = /* @__PURE__ */ new Map();
10783
+ /**
10784
+ * Track which nodes have acknowledged subscription registration.
10785
+ * subscriptionId → Set<nodeId>
10786
+ */
10787
+ this.nodeAcks = /* @__PURE__ */ new Map();
10788
+ /**
10789
+ * Pending ACK promises for subscription registration.
10790
+ * subscriptionId → { resolve, reject, timeout, startTime }
10791
+ */
10792
+ this.pendingAcks = /* @__PURE__ */ new Map();
10793
+ this.clusterManager = clusterManager;
10794
+ this.localQueryRegistry = queryRegistry;
10795
+ this.localSearchCoordinator = searchCoordinator;
10796
+ this.config = { ...DEFAULT_CONFIG7, ...config };
10797
+ this.rrf = new import_core21.ReciprocalRankFusion({ k: this.config.rrfK });
10798
+ this.metricsService = metricsService;
10799
+ this.clusterManager.on("message", this.handleClusterMessage.bind(this));
10800
+ this.localSearchCoordinator.on("distributedUpdate", this.handleLocalSearchUpdate.bind(this));
10801
+ this.clusterManager.on("memberLeft", this.handleMemberLeft.bind(this));
10802
+ logger.debug("DistributedSubscriptionCoordinator initialized");
10803
+ }
10804
+ /**
10805
+ * Create a new distributed search subscription.
10806
+ *
10807
+ * @param subscriptionId - Unique subscription ID
10808
+ * @param clientSocket - Client WebSocket for sending updates
10809
+ * @param mapName - Map name to search
10810
+ * @param query - Search query string
10811
+ * @param options - Search options
10812
+ * @returns Promise resolving to initial results
10813
+ */
10814
+ async subscribeSearch(subscriptionId, clientSocket, mapName, query, options = {}) {
10815
+ const myNodeId = this.clusterManager.config.nodeId;
10816
+ const allNodes = new Set(this.clusterManager.getMembers());
10817
+ logger.debug(
10818
+ { subscriptionId, mapName, query, nodes: Array.from(allNodes) },
10819
+ "Creating distributed search subscription"
10820
+ );
10821
+ const subscription = {
10822
+ id: subscriptionId,
10823
+ type: "SEARCH",
10824
+ coordinatorNodeId: myNodeId,
10825
+ clientSocket,
10826
+ mapName,
10827
+ searchQuery: query,
10828
+ searchOptions: options,
10829
+ registeredNodes: /* @__PURE__ */ new Set(),
10830
+ pendingResults: /* @__PURE__ */ new Map(),
10831
+ createdAt: Date.now(),
10832
+ currentResults: /* @__PURE__ */ new Map()
10833
+ };
10834
+ this.subscriptions.set(subscriptionId, subscription);
10835
+ this.nodeAcks.set(subscriptionId, /* @__PURE__ */ new Set());
10836
+ const registerPayload = {
10837
+ subscriptionId,
10838
+ coordinatorNodeId: myNodeId,
10839
+ mapName,
10840
+ type: "SEARCH",
10841
+ searchQuery: query,
10842
+ searchOptions: {
10843
+ limit: options.limit,
10844
+ minScore: options.minScore,
10845
+ boost: options.boost
10846
+ }
10847
+ };
10848
+ const localResult = this.registerLocalSearchSubscription(subscription);
10849
+ this.handleLocalAck(subscriptionId, myNodeId, localResult);
10850
+ for (const nodeId of allNodes) {
10851
+ if (nodeId !== myNodeId) {
10852
+ this.clusterManager.send(nodeId, "CLUSTER_SUB_REGISTER", registerPayload);
10853
+ }
10854
+ }
10855
+ return this.waitForAcks(subscriptionId, allNodes);
10856
+ }
10857
+ /**
10858
+ * Create a new distributed query subscription.
10859
+ *
10860
+ * @param subscriptionId - Unique subscription ID
10861
+ * @param clientSocket - Client WebSocket for sending updates
10862
+ * @param mapName - Map name to query
10863
+ * @param query - Query predicate
10864
+ * @returns Promise resolving to initial results
10865
+ */
10866
+ async subscribeQuery(subscriptionId, clientSocket, mapName, query) {
10867
+ const myNodeId = this.clusterManager.config.nodeId;
10868
+ const allNodes = new Set(this.clusterManager.getMembers());
10869
+ logger.debug(
10870
+ { subscriptionId, mapName, nodes: Array.from(allNodes) },
10871
+ "Creating distributed query subscription"
10872
+ );
10873
+ const subscription = {
10874
+ id: subscriptionId,
10875
+ type: "QUERY",
10876
+ coordinatorNodeId: myNodeId,
10877
+ clientSocket,
10878
+ mapName,
10879
+ queryPredicate: query,
10880
+ registeredNodes: /* @__PURE__ */ new Set(),
10881
+ pendingResults: /* @__PURE__ */ new Map(),
10882
+ createdAt: Date.now(),
10883
+ currentResults: /* @__PURE__ */ new Map()
10884
+ };
10885
+ this.subscriptions.set(subscriptionId, subscription);
10886
+ this.nodeAcks.set(subscriptionId, /* @__PURE__ */ new Set());
10887
+ const registerPayload = {
10888
+ subscriptionId,
10889
+ coordinatorNodeId: myNodeId,
10890
+ mapName,
10891
+ type: "QUERY",
10892
+ queryPredicate: query
10893
+ };
10894
+ const localResults = this.registerLocalQuerySubscription(subscription);
10895
+ this.handleLocalAck(subscriptionId, myNodeId, { results: localResults, totalHits: localResults.length });
10896
+ for (const nodeId of allNodes) {
10897
+ if (nodeId !== myNodeId) {
10898
+ this.clusterManager.send(nodeId, "CLUSTER_SUB_REGISTER", registerPayload);
10899
+ }
10900
+ }
10901
+ return this.waitForAcks(subscriptionId, allNodes);
10902
+ }
10903
+ /**
10904
+ * Unsubscribe from a distributed subscription.
10905
+ *
10906
+ * @param subscriptionId - Subscription ID to unsubscribe
10907
+ */
10908
+ async unsubscribe(subscriptionId) {
10909
+ const subscription = this.subscriptions.get(subscriptionId);
10910
+ if (!subscription) {
10911
+ logger.warn({ subscriptionId }, "Attempt to unsubscribe from unknown subscription");
10912
+ return;
10913
+ }
10914
+ logger.debug({ subscriptionId }, "Unsubscribing from distributed subscription");
10915
+ const myNodeId = this.clusterManager.config.nodeId;
10916
+ const payload = { subscriptionId };
10917
+ for (const nodeId of subscription.registeredNodes) {
10918
+ if (nodeId !== myNodeId) {
10919
+ this.clusterManager.send(nodeId, "CLUSTER_SUB_UNREGISTER", payload);
10920
+ }
10921
+ }
10922
+ this.unregisterLocalSubscription(subscription);
10923
+ this.subscriptions.delete(subscriptionId);
10924
+ this.nodeAcks.delete(subscriptionId);
10925
+ const pendingAck = this.pendingAcks.get(subscriptionId);
10926
+ if (pendingAck) {
10927
+ clearTimeout(pendingAck.timeoutHandle);
10928
+ this.pendingAcks.delete(subscriptionId);
10929
+ }
10930
+ this.metricsService?.incDistributedSubUnsubscribe(subscription.type);
10931
+ this.metricsService?.decDistributedSubActive(subscription.type);
10932
+ this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
10933
+ }
10934
+ /**
10935
+ * Handle client disconnect - unsubscribe all their subscriptions.
10936
+ */
10937
+ unsubscribeClient(clientSocket) {
10938
+ const subscriptionsToRemove = [];
10939
+ for (const [subId, sub] of this.subscriptions) {
10940
+ if (sub.clientSocket === clientSocket) {
10941
+ subscriptionsToRemove.push(subId);
10942
+ }
10943
+ }
10944
+ for (const subId of subscriptionsToRemove) {
10945
+ this.unsubscribe(subId);
10946
+ }
10947
+ }
10948
+ /**
10949
+ * Get active subscription count.
10950
+ */
10951
+ getActiveSubscriptionCount() {
10952
+ return this.subscriptions.size;
10953
+ }
10954
+ /**
10955
+ * Handle incoming cluster messages with Zod validation.
10956
+ */
10957
+ handleClusterMessage(msg) {
10958
+ switch (msg.type) {
10959
+ case "CLUSTER_SUB_REGISTER": {
10960
+ const parsed = import_core21.ClusterSubRegisterPayloadSchema.safeParse(msg.payload);
10961
+ if (!parsed.success) {
10962
+ logger.warn(
10963
+ { senderId: msg.senderId, error: parsed.error.message },
10964
+ "Invalid CLUSTER_SUB_REGISTER payload"
10965
+ );
10966
+ return;
10967
+ }
10968
+ this.handleSubRegister(msg.senderId, parsed.data);
10969
+ break;
10970
+ }
10971
+ case "CLUSTER_SUB_ACK": {
10972
+ const parsed = import_core21.ClusterSubAckPayloadSchema.safeParse(msg.payload);
10973
+ if (!parsed.success) {
10974
+ logger.warn(
10975
+ { senderId: msg.senderId, error: parsed.error.message },
10976
+ "Invalid CLUSTER_SUB_ACK payload"
10977
+ );
10978
+ return;
10979
+ }
10980
+ this.handleSubAck(msg.senderId, parsed.data);
10981
+ break;
10982
+ }
10983
+ case "CLUSTER_SUB_UPDATE": {
10984
+ const parsed = import_core21.ClusterSubUpdatePayloadSchema.safeParse(msg.payload);
10985
+ if (!parsed.success) {
10986
+ logger.warn(
10987
+ { senderId: msg.senderId, error: parsed.error.message },
10988
+ "Invalid CLUSTER_SUB_UPDATE payload"
10989
+ );
10990
+ return;
10991
+ }
10992
+ this.handleSubUpdate(msg.senderId, parsed.data);
10993
+ break;
10994
+ }
10995
+ case "CLUSTER_SUB_UNREGISTER": {
10996
+ const parsed = import_core21.ClusterSubUnregisterPayloadSchema.safeParse(msg.payload);
10997
+ if (!parsed.success) {
10998
+ logger.warn(
10999
+ { senderId: msg.senderId, error: parsed.error.message },
11000
+ "Invalid CLUSTER_SUB_UNREGISTER payload"
11001
+ );
11002
+ return;
11003
+ }
11004
+ this.handleSubUnregister(msg.senderId, parsed.data);
11005
+ break;
11006
+ }
11007
+ }
11008
+ }
11009
+ /**
11010
+ * Handle cluster node disconnect - cleanup subscriptions involving this node.
11011
+ */
11012
+ handleMemberLeft(nodeId) {
11013
+ logger.debug({ nodeId }, "Handling member left for distributed subscriptions");
11014
+ const subscriptionsToRemove = [];
11015
+ const myNodeId = this.clusterManager.config.nodeId;
11016
+ for (const [subId, subscription] of this.subscriptions) {
11017
+ if (subscription.registeredNodes.has(nodeId)) {
11018
+ subscription.registeredNodes.delete(nodeId);
11019
+ for (const [key, result] of subscription.currentResults) {
11020
+ if (result.sourceNode === nodeId) {
11021
+ subscription.currentResults.delete(key);
11022
+ }
11023
+ }
11024
+ logger.debug(
11025
+ { subscriptionId: subId, nodeId, remainingNodes: subscription.registeredNodes.size },
11026
+ "Removed disconnected node from subscription"
11027
+ );
11028
+ }
11029
+ }
11030
+ for (const [subId, pending] of this.pendingAcks) {
11031
+ const acks = this.nodeAcks.get(subId);
11032
+ if (acks && !acks.has(nodeId)) {
11033
+ acks.add(nodeId);
11034
+ this.checkAcksComplete(subId);
11035
+ }
11036
+ }
11037
+ this.localSearchCoordinator.unsubscribeByCoordinator(nodeId);
11038
+ this.localQueryRegistry.unregisterByCoordinator(nodeId);
11039
+ this.metricsService?.incDistributedSubNodeDisconnect();
11040
+ }
11041
+ /**
11042
+ * Handle CLUSTER_SUB_REGISTER from coordinator (we are a data node).
11043
+ */
11044
+ handleSubRegister(senderId, payload) {
11045
+ const myNodeId = this.clusterManager.config.nodeId;
11046
+ logger.debug(
11047
+ { subscriptionId: payload.subscriptionId, coordinator: payload.coordinatorNodeId },
11048
+ "Received distributed subscription registration"
11049
+ );
11050
+ let ackPayload;
11051
+ try {
11052
+ if (payload.type === "SEARCH") {
11053
+ const result = this.localSearchCoordinator.registerDistributedSubscription(
11054
+ payload.subscriptionId,
11055
+ payload.mapName,
11056
+ payload.searchQuery,
11057
+ payload.searchOptions || {},
11058
+ payload.coordinatorNodeId
11059
+ );
11060
+ ackPayload = {
11061
+ subscriptionId: payload.subscriptionId,
11062
+ nodeId: myNodeId,
11063
+ success: true,
11064
+ initialResults: result.results.map((r) => ({
11065
+ key: r.key,
11066
+ value: r.value,
11067
+ score: r.score,
11068
+ matchedTerms: r.matchedTerms
11069
+ })),
11070
+ totalHits: result.totalHits
11071
+ };
11072
+ } else {
11073
+ const results = this.localQueryRegistry.registerDistributed(
11074
+ payload.subscriptionId,
11075
+ payload.mapName,
11076
+ payload.queryPredicate,
11077
+ payload.coordinatorNodeId
11078
+ );
11079
+ ackPayload = {
11080
+ subscriptionId: payload.subscriptionId,
11081
+ nodeId: myNodeId,
11082
+ success: true,
11083
+ initialResults: results.map((r) => ({
11084
+ key: r.key,
11085
+ value: r.value
11086
+ })),
11087
+ totalHits: results.length
11088
+ };
11089
+ }
11090
+ } catch (error) {
11091
+ logger.error(
11092
+ { subscriptionId: payload.subscriptionId, error },
11093
+ "Failed to register distributed subscription locally"
11094
+ );
11095
+ ackPayload = {
11096
+ subscriptionId: payload.subscriptionId,
11097
+ nodeId: myNodeId,
11098
+ success: false,
11099
+ error: error instanceof Error ? error.message : "Unknown error"
11100
+ };
11101
+ }
11102
+ this.clusterManager.send(payload.coordinatorNodeId, "CLUSTER_SUB_ACK", ackPayload);
11103
+ }
11104
+ /**
11105
+ * Handle CLUSTER_SUB_ACK from a data node.
11106
+ */
11107
+ handleSubAck(senderId, payload) {
11108
+ const subscription = this.subscriptions.get(payload.subscriptionId);
11109
+ if (!subscription) {
11110
+ logger.warn(
11111
+ { subscriptionId: payload.subscriptionId, nodeId: payload.nodeId },
11112
+ "Received ACK for unknown subscription"
11113
+ );
11114
+ return;
11115
+ }
11116
+ const acks = this.nodeAcks.get(payload.subscriptionId);
11117
+ if (!acks) return;
11118
+ logger.debug(
11119
+ { subscriptionId: payload.subscriptionId, nodeId: payload.nodeId, success: payload.success },
11120
+ "Received subscription ACK"
11121
+ );
11122
+ if (payload.success) {
11123
+ subscription.registeredNodes.add(payload.nodeId);
11124
+ subscription.pendingResults.set(payload.nodeId, payload);
11125
+ }
11126
+ acks.add(payload.nodeId);
11127
+ this.checkAcksComplete(payload.subscriptionId);
11128
+ }
11129
+ /**
11130
+ * Handle CLUSTER_SUB_UPDATE from a data node.
11131
+ */
11132
+ handleSubUpdate(senderId, payload) {
11133
+ const subscription = this.subscriptions.get(payload.subscriptionId);
11134
+ if (!subscription) {
11135
+ logger.warn(
11136
+ { subscriptionId: payload.subscriptionId },
11137
+ "Update for unknown subscription"
11138
+ );
11139
+ return;
11140
+ }
11141
+ logger.debug(
11142
+ { subscriptionId: payload.subscriptionId, key: payload.key, changeType: payload.changeType },
11143
+ "Received subscription update"
11144
+ );
11145
+ if (payload.changeType === "LEAVE") {
11146
+ subscription.currentResults.delete(payload.key);
11147
+ } else {
11148
+ subscription.currentResults.set(payload.key, {
11149
+ value: payload.value,
11150
+ score: payload.score,
11151
+ sourceNode: payload.sourceNodeId
11152
+ });
11153
+ }
11154
+ this.forwardUpdateToClient(subscription, payload);
11155
+ this.metricsService?.incDistributedSubUpdates("received", payload.changeType);
11156
+ if (payload.timestamp) {
11157
+ const latencyMs = Date.now() - payload.timestamp;
11158
+ this.metricsService?.recordDistributedSubUpdateLatency(subscription.type, latencyMs);
11159
+ }
11160
+ }
11161
+ /**
11162
+ * Handle CLUSTER_SUB_UNREGISTER from coordinator.
11163
+ */
11164
+ handleSubUnregister(senderId, payload) {
11165
+ logger.debug(
11166
+ { subscriptionId: payload.subscriptionId },
11167
+ "Received subscription unregister request"
11168
+ );
11169
+ this.localSearchCoordinator.unsubscribe(payload.subscriptionId);
11170
+ this.localQueryRegistry.unregister(payload.subscriptionId);
11171
+ }
11172
+ /**
11173
+ * Handle local search update (emitted by SearchCoordinator for distributed subscriptions).
11174
+ */
11175
+ handleLocalSearchUpdate(payload) {
11176
+ const coordinatorNodeId = this.getCoordinatorForSubscription(payload.subscriptionId);
11177
+ if (!coordinatorNodeId) return;
11178
+ const myNodeId = this.clusterManager.config.nodeId;
11179
+ if (coordinatorNodeId === myNodeId) {
11180
+ this.handleSubUpdate(myNodeId, payload);
11181
+ } else {
11182
+ this.clusterManager.send(coordinatorNodeId, "CLUSTER_SUB_UPDATE", payload);
11183
+ }
11184
+ this.metricsService?.incDistributedSubUpdates("sent", payload.changeType);
11185
+ }
11186
+ /**
11187
+ * Register a local search subscription for a distributed coordinator.
11188
+ */
11189
+ registerLocalSearchSubscription(subscription) {
11190
+ return this.localSearchCoordinator.registerDistributedSubscription(
11191
+ subscription.id,
11192
+ subscription.mapName,
11193
+ subscription.searchQuery,
11194
+ subscription.searchOptions || {},
11195
+ subscription.coordinatorNodeId
11196
+ );
11197
+ }
11198
+ /**
11199
+ * Register a local query subscription for a distributed coordinator.
11200
+ */
11201
+ registerLocalQuerySubscription(subscription) {
11202
+ return this.localQueryRegistry.registerDistributed(
11203
+ subscription.id,
11204
+ subscription.mapName,
11205
+ subscription.queryPredicate,
11206
+ subscription.coordinatorNodeId
11207
+ );
11208
+ }
11209
+ /**
11210
+ * Unregister local subscription.
11211
+ */
11212
+ unregisterLocalSubscription(subscription) {
11213
+ if (subscription.type === "SEARCH") {
11214
+ this.localSearchCoordinator.unsubscribe(subscription.id);
11215
+ } else {
11216
+ this.localQueryRegistry.unregister(subscription.id);
11217
+ }
11218
+ }
11219
+ /**
11220
+ * Handle local ACK (from this node's registration).
11221
+ */
11222
+ handleLocalAck(subscriptionId, nodeId, result) {
11223
+ const subscription = this.subscriptions.get(subscriptionId);
11224
+ if (!subscription) return;
11225
+ const acks = this.nodeAcks.get(subscriptionId);
11226
+ if (!acks) return;
11227
+ subscription.registeredNodes.add(nodeId);
11228
+ subscription.pendingResults.set(nodeId, {
11229
+ subscriptionId,
11230
+ nodeId,
11231
+ success: true,
11232
+ initialResults: (result.results || []).map((r) => ({
11233
+ key: r.key,
11234
+ value: r.value,
11235
+ score: r.score,
11236
+ matchedTerms: r.matchedTerms
11237
+ })),
11238
+ totalHits: result.totalHits || (result.results?.length ?? 0)
11239
+ });
11240
+ acks.add(nodeId);
11241
+ this.checkAcksComplete(subscriptionId);
11242
+ }
11243
+ /**
11244
+ * Wait for all node ACKs with timeout.
11245
+ */
11246
+ waitForAcks(subscriptionId, expectedNodes) {
11247
+ return new Promise((resolve, reject) => {
11248
+ const startTime = performance.now();
11249
+ const timeoutHandle = setTimeout(() => {
11250
+ this.resolveWithPartialAcks(subscriptionId);
11251
+ }, this.config.ackTimeoutMs);
11252
+ this.pendingAcks.set(subscriptionId, { resolve, reject, timeoutHandle, startTime });
11253
+ this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
11254
+ this.checkAcksComplete(subscriptionId);
11255
+ });
11256
+ }
11257
+ /**
11258
+ * Check if all ACKs have been received.
11259
+ */
11260
+ checkAcksComplete(subscriptionId) {
11261
+ const subscription = this.subscriptions.get(subscriptionId);
11262
+ const acks = this.nodeAcks.get(subscriptionId);
11263
+ const pendingAck = this.pendingAcks.get(subscriptionId);
11264
+ if (!subscription || !acks || !pendingAck) return;
11265
+ const allNodes = new Set(this.clusterManager.getMembers());
11266
+ if (acks.size >= allNodes.size) {
11267
+ clearTimeout(pendingAck.timeoutHandle);
11268
+ this.pendingAcks.delete(subscriptionId);
11269
+ const duration = performance.now() - pendingAck.startTime;
11270
+ const result = this.mergeInitialResults(subscription);
11271
+ const hasFailures = result.failedNodes.length > 0;
11272
+ this.recordCompletionMetrics(
11273
+ subscription,
11274
+ result,
11275
+ duration,
11276
+ hasFailures ? "timeout" : "success"
11277
+ );
11278
+ pendingAck.resolve(result);
11279
+ }
11280
+ }
11281
+ /**
11282
+ * Resolve with partial ACKs (on timeout).
11283
+ */
11284
+ resolveWithPartialAcks(subscriptionId) {
11285
+ const subscription = this.subscriptions.get(subscriptionId);
11286
+ const pendingAck = this.pendingAcks.get(subscriptionId);
11287
+ if (!subscription || !pendingAck) return;
11288
+ this.pendingAcks.delete(subscriptionId);
11289
+ logger.warn(
11290
+ { subscriptionId, registeredNodes: Array.from(subscription.registeredNodes) },
11291
+ "Subscription ACK timeout, resolving with partial results"
11292
+ );
11293
+ const duration = performance.now() - pendingAck.startTime;
11294
+ const result = this.mergeInitialResults(subscription);
11295
+ this.recordCompletionMetrics(subscription, result, duration, "timeout");
11296
+ pendingAck.resolve(result);
11297
+ }
11298
+ /**
11299
+ * Record metrics when subscription registration completes.
11300
+ */
11301
+ recordCompletionMetrics(subscription, result, durationMs, status) {
11302
+ this.metricsService?.incDistributedSub(subscription.type, status);
11303
+ this.metricsService?.recordDistributedSubRegistration(subscription.type, durationMs);
11304
+ this.metricsService?.recordDistributedSubInitialResultsCount(subscription.type, result.results.length);
11305
+ this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
11306
+ this.metricsService?.incDistributedSubAck("success", subscription.registeredNodes.size);
11307
+ this.metricsService?.incDistributedSubAck("timeout", result.failedNodes.length);
11308
+ }
11309
+ /**
11310
+ * Merge initial results from all nodes.
11311
+ */
11312
+ mergeInitialResults(subscription) {
11313
+ const allNodes = new Set(this.clusterManager.getMembers());
11314
+ const failedNodes = Array.from(allNodes).filter((n) => !subscription.registeredNodes.has(n));
11315
+ if (subscription.type === "SEARCH") {
11316
+ return this.mergeSearchResults(subscription, failedNodes);
11317
+ } else {
11318
+ return this.mergeQueryResults(subscription, failedNodes);
11319
+ }
11320
+ }
11321
+ /**
11322
+ * Merge search results using RRF.
11323
+ */
11324
+ mergeSearchResults(subscription, failedNodes) {
11325
+ const resultSets = [];
11326
+ const resultDataMap = /* @__PURE__ */ new Map();
11327
+ for (const [nodeId, ack] of subscription.pendingResults) {
11328
+ if (ack.success && ack.initialResults) {
11329
+ const rankedResults = [];
11330
+ for (const r of ack.initialResults) {
11331
+ rankedResults.push({
11332
+ docId: r.key,
11333
+ score: r.score ?? 0,
11334
+ source: nodeId
11335
+ });
11336
+ if (!resultDataMap.has(r.key)) {
11337
+ resultDataMap.set(r.key, {
11338
+ key: r.key,
11339
+ value: r.value,
11340
+ score: r.score ?? 0,
11341
+ matchedTerms: r.matchedTerms,
11342
+ nodeId
11343
+ });
11344
+ }
11345
+ }
11346
+ if (rankedResults.length > 0) {
11347
+ resultSets.push(rankedResults);
11348
+ }
11349
+ }
11350
+ }
11351
+ const mergedResults = this.rrf.merge(resultSets);
11352
+ const limit = subscription.searchOptions?.limit ?? 10;
11353
+ const results = [];
11354
+ for (const merged of mergedResults.slice(0, limit)) {
11355
+ const original = resultDataMap.get(merged.docId);
11356
+ if (original) {
11357
+ results.push({
11358
+ key: original.key,
11359
+ value: original.value,
11360
+ score: merged.score,
11361
+ // Use RRF score
11362
+ matchedTerms: original.matchedTerms
11363
+ });
11364
+ subscription.currentResults.set(original.key, {
11365
+ value: original.value,
11366
+ score: merged.score,
11367
+ sourceNode: original.nodeId
11368
+ });
11369
+ }
11370
+ }
11371
+ let totalHits = 0;
11372
+ for (const ack of subscription.pendingResults.values()) {
11373
+ totalHits += ack.totalHits ?? 0;
11374
+ }
11375
+ return {
11376
+ subscriptionId: subscription.id,
11377
+ results,
11378
+ totalHits,
11379
+ registeredNodes: Array.from(subscription.registeredNodes),
11380
+ failedNodes
11381
+ };
11382
+ }
11383
+ /**
11384
+ * Merge query results (simple dedupe by key).
11385
+ */
11386
+ mergeQueryResults(subscription, failedNodes) {
11387
+ const resultMap = /* @__PURE__ */ new Map();
11388
+ for (const [nodeId, ack] of subscription.pendingResults) {
11389
+ if (ack.success && ack.initialResults) {
11390
+ for (const result of ack.initialResults) {
11391
+ if (!resultMap.has(result.key)) {
11392
+ resultMap.set(result.key, { key: result.key, value: result.value });
11393
+ subscription.currentResults.set(result.key, {
11394
+ value: result.value,
11395
+ sourceNode: nodeId
11396
+ });
11397
+ }
11398
+ }
11399
+ }
11400
+ }
11401
+ const results = Array.from(resultMap.values());
11402
+ return {
11403
+ subscriptionId: subscription.id,
11404
+ results,
11405
+ totalHits: results.length,
11406
+ registeredNodes: Array.from(subscription.registeredNodes),
11407
+ failedNodes
11408
+ };
11409
+ }
11410
+ /**
11411
+ * Forward update to client WebSocket.
11412
+ */
11413
+ forwardUpdateToClient(subscription, payload) {
11414
+ if (subscription.clientSocket.readyState !== import_ws4.WebSocket.OPEN) {
11415
+ logger.warn(
11416
+ { subscriptionId: subscription.id },
11417
+ "Cannot forward update, client socket not open"
11418
+ );
11419
+ return;
11420
+ }
11421
+ const message = subscription.type === "SEARCH" ? {
11422
+ type: "SEARCH_UPDATE",
11423
+ payload: {
11424
+ subscriptionId: payload.subscriptionId,
11425
+ key: payload.key,
11426
+ value: payload.value,
11427
+ score: payload.score,
11428
+ matchedTerms: payload.matchedTerms,
11429
+ type: payload.changeType
11430
+ }
11431
+ } : {
11432
+ type: "QUERY_UPDATE",
11433
+ payload: {
11434
+ queryId: payload.subscriptionId,
11435
+ key: payload.key,
11436
+ value: payload.value,
11437
+ type: payload.changeType
11438
+ }
11439
+ };
11440
+ try {
11441
+ subscription.clientSocket.send(JSON.stringify(message));
11442
+ } catch (error) {
11443
+ logger.error(
11444
+ { subscriptionId: subscription.id, error },
11445
+ "Failed to send update to client"
11446
+ );
11447
+ }
11448
+ }
11449
+ /**
11450
+ * Get coordinator node for a subscription.
11451
+ * For remote subscriptions (where we are a data node), this returns the coordinator.
11452
+ */
11453
+ getCoordinatorForSubscription(subscriptionId) {
11454
+ if (this.subscriptions.has(subscriptionId)) {
11455
+ return this.clusterManager.config.nodeId;
11456
+ }
11457
+ const searchSub = this.localSearchCoordinator.getDistributedSubscription(subscriptionId);
11458
+ if (searchSub?.coordinatorNodeId) {
11459
+ return searchSub.coordinatorNodeId;
11460
+ }
11461
+ const querySub = this.localQueryRegistry.getDistributedSubscription(subscriptionId);
11462
+ if (querySub?.coordinatorNodeId) {
11463
+ return querySub.coordinatorNodeId;
11464
+ }
11465
+ return null;
11466
+ }
11467
+ /**
11468
+ * Cleanup on destroy.
11469
+ */
11470
+ destroy() {
11471
+ for (const subscriptionId of this.subscriptions.keys()) {
11472
+ this.unsubscribe(subscriptionId);
11473
+ }
11474
+ for (const pending of this.pendingAcks.values()) {
11475
+ clearTimeout(pending.timeoutHandle);
11476
+ }
11477
+ this.pendingAcks.clear();
11478
+ this.removeAllListeners();
11479
+ }
11480
+ };
11481
+
11482
+ // src/ServerCoordinator.ts
11483
+ var GC_INTERVAL_MS = 60 * 60 * 1e3;
11484
+ var GC_AGE_MS = 30 * 24 * 60 * 60 * 1e3;
11485
+ var CLIENT_HEARTBEAT_TIMEOUT_MS = 2e4;
11486
+ var CLIENT_HEARTBEAT_CHECK_INTERVAL_MS = 5e3;
11487
+ var ServerCoordinator = class {
11488
+ constructor(config) {
11489
+ this.clients = /* @__PURE__ */ new Map();
11490
+ // Interceptors
11491
+ this.interceptors = [];
11492
+ // In-memory storage (partitioned later)
11493
+ this.maps = /* @__PURE__ */ new Map();
11494
+ this.pendingClusterQueries = /* @__PURE__ */ new Map();
11495
+ // GC Consensus State
11496
+ this.gcReports = /* @__PURE__ */ new Map();
11497
+ // Track map loading state to avoid returning empty results during async load
11498
+ this.mapLoadingPromises = /* @__PURE__ */ new Map();
11499
+ // Track pending batch operations for testing purposes
11500
+ this.pendingBatchOperations = /* @__PURE__ */ new Set();
11501
+ this.journalSubscriptions = /* @__PURE__ */ new Map();
11502
+ this._actualPort = 0;
11503
+ this._actualClusterPort = 0;
11504
+ this._readyPromise = new Promise((resolve) => {
11505
+ this._readyResolve = resolve;
11506
+ });
11507
+ this._nodeId = config.nodeId;
11508
+ this.hlc = new import_core22.HLC(config.nodeId);
11509
+ this.storage = config.storage;
11510
+ const rawSecret = config.jwtSecret || process.env.JWT_SECRET || "topgun-secret-dev";
11511
+ this.jwtSecret = rawSecret.replace(/\\n/g, "\n");
11512
+ this.queryRegistry = new QueryRegistry();
9838
11513
  this.securityManager = new SecurityManager(config.securityPolicies || []);
9839
11514
  this.interceptors = config.interceptors || [];
9840
11515
  this.metricsService = new MetricsService();
@@ -9930,7 +11605,7 @@ var ServerCoordinator = class {
9930
11605
  this.metricsServer.on("error", (err) => {
9931
11606
  logger.error({ err, port: metricsPort }, "Metrics server failed to start");
9932
11607
  });
9933
- this.wss = new import_ws3.WebSocketServer({
11608
+ this.wss = new import_ws5.WebSocketServer({
9934
11609
  server: this.httpServer,
9935
11610
  // Increase backlog for pending connections (default Linux is 128)
9936
11611
  backlog: config.wsBacklog ?? 511,
@@ -9972,8 +11647,8 @@ var ServerCoordinator = class {
9972
11647
  this.cluster,
9973
11648
  this.partitionService,
9974
11649
  {
9975
- ...import_core20.DEFAULT_REPLICATION_CONFIG,
9976
- defaultConsistency: config.defaultConsistency ?? import_core20.ConsistencyLevel.EVENTUAL,
11650
+ ...import_core22.DEFAULT_REPLICATION_CONFIG,
11651
+ defaultConsistency: config.defaultConsistency ?? import_core22.ConsistencyLevel.EVENTUAL,
9977
11652
  ...config.replicationConfig
9978
11653
  }
9979
11654
  );
@@ -9989,7 +11664,7 @@ var ServerCoordinator = class {
9989
11664
  cluster: this.cluster,
9990
11665
  sendToClient: (clientId, message) => {
9991
11666
  const client = this.clients.get(clientId);
9992
- if (client && client.socket.readyState === import_ws3.WebSocket.OPEN) {
11667
+ if (client && client.socket.readyState === import_ws5.WebSocket.OPEN) {
9993
11668
  client.writer.write(message);
9994
11669
  }
9995
11670
  }
@@ -10036,7 +11711,7 @@ var ServerCoordinator = class {
10036
11711
  void 0,
10037
11712
  // LagTracker - can be added later
10038
11713
  {
10039
- defaultConsistency: config.defaultConsistency ?? import_core20.ConsistencyLevel.STRONG,
11714
+ defaultConsistency: config.defaultConsistency ?? import_core22.ConsistencyLevel.STRONG,
10040
11715
  preferLocalReplica: true,
10041
11716
  loadBalancing: "latency-based"
10042
11717
  }
@@ -10089,6 +11764,26 @@ var ServerCoordinator = class {
10089
11764
  logger.info({ mapName, fields: ftsConfig.fields }, "FTS enabled for map");
10090
11765
  }
10091
11766
  }
11767
+ this.clusterSearchCoordinator = new ClusterSearchCoordinator(
11768
+ this.cluster,
11769
+ this.partitionService,
11770
+ this.searchCoordinator,
11771
+ config.distributedSearch,
11772
+ this.metricsService
11773
+ );
11774
+ logger.info("ClusterSearchCoordinator initialized for distributed search");
11775
+ this.distributedSubCoordinator = new DistributedSubscriptionCoordinator(
11776
+ this.cluster,
11777
+ this.queryRegistry,
11778
+ this.searchCoordinator,
11779
+ void 0,
11780
+ // Use default config
11781
+ this.metricsService
11782
+ );
11783
+ logger.info("DistributedSubscriptionCoordinator initialized for distributed live subscriptions");
11784
+ this.searchCoordinator.setNodeId(config.nodeId);
11785
+ this.queryRegistry.setClusterManager(this.cluster, config.nodeId);
11786
+ this.queryRegistry.setMapGetter((name) => this.getMap(name));
10092
11787
  this.systemManager = new SystemManager(
10093
11788
  this.cluster,
10094
11789
  this.metricsService,
@@ -10131,7 +11826,7 @@ var ServerCoordinator = class {
10131
11826
  await this.getMapAsync(mapName);
10132
11827
  const map = this.maps.get(mapName);
10133
11828
  if (!map) return;
10134
- if (map instanceof import_core20.LWWMap) {
11829
+ if (map instanceof import_core22.LWWMap) {
10135
11830
  const entries = Array.from(map.entries());
10136
11831
  if (entries.length > 0) {
10137
11832
  logger.info({ mapName, count: entries.length }, "Backfilling FTS index");
@@ -10229,11 +11924,11 @@ var ServerCoordinator = class {
10229
11924
  const map = this.maps.get(mapName);
10230
11925
  if (map) {
10231
11926
  const entries = [];
10232
- if (map instanceof import_core20.LWWMap) {
11927
+ if (map instanceof import_core22.LWWMap) {
10233
11928
  for (const [key, value] of map.entries()) {
10234
11929
  entries.push([key, value]);
10235
11930
  }
10236
- } else if (map instanceof import_core20.ORMap) {
11931
+ } else if (map instanceof import_core22.ORMap) {
10237
11932
  for (const key of map.allKeys()) {
10238
11933
  const values = map.get(key);
10239
11934
  const value = values.length > 0 ? values[0] : null;
@@ -10354,10 +12049,10 @@ var ServerCoordinator = class {
10354
12049
  this.metricsService.destroy();
10355
12050
  this.wss.close();
10356
12051
  logger.info(`Closing ${this.clients.size} client connections...`);
10357
- const shutdownMsg = (0, import_core20.serialize)({ type: "SHUTDOWN_PENDING", retryAfter: 5e3 });
12052
+ const shutdownMsg = (0, import_core22.serialize)({ type: "SHUTDOWN_PENDING", retryAfter: 5e3 });
10358
12053
  for (const client of this.clients.values()) {
10359
12054
  try {
10360
- if (client.socket.readyState === import_ws3.WebSocket.OPEN) {
12055
+ if (client.socket.readyState === import_ws5.WebSocket.OPEN) {
10361
12056
  client.socket.send(shutdownMsg);
10362
12057
  if (client.writer) {
10363
12058
  client.writer.close();
@@ -10420,6 +12115,12 @@ var ServerCoordinator = class {
10420
12115
  if (this.eventJournalService) {
10421
12116
  this.eventJournalService.dispose();
10422
12117
  }
12118
+ if (this.clusterSearchCoordinator) {
12119
+ this.clusterSearchCoordinator.destroy();
12120
+ }
12121
+ if (this.distributedSubCoordinator) {
12122
+ this.distributedSubCoordinator.destroy();
12123
+ }
10423
12124
  logger.info("Server Coordinator shutdown complete.");
10424
12125
  }
10425
12126
  async handleConnection(ws) {
@@ -10487,7 +12188,7 @@ var ServerCoordinator = class {
10487
12188
  buf = Buffer.from(message);
10488
12189
  }
10489
12190
  try {
10490
- data = (0, import_core20.deserialize)(buf);
12191
+ data = (0, import_core22.deserialize)(buf);
10491
12192
  } catch (e) {
10492
12193
  try {
10493
12194
  const text = Buffer.isBuffer(buf) ? buf.toString() : new TextDecoder().decode(buf);
@@ -10528,6 +12229,9 @@ var ServerCoordinator = class {
10528
12229
  this.topicManager.unsubscribeAll(clientId);
10529
12230
  this.counterHandler.unsubscribeAll(clientId);
10530
12231
  this.searchCoordinator.unsubscribeClient(clientId);
12232
+ if (this.distributedSubCoordinator && connection) {
12233
+ this.distributedSubCoordinator.unsubscribeClient(connection.socket);
12234
+ }
10531
12235
  const members = this.cluster.getMembers();
10532
12236
  for (const memberId of members) {
10533
12237
  if (!this.cluster.isLocal(memberId)) {
@@ -10540,10 +12244,10 @@ var ServerCoordinator = class {
10540
12244
  this.clients.delete(clientId);
10541
12245
  this.metricsService.setConnectedClients(this.clients.size);
10542
12246
  });
10543
- ws.send((0, import_core20.serialize)({ type: "AUTH_REQUIRED" }));
12247
+ ws.send((0, import_core22.serialize)({ type: "AUTH_REQUIRED" }));
10544
12248
  }
10545
12249
  async handleMessage(client, rawMessage) {
10546
- const parseResult = import_core20.MessageSchema.safeParse(rawMessage);
12250
+ const parseResult = import_core22.MessageSchema.safeParse(rawMessage);
10547
12251
  if (!parseResult.success) {
10548
12252
  logger.error({ clientId: client.id, error: parseResult.error }, "Invalid message format from client");
10549
12253
  client.writer.write({
@@ -10602,70 +12306,115 @@ var ServerCoordinator = class {
10602
12306
  }
10603
12307
  logger.info({ clientId: client.id, mapName, query }, "Client subscribed");
10604
12308
  this.metricsService.incOp("SUBSCRIBE", mapName);
10605
- const allMembers = this.cluster.getMembers();
10606
- let remoteMembers = allMembers.filter((id) => !this.cluster.isLocal(id));
10607
- const queryKey = query._id || query.where?._id;
10608
- if (queryKey && typeof queryKey === "string" && this.readReplicaHandler) {
10609
- try {
10610
- const targetNode = this.readReplicaHandler.selectReadNode({
12309
+ if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
12310
+ this.distributedSubCoordinator.subscribeQuery(
12311
+ queryId,
12312
+ client.socket,
12313
+ mapName,
12314
+ query
12315
+ ).then((result) => {
12316
+ const filteredResults = result.results.map((res) => {
12317
+ const filteredValue = this.securityManager.filterObject(res.value, client.principal, mapName);
12318
+ return { ...res, value: filteredValue };
12319
+ });
12320
+ client.writer.write({
12321
+ type: "QUERY_RESP",
12322
+ payload: {
12323
+ queryId,
12324
+ results: filteredResults
12325
+ }
12326
+ });
12327
+ client.subscriptions.add(queryId);
12328
+ logger.debug({
12329
+ clientId: client.id,
12330
+ queryId,
10611
12331
  mapName,
10612
- key: queryKey,
10613
- options: {
10614
- // Default to EVENTUAL for read scaling unless specified otherwise
10615
- // In future, we could extract consistency from query options if available
10616
- consistency: import_core20.ConsistencyLevel.EVENTUAL
12332
+ resultCount: result.results.length,
12333
+ totalHits: result.totalHits,
12334
+ nodes: result.registeredNodes
12335
+ }, "Distributed query subscription created");
12336
+ }).catch((err) => {
12337
+ logger.error({ err, queryId }, "Distributed query subscription failed");
12338
+ client.writer.write({
12339
+ type: "QUERY_RESP",
12340
+ payload: {
12341
+ queryId,
12342
+ results: [],
12343
+ error: "Failed to create distributed subscription"
10617
12344
  }
10618
12345
  });
10619
- if (targetNode) {
10620
- if (this.cluster.isLocal(targetNode)) {
10621
- remoteMembers = [];
10622
- logger.debug({ clientId: client.id, mapName, key: queryKey }, "Read optimization: Serving locally");
10623
- } else if (remoteMembers.includes(targetNode)) {
10624
- remoteMembers = [targetNode];
10625
- logger.debug({ clientId: client.id, mapName, key: queryKey, targetNode }, "Read optimization: Routing to replica");
12346
+ });
12347
+ } else {
12348
+ const allMembers = this.cluster.getMembers();
12349
+ let remoteMembers = allMembers.filter((id) => !this.cluster.isLocal(id));
12350
+ const queryKey = query._id || query.where?._id;
12351
+ if (queryKey && typeof queryKey === "string" && this.readReplicaHandler) {
12352
+ try {
12353
+ const targetNode = this.readReplicaHandler.selectReadNode({
12354
+ mapName,
12355
+ key: queryKey,
12356
+ options: {
12357
+ // Default to EVENTUAL for read scaling unless specified otherwise
12358
+ // In future, we could extract consistency from query options if available
12359
+ consistency: import_core22.ConsistencyLevel.EVENTUAL
12360
+ }
12361
+ });
12362
+ if (targetNode) {
12363
+ if (this.cluster.isLocal(targetNode)) {
12364
+ remoteMembers = [];
12365
+ logger.debug({ clientId: client.id, mapName, key: queryKey }, "Read optimization: Serving locally");
12366
+ } else if (remoteMembers.includes(targetNode)) {
12367
+ remoteMembers = [targetNode];
12368
+ logger.debug({ clientId: client.id, mapName, key: queryKey, targetNode }, "Read optimization: Routing to replica");
12369
+ }
10626
12370
  }
12371
+ } catch (e) {
12372
+ logger.warn({ err: e }, "Error in ReadReplicaHandler selection");
10627
12373
  }
10628
- } catch (e) {
10629
- logger.warn({ err: e }, "Error in ReadReplicaHandler selection");
10630
12374
  }
10631
- }
10632
- const requestId = crypto.randomUUID();
10633
- const pending = {
10634
- requestId,
10635
- client,
10636
- queryId,
10637
- mapName,
10638
- query,
10639
- results: [],
10640
- // Will populate with local results first
10641
- expectedNodes: new Set(remoteMembers),
10642
- respondedNodes: /* @__PURE__ */ new Set(),
10643
- timer: setTimeout(() => this.finalizeClusterQuery(requestId, true), 5e3)
10644
- // 5s timeout
10645
- };
10646
- this.pendingClusterQueries.set(requestId, pending);
10647
- try {
10648
- const localResults = await this.executeLocalQuery(mapName, query);
10649
- pending.results.push(...localResults);
10650
- if (remoteMembers.length > 0) {
10651
- for (const nodeId of remoteMembers) {
10652
- this.cluster.send(nodeId, "CLUSTER_QUERY_EXEC", {
10653
- requestId,
10654
- mapName,
10655
- query
10656
- });
12375
+ const requestId = crypto.randomUUID();
12376
+ const pending = {
12377
+ requestId,
12378
+ client,
12379
+ queryId,
12380
+ mapName,
12381
+ query,
12382
+ results: [],
12383
+ // Will populate with local results first
12384
+ expectedNodes: new Set(remoteMembers),
12385
+ respondedNodes: /* @__PURE__ */ new Set(),
12386
+ timer: setTimeout(() => this.finalizeClusterQuery(requestId, true), 5e3)
12387
+ // 5s timeout
12388
+ };
12389
+ this.pendingClusterQueries.set(requestId, pending);
12390
+ try {
12391
+ const localResults = await this.executeLocalQuery(mapName, query);
12392
+ pending.results.push(...localResults);
12393
+ if (remoteMembers.length > 0) {
12394
+ for (const nodeId of remoteMembers) {
12395
+ this.cluster.send(nodeId, "CLUSTER_QUERY_EXEC", {
12396
+ requestId,
12397
+ mapName,
12398
+ query
12399
+ });
12400
+ }
12401
+ } else {
12402
+ this.finalizeClusterQuery(requestId);
10657
12403
  }
10658
- } else {
12404
+ } catch (err) {
12405
+ logger.error({ err, mapName }, "Failed to execute local query");
10659
12406
  this.finalizeClusterQuery(requestId);
10660
12407
  }
10661
- } catch (err) {
10662
- logger.error({ err, mapName }, "Failed to execute local query");
10663
- this.finalizeClusterQuery(requestId);
10664
12408
  }
10665
12409
  break;
10666
12410
  }
10667
12411
  case "QUERY_UNSUB": {
10668
12412
  const { queryId: unsubId } = message.payload;
12413
+ if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
12414
+ this.distributedSubCoordinator.unsubscribe(unsubId).catch((err) => {
12415
+ logger.warn({ err, queryId: unsubId }, "Failed to unsubscribe from distributed coordinator");
12416
+ });
12417
+ }
10669
12418
  this.queryRegistry.unregister(unsubId);
10670
12419
  client.subscriptions.delete(unsubId);
10671
12420
  break;
@@ -10808,7 +12557,7 @@ var ServerCoordinator = class {
10808
12557
  this.metricsService.incOp("GET", message.mapName);
10809
12558
  try {
10810
12559
  const mapForSync = await this.getMapAsync(message.mapName);
10811
- if (mapForSync instanceof import_core20.LWWMap) {
12560
+ if (mapForSync instanceof import_core22.LWWMap) {
10812
12561
  const tree = mapForSync.getMerkleTree();
10813
12562
  const rootHash = tree.getRootHash();
10814
12563
  client.writer.write({
@@ -10846,7 +12595,7 @@ var ServerCoordinator = class {
10846
12595
  const { mapName, path } = message.payload;
10847
12596
  try {
10848
12597
  const mapForBucket = await this.getMapAsync(mapName);
10849
- if (mapForBucket instanceof import_core20.LWWMap) {
12598
+ if (mapForBucket instanceof import_core22.LWWMap) {
10850
12599
  const treeForBucket = mapForBucket.getMerkleTree();
10851
12600
  const buckets = treeForBucket.getBuckets(path);
10852
12601
  const node = treeForBucket.getNode(path);
@@ -10989,7 +12738,7 @@ var ServerCoordinator = class {
10989
12738
  client.writer.write(result.response);
10990
12739
  for (const targetClientId of result.broadcastTo) {
10991
12740
  const targetClient = this.clients.get(targetClientId);
10992
- if (targetClient && targetClient.socket.readyState === import_ws3.WebSocket.OPEN) {
12741
+ if (targetClient && targetClient.socket.readyState === import_ws5.WebSocket.OPEN) {
10993
12742
  targetClient.writer.write(result.broadcastMessage);
10994
12743
  }
10995
12744
  }
@@ -11228,7 +12977,7 @@ var ServerCoordinator = class {
11228
12977
  this.metricsService.incOp("GET", message.mapName);
11229
12978
  try {
11230
12979
  const mapForSync = await this.getMapAsync(message.mapName, "OR");
11231
- if (mapForSync instanceof import_core20.ORMap) {
12980
+ if (mapForSync instanceof import_core22.ORMap) {
11232
12981
  const tree = mapForSync.getMerkleTree();
11233
12982
  const rootHash = tree.getRootHash();
11234
12983
  client.writer.write({
@@ -11265,7 +13014,7 @@ var ServerCoordinator = class {
11265
13014
  const { mapName, path } = message.payload;
11266
13015
  try {
11267
13016
  const mapForBucket = await this.getMapAsync(mapName, "OR");
11268
- if (mapForBucket instanceof import_core20.ORMap) {
13017
+ if (mapForBucket instanceof import_core22.ORMap) {
11269
13018
  const tree = mapForBucket.getMerkleTree();
11270
13019
  const buckets = tree.getBuckets(path);
11271
13020
  const isLeaf = tree.isLeaf(path);
@@ -11309,7 +13058,7 @@ var ServerCoordinator = class {
11309
13058
  const { mapName: diffMapName, keys } = message.payload;
11310
13059
  try {
11311
13060
  const mapForDiff = await this.getMapAsync(diffMapName, "OR");
11312
- if (mapForDiff instanceof import_core20.ORMap) {
13061
+ if (mapForDiff instanceof import_core22.ORMap) {
11313
13062
  const entries = [];
11314
13063
  const allTombstones = mapForDiff.getTombstones();
11315
13064
  for (const key of keys) {
@@ -11341,7 +13090,7 @@ var ServerCoordinator = class {
11341
13090
  const { mapName: pushMapName, entries: pushEntries } = message.payload;
11342
13091
  try {
11343
13092
  const mapForPush = await this.getMapAsync(pushMapName, "OR");
11344
- if (mapForPush instanceof import_core20.ORMap) {
13093
+ if (mapForPush instanceof import_core22.ORMap) {
11345
13094
  let totalAdded = 0;
11346
13095
  let totalUpdated = 0;
11347
13096
  for (const entry of pushEntries) {
@@ -11469,7 +13218,7 @@ var ServerCoordinator = class {
11469
13218
  });
11470
13219
  break;
11471
13220
  }
11472
- // Phase 11.1: Full-Text Search
13221
+ // Phase 11.1: Full-Text Search (Phase 14: Distributed Search)
11473
13222
  case "SEARCH": {
11474
13223
  const { requestId: searchReqId, mapName: searchMapName, query: searchQuery, options: searchOptions } = message.payload;
11475
13224
  if (!this.securityManager.checkPermission(client.principal, searchMapName, "READ")) {
@@ -11497,18 +13246,58 @@ var ServerCoordinator = class {
11497
13246
  });
11498
13247
  break;
11499
13248
  }
11500
- const searchResult = this.searchCoordinator.search(searchMapName, searchQuery, searchOptions);
11501
- searchResult.requestId = searchReqId;
11502
- logger.debug({
11503
- clientId: client.id,
11504
- mapName: searchMapName,
11505
- query: searchQuery,
11506
- resultCount: searchResult.results.length
11507
- }, "Search executed");
11508
- client.writer.write({
11509
- type: "SEARCH_RESP",
11510
- payload: searchResult
11511
- });
13249
+ if (this.clusterSearchCoordinator && this.cluster.getMembers().length > 1) {
13250
+ this.clusterSearchCoordinator.search(searchMapName, searchQuery, {
13251
+ limit: searchOptions?.limit ?? 10,
13252
+ minScore: searchOptions?.minScore,
13253
+ boost: searchOptions?.boost
13254
+ }).then((distributedResult) => {
13255
+ logger.debug({
13256
+ clientId: client.id,
13257
+ mapName: searchMapName,
13258
+ query: searchQuery,
13259
+ resultCount: distributedResult.results.length,
13260
+ totalHits: distributedResult.totalHits,
13261
+ respondedNodes: distributedResult.respondedNodes.length,
13262
+ failedNodes: distributedResult.failedNodes.length,
13263
+ executionTimeMs: distributedResult.executionTimeMs
13264
+ }, "Distributed search executed");
13265
+ client.writer.write({
13266
+ type: "SEARCH_RESP",
13267
+ payload: {
13268
+ requestId: searchReqId,
13269
+ results: distributedResult.results,
13270
+ totalCount: distributedResult.totalHits,
13271
+ // Include cursor for pagination if available
13272
+ nextCursor: distributedResult.nextCursor
13273
+ }
13274
+ });
13275
+ }).catch((err) => {
13276
+ logger.error({ err, mapName: searchMapName, query: searchQuery }, "Distributed search failed");
13277
+ client.writer.write({
13278
+ type: "SEARCH_RESP",
13279
+ payload: {
13280
+ requestId: searchReqId,
13281
+ results: [],
13282
+ totalCount: 0,
13283
+ error: `Distributed search failed: ${err.message}`
13284
+ }
13285
+ });
13286
+ });
13287
+ } else {
13288
+ const searchResult = this.searchCoordinator.search(searchMapName, searchQuery, searchOptions);
13289
+ searchResult.requestId = searchReqId;
13290
+ logger.debug({
13291
+ clientId: client.id,
13292
+ mapName: searchMapName,
13293
+ query: searchQuery,
13294
+ resultCount: searchResult.results.length
13295
+ }, "Local search executed");
13296
+ client.writer.write({
13297
+ type: "SEARCH_RESP",
13298
+ payload: searchResult
13299
+ });
13300
+ }
11512
13301
  break;
11513
13302
  }
11514
13303
  // Phase 11.1b: Live Search Subscriptions
@@ -11539,33 +13328,75 @@ var ServerCoordinator = class {
11539
13328
  });
11540
13329
  break;
11541
13330
  }
11542
- const initialResults = this.searchCoordinator.subscribe(
11543
- client.id,
11544
- subscriptionId,
11545
- subMapName,
11546
- subQuery,
11547
- subOptions
11548
- );
11549
- logger.debug({
11550
- clientId: client.id,
11551
- subscriptionId,
11552
- mapName: subMapName,
11553
- query: subQuery,
11554
- resultCount: initialResults.length
11555
- }, "Search subscription created");
11556
- client.writer.write({
11557
- type: "SEARCH_RESP",
11558
- payload: {
11559
- requestId: subscriptionId,
11560
- results: initialResults,
11561
- totalCount: initialResults.length
11562
- }
11563
- });
13331
+ if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
13332
+ this.distributedSubCoordinator.subscribeSearch(
13333
+ subscriptionId,
13334
+ client.socket,
13335
+ subMapName,
13336
+ subQuery,
13337
+ subOptions || {}
13338
+ ).then((result) => {
13339
+ client.writer.write({
13340
+ type: "SEARCH_RESP",
13341
+ payload: {
13342
+ requestId: subscriptionId,
13343
+ results: result.results,
13344
+ totalCount: result.totalHits
13345
+ }
13346
+ });
13347
+ logger.debug({
13348
+ clientId: client.id,
13349
+ subscriptionId,
13350
+ mapName: subMapName,
13351
+ query: subQuery,
13352
+ resultCount: result.results.length,
13353
+ totalHits: result.totalHits,
13354
+ nodes: result.registeredNodes
13355
+ }, "Distributed search subscription created");
13356
+ }).catch((err) => {
13357
+ logger.error({ err, subscriptionId }, "Distributed search subscription failed");
13358
+ client.writer.write({
13359
+ type: "SEARCH_RESP",
13360
+ payload: {
13361
+ requestId: subscriptionId,
13362
+ results: [],
13363
+ totalCount: 0,
13364
+ error: "Failed to create distributed subscription"
13365
+ }
13366
+ });
13367
+ });
13368
+ } else {
13369
+ const initialResults = this.searchCoordinator.subscribe(
13370
+ client.id,
13371
+ subscriptionId,
13372
+ subMapName,
13373
+ subQuery,
13374
+ subOptions
13375
+ );
13376
+ logger.debug({
13377
+ clientId: client.id,
13378
+ subscriptionId,
13379
+ mapName: subMapName,
13380
+ query: subQuery,
13381
+ resultCount: initialResults.length
13382
+ }, "Search subscription created (local)");
13383
+ client.writer.write({
13384
+ type: "SEARCH_RESP",
13385
+ payload: {
13386
+ requestId: subscriptionId,
13387
+ results: initialResults,
13388
+ totalCount: initialResults.length
13389
+ }
13390
+ });
13391
+ }
11564
13392
  break;
11565
13393
  }
11566
13394
  case "SEARCH_UNSUB": {
11567
13395
  const { subscriptionId: unsubId } = message.payload;
11568
13396
  this.searchCoordinator.unsubscribe(unsubId);
13397
+ if (this.distributedSubCoordinator) {
13398
+ this.distributedSubCoordinator.unsubscribe(unsubId);
13399
+ }
11569
13400
  logger.debug({ clientId: client.id, subscriptionId: unsubId }, "Search unsubscription");
11570
13401
  break;
11571
13402
  }
@@ -11582,7 +13413,7 @@ var ServerCoordinator = class {
11582
13413
  } else if (op.orRecord && op.orRecord.timestamp) {
11583
13414
  } else if (op.orTag) {
11584
13415
  try {
11585
- ts = import_core20.HLC.parse(op.orTag);
13416
+ ts = import_core22.HLC.parse(op.orTag);
11586
13417
  } catch (e) {
11587
13418
  }
11588
13419
  }
@@ -11606,7 +13437,7 @@ var ServerCoordinator = class {
11606
13437
  };
11607
13438
  let broadcastCount = 0;
11608
13439
  for (const client of this.clients.values()) {
11609
- if (client.isAuthenticated && client.socket.readyState === import_ws3.WebSocket.OPEN && client.writer) {
13440
+ if (client.isAuthenticated && client.socket.readyState === import_ws5.WebSocket.OPEN && client.writer) {
11610
13441
  client.writer.write(message);
11611
13442
  broadcastCount++;
11612
13443
  }
@@ -11679,7 +13510,7 @@ var ServerCoordinator = class {
11679
13510
  client.writer.write({ ...message, payload: newPayload });
11680
13511
  }
11681
13512
  } else {
11682
- const msgData = (0, import_core20.serialize)(message);
13513
+ const msgData = (0, import_core22.serialize)(message);
11683
13514
  for (const [id, client] of this.clients) {
11684
13515
  if (id !== excludeClientId && client.socket.readyState === 1) {
11685
13516
  client.writer.writeRaw(msgData);
@@ -11757,7 +13588,7 @@ var ServerCoordinator = class {
11757
13588
  payload: { events: filteredEvents },
11758
13589
  timestamp: this.hlc.now()
11759
13590
  };
11760
- const serializedBatch = (0, import_core20.serialize)(batchMessage);
13591
+ const serializedBatch = (0, import_core22.serialize)(batchMessage);
11761
13592
  for (const client of clients) {
11762
13593
  try {
11763
13594
  client.writer.writeRaw(serializedBatch);
@@ -11842,7 +13673,7 @@ var ServerCoordinator = class {
11842
13673
  payload: { events: filteredEvents },
11843
13674
  timestamp: this.hlc.now()
11844
13675
  };
11845
- const serializedBatch = (0, import_core20.serialize)(batchMessage);
13676
+ const serializedBatch = (0, import_core22.serialize)(batchMessage);
11846
13677
  for (const client of clients) {
11847
13678
  sendPromises.push(new Promise((resolve, reject) => {
11848
13679
  try {
@@ -12038,9 +13869,9 @@ var ServerCoordinator = class {
12038
13869
  async executeLocalQuery(mapName, query) {
12039
13870
  const map = await this.getMapAsync(mapName);
12040
13871
  const localQuery = { ...query };
12041
- delete localQuery.offset;
13872
+ delete localQuery.cursor;
12042
13873
  delete localQuery.limit;
12043
- if (map instanceof import_core20.IndexedLWWMap) {
13874
+ if (map instanceof import_core22.IndexedLWWMap) {
12044
13875
  const coreQuery = this.convertToCoreQuery(localQuery);
12045
13876
  if (coreQuery) {
12046
13877
  const entries = map.queryEntries(coreQuery);
@@ -12050,7 +13881,7 @@ var ServerCoordinator = class {
12050
13881
  });
12051
13882
  }
12052
13883
  }
12053
- if (map instanceof import_core20.IndexedORMap) {
13884
+ if (map instanceof import_core22.IndexedORMap) {
12054
13885
  const coreQuery = this.convertToCoreQuery(localQuery);
12055
13886
  if (coreQuery) {
12056
13887
  const results = map.query(coreQuery);
@@ -12058,14 +13889,14 @@ var ServerCoordinator = class {
12058
13889
  }
12059
13890
  }
12060
13891
  const records = /* @__PURE__ */ new Map();
12061
- if (map instanceof import_core20.LWWMap) {
13892
+ if (map instanceof import_core22.LWWMap) {
12062
13893
  for (const key of map.allKeys()) {
12063
13894
  const rec = map.getRecord(key);
12064
13895
  if (rec && rec.value !== null) {
12065
13896
  records.set(key, rec);
12066
13897
  }
12067
13898
  }
12068
- } else if (map instanceof import_core20.ORMap) {
13899
+ } else if (map instanceof import_core22.ORMap) {
12069
13900
  const items = map.items;
12070
13901
  for (const key of items.keys()) {
12071
13902
  const values = map.get(key);
@@ -12157,7 +13988,7 @@ var ServerCoordinator = class {
12157
13988
  };
12158
13989
  return mapping[op] || null;
12159
13990
  }
12160
- finalizeClusterQuery(requestId, timeout = false) {
13991
+ async finalizeClusterQuery(requestId, timeout = false) {
12161
13992
  const pending = this.pendingClusterQueries.get(requestId);
12162
13993
  if (!pending) return;
12163
13994
  if (timeout) {
@@ -12182,7 +14013,49 @@ var ServerCoordinator = class {
12182
14013
  return 0;
12183
14014
  });
12184
14015
  }
12185
- const slicedResults = query.offset || query.limit ? finalResults.slice(query.offset || 0, (query.offset || 0) + (query.limit || finalResults.length)) : finalResults;
14016
+ let slicedResults = finalResults;
14017
+ let nextCursor;
14018
+ let hasMore = false;
14019
+ let cursorStatus = "none";
14020
+ if (query.cursor || query.limit) {
14021
+ const sort = query.sort || {};
14022
+ const sortEntries = Object.entries(sort);
14023
+ const sortField = sortEntries.length > 0 ? sortEntries[0][0] : "_key";
14024
+ if (query.cursor) {
14025
+ const cursorData = import_core22.QueryCursor.decode(query.cursor);
14026
+ if (!cursorData) {
14027
+ cursorStatus = "invalid";
14028
+ } else if (!import_core22.QueryCursor.isValid(cursorData, query.predicate ?? query.where, sort)) {
14029
+ if (Date.now() - cursorData.timestamp > import_core22.DEFAULT_QUERY_CURSOR_MAX_AGE_MS) {
14030
+ cursorStatus = "expired";
14031
+ } else {
14032
+ cursorStatus = "invalid";
14033
+ }
14034
+ } else {
14035
+ cursorStatus = "valid";
14036
+ slicedResults = finalResults.filter((r) => {
14037
+ const sortValue = r.value[sortField];
14038
+ return import_core22.QueryCursor.isAfterCursor(
14039
+ { key: r.key, sortValue },
14040
+ cursorData
14041
+ );
14042
+ });
14043
+ }
14044
+ }
14045
+ if (query.limit) {
14046
+ hasMore = slicedResults.length > query.limit;
14047
+ slicedResults = slicedResults.slice(0, query.limit);
14048
+ if (hasMore && slicedResults.length > 0) {
14049
+ const lastResult = slicedResults[slicedResults.length - 1];
14050
+ const sortValue = lastResult.value[sortField];
14051
+ nextCursor = import_core22.QueryCursor.fromLastResult(
14052
+ { key: lastResult.key, sortValue },
14053
+ sort,
14054
+ query.predicate ?? query.where
14055
+ );
14056
+ }
14057
+ }
14058
+ }
12186
14059
  const resultKeys = new Set(slicedResults.map((r) => r.key));
12187
14060
  const sub = {
12188
14061
  id: queryId,
@@ -12201,7 +14074,7 @@ var ServerCoordinator = class {
12201
14074
  });
12202
14075
  client.writer.write({
12203
14076
  type: "QUERY_RESP",
12204
- payload: { queryId, results: filteredResults }
14077
+ payload: { queryId, results: filteredResults, nextCursor, hasMore, cursorStatus }
12205
14078
  });
12206
14079
  }
12207
14080
  /**
@@ -12213,11 +14086,11 @@ var ServerCoordinator = class {
12213
14086
  async applyOpToMap(op, remoteNodeId) {
12214
14087
  const typeHint = op.opType === "OR_ADD" || op.opType === "OR_REMOVE" ? "OR" : "LWW";
12215
14088
  const map = this.getMap(op.mapName, typeHint);
12216
- if (typeHint === "OR" && map instanceof import_core20.LWWMap) {
14089
+ if (typeHint === "OR" && map instanceof import_core22.LWWMap) {
12217
14090
  logger.error({ mapName: op.mapName }, "Map type mismatch: LWWMap but received OR op");
12218
14091
  throw new Error("Map type mismatch: LWWMap but received OR op");
12219
14092
  }
12220
- if (typeHint === "LWW" && map instanceof import_core20.ORMap) {
14093
+ if (typeHint === "LWW" && map instanceof import_core22.ORMap) {
12221
14094
  logger.error({ mapName: op.mapName }, "Map type mismatch: ORMap but received LWW op");
12222
14095
  throw new Error("Map type mismatch: ORMap but received LWW op");
12223
14096
  }
@@ -12228,7 +14101,7 @@ var ServerCoordinator = class {
12228
14101
  mapName: op.mapName,
12229
14102
  key: op.key
12230
14103
  };
12231
- if (map instanceof import_core20.LWWMap) {
14104
+ if (map instanceof import_core22.LWWMap) {
12232
14105
  oldRecord = map.getRecord(op.key);
12233
14106
  if (this.conflictResolverHandler.hasResolvers(op.mapName)) {
12234
14107
  const mergeResult = await this.conflictResolverHandler.mergeWithResolver(
@@ -12256,7 +14129,7 @@ var ServerCoordinator = class {
12256
14129
  eventPayload.eventType = "UPDATED";
12257
14130
  eventPayload.record = op.record;
12258
14131
  }
12259
- } else if (map instanceof import_core20.ORMap) {
14132
+ } else if (map instanceof import_core22.ORMap) {
12260
14133
  oldRecord = map.getRecords(op.key);
12261
14134
  if (op.opType === "OR_ADD") {
12262
14135
  map.apply(op.key, op.orRecord);
@@ -12272,7 +14145,7 @@ var ServerCoordinator = class {
12272
14145
  }
12273
14146
  }
12274
14147
  this.queryRegistry.processChange(op.mapName, map, op.key, op.record || op.orRecord, oldRecord);
12275
- const mapSize = map instanceof import_core20.ORMap ? map.totalRecords : map.size;
14148
+ const mapSize = map instanceof import_core22.ORMap ? map.totalRecords : map.size;
12276
14149
  this.metricsService.setMapSize(op.mapName, mapSize);
12277
14150
  if (this.storage) {
12278
14151
  if (recordToStore) {
@@ -12313,17 +14186,6 @@ var ServerCoordinator = class {
12313
14186
  }
12314
14187
  return { eventPayload, oldRecord };
12315
14188
  }
12316
- /**
12317
- * Broadcast event to cluster members (excluding self).
12318
- */
12319
- broadcastToCluster(eventPayload) {
12320
- const members = this.cluster.getMembers();
12321
- for (const memberId of members) {
12322
- if (!this.cluster.isLocal(memberId)) {
12323
- this.cluster.send(memberId, "CLUSTER_EVENT", eventPayload);
12324
- }
12325
- }
12326
- }
12327
14189
  /**
12328
14190
  * Apply replicated operation from another node (callback for ReplicationPipeline)
12329
14191
  * This is called when we receive a replicated operation as a backup node
@@ -12449,7 +14311,6 @@ var ServerCoordinator = class {
12449
14311
  payload: eventPayload,
12450
14312
  timestamp: this.hlc.now()
12451
14313
  }, originalSenderId);
12452
- this.broadcastToCluster(eventPayload);
12453
14314
  this.runAfterInterceptors(op, context);
12454
14315
  }
12455
14316
  /**
@@ -12576,7 +14437,6 @@ var ServerCoordinator = class {
12576
14437
  });
12577
14438
  }
12578
14439
  batchedEvents.push(eventPayload);
12579
- this.broadcastToCluster(eventPayload);
12580
14440
  this.runAfterInterceptors(op, context);
12581
14441
  }
12582
14442
  handleClusterEvent(payload) {
@@ -12586,11 +14446,11 @@ var ServerCoordinator = class {
12586
14446
  return;
12587
14447
  }
12588
14448
  const map = this.getMap(mapName, eventType === "OR_ADD" || eventType === "OR_REMOVE" ? "OR" : "LWW");
12589
- const oldRecord = map instanceof import_core20.LWWMap ? map.getRecord(key) : null;
14449
+ const oldRecord = map instanceof import_core22.LWWMap ? map.getRecord(key) : null;
12590
14450
  if (this.partitionService.isRelated(key)) {
12591
- if (map instanceof import_core20.LWWMap && payload.record) {
14451
+ if (map instanceof import_core22.LWWMap && payload.record) {
12592
14452
  map.merge(key, payload.record);
12593
- } else if (map instanceof import_core20.ORMap) {
14453
+ } else if (map instanceof import_core22.ORMap) {
12594
14454
  if (eventType === "OR_ADD" && payload.orRecord) {
12595
14455
  map.apply(key, payload.orRecord);
12596
14456
  } else if (eventType === "OR_REMOVE" && payload.orTag) {
@@ -12609,9 +14469,9 @@ var ServerCoordinator = class {
12609
14469
  if (!this.maps.has(name)) {
12610
14470
  let map;
12611
14471
  if (typeHint === "OR") {
12612
- map = new import_core20.ORMap(this.hlc);
14472
+ map = new import_core22.ORMap(this.hlc);
12613
14473
  } else {
12614
- map = new import_core20.LWWMap(this.hlc);
14474
+ map = new import_core22.LWWMap(this.hlc);
12615
14475
  }
12616
14476
  this.maps.set(name, map);
12617
14477
  if (this.storage) {
@@ -12634,7 +14494,7 @@ var ServerCoordinator = class {
12634
14494
  this.getMap(name, typeHint);
12635
14495
  const loadingPromise = this.mapLoadingPromises.get(name);
12636
14496
  const map = this.maps.get(name);
12637
- const mapSize = map instanceof import_core20.LWWMap ? Array.from(map.entries()).length : map instanceof import_core20.ORMap ? map.size : 0;
14497
+ const mapSize = map instanceof import_core22.LWWMap ? Array.from(map.entries()).length : map instanceof import_core22.ORMap ? map.size : 0;
12638
14498
  logger.info({
12639
14499
  mapName: name,
12640
14500
  mapExisted,
@@ -12644,7 +14504,7 @@ var ServerCoordinator = class {
12644
14504
  if (loadingPromise) {
12645
14505
  logger.info({ mapName: name }, "[getMapAsync] Waiting for loadMapFromStorage...");
12646
14506
  await loadingPromise;
12647
- const newMapSize = map instanceof import_core20.LWWMap ? Array.from(map.entries()).length : map instanceof import_core20.ORMap ? map.size : 0;
14507
+ const newMapSize = map instanceof import_core22.LWWMap ? Array.from(map.entries()).length : map instanceof import_core22.ORMap ? map.size : 0;
12648
14508
  logger.info({ mapName: name, mapSizeAfterLoad: newMapSize }, "[getMapAsync] Load completed");
12649
14509
  }
12650
14510
  return this.maps.get(name);
@@ -12661,7 +14521,7 @@ var ServerCoordinator = class {
12661
14521
  const mapName = key.substring(0, separatorIndex);
12662
14522
  const actualKey = key.substring(separatorIndex + 1);
12663
14523
  const map = this.maps.get(mapName);
12664
- if (!map || !(map instanceof import_core20.LWWMap)) {
14524
+ if (!map || !(map instanceof import_core22.LWWMap)) {
12665
14525
  return null;
12666
14526
  }
12667
14527
  return map.getRecord(actualKey) ?? null;
@@ -12715,16 +14575,16 @@ var ServerCoordinator = class {
12715
14575
  const currentMap = this.maps.get(name);
12716
14576
  if (!currentMap) return;
12717
14577
  let targetMap = currentMap;
12718
- if (isOR && currentMap instanceof import_core20.LWWMap) {
14578
+ if (isOR && currentMap instanceof import_core22.LWWMap) {
12719
14579
  logger.info({ mapName: name }, "Map auto-detected as ORMap. Switching type.");
12720
- targetMap = new import_core20.ORMap(this.hlc);
14580
+ targetMap = new import_core22.ORMap(this.hlc);
12721
14581
  this.maps.set(name, targetMap);
12722
- } else if (!isOR && currentMap instanceof import_core20.ORMap && typeHint !== "OR") {
14582
+ } else if (!isOR && currentMap instanceof import_core22.ORMap && typeHint !== "OR") {
12723
14583
  logger.info({ mapName: name }, "Map auto-detected as LWWMap. Switching type.");
12724
- targetMap = new import_core20.LWWMap(this.hlc);
14584
+ targetMap = new import_core22.LWWMap(this.hlc);
12725
14585
  this.maps.set(name, targetMap);
12726
14586
  }
12727
- if (targetMap instanceof import_core20.ORMap) {
14587
+ if (targetMap instanceof import_core22.ORMap) {
12728
14588
  for (const [key, record] of records) {
12729
14589
  if (key === "__tombstones__") {
12730
14590
  const t = record;
@@ -12737,7 +14597,7 @@ var ServerCoordinator = class {
12737
14597
  }
12738
14598
  }
12739
14599
  }
12740
- } else if (targetMap instanceof import_core20.LWWMap) {
14600
+ } else if (targetMap instanceof import_core22.LWWMap) {
12741
14601
  for (const [key, record] of records) {
12742
14602
  if (!record.type) {
12743
14603
  targetMap.merge(key, record);
@@ -12748,7 +14608,7 @@ var ServerCoordinator = class {
12748
14608
  if (count > 0) {
12749
14609
  logger.info({ mapName: name, count }, "Loaded records for map");
12750
14610
  this.queryRegistry.refreshSubscriptions(name, targetMap);
12751
- const mapSize = targetMap instanceof import_core20.ORMap ? targetMap.totalRecords : targetMap.size;
14611
+ const mapSize = targetMap instanceof import_core22.ORMap ? targetMap.totalRecords : targetMap.size;
12752
14612
  this.metricsService.setMapSize(name, mapSize);
12753
14613
  }
12754
14614
  } catch (err) {
@@ -12821,7 +14681,7 @@ var ServerCoordinator = class {
12821
14681
  idleTime: now - client.lastPingReceived,
12822
14682
  timeoutMs: CLIENT_HEARTBEAT_TIMEOUT_MS
12823
14683
  }, "Evicting dead client (heartbeat timeout)");
12824
- if (client.socket.readyState === import_ws3.WebSocket.OPEN) {
14684
+ if (client.socket.readyState === import_ws5.WebSocket.OPEN) {
12825
14685
  client.socket.close(4002, "Heartbeat timeout");
12826
14686
  }
12827
14687
  }
@@ -12830,7 +14690,7 @@ var ServerCoordinator = class {
12830
14690
  reportLocalHlc() {
12831
14691
  let minHlc = this.hlc.now();
12832
14692
  for (const client of this.clients.values()) {
12833
- if (import_core20.HLC.compare(client.lastActiveHlc, minHlc) < 0) {
14693
+ if (import_core22.HLC.compare(client.lastActiveHlc, minHlc) < 0) {
12834
14694
  minHlc = client.lastActiveHlc;
12835
14695
  }
12836
14696
  }
@@ -12851,7 +14711,7 @@ var ServerCoordinator = class {
12851
14711
  let globalSafe = this.hlc.now();
12852
14712
  let initialized = false;
12853
14713
  for (const ts of this.gcReports.values()) {
12854
- if (!initialized || import_core20.HLC.compare(ts, globalSafe) < 0) {
14714
+ if (!initialized || import_core22.HLC.compare(ts, globalSafe) < 0) {
12855
14715
  globalSafe = ts;
12856
14716
  initialized = true;
12857
14717
  }
@@ -12886,7 +14746,7 @@ var ServerCoordinator = class {
12886
14746
  logger.info({ olderThanMillis: olderThan.millis }, "Performing Garbage Collection");
12887
14747
  const now = Date.now();
12888
14748
  for (const [name, map] of this.maps) {
12889
- if (map instanceof import_core20.LWWMap) {
14749
+ if (map instanceof import_core22.LWWMap) {
12890
14750
  for (const key of map.allKeys()) {
12891
14751
  const record = map.getRecord(key);
12892
14752
  if (record && record.value !== null && record.ttlMs) {
@@ -12919,11 +14779,18 @@ var ServerCoordinator = class {
12919
14779
  payload: eventPayload,
12920
14780
  timestamp: this.hlc.now()
12921
14781
  });
12922
- const members = this.cluster.getMembers();
12923
- for (const memberId of members) {
12924
- if (!this.cluster.isLocal(memberId)) {
12925
- this.cluster.send(memberId, "CLUSTER_EVENT", eventPayload);
12926
- }
14782
+ this.queryRegistry.processChange(name, map, key, tombstone, record);
14783
+ if (this.replicationPipeline) {
14784
+ const op = {
14785
+ opType: "set",
14786
+ mapName: name,
14787
+ key,
14788
+ record: tombstone
14789
+ };
14790
+ const opId = `ttl:${name}:${key}:${Date.now()}`;
14791
+ this.replicationPipeline.replicate(op, opId, key).catch((err) => {
14792
+ logger.warn({ opId, key, err }, "TTL expiration replication failed (non-fatal)");
14793
+ });
12927
14794
  }
12928
14795
  }
12929
14796
  }
@@ -12938,7 +14805,7 @@ var ServerCoordinator = class {
12938
14805
  });
12939
14806
  }
12940
14807
  }
12941
- } else if (map instanceof import_core20.ORMap) {
14808
+ } else if (map instanceof import_core22.ORMap) {
12942
14809
  const items = map.items;
12943
14810
  const tombstonesSet = map.tombstones;
12944
14811
  const tagsToExpire = [];
@@ -12956,6 +14823,7 @@ var ServerCoordinator = class {
12956
14823
  }
12957
14824
  for (const { key, tag } of tagsToExpire) {
12958
14825
  logger.info({ mapName: name, key, tag }, "ORMap Record expired (TTL). Removing.");
14826
+ const oldRecords = map.getRecords(key);
12959
14827
  map.applyTombstone(tag);
12960
14828
  if (this.storage) {
12961
14829
  const records = map.getRecords(key);
@@ -12981,11 +14849,19 @@ var ServerCoordinator = class {
12981
14849
  payload: eventPayload,
12982
14850
  timestamp: this.hlc.now()
12983
14851
  });
12984
- const members = this.cluster.getMembers();
12985
- for (const memberId of members) {
12986
- if (!this.cluster.isLocal(memberId)) {
12987
- this.cluster.send(memberId, "CLUSTER_EVENT", eventPayload);
12988
- }
14852
+ const newRecords = map.getRecords(key);
14853
+ this.queryRegistry.processChange(name, map, key, newRecords, oldRecords);
14854
+ if (this.replicationPipeline) {
14855
+ const op = {
14856
+ opType: "OR_REMOVE",
14857
+ mapName: name,
14858
+ key,
14859
+ orTag: tag
14860
+ };
14861
+ const opId = `ttl:${name}:${key}:${tag}:${Date.now()}`;
14862
+ this.replicationPipeline.replicate(op, opId, key).catch((err) => {
14863
+ logger.warn({ opId, key, err }, "ORMap TTL expiration replication failed (non-fatal)");
14864
+ });
12989
14865
  }
12990
14866
  }
12991
14867
  const removedTags = map.prune(olderThan);
@@ -13041,17 +14917,17 @@ var ServerCoordinator = class {
13041
14917
  stringToWriteConcern(value) {
13042
14918
  switch (value) {
13043
14919
  case "FIRE_AND_FORGET":
13044
- return import_core20.WriteConcern.FIRE_AND_FORGET;
14920
+ return import_core22.WriteConcern.FIRE_AND_FORGET;
13045
14921
  case "MEMORY":
13046
- return import_core20.WriteConcern.MEMORY;
14922
+ return import_core22.WriteConcern.MEMORY;
13047
14923
  case "APPLIED":
13048
- return import_core20.WriteConcern.APPLIED;
14924
+ return import_core22.WriteConcern.APPLIED;
13049
14925
  case "REPLICATED":
13050
- return import_core20.WriteConcern.REPLICATED;
14926
+ return import_core22.WriteConcern.REPLICATED;
13051
14927
  case "PERSISTED":
13052
- return import_core20.WriteConcern.PERSISTED;
14928
+ return import_core22.WriteConcern.PERSISTED;
13053
14929
  default:
13054
- return import_core20.WriteConcern.MEMORY;
14930
+ return import_core22.WriteConcern.MEMORY;
13055
14931
  }
13056
14932
  }
13057
14933
  /**
@@ -13108,7 +14984,7 @@ var ServerCoordinator = class {
13108
14984
  }
13109
14985
  });
13110
14986
  if (op.id) {
13111
- this.writeAckManager.notifyLevel(op.id, import_core20.WriteConcern.REPLICATED);
14987
+ this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.REPLICATED);
13112
14988
  }
13113
14989
  }
13114
14990
  }
@@ -13116,7 +14992,7 @@ var ServerCoordinator = class {
13116
14992
  this.broadcastBatch(batchedEvents, clientId);
13117
14993
  for (const op of ops) {
13118
14994
  if (op.id && this.partitionService.isLocalOwner(op.key)) {
13119
- this.writeAckManager.notifyLevel(op.id, import_core20.WriteConcern.REPLICATED);
14995
+ this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.REPLICATED);
13120
14996
  }
13121
14997
  }
13122
14998
  }
@@ -13144,7 +15020,7 @@ var ServerCoordinator = class {
13144
15020
  const owner = this.partitionService.getOwner(op.key);
13145
15021
  await this.forwardOpAndWait(op, owner);
13146
15022
  if (op.id) {
13147
- this.writeAckManager.notifyLevel(op.id, import_core20.WriteConcern.REPLICATED);
15023
+ this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.REPLICATED);
13148
15024
  }
13149
15025
  }
13150
15026
  }
@@ -13152,7 +15028,7 @@ var ServerCoordinator = class {
13152
15028
  await this.broadcastBatchSync(batchedEvents, clientId);
13153
15029
  for (const op of ops) {
13154
15030
  if (op.id && this.partitionService.isLocalOwner(op.key)) {
13155
- this.writeAckManager.notifyLevel(op.id, import_core20.WriteConcern.REPLICATED);
15031
+ this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.REPLICATED);
13156
15032
  }
13157
15033
  }
13158
15034
  }
@@ -13186,7 +15062,7 @@ var ServerCoordinator = class {
13186
15062
  return;
13187
15063
  }
13188
15064
  if (op.id) {
13189
- this.writeAckManager.notifyLevel(op.id, import_core20.WriteConcern.APPLIED);
15065
+ this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.APPLIED);
13190
15066
  }
13191
15067
  if (eventPayload) {
13192
15068
  batchedEvents.push({
@@ -13200,7 +15076,7 @@ var ServerCoordinator = class {
13200
15076
  try {
13201
15077
  await this.persistOpSync(op);
13202
15078
  if (op.id) {
13203
- this.writeAckManager.notifyLevel(op.id, import_core20.WriteConcern.PERSISTED);
15079
+ this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.PERSISTED);
13204
15080
  }
13205
15081
  } catch (err) {
13206
15082
  logger.error({ opId: op.id, err }, "Persistence failed");
@@ -13543,10 +15419,10 @@ var RateLimitInterceptor = class {
13543
15419
  };
13544
15420
 
13545
15421
  // src/utils/nativeStats.ts
13546
- var import_core21 = require("@topgunbuild/core");
15422
+ var import_core23 = require("@topgunbuild/core");
13547
15423
  function getNativeModuleStatus() {
13548
15424
  return {
13549
- nativeHash: (0, import_core21.isUsingNativeHash)(),
15425
+ nativeHash: (0, import_core23.isUsingNativeHash)(),
13550
15426
  sharedArrayBuffer: SharedMemoryManager.isAvailable()
13551
15427
  };
13552
15428
  }
@@ -13579,15 +15455,15 @@ function logNativeStatus() {
13579
15455
  }
13580
15456
 
13581
15457
  // src/cluster/ClusterCoordinator.ts
13582
- var import_events13 = require("events");
13583
- var import_core22 = require("@topgunbuild/core");
15458
+ var import_events16 = require("events");
15459
+ var import_core24 = require("@topgunbuild/core");
13584
15460
  var DEFAULT_CLUSTER_COORDINATOR_CONFIG = {
13585
15461
  gradualRebalancing: true,
13586
- migration: import_core22.DEFAULT_MIGRATION_CONFIG,
13587
- replication: import_core22.DEFAULT_REPLICATION_CONFIG,
15462
+ migration: import_core24.DEFAULT_MIGRATION_CONFIG,
15463
+ replication: import_core24.DEFAULT_REPLICATION_CONFIG,
13588
15464
  replicationEnabled: true
13589
15465
  };
13590
- var ClusterCoordinator = class extends import_events13.EventEmitter {
15466
+ var ClusterCoordinator = class extends import_events16.EventEmitter {
13591
15467
  constructor(config) {
13592
15468
  super();
13593
15469
  this.replicationPipeline = null;
@@ -13952,12 +15828,12 @@ var ClusterCoordinator = class extends import_events13.EventEmitter {
13952
15828
  };
13953
15829
 
13954
15830
  // src/MapWithResolver.ts
13955
- var import_core23 = require("@topgunbuild/core");
15831
+ var import_core25 = require("@topgunbuild/core");
13956
15832
  var MapWithResolver = class {
13957
15833
  constructor(config) {
13958
15834
  this.mapName = config.name;
13959
- this.hlc = new import_core23.HLC(config.nodeId);
13960
- this.map = new import_core23.LWWMap(this.hlc);
15835
+ this.hlc = new import_core25.HLC(config.nodeId);
15836
+ this.map = new import_core25.LWWMap(this.hlc);
13961
15837
  this.resolverService = config.resolverService;
13962
15838
  this.onRejection = config.onRejection;
13963
15839
  }
@@ -14213,7 +16089,7 @@ function mergeWithDefaults(userConfig) {
14213
16089
  }
14214
16090
 
14215
16091
  // src/config/MapFactory.ts
14216
- var import_core24 = require("@topgunbuild/core");
16092
+ var import_core26 = require("@topgunbuild/core");
14217
16093
  var MapFactory = class {
14218
16094
  /**
14219
16095
  * Create a MapFactory.
@@ -14237,9 +16113,9 @@ var MapFactory = class {
14237
16113
  createLWWMap(mapName, hlc) {
14238
16114
  const mapConfig = this.mapConfigs.get(mapName);
14239
16115
  if (!mapConfig || mapConfig.indexes.length === 0) {
14240
- return new import_core24.LWWMap(hlc);
16116
+ return new import_core26.LWWMap(hlc);
14241
16117
  }
14242
- const map = new import_core24.IndexedLWWMap(hlc);
16118
+ const map = new import_core26.IndexedLWWMap(hlc);
14243
16119
  for (const indexDef of mapConfig.indexes) {
14244
16120
  this.addIndexToLWWMap(map, indexDef);
14245
16121
  }
@@ -14255,9 +16131,9 @@ var MapFactory = class {
14255
16131
  createORMap(mapName, hlc) {
14256
16132
  const mapConfig = this.mapConfigs.get(mapName);
14257
16133
  if (!mapConfig || mapConfig.indexes.length === 0) {
14258
- return new import_core24.ORMap(hlc);
16134
+ return new import_core26.ORMap(hlc);
14259
16135
  }
14260
- const map = new import_core24.IndexedORMap(hlc);
16136
+ const map = new import_core26.IndexedORMap(hlc);
14261
16137
  for (const indexDef of mapConfig.indexes) {
14262
16138
  this.addIndexToORMap(map, indexDef);
14263
16139
  }
@@ -14294,7 +16170,7 @@ var MapFactory = class {
14294
16170
  * Supports dot notation for nested paths.
14295
16171
  */
14296
16172
  createAttribute(path) {
14297
- return (0, import_core24.simpleAttribute)(path, (record) => {
16173
+ return (0, import_core26.simpleAttribute)(path, (record) => {
14298
16174
  return this.getNestedValue(record, path);
14299
16175
  });
14300
16176
  }