@topgunbuild/server 0.8.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +97 -0
- package/dist/index.d.mts +1569 -1461
- package/dist/index.d.ts +1569 -1461
- package/dist/index.js +2138 -262
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2081 -194
- package/dist/index.mjs.map +1 -1
- package/package.json +14 -12
package/dist/index.js
CHANGED
|
@@ -105,8 +105,8 @@ module.exports = __toCommonJS(index_exports);
|
|
|
105
105
|
var import_http = require("http");
|
|
106
106
|
var import_https = require("https");
|
|
107
107
|
var import_fs2 = require("fs");
|
|
108
|
-
var
|
|
109
|
-
var
|
|
108
|
+
var import_ws5 = require("ws");
|
|
109
|
+
var import_core22 = require("@topgunbuild/core");
|
|
110
110
|
var jwt = __toESM(require("jsonwebtoken"));
|
|
111
111
|
var crypto = __toESM(require("crypto"));
|
|
112
112
|
|
|
@@ -160,6 +160,10 @@ function matchesQuery(record, query) {
|
|
|
160
160
|
return true;
|
|
161
161
|
}
|
|
162
162
|
function executeQuery(records, query) {
|
|
163
|
+
const result = executeQueryWithCursor(records, query);
|
|
164
|
+
return result.results;
|
|
165
|
+
}
|
|
166
|
+
function executeQueryWithCursor(records, query) {
|
|
163
167
|
if (!query) {
|
|
164
168
|
query = {};
|
|
165
169
|
}
|
|
@@ -177,9 +181,13 @@ function executeQuery(records, query) {
|
|
|
177
181
|
}
|
|
178
182
|
}
|
|
179
183
|
}
|
|
180
|
-
|
|
184
|
+
const sort = query.sort || {};
|
|
185
|
+
const sortEntries = Object.entries(sort);
|
|
186
|
+
const sortField = sortEntries.length > 0 ? sortEntries[0][0] : "_key";
|
|
187
|
+
const sortDirection = sortEntries.length > 0 ? sortEntries[0][1] : "asc";
|
|
188
|
+
if (sortEntries.length > 0) {
|
|
181
189
|
results.sort((a, b) => {
|
|
182
|
-
for (const [field, direction] of
|
|
190
|
+
for (const [field, direction] of sortEntries) {
|
|
183
191
|
const valA = a.record.value[field];
|
|
184
192
|
const valB = b.record.value[field];
|
|
185
193
|
if (valA < valB) return direction === "asc" ? -1 : 1;
|
|
@@ -188,16 +196,55 @@ function executeQuery(records, query) {
|
|
|
188
196
|
return 0;
|
|
189
197
|
});
|
|
190
198
|
}
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
const
|
|
194
|
-
|
|
199
|
+
let cursorStatus = "none";
|
|
200
|
+
if (query.cursor) {
|
|
201
|
+
const cursorData = import_core.QueryCursor.decode(query.cursor);
|
|
202
|
+
if (!cursorData) {
|
|
203
|
+
cursorStatus = "invalid";
|
|
204
|
+
} else if (!import_core.QueryCursor.isValid(cursorData, query.predicate ?? query.where, sort)) {
|
|
205
|
+
if (Date.now() - cursorData.timestamp > import_core.DEFAULT_QUERY_CURSOR_MAX_AGE_MS) {
|
|
206
|
+
cursorStatus = "expired";
|
|
207
|
+
} else {
|
|
208
|
+
cursorStatus = "invalid";
|
|
209
|
+
}
|
|
210
|
+
} else {
|
|
211
|
+
cursorStatus = "valid";
|
|
212
|
+
results = results.filter((r) => {
|
|
213
|
+
const sortValue = r.record.value[sortField];
|
|
214
|
+
return import_core.QueryCursor.isAfterCursor(
|
|
215
|
+
{ key: r.key, sortValue },
|
|
216
|
+
cursorData
|
|
217
|
+
);
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
const hasLimit = query.limit !== void 0 && query.limit > 0;
|
|
222
|
+
const totalBeforeLimit = results.length;
|
|
223
|
+
if (hasLimit) {
|
|
224
|
+
results = results.slice(0, query.limit);
|
|
225
|
+
}
|
|
226
|
+
const hasMore = hasLimit && totalBeforeLimit > query.limit;
|
|
227
|
+
let nextCursor;
|
|
228
|
+
if (hasMore && results.length > 0) {
|
|
229
|
+
const lastResult = results[results.length - 1];
|
|
230
|
+
const sortValue = lastResult.record.value[sortField];
|
|
231
|
+
nextCursor = import_core.QueryCursor.fromLastResult(
|
|
232
|
+
{ key: lastResult.key, sortValue },
|
|
233
|
+
sort,
|
|
234
|
+
query.predicate ?? query.where
|
|
235
|
+
);
|
|
195
236
|
}
|
|
196
|
-
return
|
|
237
|
+
return {
|
|
238
|
+
results: results.map((r) => ({ key: r.key, value: r.record.value })),
|
|
239
|
+
nextCursor,
|
|
240
|
+
hasMore,
|
|
241
|
+
cursorStatus
|
|
242
|
+
};
|
|
197
243
|
}
|
|
198
244
|
|
|
199
245
|
// src/query/QueryRegistry.ts
|
|
200
246
|
var import_core2 = require("@topgunbuild/core");
|
|
247
|
+
var import_ws = require("ws");
|
|
201
248
|
|
|
202
249
|
// src/utils/logger.ts
|
|
203
250
|
var import_pino = __toESM(require("pino"));
|
|
@@ -386,6 +433,106 @@ var QueryRegistry = class {
|
|
|
386
433
|
}
|
|
387
434
|
}
|
|
388
435
|
}
|
|
436
|
+
/**
|
|
437
|
+
* Set the ClusterManager for distributed subscriptions.
|
|
438
|
+
*/
|
|
439
|
+
setClusterManager(clusterManager, nodeId) {
|
|
440
|
+
this.clusterManager = clusterManager;
|
|
441
|
+
this.nodeId = nodeId;
|
|
442
|
+
}
|
|
443
|
+
/**
|
|
444
|
+
* Set the callback for getting maps by name.
|
|
445
|
+
* Required for distributed subscriptions to return initial results.
|
|
446
|
+
*/
|
|
447
|
+
setMapGetter(getter) {
|
|
448
|
+
this.getMap = getter;
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* Register a distributed subscription from a remote coordinator.
|
|
452
|
+
* Called when receiving CLUSTER_SUB_REGISTER message.
|
|
453
|
+
*
|
|
454
|
+
* @param subscriptionId - Unique subscription ID
|
|
455
|
+
* @param mapName - Map name to query
|
|
456
|
+
* @param query - Query predicate
|
|
457
|
+
* @param coordinatorNodeId - Node ID of the coordinator (receives updates)
|
|
458
|
+
* @returns Initial query results from this node
|
|
459
|
+
*/
|
|
460
|
+
registerDistributed(subscriptionId, mapName, query, coordinatorNodeId) {
|
|
461
|
+
const dummySocket = {
|
|
462
|
+
readyState: 1,
|
|
463
|
+
send: () => {
|
|
464
|
+
}
|
|
465
|
+
// Updates go via cluster messages, not socket
|
|
466
|
+
};
|
|
467
|
+
let initialResults = [];
|
|
468
|
+
const previousResultKeys = /* @__PURE__ */ new Set();
|
|
469
|
+
if (this.getMap) {
|
|
470
|
+
const map = this.getMap(mapName);
|
|
471
|
+
if (map) {
|
|
472
|
+
const records = this.getMapRecords(map);
|
|
473
|
+
const queryResults = executeQuery(records, query);
|
|
474
|
+
initialResults = queryResults.map((r) => {
|
|
475
|
+
previousResultKeys.add(r.key);
|
|
476
|
+
return { key: r.key, value: r.value };
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
const sub = {
|
|
481
|
+
id: subscriptionId,
|
|
482
|
+
clientId: `cluster:${coordinatorNodeId}`,
|
|
483
|
+
mapName,
|
|
484
|
+
query,
|
|
485
|
+
socket: dummySocket,
|
|
486
|
+
previousResultKeys,
|
|
487
|
+
coordinatorNodeId,
|
|
488
|
+
isDistributed: true
|
|
489
|
+
};
|
|
490
|
+
this.register(sub);
|
|
491
|
+
logger.debug(
|
|
492
|
+
{ subscriptionId, mapName, coordinatorNodeId, resultCount: initialResults.length },
|
|
493
|
+
"Distributed query subscription registered"
|
|
494
|
+
);
|
|
495
|
+
return initialResults;
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* Get a distributed subscription by ID.
|
|
499
|
+
* Returns undefined if not found or not distributed.
|
|
500
|
+
*/
|
|
501
|
+
getDistributedSubscription(subscriptionId) {
|
|
502
|
+
for (const subs of this.subscriptions.values()) {
|
|
503
|
+
for (const sub of subs) {
|
|
504
|
+
if (sub.id === subscriptionId && sub.isDistributed) {
|
|
505
|
+
return sub;
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
return void 0;
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* Unregister all distributed subscriptions where the given node was the coordinator.
|
|
513
|
+
* Called when a cluster node disconnects.
|
|
514
|
+
*
|
|
515
|
+
* @param coordinatorNodeId - Node ID of the disconnected coordinator
|
|
516
|
+
*/
|
|
517
|
+
unregisterByCoordinator(coordinatorNodeId) {
|
|
518
|
+
const subscriptionsToRemove = [];
|
|
519
|
+
for (const subs of this.subscriptions.values()) {
|
|
520
|
+
for (const sub of subs) {
|
|
521
|
+
if (sub.isDistributed && sub.coordinatorNodeId === coordinatorNodeId) {
|
|
522
|
+
subscriptionsToRemove.push(sub.id);
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
for (const subId of subscriptionsToRemove) {
|
|
527
|
+
this.unregister(subId);
|
|
528
|
+
}
|
|
529
|
+
if (subscriptionsToRemove.length > 0) {
|
|
530
|
+
logger.debug(
|
|
531
|
+
{ coordinatorNodeId, count: subscriptionsToRemove.length },
|
|
532
|
+
"Cleaned up distributed query subscriptions for disconnected coordinator"
|
|
533
|
+
);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
389
536
|
/**
|
|
390
537
|
* Returns all active subscriptions for a specific map.
|
|
391
538
|
* Used for subscription-based event routing to avoid broadcasting to all clients.
|
|
@@ -668,7 +815,9 @@ var QueryRegistry = class {
|
|
|
668
815
|
return record.value;
|
|
669
816
|
}
|
|
670
817
|
sendUpdate(sub, key, value, type) {
|
|
671
|
-
if (sub.
|
|
818
|
+
if (sub.isDistributed && sub.coordinatorNodeId && this.clusterManager) {
|
|
819
|
+
this.sendDistributedUpdate(sub, key, value, type);
|
|
820
|
+
} else if (sub.socket.readyState === import_ws.WebSocket.OPEN) {
|
|
672
821
|
sub.socket.send((0, import_core2.serialize)({
|
|
673
822
|
type: "QUERY_UPDATE",
|
|
674
823
|
payload: {
|
|
@@ -680,6 +829,26 @@ var QueryRegistry = class {
|
|
|
680
829
|
}));
|
|
681
830
|
}
|
|
682
831
|
}
|
|
832
|
+
/**
|
|
833
|
+
* Send update to remote coordinator node for a distributed subscription.
|
|
834
|
+
*/
|
|
835
|
+
sendDistributedUpdate(sub, key, value, type) {
|
|
836
|
+
if (!this.clusterManager || !sub.coordinatorNodeId) return;
|
|
837
|
+
const changeType = type === "UPDATE" ? sub.previousResultKeys.has(key) ? "UPDATE" : "ENTER" : "LEAVE";
|
|
838
|
+
const payload = {
|
|
839
|
+
subscriptionId: sub.id,
|
|
840
|
+
sourceNodeId: this.nodeId || "unknown",
|
|
841
|
+
key,
|
|
842
|
+
value,
|
|
843
|
+
changeType,
|
|
844
|
+
timestamp: Date.now()
|
|
845
|
+
};
|
|
846
|
+
this.clusterManager.send(sub.coordinatorNodeId, "CLUSTER_SUB_UPDATE", payload);
|
|
847
|
+
logger.debug(
|
|
848
|
+
{ subscriptionId: sub.id, key, changeType, coordinator: sub.coordinatorNodeId },
|
|
849
|
+
"Sent distributed query update"
|
|
850
|
+
);
|
|
851
|
+
}
|
|
683
852
|
analyzeQueryFields(query) {
|
|
684
853
|
const fields = /* @__PURE__ */ new Set();
|
|
685
854
|
try {
|
|
@@ -818,7 +987,7 @@ var TopicManager = class {
|
|
|
818
987
|
};
|
|
819
988
|
|
|
820
989
|
// src/cluster/ClusterManager.ts
|
|
821
|
-
var
|
|
990
|
+
var import_ws2 = require("ws");
|
|
822
991
|
var import_events2 = require("events");
|
|
823
992
|
var dns = __toESM(require("dns"));
|
|
824
993
|
var import_fs = require("fs");
|
|
@@ -1123,7 +1292,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
|
|
|
1123
1292
|
if (this.config.tls?.enabled) {
|
|
1124
1293
|
const tlsOptions = this.buildClusterTLSOptions();
|
|
1125
1294
|
const httpsServer = https.createServer(tlsOptions);
|
|
1126
|
-
this.server = new
|
|
1295
|
+
this.server = new import_ws2.WebSocketServer({ server: httpsServer });
|
|
1127
1296
|
httpsServer.listen(this.config.port, () => {
|
|
1128
1297
|
const addr = httpsServer.address();
|
|
1129
1298
|
this._actualPort = typeof addr === "object" && addr ? addr.port : this.config.port;
|
|
@@ -1131,7 +1300,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
|
|
|
1131
1300
|
this.onServerReady(resolve);
|
|
1132
1301
|
});
|
|
1133
1302
|
} else {
|
|
1134
|
-
this.server = new
|
|
1303
|
+
this.server = new import_ws2.WebSocketServer({ port: this.config.port });
|
|
1135
1304
|
this.server.on("listening", () => {
|
|
1136
1305
|
const addr = this.server.address();
|
|
1137
1306
|
this._actualPort = typeof addr === "object" && addr ? addr.port : this.config.port;
|
|
@@ -1211,7 +1380,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
|
|
|
1211
1380
|
sendHeartbeatToAll() {
|
|
1212
1381
|
for (const [nodeId, member] of this.members) {
|
|
1213
1382
|
if (member.isSelf) continue;
|
|
1214
|
-
if (member.socket && member.socket.readyState ===
|
|
1383
|
+
if (member.socket && member.socket.readyState === import_ws2.WebSocket.OPEN) {
|
|
1215
1384
|
this.send(nodeId, "HEARTBEAT", { timestamp: Date.now() });
|
|
1216
1385
|
}
|
|
1217
1386
|
}
|
|
@@ -1245,7 +1414,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
|
|
|
1245
1414
|
broadcastMemberList() {
|
|
1246
1415
|
for (const [nodeId, member] of this.members) {
|
|
1247
1416
|
if (member.isSelf) continue;
|
|
1248
|
-
if (member.socket && member.socket.readyState ===
|
|
1417
|
+
if (member.socket && member.socket.readyState === import_ws2.WebSocket.OPEN) {
|
|
1249
1418
|
this.sendMemberList(nodeId);
|
|
1250
1419
|
}
|
|
1251
1420
|
}
|
|
@@ -1270,7 +1439,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
|
|
|
1270
1439
|
const member = this.members.get(nodeId);
|
|
1271
1440
|
if (!member) return;
|
|
1272
1441
|
logger.warn({ nodeId }, "Removing failed node from cluster");
|
|
1273
|
-
if (member.socket && member.socket.readyState !==
|
|
1442
|
+
if (member.socket && member.socket.readyState !== import_ws2.WebSocket.CLOSED) {
|
|
1274
1443
|
try {
|
|
1275
1444
|
member.socket.terminate();
|
|
1276
1445
|
} catch (e) {
|
|
@@ -1344,9 +1513,9 @@ var ClusterManager = class extends import_events2.EventEmitter {
|
|
|
1344
1513
|
if (this.config.tls.caCertPath) {
|
|
1345
1514
|
wsOptions.ca = (0, import_fs.readFileSync)(this.config.tls.caCertPath);
|
|
1346
1515
|
}
|
|
1347
|
-
ws = new
|
|
1516
|
+
ws = new import_ws2.WebSocket(`${protocol}${peerAddress}`, wsOptions);
|
|
1348
1517
|
} else {
|
|
1349
|
-
ws = new
|
|
1518
|
+
ws = new import_ws2.WebSocket(`ws://${peerAddress}`);
|
|
1350
1519
|
}
|
|
1351
1520
|
ws.on("open", () => {
|
|
1352
1521
|
this.pendingConnections.delete(peerAddress);
|
|
@@ -1433,7 +1602,7 @@ var ClusterManager = class extends import_events2.EventEmitter {
|
|
|
1433
1602
|
}
|
|
1434
1603
|
send(nodeId, type, payload) {
|
|
1435
1604
|
const member = this.members.get(nodeId);
|
|
1436
|
-
if (member && member.socket && member.socket.readyState ===
|
|
1605
|
+
if (member && member.socket && member.socket.readyState === import_ws2.WebSocket.OPEN) {
|
|
1437
1606
|
const msg = {
|
|
1438
1607
|
type,
|
|
1439
1608
|
senderId: this.config.nodeId,
|
|
@@ -2652,6 +2821,90 @@ var MetricsService = class {
|
|
|
2652
2821
|
help: "Current connection rate per second",
|
|
2653
2822
|
registers: [this.registry]
|
|
2654
2823
|
});
|
|
2824
|
+
this.distributedSearchTotal = new import_prom_client.Counter({
|
|
2825
|
+
name: "topgun_distributed_search_total",
|
|
2826
|
+
help: "Total number of distributed search requests",
|
|
2827
|
+
labelNames: ["map", "status"],
|
|
2828
|
+
registers: [this.registry]
|
|
2829
|
+
});
|
|
2830
|
+
this.distributedSearchDuration = new import_prom_client.Summary({
|
|
2831
|
+
name: "topgun_distributed_search_duration_ms",
|
|
2832
|
+
help: "Distribution of distributed search execution times in milliseconds",
|
|
2833
|
+
labelNames: ["map"],
|
|
2834
|
+
percentiles: [0.5, 0.9, 0.95, 0.99],
|
|
2835
|
+
registers: [this.registry]
|
|
2836
|
+
});
|
|
2837
|
+
this.distributedSearchFailedNodes = new import_prom_client.Counter({
|
|
2838
|
+
name: "topgun_distributed_search_failed_nodes_total",
|
|
2839
|
+
help: "Total number of node failures during distributed search",
|
|
2840
|
+
registers: [this.registry]
|
|
2841
|
+
});
|
|
2842
|
+
this.distributedSearchPartialResults = new import_prom_client.Counter({
|
|
2843
|
+
name: "topgun_distributed_search_partial_results_total",
|
|
2844
|
+
help: "Total number of searches that returned partial results due to node failures",
|
|
2845
|
+
registers: [this.registry]
|
|
2846
|
+
});
|
|
2847
|
+
this.distributedSubTotal = new import_prom_client.Counter({
|
|
2848
|
+
name: "topgun_distributed_sub_total",
|
|
2849
|
+
help: "Total distributed subscriptions created",
|
|
2850
|
+
labelNames: ["type", "status"],
|
|
2851
|
+
registers: [this.registry]
|
|
2852
|
+
});
|
|
2853
|
+
this.distributedSubUnsubscribeTotal = new import_prom_client.Counter({
|
|
2854
|
+
name: "topgun_distributed_sub_unsubscribe_total",
|
|
2855
|
+
help: "Total unsubscriptions from distributed subscriptions",
|
|
2856
|
+
labelNames: ["type"],
|
|
2857
|
+
registers: [this.registry]
|
|
2858
|
+
});
|
|
2859
|
+
this.distributedSubActive = new import_prom_client.Gauge({
|
|
2860
|
+
name: "topgun_distributed_sub_active",
|
|
2861
|
+
help: "Currently active distributed subscriptions",
|
|
2862
|
+
labelNames: ["type"],
|
|
2863
|
+
registers: [this.registry]
|
|
2864
|
+
});
|
|
2865
|
+
this.distributedSubPendingAcks = new import_prom_client.Gauge({
|
|
2866
|
+
name: "topgun_distributed_sub_pending_acks",
|
|
2867
|
+
help: "Subscriptions waiting for ACKs from cluster nodes",
|
|
2868
|
+
registers: [this.registry]
|
|
2869
|
+
});
|
|
2870
|
+
this.distributedSubUpdates = new import_prom_client.Counter({
|
|
2871
|
+
name: "topgun_distributed_sub_updates_total",
|
|
2872
|
+
help: "Delta updates processed for distributed subscriptions",
|
|
2873
|
+
labelNames: ["direction", "change_type"],
|
|
2874
|
+
registers: [this.registry]
|
|
2875
|
+
});
|
|
2876
|
+
this.distributedSubAckTotal = new import_prom_client.Counter({
|
|
2877
|
+
name: "topgun_distributed_sub_ack_total",
|
|
2878
|
+
help: "Node ACK responses for distributed subscriptions",
|
|
2879
|
+
labelNames: ["status"],
|
|
2880
|
+
registers: [this.registry]
|
|
2881
|
+
});
|
|
2882
|
+
this.distributedSubNodeDisconnect = new import_prom_client.Counter({
|
|
2883
|
+
name: "topgun_distributed_sub_node_disconnect_total",
|
|
2884
|
+
help: "Node disconnects affecting distributed subscriptions",
|
|
2885
|
+
registers: [this.registry]
|
|
2886
|
+
});
|
|
2887
|
+
this.distributedSubRegistrationDuration = new import_prom_client.Histogram({
|
|
2888
|
+
name: "topgun_distributed_sub_registration_duration_ms",
|
|
2889
|
+
help: "Time to register subscription on all nodes",
|
|
2890
|
+
labelNames: ["type"],
|
|
2891
|
+
buckets: [10, 50, 100, 250, 500, 1e3, 2500],
|
|
2892
|
+
registers: [this.registry]
|
|
2893
|
+
});
|
|
2894
|
+
this.distributedSubUpdateLatency = new import_prom_client.Histogram({
|
|
2895
|
+
name: "topgun_distributed_sub_update_latency_ms",
|
|
2896
|
+
help: "Latency from data change to client notification",
|
|
2897
|
+
labelNames: ["type"],
|
|
2898
|
+
buckets: [1, 5, 10, 25, 50, 100, 250],
|
|
2899
|
+
registers: [this.registry]
|
|
2900
|
+
});
|
|
2901
|
+
this.distributedSubInitialResultsCount = new import_prom_client.Histogram({
|
|
2902
|
+
name: "topgun_distributed_sub_initial_results_count",
|
|
2903
|
+
help: "Initial result set size for distributed subscriptions",
|
|
2904
|
+
labelNames: ["type"],
|
|
2905
|
+
buckets: [0, 1, 5, 10, 25, 50, 100],
|
|
2906
|
+
registers: [this.registry]
|
|
2907
|
+
});
|
|
2655
2908
|
}
|
|
2656
2909
|
destroy() {
|
|
2657
2910
|
this.registry.clear();
|
|
@@ -2762,6 +3015,115 @@ var MetricsService = class {
|
|
|
2762
3015
|
setConnectionRatePerSecond(rate) {
|
|
2763
3016
|
this.connectionRatePerSecond.set(rate);
|
|
2764
3017
|
}
|
|
3018
|
+
// === Distributed search metric methods (Phase 14) ===
|
|
3019
|
+
/**
|
|
3020
|
+
* Record a distributed search request.
|
|
3021
|
+
* @param mapName - Name of the map being searched
|
|
3022
|
+
* @param status - 'success', 'partial', or 'error'
|
|
3023
|
+
*/
|
|
3024
|
+
incDistributedSearch(mapName, status) {
|
|
3025
|
+
this.distributedSearchTotal.inc({ map: mapName, status });
|
|
3026
|
+
}
|
|
3027
|
+
/**
|
|
3028
|
+
* Record the duration of a distributed search.
|
|
3029
|
+
* @param mapName - Name of the map being searched
|
|
3030
|
+
* @param durationMs - Duration in milliseconds
|
|
3031
|
+
*/
|
|
3032
|
+
recordDistributedSearchDuration(mapName, durationMs) {
|
|
3033
|
+
this.distributedSearchDuration.observe({ map: mapName }, durationMs);
|
|
3034
|
+
}
|
|
3035
|
+
/**
|
|
3036
|
+
* Increment counter for failed nodes during distributed search.
|
|
3037
|
+
* @param count - Number of nodes that failed (default 1)
|
|
3038
|
+
*/
|
|
3039
|
+
incDistributedSearchFailedNodes(count = 1) {
|
|
3040
|
+
this.distributedSearchFailedNodes.inc(count);
|
|
3041
|
+
}
|
|
3042
|
+
/**
|
|
3043
|
+
* Increment counter for searches returning partial results.
|
|
3044
|
+
*/
|
|
3045
|
+
incDistributedSearchPartialResults() {
|
|
3046
|
+
this.distributedSearchPartialResults.inc();
|
|
3047
|
+
}
|
|
3048
|
+
// === Distributed subscription metric methods (Phase 14.2) ===
|
|
3049
|
+
/**
|
|
3050
|
+
* Record a distributed subscription creation.
|
|
3051
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
3052
|
+
* @param status - Result status (success, failed, timeout)
|
|
3053
|
+
*/
|
|
3054
|
+
incDistributedSub(type, status) {
|
|
3055
|
+
this.distributedSubTotal.inc({ type, status });
|
|
3056
|
+
if (status === "success") {
|
|
3057
|
+
this.distributedSubActive.inc({ type });
|
|
3058
|
+
}
|
|
3059
|
+
}
|
|
3060
|
+
/**
|
|
3061
|
+
* Record a distributed subscription unsubscription.
|
|
3062
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
3063
|
+
*/
|
|
3064
|
+
incDistributedSubUnsubscribe(type) {
|
|
3065
|
+
this.distributedSubUnsubscribeTotal.inc({ type });
|
|
3066
|
+
}
|
|
3067
|
+
/**
|
|
3068
|
+
* Decrement the active distributed subscriptions gauge.
|
|
3069
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
3070
|
+
*/
|
|
3071
|
+
decDistributedSubActive(type) {
|
|
3072
|
+
this.distributedSubActive.dec({ type });
|
|
3073
|
+
}
|
|
3074
|
+
/**
|
|
3075
|
+
* Set the number of subscriptions waiting for ACKs.
|
|
3076
|
+
* @param count - Number of pending ACKs
|
|
3077
|
+
*/
|
|
3078
|
+
setDistributedSubPendingAcks(count) {
|
|
3079
|
+
this.distributedSubPendingAcks.set(count);
|
|
3080
|
+
}
|
|
3081
|
+
/**
|
|
3082
|
+
* Record a delta update for distributed subscriptions.
|
|
3083
|
+
* @param direction - Direction of update (sent or received)
|
|
3084
|
+
* @param changeType - Type of change (ENTER, UPDATE, LEAVE)
|
|
3085
|
+
*/
|
|
3086
|
+
incDistributedSubUpdates(direction, changeType) {
|
|
3087
|
+
this.distributedSubUpdates.inc({ direction, change_type: changeType });
|
|
3088
|
+
}
|
|
3089
|
+
/**
|
|
3090
|
+
* Record node ACK responses.
|
|
3091
|
+
* @param status - ACK status (success, failed, timeout)
|
|
3092
|
+
* @param count - Number of ACKs to record (default 1)
|
|
3093
|
+
*/
|
|
3094
|
+
incDistributedSubAck(status, count = 1) {
|
|
3095
|
+
this.distributedSubAckTotal.inc({ status }, count);
|
|
3096
|
+
}
|
|
3097
|
+
/**
|
|
3098
|
+
* Record a node disconnect affecting distributed subscriptions.
|
|
3099
|
+
*/
|
|
3100
|
+
incDistributedSubNodeDisconnect() {
|
|
3101
|
+
this.distributedSubNodeDisconnect.inc();
|
|
3102
|
+
}
|
|
3103
|
+
/**
|
|
3104
|
+
* Record the time to register a subscription on all nodes.
|
|
3105
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
3106
|
+
* @param durationMs - Duration in milliseconds
|
|
3107
|
+
*/
|
|
3108
|
+
recordDistributedSubRegistration(type, durationMs) {
|
|
3109
|
+
this.distributedSubRegistrationDuration.observe({ type }, durationMs);
|
|
3110
|
+
}
|
|
3111
|
+
/**
|
|
3112
|
+
* Record the latency from data change to client notification.
|
|
3113
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
3114
|
+
* @param latencyMs - Latency in milliseconds
|
|
3115
|
+
*/
|
|
3116
|
+
recordDistributedSubUpdateLatency(type, latencyMs) {
|
|
3117
|
+
this.distributedSubUpdateLatency.observe({ type }, latencyMs);
|
|
3118
|
+
}
|
|
3119
|
+
/**
|
|
3120
|
+
* Record the initial result set size for a subscription.
|
|
3121
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
3122
|
+
* @param count - Number of initial results
|
|
3123
|
+
*/
|
|
3124
|
+
recordDistributedSubInitialResultsCount(type, count) {
|
|
3125
|
+
this.distributedSubInitialResultsCount.observe({ type }, count);
|
|
3126
|
+
}
|
|
2765
3127
|
async getMetrics() {
|
|
2766
3128
|
return this.registry.metrics();
|
|
2767
3129
|
}
|
|
@@ -3292,7 +3654,7 @@ var BackpressureRegulator = class {
|
|
|
3292
3654
|
};
|
|
3293
3655
|
|
|
3294
3656
|
// src/utils/CoalescingWriter.ts
|
|
3295
|
-
var
|
|
3657
|
+
var import_ws3 = require("ws");
|
|
3296
3658
|
var import_core5 = require("@topgunbuild/core");
|
|
3297
3659
|
|
|
3298
3660
|
// src/memory/BufferPool.ts
|
|
@@ -3862,7 +4224,7 @@ var CoalescingWriter = class {
|
|
|
3862
4224
|
}
|
|
3863
4225
|
this.state = 2 /* FLUSHING */;
|
|
3864
4226
|
try {
|
|
3865
|
-
if (this.socket.readyState !==
|
|
4227
|
+
if (this.socket.readyState !== import_ws3.WebSocket.OPEN) {
|
|
3866
4228
|
this.queue = [];
|
|
3867
4229
|
this.pendingBytes = 0;
|
|
3868
4230
|
this.state = 0 /* IDLE */;
|
|
@@ -3935,7 +4297,7 @@ var CoalescingWriter = class {
|
|
|
3935
4297
|
* Send a message immediately without batching.
|
|
3936
4298
|
*/
|
|
3937
4299
|
sendImmediate(data) {
|
|
3938
|
-
if (this.socket.readyState !==
|
|
4300
|
+
if (this.socket.readyState !== import_ws3.WebSocket.OPEN) {
|
|
3939
4301
|
return;
|
|
3940
4302
|
}
|
|
3941
4303
|
try {
|
|
@@ -7517,7 +7879,7 @@ var RepairScheduler = class extends import_events12.EventEmitter {
|
|
|
7517
7879
|
this.processTimer = setInterval(() => {
|
|
7518
7880
|
this.processRepairQueue();
|
|
7519
7881
|
}, 1e3);
|
|
7520
|
-
setTimeout(() => {
|
|
7882
|
+
this.initialScanTimer = setTimeout(() => {
|
|
7521
7883
|
this.scheduleFullScan();
|
|
7522
7884
|
}, 6e4);
|
|
7523
7885
|
}
|
|
@@ -7535,9 +7897,13 @@ var RepairScheduler = class extends import_events12.EventEmitter {
|
|
|
7535
7897
|
clearInterval(this.processTimer);
|
|
7536
7898
|
this.processTimer = void 0;
|
|
7537
7899
|
}
|
|
7900
|
+
if (this.initialScanTimer) {
|
|
7901
|
+
clearTimeout(this.initialScanTimer);
|
|
7902
|
+
this.initialScanTimer = void 0;
|
|
7903
|
+
}
|
|
7538
7904
|
this.repairQueue = [];
|
|
7539
7905
|
this.activeRepairs.clear();
|
|
7540
|
-
for (const [
|
|
7906
|
+
for (const [, req] of this.pendingRequests) {
|
|
7541
7907
|
clearTimeout(req.timer);
|
|
7542
7908
|
req.reject(new Error("Scheduler stopped"));
|
|
7543
7909
|
}
|
|
@@ -9257,9 +9623,11 @@ var EventJournalService = class extends import_core18.EventJournalImpl {
|
|
|
9257
9623
|
};
|
|
9258
9624
|
|
|
9259
9625
|
// src/search/SearchCoordinator.ts
|
|
9626
|
+
var import_events13 = require("events");
|
|
9260
9627
|
var import_core19 = require("@topgunbuild/core");
|
|
9261
|
-
var SearchCoordinator = class {
|
|
9628
|
+
var SearchCoordinator = class extends import_events13.EventEmitter {
|
|
9262
9629
|
constructor() {
|
|
9630
|
+
super();
|
|
9263
9631
|
/** Map name → FullTextIndex */
|
|
9264
9632
|
this.indexes = /* @__PURE__ */ new Map();
|
|
9265
9633
|
/** Map name → FullTextIndexConfig (for reference) */
|
|
@@ -9284,6 +9652,13 @@ var SearchCoordinator = class {
|
|
|
9284
9652
|
this.BATCH_INTERVAL = 16;
|
|
9285
9653
|
logger.debug("SearchCoordinator initialized");
|
|
9286
9654
|
}
|
|
9655
|
+
/**
|
|
9656
|
+
* Set the node ID for this server.
|
|
9657
|
+
* Required for distributed subscriptions.
|
|
9658
|
+
*/
|
|
9659
|
+
setNodeId(nodeId) {
|
|
9660
|
+
this.nodeId = nodeId;
|
|
9661
|
+
}
|
|
9287
9662
|
/**
|
|
9288
9663
|
* Set the callback for sending updates to clients.
|
|
9289
9664
|
* Called by ServerCoordinator during initialization.
|
|
@@ -9585,9 +9960,112 @@ var SearchCoordinator = class {
|
|
|
9585
9960
|
getSubscriptionCount() {
|
|
9586
9961
|
return this.subscriptions.size;
|
|
9587
9962
|
}
|
|
9963
|
+
// ============================================
|
|
9964
|
+
// Phase 14.2: Distributed Subscription Methods
|
|
9965
|
+
// ============================================
|
|
9966
|
+
/**
|
|
9967
|
+
* Register a distributed subscription from a remote coordinator.
|
|
9968
|
+
* Called when receiving CLUSTER_SUB_REGISTER message.
|
|
9969
|
+
*
|
|
9970
|
+
* @param subscriptionId - Unique subscription ID
|
|
9971
|
+
* @param mapName - Map name to search
|
|
9972
|
+
* @param query - Search query string
|
|
9973
|
+
* @param options - Search options
|
|
9974
|
+
* @param coordinatorNodeId - Node ID of the coordinator (receives updates)
|
|
9975
|
+
* @returns Initial search results from this node
|
|
9976
|
+
*/
|
|
9977
|
+
registerDistributedSubscription(subscriptionId, mapName, query, options, coordinatorNodeId) {
|
|
9978
|
+
const index = this.indexes.get(mapName);
|
|
9979
|
+
if (!index) {
|
|
9980
|
+
logger.warn({ mapName }, "Distributed subscription for map without FTS enabled");
|
|
9981
|
+
return { results: [], totalHits: 0 };
|
|
9982
|
+
}
|
|
9983
|
+
const queryTerms = index.tokenizeQuery(query);
|
|
9984
|
+
const searchResults = index.search(query, options);
|
|
9985
|
+
const currentResults = /* @__PURE__ */ new Map();
|
|
9986
|
+
const results = [];
|
|
9987
|
+
for (const result of searchResults) {
|
|
9988
|
+
const value = this.getDocumentValue ? this.getDocumentValue(mapName, result.docId) : void 0;
|
|
9989
|
+
currentResults.set(result.docId, {
|
|
9990
|
+
score: result.score,
|
|
9991
|
+
matchedTerms: result.matchedTerms || []
|
|
9992
|
+
});
|
|
9993
|
+
results.push({
|
|
9994
|
+
key: result.docId,
|
|
9995
|
+
value,
|
|
9996
|
+
score: result.score,
|
|
9997
|
+
matchedTerms: result.matchedTerms || []
|
|
9998
|
+
});
|
|
9999
|
+
}
|
|
10000
|
+
const subscription = {
|
|
10001
|
+
id: subscriptionId,
|
|
10002
|
+
clientId: `cluster:${coordinatorNodeId}`,
|
|
10003
|
+
// Virtual client ID
|
|
10004
|
+
mapName,
|
|
10005
|
+
query,
|
|
10006
|
+
queryTerms,
|
|
10007
|
+
options: options || {},
|
|
10008
|
+
currentResults,
|
|
10009
|
+
// Distributed fields
|
|
10010
|
+
coordinatorNodeId,
|
|
10011
|
+
isDistributed: true
|
|
10012
|
+
};
|
|
10013
|
+
this.subscriptions.set(subscriptionId, subscription);
|
|
10014
|
+
if (!this.subscriptionsByMap.has(mapName)) {
|
|
10015
|
+
this.subscriptionsByMap.set(mapName, /* @__PURE__ */ new Set());
|
|
10016
|
+
}
|
|
10017
|
+
this.subscriptionsByMap.get(mapName).add(subscriptionId);
|
|
10018
|
+
if (!this.subscriptionsByClient.has(subscription.clientId)) {
|
|
10019
|
+
this.subscriptionsByClient.set(subscription.clientId, /* @__PURE__ */ new Set());
|
|
10020
|
+
}
|
|
10021
|
+
this.subscriptionsByClient.get(subscription.clientId).add(subscriptionId);
|
|
10022
|
+
logger.debug(
|
|
10023
|
+
{ subscriptionId, mapName, query, coordinatorNodeId, resultCount: results.length },
|
|
10024
|
+
"Distributed search subscription registered"
|
|
10025
|
+
);
|
|
10026
|
+
return {
|
|
10027
|
+
results,
|
|
10028
|
+
totalHits: results.length
|
|
10029
|
+
};
|
|
10030
|
+
}
|
|
10031
|
+
/**
|
|
10032
|
+
* Get a distributed subscription by ID.
|
|
10033
|
+
* Returns undefined if not found or not distributed.
|
|
10034
|
+
*/
|
|
10035
|
+
getDistributedSubscription(subscriptionId) {
|
|
10036
|
+
const sub = this.subscriptions.get(subscriptionId);
|
|
10037
|
+
if (sub?.isDistributed) {
|
|
10038
|
+
return sub;
|
|
10039
|
+
}
|
|
10040
|
+
return void 0;
|
|
10041
|
+
}
|
|
10042
|
+
/**
|
|
10043
|
+
* Unsubscribe all distributed subscriptions where the given node was the coordinator.
|
|
10044
|
+
* Called when a cluster node disconnects.
|
|
10045
|
+
*
|
|
10046
|
+
* @param coordinatorNodeId - Node ID of the disconnected coordinator
|
|
10047
|
+
*/
|
|
10048
|
+
unsubscribeByCoordinator(coordinatorNodeId) {
|
|
10049
|
+
const subscriptionsToRemove = [];
|
|
10050
|
+
for (const [subId, sub] of this.subscriptions) {
|
|
10051
|
+
if (sub.isDistributed && sub.coordinatorNodeId === coordinatorNodeId) {
|
|
10052
|
+
subscriptionsToRemove.push(subId);
|
|
10053
|
+
}
|
|
10054
|
+
}
|
|
10055
|
+
for (const subId of subscriptionsToRemove) {
|
|
10056
|
+
this.unsubscribe(subId);
|
|
10057
|
+
}
|
|
10058
|
+
if (subscriptionsToRemove.length > 0) {
|
|
10059
|
+
logger.debug(
|
|
10060
|
+
{ coordinatorNodeId, count: subscriptionsToRemove.length },
|
|
10061
|
+
"Cleaned up distributed subscriptions for disconnected coordinator"
|
|
10062
|
+
);
|
|
10063
|
+
}
|
|
10064
|
+
}
|
|
9588
10065
|
/**
|
|
9589
10066
|
* Notify subscribers about a document change.
|
|
9590
10067
|
* Computes delta (ENTER/UPDATE/LEAVE) for each affected subscription.
|
|
10068
|
+
* For distributed subscriptions, emits 'distributedUpdate' event instead of sending to client.
|
|
9591
10069
|
*
|
|
9592
10070
|
* @param mapName - Name of the map that changed
|
|
9593
10071
|
* @param key - Document key that changed
|
|
@@ -9595,9 +10073,6 @@ var SearchCoordinator = class {
|
|
|
9595
10073
|
* @param changeType - Type of change
|
|
9596
10074
|
*/
|
|
9597
10075
|
notifySubscribers(mapName, key, value, changeType) {
|
|
9598
|
-
if (!this.sendUpdate) {
|
|
9599
|
-
return;
|
|
9600
|
-
}
|
|
9601
10076
|
const subscriptionIds = this.subscriptionsByMap.get(mapName);
|
|
9602
10077
|
if (!subscriptionIds || subscriptionIds.size === 0) {
|
|
9603
10078
|
return;
|
|
@@ -9638,18 +10113,50 @@ var SearchCoordinator = class {
|
|
|
9638
10113
|
}
|
|
9639
10114
|
logger.debug({ subId, key, wasInResults, isInResults, updateType, newScore }, "Update decision");
|
|
9640
10115
|
if (updateType) {
|
|
9641
|
-
|
|
9642
|
-
sub
|
|
9643
|
-
|
|
9644
|
-
|
|
9645
|
-
|
|
9646
|
-
|
|
9647
|
-
|
|
9648
|
-
|
|
9649
|
-
|
|
10116
|
+
if (sub.isDistributed && sub.coordinatorNodeId) {
|
|
10117
|
+
this.sendDistributedUpdate(sub, key, value, newScore, matchedTerms, updateType);
|
|
10118
|
+
} else if (this.sendUpdate) {
|
|
10119
|
+
this.sendUpdate(
|
|
10120
|
+
sub.clientId,
|
|
10121
|
+
subId,
|
|
10122
|
+
key,
|
|
10123
|
+
value,
|
|
10124
|
+
newScore,
|
|
10125
|
+
matchedTerms,
|
|
10126
|
+
updateType
|
|
10127
|
+
);
|
|
10128
|
+
}
|
|
9650
10129
|
}
|
|
9651
10130
|
}
|
|
9652
10131
|
}
|
|
10132
|
+
/**
|
|
10133
|
+
* Send update to remote coordinator node for a distributed subscription.
|
|
10134
|
+
* Emits 'distributedUpdate' event with ClusterSubUpdatePayload.
|
|
10135
|
+
*
|
|
10136
|
+
* @param sub - The distributed subscription
|
|
10137
|
+
* @param key - Document key
|
|
10138
|
+
* @param value - Document value
|
|
10139
|
+
* @param score - Search score
|
|
10140
|
+
* @param matchedTerms - Matched search terms
|
|
10141
|
+
* @param changeType - Change type (ENTER/UPDATE/LEAVE)
|
|
10142
|
+
*/
|
|
10143
|
+
sendDistributedUpdate(sub, key, value, score, matchedTerms, changeType) {
|
|
10144
|
+
const payload = {
|
|
10145
|
+
subscriptionId: sub.id,
|
|
10146
|
+
sourceNodeId: this.nodeId || "unknown",
|
|
10147
|
+
key,
|
|
10148
|
+
value,
|
|
10149
|
+
score,
|
|
10150
|
+
matchedTerms,
|
|
10151
|
+
changeType,
|
|
10152
|
+
timestamp: Date.now()
|
|
10153
|
+
};
|
|
10154
|
+
this.emit("distributedUpdate", payload);
|
|
10155
|
+
logger.debug(
|
|
10156
|
+
{ subscriptionId: sub.id, key, changeType, coordinator: sub.coordinatorNodeId },
|
|
10157
|
+
"Emitted distributed update"
|
|
10158
|
+
);
|
|
10159
|
+
}
|
|
9653
10160
|
/**
|
|
9654
10161
|
* Score a single document against a subscription's query.
|
|
9655
10162
|
*
|
|
@@ -9804,37 +10311,1205 @@ var SearchCoordinator = class {
|
|
|
9804
10311
|
}
|
|
9805
10312
|
};
|
|
9806
10313
|
|
|
9807
|
-
// src/
|
|
9808
|
-
var
|
|
9809
|
-
var
|
|
9810
|
-
var
|
|
9811
|
-
|
|
9812
|
-
|
|
9813
|
-
|
|
9814
|
-
|
|
9815
|
-
|
|
9816
|
-
|
|
9817
|
-
|
|
9818
|
-
|
|
9819
|
-
this.
|
|
9820
|
-
|
|
9821
|
-
this.
|
|
9822
|
-
|
|
9823
|
-
this.
|
|
9824
|
-
|
|
9825
|
-
this.
|
|
9826
|
-
this.
|
|
9827
|
-
|
|
9828
|
-
|
|
9829
|
-
|
|
9830
|
-
|
|
9831
|
-
|
|
9832
|
-
|
|
9833
|
-
|
|
9834
|
-
|
|
9835
|
-
|
|
9836
|
-
|
|
9837
|
-
|
|
10314
|
+
// src/search/ClusterSearchCoordinator.ts
|
|
10315
|
+
var import_events14 = require("events");
|
|
10316
|
+
var import_core20 = require("@topgunbuild/core");
|
|
10317
|
+
var DEFAULT_CONFIG6 = {
|
|
10318
|
+
rrfK: 60,
|
|
10319
|
+
defaultTimeoutMs: 5e3,
|
|
10320
|
+
defaultMinResponses: 0
|
|
10321
|
+
};
|
|
10322
|
+
var ClusterSearchCoordinator = class extends import_events14.EventEmitter {
|
|
10323
|
+
constructor(clusterManager, partitionService, localSearchCoordinator, config, metricsService) {
|
|
10324
|
+
super();
|
|
10325
|
+
/** Pending requests awaiting responses */
|
|
10326
|
+
this.pendingRequests = /* @__PURE__ */ new Map();
|
|
10327
|
+
this.clusterManager = clusterManager;
|
|
10328
|
+
this.partitionService = partitionService;
|
|
10329
|
+
this.localSearchCoordinator = localSearchCoordinator;
|
|
10330
|
+
this.config = { ...DEFAULT_CONFIG6, ...config };
|
|
10331
|
+
this.rrf = new import_core20.ReciprocalRankFusion({ k: this.config.rrfK });
|
|
10332
|
+
this.metricsService = metricsService;
|
|
10333
|
+
this.clusterManager.on("message", this.handleClusterMessage.bind(this));
|
|
10334
|
+
}
|
|
10335
|
+
/**
|
|
10336
|
+
* Execute a distributed search across the cluster.
|
|
10337
|
+
*
|
|
10338
|
+
* @param mapName - Name of the map to search
|
|
10339
|
+
* @param query - Search query text
|
|
10340
|
+
* @param options - Search options
|
|
10341
|
+
* @returns Promise resolving to merged search results
|
|
10342
|
+
*/
|
|
10343
|
+
async search(mapName, query, options = { limit: 10 }) {
|
|
10344
|
+
const startTime = performance.now();
|
|
10345
|
+
const requestId = this.generateRequestId();
|
|
10346
|
+
const timeoutMs = options.timeoutMs ?? this.config.defaultTimeoutMs;
|
|
10347
|
+
const allNodes = new Set(this.clusterManager.getMembers());
|
|
10348
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10349
|
+
if (allNodes.size === 1 && allNodes.has(myNodeId)) {
|
|
10350
|
+
return this.executeLocalSearch(mapName, query, options, startTime);
|
|
10351
|
+
}
|
|
10352
|
+
const perNodeLimit = this.calculatePerNodeLimit(options.limit, options.cursor);
|
|
10353
|
+
let cursorData = null;
|
|
10354
|
+
if (options.cursor) {
|
|
10355
|
+
cursorData = import_core20.SearchCursor.decode(options.cursor);
|
|
10356
|
+
if (cursorData && !import_core20.SearchCursor.isValid(cursorData, query)) {
|
|
10357
|
+
cursorData = null;
|
|
10358
|
+
logger.warn({ requestId }, "Invalid or expired cursor, ignoring");
|
|
10359
|
+
}
|
|
10360
|
+
}
|
|
10361
|
+
const promise = new Promise((resolve, reject) => {
|
|
10362
|
+
const timeoutHandle = setTimeout(() => {
|
|
10363
|
+
this.resolvePartialResults(requestId);
|
|
10364
|
+
}, timeoutMs);
|
|
10365
|
+
this.pendingRequests.set(requestId, {
|
|
10366
|
+
resolve,
|
|
10367
|
+
reject,
|
|
10368
|
+
responses: /* @__PURE__ */ new Map(),
|
|
10369
|
+
expectedNodes: allNodes,
|
|
10370
|
+
startTime,
|
|
10371
|
+
timeoutHandle,
|
|
10372
|
+
options,
|
|
10373
|
+
mapName,
|
|
10374
|
+
query
|
|
10375
|
+
});
|
|
10376
|
+
});
|
|
10377
|
+
const payload = {
|
|
10378
|
+
requestId,
|
|
10379
|
+
mapName,
|
|
10380
|
+
query,
|
|
10381
|
+
options: {
|
|
10382
|
+
limit: perNodeLimit,
|
|
10383
|
+
minScore: options.minScore,
|
|
10384
|
+
boost: options.boost,
|
|
10385
|
+
includeMatchedTerms: true,
|
|
10386
|
+
// Add cursor position if available
|
|
10387
|
+
...cursorData ? {
|
|
10388
|
+
afterScore: cursorData.nodeScores[myNodeId],
|
|
10389
|
+
afterKey: cursorData.nodeKeys[myNodeId]
|
|
10390
|
+
} : {}
|
|
10391
|
+
},
|
|
10392
|
+
timeoutMs
|
|
10393
|
+
};
|
|
10394
|
+
for (const nodeId of allNodes) {
|
|
10395
|
+
if (nodeId === myNodeId) {
|
|
10396
|
+
this.executeLocalAndRespond(requestId, mapName, query, perNodeLimit, cursorData);
|
|
10397
|
+
} else {
|
|
10398
|
+
this.clusterManager.send(nodeId, "CLUSTER_SEARCH_REQ", payload);
|
|
10399
|
+
}
|
|
10400
|
+
}
|
|
10401
|
+
return promise;
|
|
10402
|
+
}
|
|
10403
|
+
/**
|
|
10404
|
+
* Handle incoming cluster messages.
|
|
10405
|
+
*/
|
|
10406
|
+
handleClusterMessage(msg) {
|
|
10407
|
+
switch (msg.type) {
|
|
10408
|
+
case "CLUSTER_SEARCH_REQ":
|
|
10409
|
+
this.handleSearchRequest(msg.senderId, msg.payload);
|
|
10410
|
+
break;
|
|
10411
|
+
case "CLUSTER_SEARCH_RESP":
|
|
10412
|
+
this.handleSearchResponse(msg.senderId, msg.payload);
|
|
10413
|
+
break;
|
|
10414
|
+
}
|
|
10415
|
+
}
|
|
10416
|
+
/**
|
|
10417
|
+
* Handle incoming search request from another node.
|
|
10418
|
+
*/
|
|
10419
|
+
async handleSearchRequest(senderId, rawPayload) {
|
|
10420
|
+
const startTime = performance.now();
|
|
10421
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10422
|
+
const parsed = import_core20.ClusterSearchReqPayloadSchema.safeParse(rawPayload);
|
|
10423
|
+
if (!parsed.success) {
|
|
10424
|
+
logger.warn(
|
|
10425
|
+
{ senderId, error: parsed.error.message },
|
|
10426
|
+
"Invalid cluster search request payload"
|
|
10427
|
+
);
|
|
10428
|
+
this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", {
|
|
10429
|
+
requestId: rawPayload?.requestId ?? "unknown",
|
|
10430
|
+
nodeId: myNodeId,
|
|
10431
|
+
results: [],
|
|
10432
|
+
totalHits: 0,
|
|
10433
|
+
executionTimeMs: performance.now() - startTime,
|
|
10434
|
+
error: "Invalid request payload"
|
|
10435
|
+
});
|
|
10436
|
+
return;
|
|
10437
|
+
}
|
|
10438
|
+
const payload = parsed.data;
|
|
10439
|
+
try {
|
|
10440
|
+
const localResult = this.localSearchCoordinator.search(
|
|
10441
|
+
payload.mapName,
|
|
10442
|
+
payload.query,
|
|
10443
|
+
{
|
|
10444
|
+
limit: payload.options.limit,
|
|
10445
|
+
minScore: payload.options.minScore,
|
|
10446
|
+
boost: payload.options.boost
|
|
10447
|
+
}
|
|
10448
|
+
);
|
|
10449
|
+
let results = localResult.results;
|
|
10450
|
+
if (payload.options.afterScore !== void 0) {
|
|
10451
|
+
results = results.filter((r) => {
|
|
10452
|
+
if (r.score < payload.options.afterScore) {
|
|
10453
|
+
return true;
|
|
10454
|
+
}
|
|
10455
|
+
if (r.score === payload.options.afterScore && payload.options.afterKey) {
|
|
10456
|
+
return r.key > payload.options.afterKey;
|
|
10457
|
+
}
|
|
10458
|
+
return false;
|
|
10459
|
+
});
|
|
10460
|
+
}
|
|
10461
|
+
const response = {
|
|
10462
|
+
requestId: payload.requestId,
|
|
10463
|
+
nodeId: myNodeId,
|
|
10464
|
+
results: results.map((r) => ({
|
|
10465
|
+
key: r.key,
|
|
10466
|
+
value: r.value,
|
|
10467
|
+
score: r.score,
|
|
10468
|
+
matchedTerms: r.matchedTerms
|
|
10469
|
+
})),
|
|
10470
|
+
totalHits: localResult.totalCount ?? results.length,
|
|
10471
|
+
executionTimeMs: performance.now() - startTime
|
|
10472
|
+
};
|
|
10473
|
+
this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", response);
|
|
10474
|
+
} catch (error) {
|
|
10475
|
+
this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", {
|
|
10476
|
+
requestId: payload.requestId,
|
|
10477
|
+
nodeId: myNodeId,
|
|
10478
|
+
results: [],
|
|
10479
|
+
totalHits: 0,
|
|
10480
|
+
executionTimeMs: performance.now() - startTime,
|
|
10481
|
+
error: error instanceof Error ? error.message : "Unknown error"
|
|
10482
|
+
});
|
|
10483
|
+
}
|
|
10484
|
+
}
|
|
10485
|
+
/**
|
|
10486
|
+
* Handle search response from a node.
|
|
10487
|
+
*/
|
|
10488
|
+
handleSearchResponse(_senderId, rawPayload) {
|
|
10489
|
+
const parsed = import_core20.ClusterSearchRespPayloadSchema.safeParse(rawPayload);
|
|
10490
|
+
if (!parsed.success) {
|
|
10491
|
+
logger.warn(
|
|
10492
|
+
{ error: parsed.error.message },
|
|
10493
|
+
"Invalid cluster search response payload"
|
|
10494
|
+
);
|
|
10495
|
+
return;
|
|
10496
|
+
}
|
|
10497
|
+
const payload = parsed.data;
|
|
10498
|
+
const pending = this.pendingRequests.get(payload.requestId);
|
|
10499
|
+
if (!pending) {
|
|
10500
|
+
logger.warn({ requestId: payload.requestId }, "Received response for unknown request");
|
|
10501
|
+
return;
|
|
10502
|
+
}
|
|
10503
|
+
pending.responses.set(payload.nodeId, payload);
|
|
10504
|
+
const minResponses = pending.options.minResponses ?? this.config.defaultMinResponses;
|
|
10505
|
+
const requiredResponses = minResponses > 0 ? minResponses : pending.expectedNodes.size;
|
|
10506
|
+
if (pending.responses.size >= requiredResponses) {
|
|
10507
|
+
clearTimeout(pending.timeoutHandle);
|
|
10508
|
+
this.mergeAndResolve(payload.requestId);
|
|
10509
|
+
}
|
|
10510
|
+
}
|
|
10511
|
+
/**
|
|
10512
|
+
* Merge results from all nodes using RRF and resolve the promise.
|
|
10513
|
+
*/
|
|
10514
|
+
mergeAndResolve(requestId) {
|
|
10515
|
+
const pending = this.pendingRequests.get(requestId);
|
|
10516
|
+
if (!pending) return;
|
|
10517
|
+
const resultSets = [];
|
|
10518
|
+
const respondedNodes = [];
|
|
10519
|
+
const failedNodes = [];
|
|
10520
|
+
let totalHits = 0;
|
|
10521
|
+
const resultsWithNodes = [];
|
|
10522
|
+
for (const [nodeId, response] of pending.responses) {
|
|
10523
|
+
if (response.error) {
|
|
10524
|
+
failedNodes.push(nodeId);
|
|
10525
|
+
logger.warn({ nodeId, error: response.error }, "Node returned error for search");
|
|
10526
|
+
} else {
|
|
10527
|
+
respondedNodes.push(nodeId);
|
|
10528
|
+
totalHits += response.totalHits;
|
|
10529
|
+
const rankedResults = response.results.map((r) => ({
|
|
10530
|
+
docId: r.key,
|
|
10531
|
+
score: r.score,
|
|
10532
|
+
source: nodeId
|
|
10533
|
+
}));
|
|
10534
|
+
resultSets.push(rankedResults);
|
|
10535
|
+
for (const r of response.results) {
|
|
10536
|
+
resultsWithNodes.push({
|
|
10537
|
+
key: r.key,
|
|
10538
|
+
score: r.score,
|
|
10539
|
+
nodeId,
|
|
10540
|
+
value: r.value,
|
|
10541
|
+
matchedTerms: r.matchedTerms || []
|
|
10542
|
+
});
|
|
10543
|
+
}
|
|
10544
|
+
}
|
|
10545
|
+
}
|
|
10546
|
+
for (const nodeId of pending.expectedNodes) {
|
|
10547
|
+
if (!pending.responses.has(nodeId)) {
|
|
10548
|
+
failedNodes.push(nodeId);
|
|
10549
|
+
}
|
|
10550
|
+
}
|
|
10551
|
+
const merged = this.rrf.merge(resultSets);
|
|
10552
|
+
const limit = pending.options.limit;
|
|
10553
|
+
const results = [];
|
|
10554
|
+
const cursorResults = [];
|
|
10555
|
+
for (const mergedResult of merged.slice(0, limit)) {
|
|
10556
|
+
const original = resultsWithNodes.find((r) => r.key === mergedResult.docId);
|
|
10557
|
+
if (original) {
|
|
10558
|
+
results.push({
|
|
10559
|
+
key: original.key,
|
|
10560
|
+
value: original.value,
|
|
10561
|
+
score: mergedResult.score,
|
|
10562
|
+
// Use RRF score
|
|
10563
|
+
matchedTerms: original.matchedTerms
|
|
10564
|
+
});
|
|
10565
|
+
cursorResults.push({
|
|
10566
|
+
key: original.key,
|
|
10567
|
+
score: original.score,
|
|
10568
|
+
// Use original score for cursor
|
|
10569
|
+
nodeId: original.nodeId
|
|
10570
|
+
});
|
|
10571
|
+
}
|
|
10572
|
+
}
|
|
10573
|
+
let nextCursor;
|
|
10574
|
+
if (merged.length > limit && cursorResults.length > 0) {
|
|
10575
|
+
nextCursor = import_core20.SearchCursor.fromResults(cursorResults, pending.query);
|
|
10576
|
+
}
|
|
10577
|
+
const executionTimeMs = performance.now() - pending.startTime;
|
|
10578
|
+
if (this.metricsService) {
|
|
10579
|
+
const status = failedNodes.length === 0 ? "success" : respondedNodes.length === 0 ? "error" : "partial";
|
|
10580
|
+
this.metricsService.incDistributedSearch(pending.mapName, status);
|
|
10581
|
+
this.metricsService.recordDistributedSearchDuration(pending.mapName, executionTimeMs);
|
|
10582
|
+
if (failedNodes.length > 0) {
|
|
10583
|
+
this.metricsService.incDistributedSearchFailedNodes(failedNodes.length);
|
|
10584
|
+
if (respondedNodes.length > 0) {
|
|
10585
|
+
this.metricsService.incDistributedSearchPartialResults();
|
|
10586
|
+
}
|
|
10587
|
+
}
|
|
10588
|
+
}
|
|
10589
|
+
pending.resolve({
|
|
10590
|
+
results,
|
|
10591
|
+
totalHits,
|
|
10592
|
+
nextCursor,
|
|
10593
|
+
respondedNodes,
|
|
10594
|
+
failedNodes,
|
|
10595
|
+
executionTimeMs
|
|
10596
|
+
});
|
|
10597
|
+
this.pendingRequests.delete(requestId);
|
|
10598
|
+
logger.debug({
|
|
10599
|
+
requestId,
|
|
10600
|
+
mapName: pending.mapName,
|
|
10601
|
+
query: pending.query,
|
|
10602
|
+
resultCount: results.length,
|
|
10603
|
+
totalHits,
|
|
10604
|
+
respondedNodes: respondedNodes.length,
|
|
10605
|
+
failedNodes: failedNodes.length,
|
|
10606
|
+
executionTimeMs
|
|
10607
|
+
}, "Distributed search completed");
|
|
10608
|
+
}
|
|
10609
|
+
/**
|
|
10610
|
+
* Resolve with partial results when timeout occurs.
|
|
10611
|
+
*/
|
|
10612
|
+
resolvePartialResults(requestId) {
|
|
10613
|
+
const pending = this.pendingRequests.get(requestId);
|
|
10614
|
+
if (!pending) return;
|
|
10615
|
+
logger.warn(
|
|
10616
|
+
{
|
|
10617
|
+
requestId,
|
|
10618
|
+
received: pending.responses.size,
|
|
10619
|
+
expected: pending.expectedNodes.size
|
|
10620
|
+
},
|
|
10621
|
+
"Search request timed out, returning partial results"
|
|
10622
|
+
);
|
|
10623
|
+
this.mergeAndResolve(requestId);
|
|
10624
|
+
}
|
|
10625
|
+
/**
|
|
10626
|
+
* Execute local search and add response to pending request.
|
|
10627
|
+
*/
|
|
10628
|
+
async executeLocalAndRespond(requestId, mapName, query, limit, cursorData) {
|
|
10629
|
+
const startTime = performance.now();
|
|
10630
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10631
|
+
const pending = this.pendingRequests.get(requestId);
|
|
10632
|
+
if (!pending) return;
|
|
10633
|
+
try {
|
|
10634
|
+
const localResult = this.localSearchCoordinator.search(mapName, query, {
|
|
10635
|
+
limit,
|
|
10636
|
+
minScore: pending.options.minScore,
|
|
10637
|
+
boost: pending.options.boost
|
|
10638
|
+
});
|
|
10639
|
+
let results = localResult.results;
|
|
10640
|
+
if (cursorData) {
|
|
10641
|
+
const position = import_core20.SearchCursor.getNodePosition(cursorData, myNodeId);
|
|
10642
|
+
if (position) {
|
|
10643
|
+
results = results.filter((r) => {
|
|
10644
|
+
if (r.score < position.afterScore) {
|
|
10645
|
+
return true;
|
|
10646
|
+
}
|
|
10647
|
+
if (r.score === position.afterScore) {
|
|
10648
|
+
return r.key > position.afterKey;
|
|
10649
|
+
}
|
|
10650
|
+
return false;
|
|
10651
|
+
});
|
|
10652
|
+
}
|
|
10653
|
+
}
|
|
10654
|
+
const response = {
|
|
10655
|
+
requestId,
|
|
10656
|
+
nodeId: myNodeId,
|
|
10657
|
+
results: results.map((r) => ({
|
|
10658
|
+
key: r.key,
|
|
10659
|
+
value: r.value,
|
|
10660
|
+
score: r.score,
|
|
10661
|
+
matchedTerms: r.matchedTerms
|
|
10662
|
+
})),
|
|
10663
|
+
totalHits: localResult.totalCount ?? results.length,
|
|
10664
|
+
executionTimeMs: performance.now() - startTime
|
|
10665
|
+
};
|
|
10666
|
+
this.handleSearchResponse(myNodeId, response);
|
|
10667
|
+
} catch (error) {
|
|
10668
|
+
this.handleSearchResponse(myNodeId, {
|
|
10669
|
+
requestId,
|
|
10670
|
+
nodeId: myNodeId,
|
|
10671
|
+
results: [],
|
|
10672
|
+
totalHits: 0,
|
|
10673
|
+
executionTimeMs: performance.now() - startTime,
|
|
10674
|
+
error: error instanceof Error ? error.message : "Unknown error"
|
|
10675
|
+
});
|
|
10676
|
+
}
|
|
10677
|
+
}
|
|
10678
|
+
/**
|
|
10679
|
+
* Execute search locally only (single-node optimization).
|
|
10680
|
+
*/
|
|
10681
|
+
async executeLocalSearch(mapName, query, options, startTime) {
|
|
10682
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10683
|
+
const localResult = this.localSearchCoordinator.search(mapName, query, {
|
|
10684
|
+
limit: options.limit,
|
|
10685
|
+
minScore: options.minScore,
|
|
10686
|
+
boost: options.boost
|
|
10687
|
+
});
|
|
10688
|
+
let results = localResult.results;
|
|
10689
|
+
if (options.cursor) {
|
|
10690
|
+
const cursorData = import_core20.SearchCursor.decode(options.cursor);
|
|
10691
|
+
if (cursorData && import_core20.SearchCursor.isValid(cursorData, query)) {
|
|
10692
|
+
const position = import_core20.SearchCursor.getNodePosition(cursorData, myNodeId);
|
|
10693
|
+
if (position) {
|
|
10694
|
+
results = results.filter((r) => {
|
|
10695
|
+
if (r.score < position.afterScore) {
|
|
10696
|
+
return true;
|
|
10697
|
+
}
|
|
10698
|
+
if (r.score === position.afterScore) {
|
|
10699
|
+
return r.key > position.afterKey;
|
|
10700
|
+
}
|
|
10701
|
+
return false;
|
|
10702
|
+
});
|
|
10703
|
+
}
|
|
10704
|
+
}
|
|
10705
|
+
}
|
|
10706
|
+
results = results.slice(0, options.limit);
|
|
10707
|
+
const totalCount = localResult.totalCount ?? localResult.results.length;
|
|
10708
|
+
let nextCursor;
|
|
10709
|
+
if (totalCount > options.limit && results.length > 0) {
|
|
10710
|
+
const lastResult = results[results.length - 1];
|
|
10711
|
+
nextCursor = import_core20.SearchCursor.fromResults(
|
|
10712
|
+
[{ key: lastResult.key, score: lastResult.score, nodeId: myNodeId }],
|
|
10713
|
+
query
|
|
10714
|
+
);
|
|
10715
|
+
}
|
|
10716
|
+
const executionTimeMs = performance.now() - startTime;
|
|
10717
|
+
if (this.metricsService) {
|
|
10718
|
+
this.metricsService.incDistributedSearch(mapName, "success");
|
|
10719
|
+
this.metricsService.recordDistributedSearchDuration(mapName, executionTimeMs);
|
|
10720
|
+
}
|
|
10721
|
+
return {
|
|
10722
|
+
results,
|
|
10723
|
+
totalHits: localResult.totalCount ?? results.length,
|
|
10724
|
+
nextCursor,
|
|
10725
|
+
respondedNodes: [myNodeId],
|
|
10726
|
+
failedNodes: [],
|
|
10727
|
+
executionTimeMs
|
|
10728
|
+
};
|
|
10729
|
+
}
|
|
10730
|
+
/**
|
|
10731
|
+
* Calculate per-node limit for distributed query.
|
|
10732
|
+
*
|
|
10733
|
+
* For quality RRF merge, each node should return more than the final limit.
|
|
10734
|
+
* Rule of thumb: 2x limit for good merge quality.
|
|
10735
|
+
*/
|
|
10736
|
+
calculatePerNodeLimit(limit, cursor) {
|
|
10737
|
+
if (cursor) {
|
|
10738
|
+
return limit;
|
|
10739
|
+
}
|
|
10740
|
+
return Math.min(limit * 2, 1e3);
|
|
10741
|
+
}
|
|
10742
|
+
/**
|
|
10743
|
+
* Generate unique request ID.
|
|
10744
|
+
*/
|
|
10745
|
+
generateRequestId() {
|
|
10746
|
+
return `search-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;
|
|
10747
|
+
}
|
|
10748
|
+
/**
|
|
10749
|
+
* Get RRF constant k.
|
|
10750
|
+
*/
|
|
10751
|
+
getRrfK() {
|
|
10752
|
+
return this.config.rrfK;
|
|
10753
|
+
}
|
|
10754
|
+
/**
|
|
10755
|
+
* Clean up resources.
|
|
10756
|
+
*/
|
|
10757
|
+
destroy() {
|
|
10758
|
+
for (const [requestId, pending] of this.pendingRequests) {
|
|
10759
|
+
clearTimeout(pending.timeoutHandle);
|
|
10760
|
+
pending.reject(new Error("ClusterSearchCoordinator destroyed"));
|
|
10761
|
+
}
|
|
10762
|
+
this.pendingRequests.clear();
|
|
10763
|
+
this.removeAllListeners();
|
|
10764
|
+
}
|
|
10765
|
+
};
|
|
10766
|
+
|
|
10767
|
+
// src/subscriptions/DistributedSubscriptionCoordinator.ts
|
|
10768
|
+
var import_events15 = require("events");
|
|
10769
|
+
var import_core21 = require("@topgunbuild/core");
|
|
10770
|
+
var import_ws4 = require("ws");
|
|
10771
|
+
var DEFAULT_CONFIG7 = {
|
|
10772
|
+
ackTimeoutMs: 5e3,
|
|
10773
|
+
rrfK: 60
|
|
10774
|
+
};
|
|
10775
|
+
var DistributedSubscriptionCoordinator = class extends import_events15.EventEmitter {
|
|
10776
|
+
constructor(clusterManager, queryRegistry, searchCoordinator, config, metricsService) {
|
|
10777
|
+
super();
|
|
10778
|
+
/**
|
|
10779
|
+
* Active subscriptions where this node is coordinator.
|
|
10780
|
+
* subscriptionId → SubscriptionState
|
|
10781
|
+
*/
|
|
10782
|
+
this.subscriptions = /* @__PURE__ */ new Map();
|
|
10783
|
+
/**
|
|
10784
|
+
* Track which nodes have acknowledged subscription registration.
|
|
10785
|
+
* subscriptionId → Set<nodeId>
|
|
10786
|
+
*/
|
|
10787
|
+
this.nodeAcks = /* @__PURE__ */ new Map();
|
|
10788
|
+
/**
|
|
10789
|
+
* Pending ACK promises for subscription registration.
|
|
10790
|
+
* subscriptionId → { resolve, reject, timeout, startTime }
|
|
10791
|
+
*/
|
|
10792
|
+
this.pendingAcks = /* @__PURE__ */ new Map();
|
|
10793
|
+
this.clusterManager = clusterManager;
|
|
10794
|
+
this.localQueryRegistry = queryRegistry;
|
|
10795
|
+
this.localSearchCoordinator = searchCoordinator;
|
|
10796
|
+
this.config = { ...DEFAULT_CONFIG7, ...config };
|
|
10797
|
+
this.rrf = new import_core21.ReciprocalRankFusion({ k: this.config.rrfK });
|
|
10798
|
+
this.metricsService = metricsService;
|
|
10799
|
+
this.clusterManager.on("message", this.handleClusterMessage.bind(this));
|
|
10800
|
+
this.localSearchCoordinator.on("distributedUpdate", this.handleLocalSearchUpdate.bind(this));
|
|
10801
|
+
this.clusterManager.on("memberLeft", this.handleMemberLeft.bind(this));
|
|
10802
|
+
logger.debug("DistributedSubscriptionCoordinator initialized");
|
|
10803
|
+
}
|
|
10804
|
+
/**
|
|
10805
|
+
* Create a new distributed search subscription.
|
|
10806
|
+
*
|
|
10807
|
+
* @param subscriptionId - Unique subscription ID
|
|
10808
|
+
* @param clientSocket - Client WebSocket for sending updates
|
|
10809
|
+
* @param mapName - Map name to search
|
|
10810
|
+
* @param query - Search query string
|
|
10811
|
+
* @param options - Search options
|
|
10812
|
+
* @returns Promise resolving to initial results
|
|
10813
|
+
*/
|
|
10814
|
+
async subscribeSearch(subscriptionId, clientSocket, mapName, query, options = {}) {
|
|
10815
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10816
|
+
const allNodes = new Set(this.clusterManager.getMembers());
|
|
10817
|
+
logger.debug(
|
|
10818
|
+
{ subscriptionId, mapName, query, nodes: Array.from(allNodes) },
|
|
10819
|
+
"Creating distributed search subscription"
|
|
10820
|
+
);
|
|
10821
|
+
const subscription = {
|
|
10822
|
+
id: subscriptionId,
|
|
10823
|
+
type: "SEARCH",
|
|
10824
|
+
coordinatorNodeId: myNodeId,
|
|
10825
|
+
clientSocket,
|
|
10826
|
+
mapName,
|
|
10827
|
+
searchQuery: query,
|
|
10828
|
+
searchOptions: options,
|
|
10829
|
+
registeredNodes: /* @__PURE__ */ new Set(),
|
|
10830
|
+
pendingResults: /* @__PURE__ */ new Map(),
|
|
10831
|
+
createdAt: Date.now(),
|
|
10832
|
+
currentResults: /* @__PURE__ */ new Map()
|
|
10833
|
+
};
|
|
10834
|
+
this.subscriptions.set(subscriptionId, subscription);
|
|
10835
|
+
this.nodeAcks.set(subscriptionId, /* @__PURE__ */ new Set());
|
|
10836
|
+
const registerPayload = {
|
|
10837
|
+
subscriptionId,
|
|
10838
|
+
coordinatorNodeId: myNodeId,
|
|
10839
|
+
mapName,
|
|
10840
|
+
type: "SEARCH",
|
|
10841
|
+
searchQuery: query,
|
|
10842
|
+
searchOptions: {
|
|
10843
|
+
limit: options.limit,
|
|
10844
|
+
minScore: options.minScore,
|
|
10845
|
+
boost: options.boost
|
|
10846
|
+
}
|
|
10847
|
+
};
|
|
10848
|
+
const localResult = this.registerLocalSearchSubscription(subscription);
|
|
10849
|
+
this.handleLocalAck(subscriptionId, myNodeId, localResult);
|
|
10850
|
+
for (const nodeId of allNodes) {
|
|
10851
|
+
if (nodeId !== myNodeId) {
|
|
10852
|
+
this.clusterManager.send(nodeId, "CLUSTER_SUB_REGISTER", registerPayload);
|
|
10853
|
+
}
|
|
10854
|
+
}
|
|
10855
|
+
return this.waitForAcks(subscriptionId, allNodes);
|
|
10856
|
+
}
|
|
10857
|
+
/**
|
|
10858
|
+
* Create a new distributed query subscription.
|
|
10859
|
+
*
|
|
10860
|
+
* @param subscriptionId - Unique subscription ID
|
|
10861
|
+
* @param clientSocket - Client WebSocket for sending updates
|
|
10862
|
+
* @param mapName - Map name to query
|
|
10863
|
+
* @param query - Query predicate
|
|
10864
|
+
* @returns Promise resolving to initial results
|
|
10865
|
+
*/
|
|
10866
|
+
async subscribeQuery(subscriptionId, clientSocket, mapName, query) {
|
|
10867
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10868
|
+
const allNodes = new Set(this.clusterManager.getMembers());
|
|
10869
|
+
logger.debug(
|
|
10870
|
+
{ subscriptionId, mapName, nodes: Array.from(allNodes) },
|
|
10871
|
+
"Creating distributed query subscription"
|
|
10872
|
+
);
|
|
10873
|
+
const subscription = {
|
|
10874
|
+
id: subscriptionId,
|
|
10875
|
+
type: "QUERY",
|
|
10876
|
+
coordinatorNodeId: myNodeId,
|
|
10877
|
+
clientSocket,
|
|
10878
|
+
mapName,
|
|
10879
|
+
queryPredicate: query,
|
|
10880
|
+
registeredNodes: /* @__PURE__ */ new Set(),
|
|
10881
|
+
pendingResults: /* @__PURE__ */ new Map(),
|
|
10882
|
+
createdAt: Date.now(),
|
|
10883
|
+
currentResults: /* @__PURE__ */ new Map()
|
|
10884
|
+
};
|
|
10885
|
+
this.subscriptions.set(subscriptionId, subscription);
|
|
10886
|
+
this.nodeAcks.set(subscriptionId, /* @__PURE__ */ new Set());
|
|
10887
|
+
const registerPayload = {
|
|
10888
|
+
subscriptionId,
|
|
10889
|
+
coordinatorNodeId: myNodeId,
|
|
10890
|
+
mapName,
|
|
10891
|
+
type: "QUERY",
|
|
10892
|
+
queryPredicate: query
|
|
10893
|
+
};
|
|
10894
|
+
const localResults = this.registerLocalQuerySubscription(subscription);
|
|
10895
|
+
this.handleLocalAck(subscriptionId, myNodeId, { results: localResults, totalHits: localResults.length });
|
|
10896
|
+
for (const nodeId of allNodes) {
|
|
10897
|
+
if (nodeId !== myNodeId) {
|
|
10898
|
+
this.clusterManager.send(nodeId, "CLUSTER_SUB_REGISTER", registerPayload);
|
|
10899
|
+
}
|
|
10900
|
+
}
|
|
10901
|
+
return this.waitForAcks(subscriptionId, allNodes);
|
|
10902
|
+
}
|
|
10903
|
+
/**
|
|
10904
|
+
* Unsubscribe from a distributed subscription.
|
|
10905
|
+
*
|
|
10906
|
+
* @param subscriptionId - Subscription ID to unsubscribe
|
|
10907
|
+
*/
|
|
10908
|
+
async unsubscribe(subscriptionId) {
|
|
10909
|
+
const subscription = this.subscriptions.get(subscriptionId);
|
|
10910
|
+
if (!subscription) {
|
|
10911
|
+
logger.warn({ subscriptionId }, "Attempt to unsubscribe from unknown subscription");
|
|
10912
|
+
return;
|
|
10913
|
+
}
|
|
10914
|
+
logger.debug({ subscriptionId }, "Unsubscribing from distributed subscription");
|
|
10915
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10916
|
+
const payload = { subscriptionId };
|
|
10917
|
+
for (const nodeId of subscription.registeredNodes) {
|
|
10918
|
+
if (nodeId !== myNodeId) {
|
|
10919
|
+
this.clusterManager.send(nodeId, "CLUSTER_SUB_UNREGISTER", payload);
|
|
10920
|
+
}
|
|
10921
|
+
}
|
|
10922
|
+
this.unregisterLocalSubscription(subscription);
|
|
10923
|
+
this.subscriptions.delete(subscriptionId);
|
|
10924
|
+
this.nodeAcks.delete(subscriptionId);
|
|
10925
|
+
const pendingAck = this.pendingAcks.get(subscriptionId);
|
|
10926
|
+
if (pendingAck) {
|
|
10927
|
+
clearTimeout(pendingAck.timeoutHandle);
|
|
10928
|
+
this.pendingAcks.delete(subscriptionId);
|
|
10929
|
+
}
|
|
10930
|
+
this.metricsService?.incDistributedSubUnsubscribe(subscription.type);
|
|
10931
|
+
this.metricsService?.decDistributedSubActive(subscription.type);
|
|
10932
|
+
this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
|
|
10933
|
+
}
|
|
10934
|
+
/**
|
|
10935
|
+
* Handle client disconnect - unsubscribe all their subscriptions.
|
|
10936
|
+
*/
|
|
10937
|
+
unsubscribeClient(clientSocket) {
|
|
10938
|
+
const subscriptionsToRemove = [];
|
|
10939
|
+
for (const [subId, sub] of this.subscriptions) {
|
|
10940
|
+
if (sub.clientSocket === clientSocket) {
|
|
10941
|
+
subscriptionsToRemove.push(subId);
|
|
10942
|
+
}
|
|
10943
|
+
}
|
|
10944
|
+
for (const subId of subscriptionsToRemove) {
|
|
10945
|
+
this.unsubscribe(subId);
|
|
10946
|
+
}
|
|
10947
|
+
}
|
|
10948
|
+
/**
|
|
10949
|
+
* Get active subscription count.
|
|
10950
|
+
*/
|
|
10951
|
+
getActiveSubscriptionCount() {
|
|
10952
|
+
return this.subscriptions.size;
|
|
10953
|
+
}
|
|
10954
|
+
/**
|
|
10955
|
+
* Handle incoming cluster messages with Zod validation.
|
|
10956
|
+
*/
|
|
10957
|
+
handleClusterMessage(msg) {
|
|
10958
|
+
switch (msg.type) {
|
|
10959
|
+
case "CLUSTER_SUB_REGISTER": {
|
|
10960
|
+
const parsed = import_core21.ClusterSubRegisterPayloadSchema.safeParse(msg.payload);
|
|
10961
|
+
if (!parsed.success) {
|
|
10962
|
+
logger.warn(
|
|
10963
|
+
{ senderId: msg.senderId, error: parsed.error.message },
|
|
10964
|
+
"Invalid CLUSTER_SUB_REGISTER payload"
|
|
10965
|
+
);
|
|
10966
|
+
return;
|
|
10967
|
+
}
|
|
10968
|
+
this.handleSubRegister(msg.senderId, parsed.data);
|
|
10969
|
+
break;
|
|
10970
|
+
}
|
|
10971
|
+
case "CLUSTER_SUB_ACK": {
|
|
10972
|
+
const parsed = import_core21.ClusterSubAckPayloadSchema.safeParse(msg.payload);
|
|
10973
|
+
if (!parsed.success) {
|
|
10974
|
+
logger.warn(
|
|
10975
|
+
{ senderId: msg.senderId, error: parsed.error.message },
|
|
10976
|
+
"Invalid CLUSTER_SUB_ACK payload"
|
|
10977
|
+
);
|
|
10978
|
+
return;
|
|
10979
|
+
}
|
|
10980
|
+
this.handleSubAck(msg.senderId, parsed.data);
|
|
10981
|
+
break;
|
|
10982
|
+
}
|
|
10983
|
+
case "CLUSTER_SUB_UPDATE": {
|
|
10984
|
+
const parsed = import_core21.ClusterSubUpdatePayloadSchema.safeParse(msg.payload);
|
|
10985
|
+
if (!parsed.success) {
|
|
10986
|
+
logger.warn(
|
|
10987
|
+
{ senderId: msg.senderId, error: parsed.error.message },
|
|
10988
|
+
"Invalid CLUSTER_SUB_UPDATE payload"
|
|
10989
|
+
);
|
|
10990
|
+
return;
|
|
10991
|
+
}
|
|
10992
|
+
this.handleSubUpdate(msg.senderId, parsed.data);
|
|
10993
|
+
break;
|
|
10994
|
+
}
|
|
10995
|
+
case "CLUSTER_SUB_UNREGISTER": {
|
|
10996
|
+
const parsed = import_core21.ClusterSubUnregisterPayloadSchema.safeParse(msg.payload);
|
|
10997
|
+
if (!parsed.success) {
|
|
10998
|
+
logger.warn(
|
|
10999
|
+
{ senderId: msg.senderId, error: parsed.error.message },
|
|
11000
|
+
"Invalid CLUSTER_SUB_UNREGISTER payload"
|
|
11001
|
+
);
|
|
11002
|
+
return;
|
|
11003
|
+
}
|
|
11004
|
+
this.handleSubUnregister(msg.senderId, parsed.data);
|
|
11005
|
+
break;
|
|
11006
|
+
}
|
|
11007
|
+
}
|
|
11008
|
+
}
|
|
11009
|
+
/**
|
|
11010
|
+
* Handle cluster node disconnect - cleanup subscriptions involving this node.
|
|
11011
|
+
*/
|
|
11012
|
+
handleMemberLeft(nodeId) {
|
|
11013
|
+
logger.debug({ nodeId }, "Handling member left for distributed subscriptions");
|
|
11014
|
+
const subscriptionsToRemove = [];
|
|
11015
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
11016
|
+
for (const [subId, subscription] of this.subscriptions) {
|
|
11017
|
+
if (subscription.registeredNodes.has(nodeId)) {
|
|
11018
|
+
subscription.registeredNodes.delete(nodeId);
|
|
11019
|
+
for (const [key, result] of subscription.currentResults) {
|
|
11020
|
+
if (result.sourceNode === nodeId) {
|
|
11021
|
+
subscription.currentResults.delete(key);
|
|
11022
|
+
}
|
|
11023
|
+
}
|
|
11024
|
+
logger.debug(
|
|
11025
|
+
{ subscriptionId: subId, nodeId, remainingNodes: subscription.registeredNodes.size },
|
|
11026
|
+
"Removed disconnected node from subscription"
|
|
11027
|
+
);
|
|
11028
|
+
}
|
|
11029
|
+
}
|
|
11030
|
+
for (const [subId, pending] of this.pendingAcks) {
|
|
11031
|
+
const acks = this.nodeAcks.get(subId);
|
|
11032
|
+
if (acks && !acks.has(nodeId)) {
|
|
11033
|
+
acks.add(nodeId);
|
|
11034
|
+
this.checkAcksComplete(subId);
|
|
11035
|
+
}
|
|
11036
|
+
}
|
|
11037
|
+
this.localSearchCoordinator.unsubscribeByCoordinator(nodeId);
|
|
11038
|
+
this.localQueryRegistry.unregisterByCoordinator(nodeId);
|
|
11039
|
+
this.metricsService?.incDistributedSubNodeDisconnect();
|
|
11040
|
+
}
|
|
11041
|
+
/**
|
|
11042
|
+
* Handle CLUSTER_SUB_REGISTER from coordinator (we are a data node).
|
|
11043
|
+
*/
|
|
11044
|
+
handleSubRegister(senderId, payload) {
|
|
11045
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
11046
|
+
logger.debug(
|
|
11047
|
+
{ subscriptionId: payload.subscriptionId, coordinator: payload.coordinatorNodeId },
|
|
11048
|
+
"Received distributed subscription registration"
|
|
11049
|
+
);
|
|
11050
|
+
let ackPayload;
|
|
11051
|
+
try {
|
|
11052
|
+
if (payload.type === "SEARCH") {
|
|
11053
|
+
const result = this.localSearchCoordinator.registerDistributedSubscription(
|
|
11054
|
+
payload.subscriptionId,
|
|
11055
|
+
payload.mapName,
|
|
11056
|
+
payload.searchQuery,
|
|
11057
|
+
payload.searchOptions || {},
|
|
11058
|
+
payload.coordinatorNodeId
|
|
11059
|
+
);
|
|
11060
|
+
ackPayload = {
|
|
11061
|
+
subscriptionId: payload.subscriptionId,
|
|
11062
|
+
nodeId: myNodeId,
|
|
11063
|
+
success: true,
|
|
11064
|
+
initialResults: result.results.map((r) => ({
|
|
11065
|
+
key: r.key,
|
|
11066
|
+
value: r.value,
|
|
11067
|
+
score: r.score,
|
|
11068
|
+
matchedTerms: r.matchedTerms
|
|
11069
|
+
})),
|
|
11070
|
+
totalHits: result.totalHits
|
|
11071
|
+
};
|
|
11072
|
+
} else {
|
|
11073
|
+
const results = this.localQueryRegistry.registerDistributed(
|
|
11074
|
+
payload.subscriptionId,
|
|
11075
|
+
payload.mapName,
|
|
11076
|
+
payload.queryPredicate,
|
|
11077
|
+
payload.coordinatorNodeId
|
|
11078
|
+
);
|
|
11079
|
+
ackPayload = {
|
|
11080
|
+
subscriptionId: payload.subscriptionId,
|
|
11081
|
+
nodeId: myNodeId,
|
|
11082
|
+
success: true,
|
|
11083
|
+
initialResults: results.map((r) => ({
|
|
11084
|
+
key: r.key,
|
|
11085
|
+
value: r.value
|
|
11086
|
+
})),
|
|
11087
|
+
totalHits: results.length
|
|
11088
|
+
};
|
|
11089
|
+
}
|
|
11090
|
+
} catch (error) {
|
|
11091
|
+
logger.error(
|
|
11092
|
+
{ subscriptionId: payload.subscriptionId, error },
|
|
11093
|
+
"Failed to register distributed subscription locally"
|
|
11094
|
+
);
|
|
11095
|
+
ackPayload = {
|
|
11096
|
+
subscriptionId: payload.subscriptionId,
|
|
11097
|
+
nodeId: myNodeId,
|
|
11098
|
+
success: false,
|
|
11099
|
+
error: error instanceof Error ? error.message : "Unknown error"
|
|
11100
|
+
};
|
|
11101
|
+
}
|
|
11102
|
+
this.clusterManager.send(payload.coordinatorNodeId, "CLUSTER_SUB_ACK", ackPayload);
|
|
11103
|
+
}
|
|
11104
|
+
/**
|
|
11105
|
+
* Handle CLUSTER_SUB_ACK from a data node.
|
|
11106
|
+
*/
|
|
11107
|
+
handleSubAck(senderId, payload) {
|
|
11108
|
+
const subscription = this.subscriptions.get(payload.subscriptionId);
|
|
11109
|
+
if (!subscription) {
|
|
11110
|
+
logger.warn(
|
|
11111
|
+
{ subscriptionId: payload.subscriptionId, nodeId: payload.nodeId },
|
|
11112
|
+
"Received ACK for unknown subscription"
|
|
11113
|
+
);
|
|
11114
|
+
return;
|
|
11115
|
+
}
|
|
11116
|
+
const acks = this.nodeAcks.get(payload.subscriptionId);
|
|
11117
|
+
if (!acks) return;
|
|
11118
|
+
logger.debug(
|
|
11119
|
+
{ subscriptionId: payload.subscriptionId, nodeId: payload.nodeId, success: payload.success },
|
|
11120
|
+
"Received subscription ACK"
|
|
11121
|
+
);
|
|
11122
|
+
if (payload.success) {
|
|
11123
|
+
subscription.registeredNodes.add(payload.nodeId);
|
|
11124
|
+
subscription.pendingResults.set(payload.nodeId, payload);
|
|
11125
|
+
}
|
|
11126
|
+
acks.add(payload.nodeId);
|
|
11127
|
+
this.checkAcksComplete(payload.subscriptionId);
|
|
11128
|
+
}
|
|
11129
|
+
/**
|
|
11130
|
+
* Handle CLUSTER_SUB_UPDATE from a data node.
|
|
11131
|
+
*/
|
|
11132
|
+
handleSubUpdate(senderId, payload) {
|
|
11133
|
+
const subscription = this.subscriptions.get(payload.subscriptionId);
|
|
11134
|
+
if (!subscription) {
|
|
11135
|
+
logger.warn(
|
|
11136
|
+
{ subscriptionId: payload.subscriptionId },
|
|
11137
|
+
"Update for unknown subscription"
|
|
11138
|
+
);
|
|
11139
|
+
return;
|
|
11140
|
+
}
|
|
11141
|
+
logger.debug(
|
|
11142
|
+
{ subscriptionId: payload.subscriptionId, key: payload.key, changeType: payload.changeType },
|
|
11143
|
+
"Received subscription update"
|
|
11144
|
+
);
|
|
11145
|
+
if (payload.changeType === "LEAVE") {
|
|
11146
|
+
subscription.currentResults.delete(payload.key);
|
|
11147
|
+
} else {
|
|
11148
|
+
subscription.currentResults.set(payload.key, {
|
|
11149
|
+
value: payload.value,
|
|
11150
|
+
score: payload.score,
|
|
11151
|
+
sourceNode: payload.sourceNodeId
|
|
11152
|
+
});
|
|
11153
|
+
}
|
|
11154
|
+
this.forwardUpdateToClient(subscription, payload);
|
|
11155
|
+
this.metricsService?.incDistributedSubUpdates("received", payload.changeType);
|
|
11156
|
+
if (payload.timestamp) {
|
|
11157
|
+
const latencyMs = Date.now() - payload.timestamp;
|
|
11158
|
+
this.metricsService?.recordDistributedSubUpdateLatency(subscription.type, latencyMs);
|
|
11159
|
+
}
|
|
11160
|
+
}
|
|
11161
|
+
/**
|
|
11162
|
+
* Handle CLUSTER_SUB_UNREGISTER from coordinator.
|
|
11163
|
+
*/
|
|
11164
|
+
handleSubUnregister(senderId, payload) {
|
|
11165
|
+
logger.debug(
|
|
11166
|
+
{ subscriptionId: payload.subscriptionId },
|
|
11167
|
+
"Received subscription unregister request"
|
|
11168
|
+
);
|
|
11169
|
+
this.localSearchCoordinator.unsubscribe(payload.subscriptionId);
|
|
11170
|
+
this.localQueryRegistry.unregister(payload.subscriptionId);
|
|
11171
|
+
}
|
|
11172
|
+
/**
|
|
11173
|
+
* Handle local search update (emitted by SearchCoordinator for distributed subscriptions).
|
|
11174
|
+
*/
|
|
11175
|
+
handleLocalSearchUpdate(payload) {
|
|
11176
|
+
const coordinatorNodeId = this.getCoordinatorForSubscription(payload.subscriptionId);
|
|
11177
|
+
if (!coordinatorNodeId) return;
|
|
11178
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
11179
|
+
if (coordinatorNodeId === myNodeId) {
|
|
11180
|
+
this.handleSubUpdate(myNodeId, payload);
|
|
11181
|
+
} else {
|
|
11182
|
+
this.clusterManager.send(coordinatorNodeId, "CLUSTER_SUB_UPDATE", payload);
|
|
11183
|
+
}
|
|
11184
|
+
this.metricsService?.incDistributedSubUpdates("sent", payload.changeType);
|
|
11185
|
+
}
|
|
11186
|
+
/**
|
|
11187
|
+
* Register a local search subscription for a distributed coordinator.
|
|
11188
|
+
*/
|
|
11189
|
+
registerLocalSearchSubscription(subscription) {
|
|
11190
|
+
return this.localSearchCoordinator.registerDistributedSubscription(
|
|
11191
|
+
subscription.id,
|
|
11192
|
+
subscription.mapName,
|
|
11193
|
+
subscription.searchQuery,
|
|
11194
|
+
subscription.searchOptions || {},
|
|
11195
|
+
subscription.coordinatorNodeId
|
|
11196
|
+
);
|
|
11197
|
+
}
|
|
11198
|
+
/**
|
|
11199
|
+
* Register a local query subscription for a distributed coordinator.
|
|
11200
|
+
*/
|
|
11201
|
+
registerLocalQuerySubscription(subscription) {
|
|
11202
|
+
return this.localQueryRegistry.registerDistributed(
|
|
11203
|
+
subscription.id,
|
|
11204
|
+
subscription.mapName,
|
|
11205
|
+
subscription.queryPredicate,
|
|
11206
|
+
subscription.coordinatorNodeId
|
|
11207
|
+
);
|
|
11208
|
+
}
|
|
11209
|
+
/**
|
|
11210
|
+
* Unregister local subscription.
|
|
11211
|
+
*/
|
|
11212
|
+
unregisterLocalSubscription(subscription) {
|
|
11213
|
+
if (subscription.type === "SEARCH") {
|
|
11214
|
+
this.localSearchCoordinator.unsubscribe(subscription.id);
|
|
11215
|
+
} else {
|
|
11216
|
+
this.localQueryRegistry.unregister(subscription.id);
|
|
11217
|
+
}
|
|
11218
|
+
}
|
|
11219
|
+
/**
|
|
11220
|
+
* Handle local ACK (from this node's registration).
|
|
11221
|
+
*/
|
|
11222
|
+
handleLocalAck(subscriptionId, nodeId, result) {
|
|
11223
|
+
const subscription = this.subscriptions.get(subscriptionId);
|
|
11224
|
+
if (!subscription) return;
|
|
11225
|
+
const acks = this.nodeAcks.get(subscriptionId);
|
|
11226
|
+
if (!acks) return;
|
|
11227
|
+
subscription.registeredNodes.add(nodeId);
|
|
11228
|
+
subscription.pendingResults.set(nodeId, {
|
|
11229
|
+
subscriptionId,
|
|
11230
|
+
nodeId,
|
|
11231
|
+
success: true,
|
|
11232
|
+
initialResults: (result.results || []).map((r) => ({
|
|
11233
|
+
key: r.key,
|
|
11234
|
+
value: r.value,
|
|
11235
|
+
score: r.score,
|
|
11236
|
+
matchedTerms: r.matchedTerms
|
|
11237
|
+
})),
|
|
11238
|
+
totalHits: result.totalHits || (result.results?.length ?? 0)
|
|
11239
|
+
});
|
|
11240
|
+
acks.add(nodeId);
|
|
11241
|
+
this.checkAcksComplete(subscriptionId);
|
|
11242
|
+
}
|
|
11243
|
+
/**
|
|
11244
|
+
* Wait for all node ACKs with timeout.
|
|
11245
|
+
*/
|
|
11246
|
+
waitForAcks(subscriptionId, expectedNodes) {
|
|
11247
|
+
return new Promise((resolve, reject) => {
|
|
11248
|
+
const startTime = performance.now();
|
|
11249
|
+
const timeoutHandle = setTimeout(() => {
|
|
11250
|
+
this.resolveWithPartialAcks(subscriptionId);
|
|
11251
|
+
}, this.config.ackTimeoutMs);
|
|
11252
|
+
this.pendingAcks.set(subscriptionId, { resolve, reject, timeoutHandle, startTime });
|
|
11253
|
+
this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
|
|
11254
|
+
this.checkAcksComplete(subscriptionId);
|
|
11255
|
+
});
|
|
11256
|
+
}
|
|
11257
|
+
/**
|
|
11258
|
+
* Check if all ACKs have been received.
|
|
11259
|
+
*/
|
|
11260
|
+
checkAcksComplete(subscriptionId) {
|
|
11261
|
+
const subscription = this.subscriptions.get(subscriptionId);
|
|
11262
|
+
const acks = this.nodeAcks.get(subscriptionId);
|
|
11263
|
+
const pendingAck = this.pendingAcks.get(subscriptionId);
|
|
11264
|
+
if (!subscription || !acks || !pendingAck) return;
|
|
11265
|
+
const allNodes = new Set(this.clusterManager.getMembers());
|
|
11266
|
+
if (acks.size >= allNodes.size) {
|
|
11267
|
+
clearTimeout(pendingAck.timeoutHandle);
|
|
11268
|
+
this.pendingAcks.delete(subscriptionId);
|
|
11269
|
+
const duration = performance.now() - pendingAck.startTime;
|
|
11270
|
+
const result = this.mergeInitialResults(subscription);
|
|
11271
|
+
const hasFailures = result.failedNodes.length > 0;
|
|
11272
|
+
this.recordCompletionMetrics(
|
|
11273
|
+
subscription,
|
|
11274
|
+
result,
|
|
11275
|
+
duration,
|
|
11276
|
+
hasFailures ? "timeout" : "success"
|
|
11277
|
+
);
|
|
11278
|
+
pendingAck.resolve(result);
|
|
11279
|
+
}
|
|
11280
|
+
}
|
|
11281
|
+
/**
|
|
11282
|
+
* Resolve with partial ACKs (on timeout).
|
|
11283
|
+
*/
|
|
11284
|
+
resolveWithPartialAcks(subscriptionId) {
|
|
11285
|
+
const subscription = this.subscriptions.get(subscriptionId);
|
|
11286
|
+
const pendingAck = this.pendingAcks.get(subscriptionId);
|
|
11287
|
+
if (!subscription || !pendingAck) return;
|
|
11288
|
+
this.pendingAcks.delete(subscriptionId);
|
|
11289
|
+
logger.warn(
|
|
11290
|
+
{ subscriptionId, registeredNodes: Array.from(subscription.registeredNodes) },
|
|
11291
|
+
"Subscription ACK timeout, resolving with partial results"
|
|
11292
|
+
);
|
|
11293
|
+
const duration = performance.now() - pendingAck.startTime;
|
|
11294
|
+
const result = this.mergeInitialResults(subscription);
|
|
11295
|
+
this.recordCompletionMetrics(subscription, result, duration, "timeout");
|
|
11296
|
+
pendingAck.resolve(result);
|
|
11297
|
+
}
|
|
11298
|
+
/**
|
|
11299
|
+
* Record metrics when subscription registration completes.
|
|
11300
|
+
*/
|
|
11301
|
+
recordCompletionMetrics(subscription, result, durationMs, status) {
|
|
11302
|
+
this.metricsService?.incDistributedSub(subscription.type, status);
|
|
11303
|
+
this.metricsService?.recordDistributedSubRegistration(subscription.type, durationMs);
|
|
11304
|
+
this.metricsService?.recordDistributedSubInitialResultsCount(subscription.type, result.results.length);
|
|
11305
|
+
this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
|
|
11306
|
+
this.metricsService?.incDistributedSubAck("success", subscription.registeredNodes.size);
|
|
11307
|
+
this.metricsService?.incDistributedSubAck("timeout", result.failedNodes.length);
|
|
11308
|
+
}
|
|
11309
|
+
/**
|
|
11310
|
+
* Merge initial results from all nodes.
|
|
11311
|
+
*/
|
|
11312
|
+
mergeInitialResults(subscription) {
|
|
11313
|
+
const allNodes = new Set(this.clusterManager.getMembers());
|
|
11314
|
+
const failedNodes = Array.from(allNodes).filter((n) => !subscription.registeredNodes.has(n));
|
|
11315
|
+
if (subscription.type === "SEARCH") {
|
|
11316
|
+
return this.mergeSearchResults(subscription, failedNodes);
|
|
11317
|
+
} else {
|
|
11318
|
+
return this.mergeQueryResults(subscription, failedNodes);
|
|
11319
|
+
}
|
|
11320
|
+
}
|
|
11321
|
+
/**
|
|
11322
|
+
* Merge search results using RRF.
|
|
11323
|
+
*/
|
|
11324
|
+
mergeSearchResults(subscription, failedNodes) {
|
|
11325
|
+
const resultSets = [];
|
|
11326
|
+
const resultDataMap = /* @__PURE__ */ new Map();
|
|
11327
|
+
for (const [nodeId, ack] of subscription.pendingResults) {
|
|
11328
|
+
if (ack.success && ack.initialResults) {
|
|
11329
|
+
const rankedResults = [];
|
|
11330
|
+
for (const r of ack.initialResults) {
|
|
11331
|
+
rankedResults.push({
|
|
11332
|
+
docId: r.key,
|
|
11333
|
+
score: r.score ?? 0,
|
|
11334
|
+
source: nodeId
|
|
11335
|
+
});
|
|
11336
|
+
if (!resultDataMap.has(r.key)) {
|
|
11337
|
+
resultDataMap.set(r.key, {
|
|
11338
|
+
key: r.key,
|
|
11339
|
+
value: r.value,
|
|
11340
|
+
score: r.score ?? 0,
|
|
11341
|
+
matchedTerms: r.matchedTerms,
|
|
11342
|
+
nodeId
|
|
11343
|
+
});
|
|
11344
|
+
}
|
|
11345
|
+
}
|
|
11346
|
+
if (rankedResults.length > 0) {
|
|
11347
|
+
resultSets.push(rankedResults);
|
|
11348
|
+
}
|
|
11349
|
+
}
|
|
11350
|
+
}
|
|
11351
|
+
const mergedResults = this.rrf.merge(resultSets);
|
|
11352
|
+
const limit = subscription.searchOptions?.limit ?? 10;
|
|
11353
|
+
const results = [];
|
|
11354
|
+
for (const merged of mergedResults.slice(0, limit)) {
|
|
11355
|
+
const original = resultDataMap.get(merged.docId);
|
|
11356
|
+
if (original) {
|
|
11357
|
+
results.push({
|
|
11358
|
+
key: original.key,
|
|
11359
|
+
value: original.value,
|
|
11360
|
+
score: merged.score,
|
|
11361
|
+
// Use RRF score
|
|
11362
|
+
matchedTerms: original.matchedTerms
|
|
11363
|
+
});
|
|
11364
|
+
subscription.currentResults.set(original.key, {
|
|
11365
|
+
value: original.value,
|
|
11366
|
+
score: merged.score,
|
|
11367
|
+
sourceNode: original.nodeId
|
|
11368
|
+
});
|
|
11369
|
+
}
|
|
11370
|
+
}
|
|
11371
|
+
let totalHits = 0;
|
|
11372
|
+
for (const ack of subscription.pendingResults.values()) {
|
|
11373
|
+
totalHits += ack.totalHits ?? 0;
|
|
11374
|
+
}
|
|
11375
|
+
return {
|
|
11376
|
+
subscriptionId: subscription.id,
|
|
11377
|
+
results,
|
|
11378
|
+
totalHits,
|
|
11379
|
+
registeredNodes: Array.from(subscription.registeredNodes),
|
|
11380
|
+
failedNodes
|
|
11381
|
+
};
|
|
11382
|
+
}
|
|
11383
|
+
/**
|
|
11384
|
+
* Merge query results (simple dedupe by key).
|
|
11385
|
+
*/
|
|
11386
|
+
mergeQueryResults(subscription, failedNodes) {
|
|
11387
|
+
const resultMap = /* @__PURE__ */ new Map();
|
|
11388
|
+
for (const [nodeId, ack] of subscription.pendingResults) {
|
|
11389
|
+
if (ack.success && ack.initialResults) {
|
|
11390
|
+
for (const result of ack.initialResults) {
|
|
11391
|
+
if (!resultMap.has(result.key)) {
|
|
11392
|
+
resultMap.set(result.key, { key: result.key, value: result.value });
|
|
11393
|
+
subscription.currentResults.set(result.key, {
|
|
11394
|
+
value: result.value,
|
|
11395
|
+
sourceNode: nodeId
|
|
11396
|
+
});
|
|
11397
|
+
}
|
|
11398
|
+
}
|
|
11399
|
+
}
|
|
11400
|
+
}
|
|
11401
|
+
const results = Array.from(resultMap.values());
|
|
11402
|
+
return {
|
|
11403
|
+
subscriptionId: subscription.id,
|
|
11404
|
+
results,
|
|
11405
|
+
totalHits: results.length,
|
|
11406
|
+
registeredNodes: Array.from(subscription.registeredNodes),
|
|
11407
|
+
failedNodes
|
|
11408
|
+
};
|
|
11409
|
+
}
|
|
11410
|
+
/**
|
|
11411
|
+
* Forward update to client WebSocket.
|
|
11412
|
+
*/
|
|
11413
|
+
forwardUpdateToClient(subscription, payload) {
|
|
11414
|
+
if (subscription.clientSocket.readyState !== import_ws4.WebSocket.OPEN) {
|
|
11415
|
+
logger.warn(
|
|
11416
|
+
{ subscriptionId: subscription.id },
|
|
11417
|
+
"Cannot forward update, client socket not open"
|
|
11418
|
+
);
|
|
11419
|
+
return;
|
|
11420
|
+
}
|
|
11421
|
+
const message = subscription.type === "SEARCH" ? {
|
|
11422
|
+
type: "SEARCH_UPDATE",
|
|
11423
|
+
payload: {
|
|
11424
|
+
subscriptionId: payload.subscriptionId,
|
|
11425
|
+
key: payload.key,
|
|
11426
|
+
value: payload.value,
|
|
11427
|
+
score: payload.score,
|
|
11428
|
+
matchedTerms: payload.matchedTerms,
|
|
11429
|
+
type: payload.changeType
|
|
11430
|
+
}
|
|
11431
|
+
} : {
|
|
11432
|
+
type: "QUERY_UPDATE",
|
|
11433
|
+
payload: {
|
|
11434
|
+
queryId: payload.subscriptionId,
|
|
11435
|
+
key: payload.key,
|
|
11436
|
+
value: payload.value,
|
|
11437
|
+
type: payload.changeType
|
|
11438
|
+
}
|
|
11439
|
+
};
|
|
11440
|
+
try {
|
|
11441
|
+
subscription.clientSocket.send(JSON.stringify(message));
|
|
11442
|
+
} catch (error) {
|
|
11443
|
+
logger.error(
|
|
11444
|
+
{ subscriptionId: subscription.id, error },
|
|
11445
|
+
"Failed to send update to client"
|
|
11446
|
+
);
|
|
11447
|
+
}
|
|
11448
|
+
}
|
|
11449
|
+
/**
|
|
11450
|
+
* Get coordinator node for a subscription.
|
|
11451
|
+
* For remote subscriptions (where we are a data node), this returns the coordinator.
|
|
11452
|
+
*/
|
|
11453
|
+
getCoordinatorForSubscription(subscriptionId) {
|
|
11454
|
+
if (this.subscriptions.has(subscriptionId)) {
|
|
11455
|
+
return this.clusterManager.config.nodeId;
|
|
11456
|
+
}
|
|
11457
|
+
const searchSub = this.localSearchCoordinator.getDistributedSubscription(subscriptionId);
|
|
11458
|
+
if (searchSub?.coordinatorNodeId) {
|
|
11459
|
+
return searchSub.coordinatorNodeId;
|
|
11460
|
+
}
|
|
11461
|
+
const querySub = this.localQueryRegistry.getDistributedSubscription(subscriptionId);
|
|
11462
|
+
if (querySub?.coordinatorNodeId) {
|
|
11463
|
+
return querySub.coordinatorNodeId;
|
|
11464
|
+
}
|
|
11465
|
+
return null;
|
|
11466
|
+
}
|
|
11467
|
+
/**
|
|
11468
|
+
* Cleanup on destroy.
|
|
11469
|
+
*/
|
|
11470
|
+
destroy() {
|
|
11471
|
+
for (const subscriptionId of this.subscriptions.keys()) {
|
|
11472
|
+
this.unsubscribe(subscriptionId);
|
|
11473
|
+
}
|
|
11474
|
+
for (const pending of this.pendingAcks.values()) {
|
|
11475
|
+
clearTimeout(pending.timeoutHandle);
|
|
11476
|
+
}
|
|
11477
|
+
this.pendingAcks.clear();
|
|
11478
|
+
this.removeAllListeners();
|
|
11479
|
+
}
|
|
11480
|
+
};
|
|
11481
|
+
|
|
11482
|
+
// src/ServerCoordinator.ts
|
|
11483
|
+
var GC_INTERVAL_MS = 60 * 60 * 1e3;
|
|
11484
|
+
var GC_AGE_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
11485
|
+
var CLIENT_HEARTBEAT_TIMEOUT_MS = 2e4;
|
|
11486
|
+
var CLIENT_HEARTBEAT_CHECK_INTERVAL_MS = 5e3;
|
|
11487
|
+
var ServerCoordinator = class {
|
|
11488
|
+
constructor(config) {
|
|
11489
|
+
this.clients = /* @__PURE__ */ new Map();
|
|
11490
|
+
// Interceptors
|
|
11491
|
+
this.interceptors = [];
|
|
11492
|
+
// In-memory storage (partitioned later)
|
|
11493
|
+
this.maps = /* @__PURE__ */ new Map();
|
|
11494
|
+
this.pendingClusterQueries = /* @__PURE__ */ new Map();
|
|
11495
|
+
// GC Consensus State
|
|
11496
|
+
this.gcReports = /* @__PURE__ */ new Map();
|
|
11497
|
+
// Track map loading state to avoid returning empty results during async load
|
|
11498
|
+
this.mapLoadingPromises = /* @__PURE__ */ new Map();
|
|
11499
|
+
// Track pending batch operations for testing purposes
|
|
11500
|
+
this.pendingBatchOperations = /* @__PURE__ */ new Set();
|
|
11501
|
+
this.journalSubscriptions = /* @__PURE__ */ new Map();
|
|
11502
|
+
this._actualPort = 0;
|
|
11503
|
+
this._actualClusterPort = 0;
|
|
11504
|
+
this._readyPromise = new Promise((resolve) => {
|
|
11505
|
+
this._readyResolve = resolve;
|
|
11506
|
+
});
|
|
11507
|
+
this._nodeId = config.nodeId;
|
|
11508
|
+
this.hlc = new import_core22.HLC(config.nodeId);
|
|
11509
|
+
this.storage = config.storage;
|
|
11510
|
+
const rawSecret = config.jwtSecret || process.env.JWT_SECRET || "topgun-secret-dev";
|
|
11511
|
+
this.jwtSecret = rawSecret.replace(/\\n/g, "\n");
|
|
11512
|
+
this.queryRegistry = new QueryRegistry();
|
|
9838
11513
|
this.securityManager = new SecurityManager(config.securityPolicies || []);
|
|
9839
11514
|
this.interceptors = config.interceptors || [];
|
|
9840
11515
|
this.metricsService = new MetricsService();
|
|
@@ -9930,7 +11605,7 @@ var ServerCoordinator = class {
|
|
|
9930
11605
|
this.metricsServer.on("error", (err) => {
|
|
9931
11606
|
logger.error({ err, port: metricsPort }, "Metrics server failed to start");
|
|
9932
11607
|
});
|
|
9933
|
-
this.wss = new
|
|
11608
|
+
this.wss = new import_ws5.WebSocketServer({
|
|
9934
11609
|
server: this.httpServer,
|
|
9935
11610
|
// Increase backlog for pending connections (default Linux is 128)
|
|
9936
11611
|
backlog: config.wsBacklog ?? 511,
|
|
@@ -9972,8 +11647,8 @@ var ServerCoordinator = class {
|
|
|
9972
11647
|
this.cluster,
|
|
9973
11648
|
this.partitionService,
|
|
9974
11649
|
{
|
|
9975
|
-
...
|
|
9976
|
-
defaultConsistency: config.defaultConsistency ??
|
|
11650
|
+
...import_core22.DEFAULT_REPLICATION_CONFIG,
|
|
11651
|
+
defaultConsistency: config.defaultConsistency ?? import_core22.ConsistencyLevel.EVENTUAL,
|
|
9977
11652
|
...config.replicationConfig
|
|
9978
11653
|
}
|
|
9979
11654
|
);
|
|
@@ -9989,7 +11664,7 @@ var ServerCoordinator = class {
|
|
|
9989
11664
|
cluster: this.cluster,
|
|
9990
11665
|
sendToClient: (clientId, message) => {
|
|
9991
11666
|
const client = this.clients.get(clientId);
|
|
9992
|
-
if (client && client.socket.readyState ===
|
|
11667
|
+
if (client && client.socket.readyState === import_ws5.WebSocket.OPEN) {
|
|
9993
11668
|
client.writer.write(message);
|
|
9994
11669
|
}
|
|
9995
11670
|
}
|
|
@@ -10036,7 +11711,7 @@ var ServerCoordinator = class {
|
|
|
10036
11711
|
void 0,
|
|
10037
11712
|
// LagTracker - can be added later
|
|
10038
11713
|
{
|
|
10039
|
-
defaultConsistency: config.defaultConsistency ??
|
|
11714
|
+
defaultConsistency: config.defaultConsistency ?? import_core22.ConsistencyLevel.STRONG,
|
|
10040
11715
|
preferLocalReplica: true,
|
|
10041
11716
|
loadBalancing: "latency-based"
|
|
10042
11717
|
}
|
|
@@ -10089,6 +11764,26 @@ var ServerCoordinator = class {
|
|
|
10089
11764
|
logger.info({ mapName, fields: ftsConfig.fields }, "FTS enabled for map");
|
|
10090
11765
|
}
|
|
10091
11766
|
}
|
|
11767
|
+
this.clusterSearchCoordinator = new ClusterSearchCoordinator(
|
|
11768
|
+
this.cluster,
|
|
11769
|
+
this.partitionService,
|
|
11770
|
+
this.searchCoordinator,
|
|
11771
|
+
config.distributedSearch,
|
|
11772
|
+
this.metricsService
|
|
11773
|
+
);
|
|
11774
|
+
logger.info("ClusterSearchCoordinator initialized for distributed search");
|
|
11775
|
+
this.distributedSubCoordinator = new DistributedSubscriptionCoordinator(
|
|
11776
|
+
this.cluster,
|
|
11777
|
+
this.queryRegistry,
|
|
11778
|
+
this.searchCoordinator,
|
|
11779
|
+
void 0,
|
|
11780
|
+
// Use default config
|
|
11781
|
+
this.metricsService
|
|
11782
|
+
);
|
|
11783
|
+
logger.info("DistributedSubscriptionCoordinator initialized for distributed live subscriptions");
|
|
11784
|
+
this.searchCoordinator.setNodeId(config.nodeId);
|
|
11785
|
+
this.queryRegistry.setClusterManager(this.cluster, config.nodeId);
|
|
11786
|
+
this.queryRegistry.setMapGetter((name) => this.getMap(name));
|
|
10092
11787
|
this.systemManager = new SystemManager(
|
|
10093
11788
|
this.cluster,
|
|
10094
11789
|
this.metricsService,
|
|
@@ -10131,7 +11826,7 @@ var ServerCoordinator = class {
|
|
|
10131
11826
|
await this.getMapAsync(mapName);
|
|
10132
11827
|
const map = this.maps.get(mapName);
|
|
10133
11828
|
if (!map) return;
|
|
10134
|
-
if (map instanceof
|
|
11829
|
+
if (map instanceof import_core22.LWWMap) {
|
|
10135
11830
|
const entries = Array.from(map.entries());
|
|
10136
11831
|
if (entries.length > 0) {
|
|
10137
11832
|
logger.info({ mapName, count: entries.length }, "Backfilling FTS index");
|
|
@@ -10229,11 +11924,11 @@ var ServerCoordinator = class {
|
|
|
10229
11924
|
const map = this.maps.get(mapName);
|
|
10230
11925
|
if (map) {
|
|
10231
11926
|
const entries = [];
|
|
10232
|
-
if (map instanceof
|
|
11927
|
+
if (map instanceof import_core22.LWWMap) {
|
|
10233
11928
|
for (const [key, value] of map.entries()) {
|
|
10234
11929
|
entries.push([key, value]);
|
|
10235
11930
|
}
|
|
10236
|
-
} else if (map instanceof
|
|
11931
|
+
} else if (map instanceof import_core22.ORMap) {
|
|
10237
11932
|
for (const key of map.allKeys()) {
|
|
10238
11933
|
const values = map.get(key);
|
|
10239
11934
|
const value = values.length > 0 ? values[0] : null;
|
|
@@ -10354,10 +12049,10 @@ var ServerCoordinator = class {
|
|
|
10354
12049
|
this.metricsService.destroy();
|
|
10355
12050
|
this.wss.close();
|
|
10356
12051
|
logger.info(`Closing ${this.clients.size} client connections...`);
|
|
10357
|
-
const shutdownMsg = (0,
|
|
12052
|
+
const shutdownMsg = (0, import_core22.serialize)({ type: "SHUTDOWN_PENDING", retryAfter: 5e3 });
|
|
10358
12053
|
for (const client of this.clients.values()) {
|
|
10359
12054
|
try {
|
|
10360
|
-
if (client.socket.readyState ===
|
|
12055
|
+
if (client.socket.readyState === import_ws5.WebSocket.OPEN) {
|
|
10361
12056
|
client.socket.send(shutdownMsg);
|
|
10362
12057
|
if (client.writer) {
|
|
10363
12058
|
client.writer.close();
|
|
@@ -10420,6 +12115,12 @@ var ServerCoordinator = class {
|
|
|
10420
12115
|
if (this.eventJournalService) {
|
|
10421
12116
|
this.eventJournalService.dispose();
|
|
10422
12117
|
}
|
|
12118
|
+
if (this.clusterSearchCoordinator) {
|
|
12119
|
+
this.clusterSearchCoordinator.destroy();
|
|
12120
|
+
}
|
|
12121
|
+
if (this.distributedSubCoordinator) {
|
|
12122
|
+
this.distributedSubCoordinator.destroy();
|
|
12123
|
+
}
|
|
10423
12124
|
logger.info("Server Coordinator shutdown complete.");
|
|
10424
12125
|
}
|
|
10425
12126
|
async handleConnection(ws) {
|
|
@@ -10487,7 +12188,7 @@ var ServerCoordinator = class {
|
|
|
10487
12188
|
buf = Buffer.from(message);
|
|
10488
12189
|
}
|
|
10489
12190
|
try {
|
|
10490
|
-
data = (0,
|
|
12191
|
+
data = (0, import_core22.deserialize)(buf);
|
|
10491
12192
|
} catch (e) {
|
|
10492
12193
|
try {
|
|
10493
12194
|
const text = Buffer.isBuffer(buf) ? buf.toString() : new TextDecoder().decode(buf);
|
|
@@ -10528,6 +12229,9 @@ var ServerCoordinator = class {
|
|
|
10528
12229
|
this.topicManager.unsubscribeAll(clientId);
|
|
10529
12230
|
this.counterHandler.unsubscribeAll(clientId);
|
|
10530
12231
|
this.searchCoordinator.unsubscribeClient(clientId);
|
|
12232
|
+
if (this.distributedSubCoordinator && connection) {
|
|
12233
|
+
this.distributedSubCoordinator.unsubscribeClient(connection.socket);
|
|
12234
|
+
}
|
|
10531
12235
|
const members = this.cluster.getMembers();
|
|
10532
12236
|
for (const memberId of members) {
|
|
10533
12237
|
if (!this.cluster.isLocal(memberId)) {
|
|
@@ -10540,10 +12244,10 @@ var ServerCoordinator = class {
|
|
|
10540
12244
|
this.clients.delete(clientId);
|
|
10541
12245
|
this.metricsService.setConnectedClients(this.clients.size);
|
|
10542
12246
|
});
|
|
10543
|
-
ws.send((0,
|
|
12247
|
+
ws.send((0, import_core22.serialize)({ type: "AUTH_REQUIRED" }));
|
|
10544
12248
|
}
|
|
10545
12249
|
async handleMessage(client, rawMessage) {
|
|
10546
|
-
const parseResult =
|
|
12250
|
+
const parseResult = import_core22.MessageSchema.safeParse(rawMessage);
|
|
10547
12251
|
if (!parseResult.success) {
|
|
10548
12252
|
logger.error({ clientId: client.id, error: parseResult.error }, "Invalid message format from client");
|
|
10549
12253
|
client.writer.write({
|
|
@@ -10602,70 +12306,115 @@ var ServerCoordinator = class {
|
|
|
10602
12306
|
}
|
|
10603
12307
|
logger.info({ clientId: client.id, mapName, query }, "Client subscribed");
|
|
10604
12308
|
this.metricsService.incOp("SUBSCRIBE", mapName);
|
|
10605
|
-
|
|
10606
|
-
|
|
10607
|
-
|
|
10608
|
-
|
|
10609
|
-
|
|
10610
|
-
|
|
12309
|
+
if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
|
|
12310
|
+
this.distributedSubCoordinator.subscribeQuery(
|
|
12311
|
+
queryId,
|
|
12312
|
+
client.socket,
|
|
12313
|
+
mapName,
|
|
12314
|
+
query
|
|
12315
|
+
).then((result) => {
|
|
12316
|
+
const filteredResults = result.results.map((res) => {
|
|
12317
|
+
const filteredValue = this.securityManager.filterObject(res.value, client.principal, mapName);
|
|
12318
|
+
return { ...res, value: filteredValue };
|
|
12319
|
+
});
|
|
12320
|
+
client.writer.write({
|
|
12321
|
+
type: "QUERY_RESP",
|
|
12322
|
+
payload: {
|
|
12323
|
+
queryId,
|
|
12324
|
+
results: filteredResults
|
|
12325
|
+
}
|
|
12326
|
+
});
|
|
12327
|
+
client.subscriptions.add(queryId);
|
|
12328
|
+
logger.debug({
|
|
12329
|
+
clientId: client.id,
|
|
12330
|
+
queryId,
|
|
10611
12331
|
mapName,
|
|
10612
|
-
|
|
10613
|
-
|
|
10614
|
-
|
|
10615
|
-
|
|
10616
|
-
|
|
12332
|
+
resultCount: result.results.length,
|
|
12333
|
+
totalHits: result.totalHits,
|
|
12334
|
+
nodes: result.registeredNodes
|
|
12335
|
+
}, "Distributed query subscription created");
|
|
12336
|
+
}).catch((err) => {
|
|
12337
|
+
logger.error({ err, queryId }, "Distributed query subscription failed");
|
|
12338
|
+
client.writer.write({
|
|
12339
|
+
type: "QUERY_RESP",
|
|
12340
|
+
payload: {
|
|
12341
|
+
queryId,
|
|
12342
|
+
results: [],
|
|
12343
|
+
error: "Failed to create distributed subscription"
|
|
10617
12344
|
}
|
|
10618
12345
|
});
|
|
10619
|
-
|
|
10620
|
-
|
|
10621
|
-
|
|
10622
|
-
|
|
10623
|
-
|
|
10624
|
-
|
|
10625
|
-
|
|
12346
|
+
});
|
|
12347
|
+
} else {
|
|
12348
|
+
const allMembers = this.cluster.getMembers();
|
|
12349
|
+
let remoteMembers = allMembers.filter((id) => !this.cluster.isLocal(id));
|
|
12350
|
+
const queryKey = query._id || query.where?._id;
|
|
12351
|
+
if (queryKey && typeof queryKey === "string" && this.readReplicaHandler) {
|
|
12352
|
+
try {
|
|
12353
|
+
const targetNode = this.readReplicaHandler.selectReadNode({
|
|
12354
|
+
mapName,
|
|
12355
|
+
key: queryKey,
|
|
12356
|
+
options: {
|
|
12357
|
+
// Default to EVENTUAL for read scaling unless specified otherwise
|
|
12358
|
+
// In future, we could extract consistency from query options if available
|
|
12359
|
+
consistency: import_core22.ConsistencyLevel.EVENTUAL
|
|
12360
|
+
}
|
|
12361
|
+
});
|
|
12362
|
+
if (targetNode) {
|
|
12363
|
+
if (this.cluster.isLocal(targetNode)) {
|
|
12364
|
+
remoteMembers = [];
|
|
12365
|
+
logger.debug({ clientId: client.id, mapName, key: queryKey }, "Read optimization: Serving locally");
|
|
12366
|
+
} else if (remoteMembers.includes(targetNode)) {
|
|
12367
|
+
remoteMembers = [targetNode];
|
|
12368
|
+
logger.debug({ clientId: client.id, mapName, key: queryKey, targetNode }, "Read optimization: Routing to replica");
|
|
12369
|
+
}
|
|
10626
12370
|
}
|
|
12371
|
+
} catch (e) {
|
|
12372
|
+
logger.warn({ err: e }, "Error in ReadReplicaHandler selection");
|
|
10627
12373
|
}
|
|
10628
|
-
} catch (e) {
|
|
10629
|
-
logger.warn({ err: e }, "Error in ReadReplicaHandler selection");
|
|
10630
12374
|
}
|
|
10631
|
-
|
|
10632
|
-
|
|
10633
|
-
|
|
10634
|
-
|
|
10635
|
-
|
|
10636
|
-
|
|
10637
|
-
|
|
10638
|
-
|
|
10639
|
-
|
|
10640
|
-
|
|
10641
|
-
|
|
10642
|
-
|
|
10643
|
-
|
|
10644
|
-
|
|
10645
|
-
|
|
10646
|
-
|
|
10647
|
-
|
|
10648
|
-
|
|
10649
|
-
|
|
10650
|
-
|
|
10651
|
-
|
|
10652
|
-
|
|
10653
|
-
|
|
10654
|
-
|
|
10655
|
-
|
|
10656
|
-
}
|
|
12375
|
+
const requestId = crypto.randomUUID();
|
|
12376
|
+
const pending = {
|
|
12377
|
+
requestId,
|
|
12378
|
+
client,
|
|
12379
|
+
queryId,
|
|
12380
|
+
mapName,
|
|
12381
|
+
query,
|
|
12382
|
+
results: [],
|
|
12383
|
+
// Will populate with local results first
|
|
12384
|
+
expectedNodes: new Set(remoteMembers),
|
|
12385
|
+
respondedNodes: /* @__PURE__ */ new Set(),
|
|
12386
|
+
timer: setTimeout(() => this.finalizeClusterQuery(requestId, true), 5e3)
|
|
12387
|
+
// 5s timeout
|
|
12388
|
+
};
|
|
12389
|
+
this.pendingClusterQueries.set(requestId, pending);
|
|
12390
|
+
try {
|
|
12391
|
+
const localResults = await this.executeLocalQuery(mapName, query);
|
|
12392
|
+
pending.results.push(...localResults);
|
|
12393
|
+
if (remoteMembers.length > 0) {
|
|
12394
|
+
for (const nodeId of remoteMembers) {
|
|
12395
|
+
this.cluster.send(nodeId, "CLUSTER_QUERY_EXEC", {
|
|
12396
|
+
requestId,
|
|
12397
|
+
mapName,
|
|
12398
|
+
query
|
|
12399
|
+
});
|
|
12400
|
+
}
|
|
12401
|
+
} else {
|
|
12402
|
+
this.finalizeClusterQuery(requestId);
|
|
10657
12403
|
}
|
|
10658
|
-
}
|
|
12404
|
+
} catch (err) {
|
|
12405
|
+
logger.error({ err, mapName }, "Failed to execute local query");
|
|
10659
12406
|
this.finalizeClusterQuery(requestId);
|
|
10660
12407
|
}
|
|
10661
|
-
} catch (err) {
|
|
10662
|
-
logger.error({ err, mapName }, "Failed to execute local query");
|
|
10663
|
-
this.finalizeClusterQuery(requestId);
|
|
10664
12408
|
}
|
|
10665
12409
|
break;
|
|
10666
12410
|
}
|
|
10667
12411
|
case "QUERY_UNSUB": {
|
|
10668
12412
|
const { queryId: unsubId } = message.payload;
|
|
12413
|
+
if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
|
|
12414
|
+
this.distributedSubCoordinator.unsubscribe(unsubId).catch((err) => {
|
|
12415
|
+
logger.warn({ err, queryId: unsubId }, "Failed to unsubscribe from distributed coordinator");
|
|
12416
|
+
});
|
|
12417
|
+
}
|
|
10669
12418
|
this.queryRegistry.unregister(unsubId);
|
|
10670
12419
|
client.subscriptions.delete(unsubId);
|
|
10671
12420
|
break;
|
|
@@ -10808,7 +12557,7 @@ var ServerCoordinator = class {
|
|
|
10808
12557
|
this.metricsService.incOp("GET", message.mapName);
|
|
10809
12558
|
try {
|
|
10810
12559
|
const mapForSync = await this.getMapAsync(message.mapName);
|
|
10811
|
-
if (mapForSync instanceof
|
|
12560
|
+
if (mapForSync instanceof import_core22.LWWMap) {
|
|
10812
12561
|
const tree = mapForSync.getMerkleTree();
|
|
10813
12562
|
const rootHash = tree.getRootHash();
|
|
10814
12563
|
client.writer.write({
|
|
@@ -10846,7 +12595,7 @@ var ServerCoordinator = class {
|
|
|
10846
12595
|
const { mapName, path } = message.payload;
|
|
10847
12596
|
try {
|
|
10848
12597
|
const mapForBucket = await this.getMapAsync(mapName);
|
|
10849
|
-
if (mapForBucket instanceof
|
|
12598
|
+
if (mapForBucket instanceof import_core22.LWWMap) {
|
|
10850
12599
|
const treeForBucket = mapForBucket.getMerkleTree();
|
|
10851
12600
|
const buckets = treeForBucket.getBuckets(path);
|
|
10852
12601
|
const node = treeForBucket.getNode(path);
|
|
@@ -10989,7 +12738,7 @@ var ServerCoordinator = class {
|
|
|
10989
12738
|
client.writer.write(result.response);
|
|
10990
12739
|
for (const targetClientId of result.broadcastTo) {
|
|
10991
12740
|
const targetClient = this.clients.get(targetClientId);
|
|
10992
|
-
if (targetClient && targetClient.socket.readyState ===
|
|
12741
|
+
if (targetClient && targetClient.socket.readyState === import_ws5.WebSocket.OPEN) {
|
|
10993
12742
|
targetClient.writer.write(result.broadcastMessage);
|
|
10994
12743
|
}
|
|
10995
12744
|
}
|
|
@@ -11228,7 +12977,7 @@ var ServerCoordinator = class {
|
|
|
11228
12977
|
this.metricsService.incOp("GET", message.mapName);
|
|
11229
12978
|
try {
|
|
11230
12979
|
const mapForSync = await this.getMapAsync(message.mapName, "OR");
|
|
11231
|
-
if (mapForSync instanceof
|
|
12980
|
+
if (mapForSync instanceof import_core22.ORMap) {
|
|
11232
12981
|
const tree = mapForSync.getMerkleTree();
|
|
11233
12982
|
const rootHash = tree.getRootHash();
|
|
11234
12983
|
client.writer.write({
|
|
@@ -11265,7 +13014,7 @@ var ServerCoordinator = class {
|
|
|
11265
13014
|
const { mapName, path } = message.payload;
|
|
11266
13015
|
try {
|
|
11267
13016
|
const mapForBucket = await this.getMapAsync(mapName, "OR");
|
|
11268
|
-
if (mapForBucket instanceof
|
|
13017
|
+
if (mapForBucket instanceof import_core22.ORMap) {
|
|
11269
13018
|
const tree = mapForBucket.getMerkleTree();
|
|
11270
13019
|
const buckets = tree.getBuckets(path);
|
|
11271
13020
|
const isLeaf = tree.isLeaf(path);
|
|
@@ -11309,7 +13058,7 @@ var ServerCoordinator = class {
|
|
|
11309
13058
|
const { mapName: diffMapName, keys } = message.payload;
|
|
11310
13059
|
try {
|
|
11311
13060
|
const mapForDiff = await this.getMapAsync(diffMapName, "OR");
|
|
11312
|
-
if (mapForDiff instanceof
|
|
13061
|
+
if (mapForDiff instanceof import_core22.ORMap) {
|
|
11313
13062
|
const entries = [];
|
|
11314
13063
|
const allTombstones = mapForDiff.getTombstones();
|
|
11315
13064
|
for (const key of keys) {
|
|
@@ -11341,7 +13090,7 @@ var ServerCoordinator = class {
|
|
|
11341
13090
|
const { mapName: pushMapName, entries: pushEntries } = message.payload;
|
|
11342
13091
|
try {
|
|
11343
13092
|
const mapForPush = await this.getMapAsync(pushMapName, "OR");
|
|
11344
|
-
if (mapForPush instanceof
|
|
13093
|
+
if (mapForPush instanceof import_core22.ORMap) {
|
|
11345
13094
|
let totalAdded = 0;
|
|
11346
13095
|
let totalUpdated = 0;
|
|
11347
13096
|
for (const entry of pushEntries) {
|
|
@@ -11469,7 +13218,7 @@ var ServerCoordinator = class {
|
|
|
11469
13218
|
});
|
|
11470
13219
|
break;
|
|
11471
13220
|
}
|
|
11472
|
-
// Phase 11.1: Full-Text Search
|
|
13221
|
+
// Phase 11.1: Full-Text Search (Phase 14: Distributed Search)
|
|
11473
13222
|
case "SEARCH": {
|
|
11474
13223
|
const { requestId: searchReqId, mapName: searchMapName, query: searchQuery, options: searchOptions } = message.payload;
|
|
11475
13224
|
if (!this.securityManager.checkPermission(client.principal, searchMapName, "READ")) {
|
|
@@ -11497,18 +13246,58 @@ var ServerCoordinator = class {
|
|
|
11497
13246
|
});
|
|
11498
13247
|
break;
|
|
11499
13248
|
}
|
|
11500
|
-
|
|
11501
|
-
|
|
11502
|
-
|
|
11503
|
-
|
|
11504
|
-
|
|
11505
|
-
|
|
11506
|
-
|
|
11507
|
-
|
|
11508
|
-
|
|
11509
|
-
|
|
11510
|
-
|
|
11511
|
-
|
|
13249
|
+
if (this.clusterSearchCoordinator && this.cluster.getMembers().length > 1) {
|
|
13250
|
+
this.clusterSearchCoordinator.search(searchMapName, searchQuery, {
|
|
13251
|
+
limit: searchOptions?.limit ?? 10,
|
|
13252
|
+
minScore: searchOptions?.minScore,
|
|
13253
|
+
boost: searchOptions?.boost
|
|
13254
|
+
}).then((distributedResult) => {
|
|
13255
|
+
logger.debug({
|
|
13256
|
+
clientId: client.id,
|
|
13257
|
+
mapName: searchMapName,
|
|
13258
|
+
query: searchQuery,
|
|
13259
|
+
resultCount: distributedResult.results.length,
|
|
13260
|
+
totalHits: distributedResult.totalHits,
|
|
13261
|
+
respondedNodes: distributedResult.respondedNodes.length,
|
|
13262
|
+
failedNodes: distributedResult.failedNodes.length,
|
|
13263
|
+
executionTimeMs: distributedResult.executionTimeMs
|
|
13264
|
+
}, "Distributed search executed");
|
|
13265
|
+
client.writer.write({
|
|
13266
|
+
type: "SEARCH_RESP",
|
|
13267
|
+
payload: {
|
|
13268
|
+
requestId: searchReqId,
|
|
13269
|
+
results: distributedResult.results,
|
|
13270
|
+
totalCount: distributedResult.totalHits,
|
|
13271
|
+
// Include cursor for pagination if available
|
|
13272
|
+
nextCursor: distributedResult.nextCursor
|
|
13273
|
+
}
|
|
13274
|
+
});
|
|
13275
|
+
}).catch((err) => {
|
|
13276
|
+
logger.error({ err, mapName: searchMapName, query: searchQuery }, "Distributed search failed");
|
|
13277
|
+
client.writer.write({
|
|
13278
|
+
type: "SEARCH_RESP",
|
|
13279
|
+
payload: {
|
|
13280
|
+
requestId: searchReqId,
|
|
13281
|
+
results: [],
|
|
13282
|
+
totalCount: 0,
|
|
13283
|
+
error: `Distributed search failed: ${err.message}`
|
|
13284
|
+
}
|
|
13285
|
+
});
|
|
13286
|
+
});
|
|
13287
|
+
} else {
|
|
13288
|
+
const searchResult = this.searchCoordinator.search(searchMapName, searchQuery, searchOptions);
|
|
13289
|
+
searchResult.requestId = searchReqId;
|
|
13290
|
+
logger.debug({
|
|
13291
|
+
clientId: client.id,
|
|
13292
|
+
mapName: searchMapName,
|
|
13293
|
+
query: searchQuery,
|
|
13294
|
+
resultCount: searchResult.results.length
|
|
13295
|
+
}, "Local search executed");
|
|
13296
|
+
client.writer.write({
|
|
13297
|
+
type: "SEARCH_RESP",
|
|
13298
|
+
payload: searchResult
|
|
13299
|
+
});
|
|
13300
|
+
}
|
|
11512
13301
|
break;
|
|
11513
13302
|
}
|
|
11514
13303
|
// Phase 11.1b: Live Search Subscriptions
|
|
@@ -11539,33 +13328,75 @@ var ServerCoordinator = class {
|
|
|
11539
13328
|
});
|
|
11540
13329
|
break;
|
|
11541
13330
|
}
|
|
11542
|
-
|
|
11543
|
-
|
|
11544
|
-
|
|
11545
|
-
|
|
11546
|
-
|
|
11547
|
-
|
|
11548
|
-
|
|
11549
|
-
|
|
11550
|
-
|
|
11551
|
-
|
|
11552
|
-
|
|
11553
|
-
|
|
11554
|
-
|
|
11555
|
-
|
|
11556
|
-
|
|
11557
|
-
|
|
11558
|
-
|
|
11559
|
-
|
|
11560
|
-
|
|
11561
|
-
|
|
11562
|
-
|
|
11563
|
-
|
|
13331
|
+
if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
|
|
13332
|
+
this.distributedSubCoordinator.subscribeSearch(
|
|
13333
|
+
subscriptionId,
|
|
13334
|
+
client.socket,
|
|
13335
|
+
subMapName,
|
|
13336
|
+
subQuery,
|
|
13337
|
+
subOptions || {}
|
|
13338
|
+
).then((result) => {
|
|
13339
|
+
client.writer.write({
|
|
13340
|
+
type: "SEARCH_RESP",
|
|
13341
|
+
payload: {
|
|
13342
|
+
requestId: subscriptionId,
|
|
13343
|
+
results: result.results,
|
|
13344
|
+
totalCount: result.totalHits
|
|
13345
|
+
}
|
|
13346
|
+
});
|
|
13347
|
+
logger.debug({
|
|
13348
|
+
clientId: client.id,
|
|
13349
|
+
subscriptionId,
|
|
13350
|
+
mapName: subMapName,
|
|
13351
|
+
query: subQuery,
|
|
13352
|
+
resultCount: result.results.length,
|
|
13353
|
+
totalHits: result.totalHits,
|
|
13354
|
+
nodes: result.registeredNodes
|
|
13355
|
+
}, "Distributed search subscription created");
|
|
13356
|
+
}).catch((err) => {
|
|
13357
|
+
logger.error({ err, subscriptionId }, "Distributed search subscription failed");
|
|
13358
|
+
client.writer.write({
|
|
13359
|
+
type: "SEARCH_RESP",
|
|
13360
|
+
payload: {
|
|
13361
|
+
requestId: subscriptionId,
|
|
13362
|
+
results: [],
|
|
13363
|
+
totalCount: 0,
|
|
13364
|
+
error: "Failed to create distributed subscription"
|
|
13365
|
+
}
|
|
13366
|
+
});
|
|
13367
|
+
});
|
|
13368
|
+
} else {
|
|
13369
|
+
const initialResults = this.searchCoordinator.subscribe(
|
|
13370
|
+
client.id,
|
|
13371
|
+
subscriptionId,
|
|
13372
|
+
subMapName,
|
|
13373
|
+
subQuery,
|
|
13374
|
+
subOptions
|
|
13375
|
+
);
|
|
13376
|
+
logger.debug({
|
|
13377
|
+
clientId: client.id,
|
|
13378
|
+
subscriptionId,
|
|
13379
|
+
mapName: subMapName,
|
|
13380
|
+
query: subQuery,
|
|
13381
|
+
resultCount: initialResults.length
|
|
13382
|
+
}, "Search subscription created (local)");
|
|
13383
|
+
client.writer.write({
|
|
13384
|
+
type: "SEARCH_RESP",
|
|
13385
|
+
payload: {
|
|
13386
|
+
requestId: subscriptionId,
|
|
13387
|
+
results: initialResults,
|
|
13388
|
+
totalCount: initialResults.length
|
|
13389
|
+
}
|
|
13390
|
+
});
|
|
13391
|
+
}
|
|
11564
13392
|
break;
|
|
11565
13393
|
}
|
|
11566
13394
|
case "SEARCH_UNSUB": {
|
|
11567
13395
|
const { subscriptionId: unsubId } = message.payload;
|
|
11568
13396
|
this.searchCoordinator.unsubscribe(unsubId);
|
|
13397
|
+
if (this.distributedSubCoordinator) {
|
|
13398
|
+
this.distributedSubCoordinator.unsubscribe(unsubId);
|
|
13399
|
+
}
|
|
11569
13400
|
logger.debug({ clientId: client.id, subscriptionId: unsubId }, "Search unsubscription");
|
|
11570
13401
|
break;
|
|
11571
13402
|
}
|
|
@@ -11582,7 +13413,7 @@ var ServerCoordinator = class {
|
|
|
11582
13413
|
} else if (op.orRecord && op.orRecord.timestamp) {
|
|
11583
13414
|
} else if (op.orTag) {
|
|
11584
13415
|
try {
|
|
11585
|
-
ts =
|
|
13416
|
+
ts = import_core22.HLC.parse(op.orTag);
|
|
11586
13417
|
} catch (e) {
|
|
11587
13418
|
}
|
|
11588
13419
|
}
|
|
@@ -11606,7 +13437,7 @@ var ServerCoordinator = class {
|
|
|
11606
13437
|
};
|
|
11607
13438
|
let broadcastCount = 0;
|
|
11608
13439
|
for (const client of this.clients.values()) {
|
|
11609
|
-
if (client.isAuthenticated && client.socket.readyState ===
|
|
13440
|
+
if (client.isAuthenticated && client.socket.readyState === import_ws5.WebSocket.OPEN && client.writer) {
|
|
11610
13441
|
client.writer.write(message);
|
|
11611
13442
|
broadcastCount++;
|
|
11612
13443
|
}
|
|
@@ -11679,7 +13510,7 @@ var ServerCoordinator = class {
|
|
|
11679
13510
|
client.writer.write({ ...message, payload: newPayload });
|
|
11680
13511
|
}
|
|
11681
13512
|
} else {
|
|
11682
|
-
const msgData = (0,
|
|
13513
|
+
const msgData = (0, import_core22.serialize)(message);
|
|
11683
13514
|
for (const [id, client] of this.clients) {
|
|
11684
13515
|
if (id !== excludeClientId && client.socket.readyState === 1) {
|
|
11685
13516
|
client.writer.writeRaw(msgData);
|
|
@@ -11757,7 +13588,7 @@ var ServerCoordinator = class {
|
|
|
11757
13588
|
payload: { events: filteredEvents },
|
|
11758
13589
|
timestamp: this.hlc.now()
|
|
11759
13590
|
};
|
|
11760
|
-
const serializedBatch = (0,
|
|
13591
|
+
const serializedBatch = (0, import_core22.serialize)(batchMessage);
|
|
11761
13592
|
for (const client of clients) {
|
|
11762
13593
|
try {
|
|
11763
13594
|
client.writer.writeRaw(serializedBatch);
|
|
@@ -11842,7 +13673,7 @@ var ServerCoordinator = class {
|
|
|
11842
13673
|
payload: { events: filteredEvents },
|
|
11843
13674
|
timestamp: this.hlc.now()
|
|
11844
13675
|
};
|
|
11845
|
-
const serializedBatch = (0,
|
|
13676
|
+
const serializedBatch = (0, import_core22.serialize)(batchMessage);
|
|
11846
13677
|
for (const client of clients) {
|
|
11847
13678
|
sendPromises.push(new Promise((resolve, reject) => {
|
|
11848
13679
|
try {
|
|
@@ -12038,9 +13869,9 @@ var ServerCoordinator = class {
|
|
|
12038
13869
|
async executeLocalQuery(mapName, query) {
|
|
12039
13870
|
const map = await this.getMapAsync(mapName);
|
|
12040
13871
|
const localQuery = { ...query };
|
|
12041
|
-
delete localQuery.
|
|
13872
|
+
delete localQuery.cursor;
|
|
12042
13873
|
delete localQuery.limit;
|
|
12043
|
-
if (map instanceof
|
|
13874
|
+
if (map instanceof import_core22.IndexedLWWMap) {
|
|
12044
13875
|
const coreQuery = this.convertToCoreQuery(localQuery);
|
|
12045
13876
|
if (coreQuery) {
|
|
12046
13877
|
const entries = map.queryEntries(coreQuery);
|
|
@@ -12050,7 +13881,7 @@ var ServerCoordinator = class {
|
|
|
12050
13881
|
});
|
|
12051
13882
|
}
|
|
12052
13883
|
}
|
|
12053
|
-
if (map instanceof
|
|
13884
|
+
if (map instanceof import_core22.IndexedORMap) {
|
|
12054
13885
|
const coreQuery = this.convertToCoreQuery(localQuery);
|
|
12055
13886
|
if (coreQuery) {
|
|
12056
13887
|
const results = map.query(coreQuery);
|
|
@@ -12058,14 +13889,14 @@ var ServerCoordinator = class {
|
|
|
12058
13889
|
}
|
|
12059
13890
|
}
|
|
12060
13891
|
const records = /* @__PURE__ */ new Map();
|
|
12061
|
-
if (map instanceof
|
|
13892
|
+
if (map instanceof import_core22.LWWMap) {
|
|
12062
13893
|
for (const key of map.allKeys()) {
|
|
12063
13894
|
const rec = map.getRecord(key);
|
|
12064
13895
|
if (rec && rec.value !== null) {
|
|
12065
13896
|
records.set(key, rec);
|
|
12066
13897
|
}
|
|
12067
13898
|
}
|
|
12068
|
-
} else if (map instanceof
|
|
13899
|
+
} else if (map instanceof import_core22.ORMap) {
|
|
12069
13900
|
const items = map.items;
|
|
12070
13901
|
for (const key of items.keys()) {
|
|
12071
13902
|
const values = map.get(key);
|
|
@@ -12157,7 +13988,7 @@ var ServerCoordinator = class {
|
|
|
12157
13988
|
};
|
|
12158
13989
|
return mapping[op] || null;
|
|
12159
13990
|
}
|
|
12160
|
-
finalizeClusterQuery(requestId, timeout = false) {
|
|
13991
|
+
async finalizeClusterQuery(requestId, timeout = false) {
|
|
12161
13992
|
const pending = this.pendingClusterQueries.get(requestId);
|
|
12162
13993
|
if (!pending) return;
|
|
12163
13994
|
if (timeout) {
|
|
@@ -12182,7 +14013,49 @@ var ServerCoordinator = class {
|
|
|
12182
14013
|
return 0;
|
|
12183
14014
|
});
|
|
12184
14015
|
}
|
|
12185
|
-
|
|
14016
|
+
let slicedResults = finalResults;
|
|
14017
|
+
let nextCursor;
|
|
14018
|
+
let hasMore = false;
|
|
14019
|
+
let cursorStatus = "none";
|
|
14020
|
+
if (query.cursor || query.limit) {
|
|
14021
|
+
const sort = query.sort || {};
|
|
14022
|
+
const sortEntries = Object.entries(sort);
|
|
14023
|
+
const sortField = sortEntries.length > 0 ? sortEntries[0][0] : "_key";
|
|
14024
|
+
if (query.cursor) {
|
|
14025
|
+
const cursorData = import_core22.QueryCursor.decode(query.cursor);
|
|
14026
|
+
if (!cursorData) {
|
|
14027
|
+
cursorStatus = "invalid";
|
|
14028
|
+
} else if (!import_core22.QueryCursor.isValid(cursorData, query.predicate ?? query.where, sort)) {
|
|
14029
|
+
if (Date.now() - cursorData.timestamp > import_core22.DEFAULT_QUERY_CURSOR_MAX_AGE_MS) {
|
|
14030
|
+
cursorStatus = "expired";
|
|
14031
|
+
} else {
|
|
14032
|
+
cursorStatus = "invalid";
|
|
14033
|
+
}
|
|
14034
|
+
} else {
|
|
14035
|
+
cursorStatus = "valid";
|
|
14036
|
+
slicedResults = finalResults.filter((r) => {
|
|
14037
|
+
const sortValue = r.value[sortField];
|
|
14038
|
+
return import_core22.QueryCursor.isAfterCursor(
|
|
14039
|
+
{ key: r.key, sortValue },
|
|
14040
|
+
cursorData
|
|
14041
|
+
);
|
|
14042
|
+
});
|
|
14043
|
+
}
|
|
14044
|
+
}
|
|
14045
|
+
if (query.limit) {
|
|
14046
|
+
hasMore = slicedResults.length > query.limit;
|
|
14047
|
+
slicedResults = slicedResults.slice(0, query.limit);
|
|
14048
|
+
if (hasMore && slicedResults.length > 0) {
|
|
14049
|
+
const lastResult = slicedResults[slicedResults.length - 1];
|
|
14050
|
+
const sortValue = lastResult.value[sortField];
|
|
14051
|
+
nextCursor = import_core22.QueryCursor.fromLastResult(
|
|
14052
|
+
{ key: lastResult.key, sortValue },
|
|
14053
|
+
sort,
|
|
14054
|
+
query.predicate ?? query.where
|
|
14055
|
+
);
|
|
14056
|
+
}
|
|
14057
|
+
}
|
|
14058
|
+
}
|
|
12186
14059
|
const resultKeys = new Set(slicedResults.map((r) => r.key));
|
|
12187
14060
|
const sub = {
|
|
12188
14061
|
id: queryId,
|
|
@@ -12201,7 +14074,7 @@ var ServerCoordinator = class {
|
|
|
12201
14074
|
});
|
|
12202
14075
|
client.writer.write({
|
|
12203
14076
|
type: "QUERY_RESP",
|
|
12204
|
-
payload: { queryId, results: filteredResults }
|
|
14077
|
+
payload: { queryId, results: filteredResults, nextCursor, hasMore, cursorStatus }
|
|
12205
14078
|
});
|
|
12206
14079
|
}
|
|
12207
14080
|
/**
|
|
@@ -12213,11 +14086,11 @@ var ServerCoordinator = class {
|
|
|
12213
14086
|
async applyOpToMap(op, remoteNodeId) {
|
|
12214
14087
|
const typeHint = op.opType === "OR_ADD" || op.opType === "OR_REMOVE" ? "OR" : "LWW";
|
|
12215
14088
|
const map = this.getMap(op.mapName, typeHint);
|
|
12216
|
-
if (typeHint === "OR" && map instanceof
|
|
14089
|
+
if (typeHint === "OR" && map instanceof import_core22.LWWMap) {
|
|
12217
14090
|
logger.error({ mapName: op.mapName }, "Map type mismatch: LWWMap but received OR op");
|
|
12218
14091
|
throw new Error("Map type mismatch: LWWMap but received OR op");
|
|
12219
14092
|
}
|
|
12220
|
-
if (typeHint === "LWW" && map instanceof
|
|
14093
|
+
if (typeHint === "LWW" && map instanceof import_core22.ORMap) {
|
|
12221
14094
|
logger.error({ mapName: op.mapName }, "Map type mismatch: ORMap but received LWW op");
|
|
12222
14095
|
throw new Error("Map type mismatch: ORMap but received LWW op");
|
|
12223
14096
|
}
|
|
@@ -12228,7 +14101,7 @@ var ServerCoordinator = class {
|
|
|
12228
14101
|
mapName: op.mapName,
|
|
12229
14102
|
key: op.key
|
|
12230
14103
|
};
|
|
12231
|
-
if (map instanceof
|
|
14104
|
+
if (map instanceof import_core22.LWWMap) {
|
|
12232
14105
|
oldRecord = map.getRecord(op.key);
|
|
12233
14106
|
if (this.conflictResolverHandler.hasResolvers(op.mapName)) {
|
|
12234
14107
|
const mergeResult = await this.conflictResolverHandler.mergeWithResolver(
|
|
@@ -12256,7 +14129,7 @@ var ServerCoordinator = class {
|
|
|
12256
14129
|
eventPayload.eventType = "UPDATED";
|
|
12257
14130
|
eventPayload.record = op.record;
|
|
12258
14131
|
}
|
|
12259
|
-
} else if (map instanceof
|
|
14132
|
+
} else if (map instanceof import_core22.ORMap) {
|
|
12260
14133
|
oldRecord = map.getRecords(op.key);
|
|
12261
14134
|
if (op.opType === "OR_ADD") {
|
|
12262
14135
|
map.apply(op.key, op.orRecord);
|
|
@@ -12272,7 +14145,7 @@ var ServerCoordinator = class {
|
|
|
12272
14145
|
}
|
|
12273
14146
|
}
|
|
12274
14147
|
this.queryRegistry.processChange(op.mapName, map, op.key, op.record || op.orRecord, oldRecord);
|
|
12275
|
-
const mapSize = map instanceof
|
|
14148
|
+
const mapSize = map instanceof import_core22.ORMap ? map.totalRecords : map.size;
|
|
12276
14149
|
this.metricsService.setMapSize(op.mapName, mapSize);
|
|
12277
14150
|
if (this.storage) {
|
|
12278
14151
|
if (recordToStore) {
|
|
@@ -12313,17 +14186,6 @@ var ServerCoordinator = class {
|
|
|
12313
14186
|
}
|
|
12314
14187
|
return { eventPayload, oldRecord };
|
|
12315
14188
|
}
|
|
12316
|
-
/**
|
|
12317
|
-
* Broadcast event to cluster members (excluding self).
|
|
12318
|
-
*/
|
|
12319
|
-
broadcastToCluster(eventPayload) {
|
|
12320
|
-
const members = this.cluster.getMembers();
|
|
12321
|
-
for (const memberId of members) {
|
|
12322
|
-
if (!this.cluster.isLocal(memberId)) {
|
|
12323
|
-
this.cluster.send(memberId, "CLUSTER_EVENT", eventPayload);
|
|
12324
|
-
}
|
|
12325
|
-
}
|
|
12326
|
-
}
|
|
12327
14189
|
/**
|
|
12328
14190
|
* Apply replicated operation from another node (callback for ReplicationPipeline)
|
|
12329
14191
|
* This is called when we receive a replicated operation as a backup node
|
|
@@ -12449,7 +14311,6 @@ var ServerCoordinator = class {
|
|
|
12449
14311
|
payload: eventPayload,
|
|
12450
14312
|
timestamp: this.hlc.now()
|
|
12451
14313
|
}, originalSenderId);
|
|
12452
|
-
this.broadcastToCluster(eventPayload);
|
|
12453
14314
|
this.runAfterInterceptors(op, context);
|
|
12454
14315
|
}
|
|
12455
14316
|
/**
|
|
@@ -12576,7 +14437,6 @@ var ServerCoordinator = class {
|
|
|
12576
14437
|
});
|
|
12577
14438
|
}
|
|
12578
14439
|
batchedEvents.push(eventPayload);
|
|
12579
|
-
this.broadcastToCluster(eventPayload);
|
|
12580
14440
|
this.runAfterInterceptors(op, context);
|
|
12581
14441
|
}
|
|
12582
14442
|
handleClusterEvent(payload) {
|
|
@@ -12586,11 +14446,11 @@ var ServerCoordinator = class {
|
|
|
12586
14446
|
return;
|
|
12587
14447
|
}
|
|
12588
14448
|
const map = this.getMap(mapName, eventType === "OR_ADD" || eventType === "OR_REMOVE" ? "OR" : "LWW");
|
|
12589
|
-
const oldRecord = map instanceof
|
|
14449
|
+
const oldRecord = map instanceof import_core22.LWWMap ? map.getRecord(key) : null;
|
|
12590
14450
|
if (this.partitionService.isRelated(key)) {
|
|
12591
|
-
if (map instanceof
|
|
14451
|
+
if (map instanceof import_core22.LWWMap && payload.record) {
|
|
12592
14452
|
map.merge(key, payload.record);
|
|
12593
|
-
} else if (map instanceof
|
|
14453
|
+
} else if (map instanceof import_core22.ORMap) {
|
|
12594
14454
|
if (eventType === "OR_ADD" && payload.orRecord) {
|
|
12595
14455
|
map.apply(key, payload.orRecord);
|
|
12596
14456
|
} else if (eventType === "OR_REMOVE" && payload.orTag) {
|
|
@@ -12609,9 +14469,9 @@ var ServerCoordinator = class {
|
|
|
12609
14469
|
if (!this.maps.has(name)) {
|
|
12610
14470
|
let map;
|
|
12611
14471
|
if (typeHint === "OR") {
|
|
12612
|
-
map = new
|
|
14472
|
+
map = new import_core22.ORMap(this.hlc);
|
|
12613
14473
|
} else {
|
|
12614
|
-
map = new
|
|
14474
|
+
map = new import_core22.LWWMap(this.hlc);
|
|
12615
14475
|
}
|
|
12616
14476
|
this.maps.set(name, map);
|
|
12617
14477
|
if (this.storage) {
|
|
@@ -12634,7 +14494,7 @@ var ServerCoordinator = class {
|
|
|
12634
14494
|
this.getMap(name, typeHint);
|
|
12635
14495
|
const loadingPromise = this.mapLoadingPromises.get(name);
|
|
12636
14496
|
const map = this.maps.get(name);
|
|
12637
|
-
const mapSize = map instanceof
|
|
14497
|
+
const mapSize = map instanceof import_core22.LWWMap ? Array.from(map.entries()).length : map instanceof import_core22.ORMap ? map.size : 0;
|
|
12638
14498
|
logger.info({
|
|
12639
14499
|
mapName: name,
|
|
12640
14500
|
mapExisted,
|
|
@@ -12644,7 +14504,7 @@ var ServerCoordinator = class {
|
|
|
12644
14504
|
if (loadingPromise) {
|
|
12645
14505
|
logger.info({ mapName: name }, "[getMapAsync] Waiting for loadMapFromStorage...");
|
|
12646
14506
|
await loadingPromise;
|
|
12647
|
-
const newMapSize = map instanceof
|
|
14507
|
+
const newMapSize = map instanceof import_core22.LWWMap ? Array.from(map.entries()).length : map instanceof import_core22.ORMap ? map.size : 0;
|
|
12648
14508
|
logger.info({ mapName: name, mapSizeAfterLoad: newMapSize }, "[getMapAsync] Load completed");
|
|
12649
14509
|
}
|
|
12650
14510
|
return this.maps.get(name);
|
|
@@ -12661,7 +14521,7 @@ var ServerCoordinator = class {
|
|
|
12661
14521
|
const mapName = key.substring(0, separatorIndex);
|
|
12662
14522
|
const actualKey = key.substring(separatorIndex + 1);
|
|
12663
14523
|
const map = this.maps.get(mapName);
|
|
12664
|
-
if (!map || !(map instanceof
|
|
14524
|
+
if (!map || !(map instanceof import_core22.LWWMap)) {
|
|
12665
14525
|
return null;
|
|
12666
14526
|
}
|
|
12667
14527
|
return map.getRecord(actualKey) ?? null;
|
|
@@ -12715,16 +14575,16 @@ var ServerCoordinator = class {
|
|
|
12715
14575
|
const currentMap = this.maps.get(name);
|
|
12716
14576
|
if (!currentMap) return;
|
|
12717
14577
|
let targetMap = currentMap;
|
|
12718
|
-
if (isOR && currentMap instanceof
|
|
14578
|
+
if (isOR && currentMap instanceof import_core22.LWWMap) {
|
|
12719
14579
|
logger.info({ mapName: name }, "Map auto-detected as ORMap. Switching type.");
|
|
12720
|
-
targetMap = new
|
|
14580
|
+
targetMap = new import_core22.ORMap(this.hlc);
|
|
12721
14581
|
this.maps.set(name, targetMap);
|
|
12722
|
-
} else if (!isOR && currentMap instanceof
|
|
14582
|
+
} else if (!isOR && currentMap instanceof import_core22.ORMap && typeHint !== "OR") {
|
|
12723
14583
|
logger.info({ mapName: name }, "Map auto-detected as LWWMap. Switching type.");
|
|
12724
|
-
targetMap = new
|
|
14584
|
+
targetMap = new import_core22.LWWMap(this.hlc);
|
|
12725
14585
|
this.maps.set(name, targetMap);
|
|
12726
14586
|
}
|
|
12727
|
-
if (targetMap instanceof
|
|
14587
|
+
if (targetMap instanceof import_core22.ORMap) {
|
|
12728
14588
|
for (const [key, record] of records) {
|
|
12729
14589
|
if (key === "__tombstones__") {
|
|
12730
14590
|
const t = record;
|
|
@@ -12737,7 +14597,7 @@ var ServerCoordinator = class {
|
|
|
12737
14597
|
}
|
|
12738
14598
|
}
|
|
12739
14599
|
}
|
|
12740
|
-
} else if (targetMap instanceof
|
|
14600
|
+
} else if (targetMap instanceof import_core22.LWWMap) {
|
|
12741
14601
|
for (const [key, record] of records) {
|
|
12742
14602
|
if (!record.type) {
|
|
12743
14603
|
targetMap.merge(key, record);
|
|
@@ -12748,7 +14608,7 @@ var ServerCoordinator = class {
|
|
|
12748
14608
|
if (count > 0) {
|
|
12749
14609
|
logger.info({ mapName: name, count }, "Loaded records for map");
|
|
12750
14610
|
this.queryRegistry.refreshSubscriptions(name, targetMap);
|
|
12751
|
-
const mapSize = targetMap instanceof
|
|
14611
|
+
const mapSize = targetMap instanceof import_core22.ORMap ? targetMap.totalRecords : targetMap.size;
|
|
12752
14612
|
this.metricsService.setMapSize(name, mapSize);
|
|
12753
14613
|
}
|
|
12754
14614
|
} catch (err) {
|
|
@@ -12821,7 +14681,7 @@ var ServerCoordinator = class {
|
|
|
12821
14681
|
idleTime: now - client.lastPingReceived,
|
|
12822
14682
|
timeoutMs: CLIENT_HEARTBEAT_TIMEOUT_MS
|
|
12823
14683
|
}, "Evicting dead client (heartbeat timeout)");
|
|
12824
|
-
if (client.socket.readyState ===
|
|
14684
|
+
if (client.socket.readyState === import_ws5.WebSocket.OPEN) {
|
|
12825
14685
|
client.socket.close(4002, "Heartbeat timeout");
|
|
12826
14686
|
}
|
|
12827
14687
|
}
|
|
@@ -12830,7 +14690,7 @@ var ServerCoordinator = class {
|
|
|
12830
14690
|
reportLocalHlc() {
|
|
12831
14691
|
let minHlc = this.hlc.now();
|
|
12832
14692
|
for (const client of this.clients.values()) {
|
|
12833
|
-
if (
|
|
14693
|
+
if (import_core22.HLC.compare(client.lastActiveHlc, minHlc) < 0) {
|
|
12834
14694
|
minHlc = client.lastActiveHlc;
|
|
12835
14695
|
}
|
|
12836
14696
|
}
|
|
@@ -12851,7 +14711,7 @@ var ServerCoordinator = class {
|
|
|
12851
14711
|
let globalSafe = this.hlc.now();
|
|
12852
14712
|
let initialized = false;
|
|
12853
14713
|
for (const ts of this.gcReports.values()) {
|
|
12854
|
-
if (!initialized ||
|
|
14714
|
+
if (!initialized || import_core22.HLC.compare(ts, globalSafe) < 0) {
|
|
12855
14715
|
globalSafe = ts;
|
|
12856
14716
|
initialized = true;
|
|
12857
14717
|
}
|
|
@@ -12886,7 +14746,7 @@ var ServerCoordinator = class {
|
|
|
12886
14746
|
logger.info({ olderThanMillis: olderThan.millis }, "Performing Garbage Collection");
|
|
12887
14747
|
const now = Date.now();
|
|
12888
14748
|
for (const [name, map] of this.maps) {
|
|
12889
|
-
if (map instanceof
|
|
14749
|
+
if (map instanceof import_core22.LWWMap) {
|
|
12890
14750
|
for (const key of map.allKeys()) {
|
|
12891
14751
|
const record = map.getRecord(key);
|
|
12892
14752
|
if (record && record.value !== null && record.ttlMs) {
|
|
@@ -12919,11 +14779,18 @@ var ServerCoordinator = class {
|
|
|
12919
14779
|
payload: eventPayload,
|
|
12920
14780
|
timestamp: this.hlc.now()
|
|
12921
14781
|
});
|
|
12922
|
-
|
|
12923
|
-
|
|
12924
|
-
|
|
12925
|
-
|
|
12926
|
-
|
|
14782
|
+
this.queryRegistry.processChange(name, map, key, tombstone, record);
|
|
14783
|
+
if (this.replicationPipeline) {
|
|
14784
|
+
const op = {
|
|
14785
|
+
opType: "set",
|
|
14786
|
+
mapName: name,
|
|
14787
|
+
key,
|
|
14788
|
+
record: tombstone
|
|
14789
|
+
};
|
|
14790
|
+
const opId = `ttl:${name}:${key}:${Date.now()}`;
|
|
14791
|
+
this.replicationPipeline.replicate(op, opId, key).catch((err) => {
|
|
14792
|
+
logger.warn({ opId, key, err }, "TTL expiration replication failed (non-fatal)");
|
|
14793
|
+
});
|
|
12927
14794
|
}
|
|
12928
14795
|
}
|
|
12929
14796
|
}
|
|
@@ -12938,7 +14805,7 @@ var ServerCoordinator = class {
|
|
|
12938
14805
|
});
|
|
12939
14806
|
}
|
|
12940
14807
|
}
|
|
12941
|
-
} else if (map instanceof
|
|
14808
|
+
} else if (map instanceof import_core22.ORMap) {
|
|
12942
14809
|
const items = map.items;
|
|
12943
14810
|
const tombstonesSet = map.tombstones;
|
|
12944
14811
|
const tagsToExpire = [];
|
|
@@ -12956,6 +14823,7 @@ var ServerCoordinator = class {
|
|
|
12956
14823
|
}
|
|
12957
14824
|
for (const { key, tag } of tagsToExpire) {
|
|
12958
14825
|
logger.info({ mapName: name, key, tag }, "ORMap Record expired (TTL). Removing.");
|
|
14826
|
+
const oldRecords = map.getRecords(key);
|
|
12959
14827
|
map.applyTombstone(tag);
|
|
12960
14828
|
if (this.storage) {
|
|
12961
14829
|
const records = map.getRecords(key);
|
|
@@ -12981,11 +14849,19 @@ var ServerCoordinator = class {
|
|
|
12981
14849
|
payload: eventPayload,
|
|
12982
14850
|
timestamp: this.hlc.now()
|
|
12983
14851
|
});
|
|
12984
|
-
const
|
|
12985
|
-
|
|
12986
|
-
|
|
12987
|
-
|
|
12988
|
-
|
|
14852
|
+
const newRecords = map.getRecords(key);
|
|
14853
|
+
this.queryRegistry.processChange(name, map, key, newRecords, oldRecords);
|
|
14854
|
+
if (this.replicationPipeline) {
|
|
14855
|
+
const op = {
|
|
14856
|
+
opType: "OR_REMOVE",
|
|
14857
|
+
mapName: name,
|
|
14858
|
+
key,
|
|
14859
|
+
orTag: tag
|
|
14860
|
+
};
|
|
14861
|
+
const opId = `ttl:${name}:${key}:${tag}:${Date.now()}`;
|
|
14862
|
+
this.replicationPipeline.replicate(op, opId, key).catch((err) => {
|
|
14863
|
+
logger.warn({ opId, key, err }, "ORMap TTL expiration replication failed (non-fatal)");
|
|
14864
|
+
});
|
|
12989
14865
|
}
|
|
12990
14866
|
}
|
|
12991
14867
|
const removedTags = map.prune(olderThan);
|
|
@@ -13041,17 +14917,17 @@ var ServerCoordinator = class {
|
|
|
13041
14917
|
stringToWriteConcern(value) {
|
|
13042
14918
|
switch (value) {
|
|
13043
14919
|
case "FIRE_AND_FORGET":
|
|
13044
|
-
return
|
|
14920
|
+
return import_core22.WriteConcern.FIRE_AND_FORGET;
|
|
13045
14921
|
case "MEMORY":
|
|
13046
|
-
return
|
|
14922
|
+
return import_core22.WriteConcern.MEMORY;
|
|
13047
14923
|
case "APPLIED":
|
|
13048
|
-
return
|
|
14924
|
+
return import_core22.WriteConcern.APPLIED;
|
|
13049
14925
|
case "REPLICATED":
|
|
13050
|
-
return
|
|
14926
|
+
return import_core22.WriteConcern.REPLICATED;
|
|
13051
14927
|
case "PERSISTED":
|
|
13052
|
-
return
|
|
14928
|
+
return import_core22.WriteConcern.PERSISTED;
|
|
13053
14929
|
default:
|
|
13054
|
-
return
|
|
14930
|
+
return import_core22.WriteConcern.MEMORY;
|
|
13055
14931
|
}
|
|
13056
14932
|
}
|
|
13057
14933
|
/**
|
|
@@ -13108,7 +14984,7 @@ var ServerCoordinator = class {
|
|
|
13108
14984
|
}
|
|
13109
14985
|
});
|
|
13110
14986
|
if (op.id) {
|
|
13111
|
-
this.writeAckManager.notifyLevel(op.id,
|
|
14987
|
+
this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.REPLICATED);
|
|
13112
14988
|
}
|
|
13113
14989
|
}
|
|
13114
14990
|
}
|
|
@@ -13116,7 +14992,7 @@ var ServerCoordinator = class {
|
|
|
13116
14992
|
this.broadcastBatch(batchedEvents, clientId);
|
|
13117
14993
|
for (const op of ops) {
|
|
13118
14994
|
if (op.id && this.partitionService.isLocalOwner(op.key)) {
|
|
13119
|
-
this.writeAckManager.notifyLevel(op.id,
|
|
14995
|
+
this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.REPLICATED);
|
|
13120
14996
|
}
|
|
13121
14997
|
}
|
|
13122
14998
|
}
|
|
@@ -13144,7 +15020,7 @@ var ServerCoordinator = class {
|
|
|
13144
15020
|
const owner = this.partitionService.getOwner(op.key);
|
|
13145
15021
|
await this.forwardOpAndWait(op, owner);
|
|
13146
15022
|
if (op.id) {
|
|
13147
|
-
this.writeAckManager.notifyLevel(op.id,
|
|
15023
|
+
this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.REPLICATED);
|
|
13148
15024
|
}
|
|
13149
15025
|
}
|
|
13150
15026
|
}
|
|
@@ -13152,7 +15028,7 @@ var ServerCoordinator = class {
|
|
|
13152
15028
|
await this.broadcastBatchSync(batchedEvents, clientId);
|
|
13153
15029
|
for (const op of ops) {
|
|
13154
15030
|
if (op.id && this.partitionService.isLocalOwner(op.key)) {
|
|
13155
|
-
this.writeAckManager.notifyLevel(op.id,
|
|
15031
|
+
this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.REPLICATED);
|
|
13156
15032
|
}
|
|
13157
15033
|
}
|
|
13158
15034
|
}
|
|
@@ -13186,7 +15062,7 @@ var ServerCoordinator = class {
|
|
|
13186
15062
|
return;
|
|
13187
15063
|
}
|
|
13188
15064
|
if (op.id) {
|
|
13189
|
-
this.writeAckManager.notifyLevel(op.id,
|
|
15065
|
+
this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.APPLIED);
|
|
13190
15066
|
}
|
|
13191
15067
|
if (eventPayload) {
|
|
13192
15068
|
batchedEvents.push({
|
|
@@ -13200,7 +15076,7 @@ var ServerCoordinator = class {
|
|
|
13200
15076
|
try {
|
|
13201
15077
|
await this.persistOpSync(op);
|
|
13202
15078
|
if (op.id) {
|
|
13203
|
-
this.writeAckManager.notifyLevel(op.id,
|
|
15079
|
+
this.writeAckManager.notifyLevel(op.id, import_core22.WriteConcern.PERSISTED);
|
|
13204
15080
|
}
|
|
13205
15081
|
} catch (err) {
|
|
13206
15082
|
logger.error({ opId: op.id, err }, "Persistence failed");
|
|
@@ -13543,10 +15419,10 @@ var RateLimitInterceptor = class {
|
|
|
13543
15419
|
};
|
|
13544
15420
|
|
|
13545
15421
|
// src/utils/nativeStats.ts
|
|
13546
|
-
var
|
|
15422
|
+
var import_core23 = require("@topgunbuild/core");
|
|
13547
15423
|
function getNativeModuleStatus() {
|
|
13548
15424
|
return {
|
|
13549
|
-
nativeHash: (0,
|
|
15425
|
+
nativeHash: (0, import_core23.isUsingNativeHash)(),
|
|
13550
15426
|
sharedArrayBuffer: SharedMemoryManager.isAvailable()
|
|
13551
15427
|
};
|
|
13552
15428
|
}
|
|
@@ -13579,15 +15455,15 @@ function logNativeStatus() {
|
|
|
13579
15455
|
}
|
|
13580
15456
|
|
|
13581
15457
|
// src/cluster/ClusterCoordinator.ts
|
|
13582
|
-
var
|
|
13583
|
-
var
|
|
15458
|
+
var import_events16 = require("events");
|
|
15459
|
+
var import_core24 = require("@topgunbuild/core");
|
|
13584
15460
|
var DEFAULT_CLUSTER_COORDINATOR_CONFIG = {
|
|
13585
15461
|
gradualRebalancing: true,
|
|
13586
|
-
migration:
|
|
13587
|
-
replication:
|
|
15462
|
+
migration: import_core24.DEFAULT_MIGRATION_CONFIG,
|
|
15463
|
+
replication: import_core24.DEFAULT_REPLICATION_CONFIG,
|
|
13588
15464
|
replicationEnabled: true
|
|
13589
15465
|
};
|
|
13590
|
-
var ClusterCoordinator = class extends
|
|
15466
|
+
var ClusterCoordinator = class extends import_events16.EventEmitter {
|
|
13591
15467
|
constructor(config) {
|
|
13592
15468
|
super();
|
|
13593
15469
|
this.replicationPipeline = null;
|
|
@@ -13952,12 +15828,12 @@ var ClusterCoordinator = class extends import_events13.EventEmitter {
|
|
|
13952
15828
|
};
|
|
13953
15829
|
|
|
13954
15830
|
// src/MapWithResolver.ts
|
|
13955
|
-
var
|
|
15831
|
+
var import_core25 = require("@topgunbuild/core");
|
|
13956
15832
|
var MapWithResolver = class {
|
|
13957
15833
|
constructor(config) {
|
|
13958
15834
|
this.mapName = config.name;
|
|
13959
|
-
this.hlc = new
|
|
13960
|
-
this.map = new
|
|
15835
|
+
this.hlc = new import_core25.HLC(config.nodeId);
|
|
15836
|
+
this.map = new import_core25.LWWMap(this.hlc);
|
|
13961
15837
|
this.resolverService = config.resolverService;
|
|
13962
15838
|
this.onRejection = config.onRejection;
|
|
13963
15839
|
}
|
|
@@ -14213,7 +16089,7 @@ function mergeWithDefaults(userConfig) {
|
|
|
14213
16089
|
}
|
|
14214
16090
|
|
|
14215
16091
|
// src/config/MapFactory.ts
|
|
14216
|
-
var
|
|
16092
|
+
var import_core26 = require("@topgunbuild/core");
|
|
14217
16093
|
var MapFactory = class {
|
|
14218
16094
|
/**
|
|
14219
16095
|
* Create a MapFactory.
|
|
@@ -14237,9 +16113,9 @@ var MapFactory = class {
|
|
|
14237
16113
|
createLWWMap(mapName, hlc) {
|
|
14238
16114
|
const mapConfig = this.mapConfigs.get(mapName);
|
|
14239
16115
|
if (!mapConfig || mapConfig.indexes.length === 0) {
|
|
14240
|
-
return new
|
|
16116
|
+
return new import_core26.LWWMap(hlc);
|
|
14241
16117
|
}
|
|
14242
|
-
const map = new
|
|
16118
|
+
const map = new import_core26.IndexedLWWMap(hlc);
|
|
14243
16119
|
for (const indexDef of mapConfig.indexes) {
|
|
14244
16120
|
this.addIndexToLWWMap(map, indexDef);
|
|
14245
16121
|
}
|
|
@@ -14255,9 +16131,9 @@ var MapFactory = class {
|
|
|
14255
16131
|
createORMap(mapName, hlc) {
|
|
14256
16132
|
const mapConfig = this.mapConfigs.get(mapName);
|
|
14257
16133
|
if (!mapConfig || mapConfig.indexes.length === 0) {
|
|
14258
|
-
return new
|
|
16134
|
+
return new import_core26.ORMap(hlc);
|
|
14259
16135
|
}
|
|
14260
|
-
const map = new
|
|
16136
|
+
const map = new import_core26.IndexedORMap(hlc);
|
|
14261
16137
|
for (const indexDef of mapConfig.indexes) {
|
|
14262
16138
|
this.addIndexToORMap(map, indexDef);
|
|
14263
16139
|
}
|
|
@@ -14294,7 +16170,7 @@ var MapFactory = class {
|
|
|
14294
16170
|
* Supports dot notation for nested paths.
|
|
14295
16171
|
*/
|
|
14296
16172
|
createAttribute(path) {
|
|
14297
|
-
return (0,
|
|
16173
|
+
return (0, import_core26.simpleAttribute)(path, (record) => {
|
|
14298
16174
|
return this.getNestedValue(record, path);
|
|
14299
16175
|
});
|
|
14300
16176
|
}
|