@topgunbuild/server 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +1569 -1461
- package/dist/index.d.ts +1569 -1461
- package/dist/index.js +2138 -262
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2081 -194
- package/dist/index.mjs.map +1 -1
- package/package.json +6 -4
package/dist/index.mjs
CHANGED
|
@@ -9,13 +9,13 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
|
|
|
9
9
|
import { createServer as createHttpServer } from "http";
|
|
10
10
|
import { createServer as createHttpsServer } from "https";
|
|
11
11
|
import { readFileSync as readFileSync2 } from "fs";
|
|
12
|
-
import { WebSocketServer as WebSocketServer2, WebSocket as
|
|
13
|
-
import { HLC as HLC2, LWWMap as LWWMap3, ORMap as ORMap2, serialize as serialize4, deserialize, MessageSchema, WriteConcern as WriteConcern2, ConsistencyLevel as ConsistencyLevel3, DEFAULT_REPLICATION_CONFIG as DEFAULT_REPLICATION_CONFIG2, IndexedLWWMap as IndexedLWWMap2, IndexedORMap as IndexedORMap2 } from "@topgunbuild/core";
|
|
12
|
+
import { WebSocketServer as WebSocketServer2, WebSocket as WebSocket5 } from "ws";
|
|
13
|
+
import { HLC as HLC2, LWWMap as LWWMap3, ORMap as ORMap2, serialize as serialize4, deserialize, MessageSchema, WriteConcern as WriteConcern2, ConsistencyLevel as ConsistencyLevel3, DEFAULT_REPLICATION_CONFIG as DEFAULT_REPLICATION_CONFIG2, IndexedLWWMap as IndexedLWWMap2, IndexedORMap as IndexedORMap2, QueryCursor as QueryCursor2, DEFAULT_QUERY_CURSOR_MAX_AGE_MS as DEFAULT_QUERY_CURSOR_MAX_AGE_MS2 } from "@topgunbuild/core";
|
|
14
14
|
import * as jwt from "jsonwebtoken";
|
|
15
15
|
import * as crypto from "crypto";
|
|
16
16
|
|
|
17
17
|
// src/query/Matcher.ts
|
|
18
|
-
import { evaluatePredicate } from "@topgunbuild/core";
|
|
18
|
+
import { evaluatePredicate, QueryCursor, DEFAULT_QUERY_CURSOR_MAX_AGE_MS } from "@topgunbuild/core";
|
|
19
19
|
function matchesQuery(record, query) {
|
|
20
20
|
const data = record.value;
|
|
21
21
|
if (!data) return false;
|
|
@@ -64,6 +64,10 @@ function matchesQuery(record, query) {
|
|
|
64
64
|
return true;
|
|
65
65
|
}
|
|
66
66
|
function executeQuery(records, query) {
|
|
67
|
+
const result = executeQueryWithCursor(records, query);
|
|
68
|
+
return result.results;
|
|
69
|
+
}
|
|
70
|
+
function executeQueryWithCursor(records, query) {
|
|
67
71
|
if (!query) {
|
|
68
72
|
query = {};
|
|
69
73
|
}
|
|
@@ -81,9 +85,13 @@ function executeQuery(records, query) {
|
|
|
81
85
|
}
|
|
82
86
|
}
|
|
83
87
|
}
|
|
84
|
-
|
|
88
|
+
const sort = query.sort || {};
|
|
89
|
+
const sortEntries = Object.entries(sort);
|
|
90
|
+
const sortField = sortEntries.length > 0 ? sortEntries[0][0] : "_key";
|
|
91
|
+
const sortDirection = sortEntries.length > 0 ? sortEntries[0][1] : "asc";
|
|
92
|
+
if (sortEntries.length > 0) {
|
|
85
93
|
results.sort((a, b) => {
|
|
86
|
-
for (const [field, direction] of
|
|
94
|
+
for (const [field, direction] of sortEntries) {
|
|
87
95
|
const valA = a.record.value[field];
|
|
88
96
|
const valB = b.record.value[field];
|
|
89
97
|
if (valA < valB) return direction === "asc" ? -1 : 1;
|
|
@@ -92,16 +100,55 @@ function executeQuery(records, query) {
|
|
|
92
100
|
return 0;
|
|
93
101
|
});
|
|
94
102
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const
|
|
98
|
-
|
|
103
|
+
let cursorStatus = "none";
|
|
104
|
+
if (query.cursor) {
|
|
105
|
+
const cursorData = QueryCursor.decode(query.cursor);
|
|
106
|
+
if (!cursorData) {
|
|
107
|
+
cursorStatus = "invalid";
|
|
108
|
+
} else if (!QueryCursor.isValid(cursorData, query.predicate ?? query.where, sort)) {
|
|
109
|
+
if (Date.now() - cursorData.timestamp > DEFAULT_QUERY_CURSOR_MAX_AGE_MS) {
|
|
110
|
+
cursorStatus = "expired";
|
|
111
|
+
} else {
|
|
112
|
+
cursorStatus = "invalid";
|
|
113
|
+
}
|
|
114
|
+
} else {
|
|
115
|
+
cursorStatus = "valid";
|
|
116
|
+
results = results.filter((r) => {
|
|
117
|
+
const sortValue = r.record.value[sortField];
|
|
118
|
+
return QueryCursor.isAfterCursor(
|
|
119
|
+
{ key: r.key, sortValue },
|
|
120
|
+
cursorData
|
|
121
|
+
);
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
const hasLimit = query.limit !== void 0 && query.limit > 0;
|
|
126
|
+
const totalBeforeLimit = results.length;
|
|
127
|
+
if (hasLimit) {
|
|
128
|
+
results = results.slice(0, query.limit);
|
|
129
|
+
}
|
|
130
|
+
const hasMore = hasLimit && totalBeforeLimit > query.limit;
|
|
131
|
+
let nextCursor;
|
|
132
|
+
if (hasMore && results.length > 0) {
|
|
133
|
+
const lastResult = results[results.length - 1];
|
|
134
|
+
const sortValue = lastResult.record.value[sortField];
|
|
135
|
+
nextCursor = QueryCursor.fromLastResult(
|
|
136
|
+
{ key: lastResult.key, sortValue },
|
|
137
|
+
sort,
|
|
138
|
+
query.predicate ?? query.where
|
|
139
|
+
);
|
|
99
140
|
}
|
|
100
|
-
return
|
|
141
|
+
return {
|
|
142
|
+
results: results.map((r) => ({ key: r.key, value: r.record.value })),
|
|
143
|
+
nextCursor,
|
|
144
|
+
hasMore,
|
|
145
|
+
cursorStatus
|
|
146
|
+
};
|
|
101
147
|
}
|
|
102
148
|
|
|
103
149
|
// src/query/QueryRegistry.ts
|
|
104
150
|
import { serialize, IndexedLWWMap } from "@topgunbuild/core";
|
|
151
|
+
import { WebSocket } from "ws";
|
|
105
152
|
|
|
106
153
|
// src/utils/logger.ts
|
|
107
154
|
import pino from "pino";
|
|
@@ -290,6 +337,106 @@ var QueryRegistry = class {
|
|
|
290
337
|
}
|
|
291
338
|
}
|
|
292
339
|
}
|
|
340
|
+
/**
|
|
341
|
+
* Set the ClusterManager for distributed subscriptions.
|
|
342
|
+
*/
|
|
343
|
+
setClusterManager(clusterManager, nodeId) {
|
|
344
|
+
this.clusterManager = clusterManager;
|
|
345
|
+
this.nodeId = nodeId;
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Set the callback for getting maps by name.
|
|
349
|
+
* Required for distributed subscriptions to return initial results.
|
|
350
|
+
*/
|
|
351
|
+
setMapGetter(getter) {
|
|
352
|
+
this.getMap = getter;
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* Register a distributed subscription from a remote coordinator.
|
|
356
|
+
* Called when receiving CLUSTER_SUB_REGISTER message.
|
|
357
|
+
*
|
|
358
|
+
* @param subscriptionId - Unique subscription ID
|
|
359
|
+
* @param mapName - Map name to query
|
|
360
|
+
* @param query - Query predicate
|
|
361
|
+
* @param coordinatorNodeId - Node ID of the coordinator (receives updates)
|
|
362
|
+
* @returns Initial query results from this node
|
|
363
|
+
*/
|
|
364
|
+
registerDistributed(subscriptionId, mapName, query, coordinatorNodeId) {
|
|
365
|
+
const dummySocket = {
|
|
366
|
+
readyState: 1,
|
|
367
|
+
send: () => {
|
|
368
|
+
}
|
|
369
|
+
// Updates go via cluster messages, not socket
|
|
370
|
+
};
|
|
371
|
+
let initialResults = [];
|
|
372
|
+
const previousResultKeys = /* @__PURE__ */ new Set();
|
|
373
|
+
if (this.getMap) {
|
|
374
|
+
const map = this.getMap(mapName);
|
|
375
|
+
if (map) {
|
|
376
|
+
const records = this.getMapRecords(map);
|
|
377
|
+
const queryResults = executeQuery(records, query);
|
|
378
|
+
initialResults = queryResults.map((r) => {
|
|
379
|
+
previousResultKeys.add(r.key);
|
|
380
|
+
return { key: r.key, value: r.value };
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
const sub = {
|
|
385
|
+
id: subscriptionId,
|
|
386
|
+
clientId: `cluster:${coordinatorNodeId}`,
|
|
387
|
+
mapName,
|
|
388
|
+
query,
|
|
389
|
+
socket: dummySocket,
|
|
390
|
+
previousResultKeys,
|
|
391
|
+
coordinatorNodeId,
|
|
392
|
+
isDistributed: true
|
|
393
|
+
};
|
|
394
|
+
this.register(sub);
|
|
395
|
+
logger.debug(
|
|
396
|
+
{ subscriptionId, mapName, coordinatorNodeId, resultCount: initialResults.length },
|
|
397
|
+
"Distributed query subscription registered"
|
|
398
|
+
);
|
|
399
|
+
return initialResults;
|
|
400
|
+
}
|
|
401
|
+
/**
|
|
402
|
+
* Get a distributed subscription by ID.
|
|
403
|
+
* Returns undefined if not found or not distributed.
|
|
404
|
+
*/
|
|
405
|
+
getDistributedSubscription(subscriptionId) {
|
|
406
|
+
for (const subs of this.subscriptions.values()) {
|
|
407
|
+
for (const sub of subs) {
|
|
408
|
+
if (sub.id === subscriptionId && sub.isDistributed) {
|
|
409
|
+
return sub;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
return void 0;
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* Unregister all distributed subscriptions where the given node was the coordinator.
|
|
417
|
+
* Called when a cluster node disconnects.
|
|
418
|
+
*
|
|
419
|
+
* @param coordinatorNodeId - Node ID of the disconnected coordinator
|
|
420
|
+
*/
|
|
421
|
+
unregisterByCoordinator(coordinatorNodeId) {
|
|
422
|
+
const subscriptionsToRemove = [];
|
|
423
|
+
for (const subs of this.subscriptions.values()) {
|
|
424
|
+
for (const sub of subs) {
|
|
425
|
+
if (sub.isDistributed && sub.coordinatorNodeId === coordinatorNodeId) {
|
|
426
|
+
subscriptionsToRemove.push(sub.id);
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
for (const subId of subscriptionsToRemove) {
|
|
431
|
+
this.unregister(subId);
|
|
432
|
+
}
|
|
433
|
+
if (subscriptionsToRemove.length > 0) {
|
|
434
|
+
logger.debug(
|
|
435
|
+
{ coordinatorNodeId, count: subscriptionsToRemove.length },
|
|
436
|
+
"Cleaned up distributed query subscriptions for disconnected coordinator"
|
|
437
|
+
);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
293
440
|
/**
|
|
294
441
|
* Returns all active subscriptions for a specific map.
|
|
295
442
|
* Used for subscription-based event routing to avoid broadcasting to all clients.
|
|
@@ -572,7 +719,9 @@ var QueryRegistry = class {
|
|
|
572
719
|
return record.value;
|
|
573
720
|
}
|
|
574
721
|
sendUpdate(sub, key, value, type) {
|
|
575
|
-
if (sub.
|
|
722
|
+
if (sub.isDistributed && sub.coordinatorNodeId && this.clusterManager) {
|
|
723
|
+
this.sendDistributedUpdate(sub, key, value, type);
|
|
724
|
+
} else if (sub.socket.readyState === WebSocket.OPEN) {
|
|
576
725
|
sub.socket.send(serialize({
|
|
577
726
|
type: "QUERY_UPDATE",
|
|
578
727
|
payload: {
|
|
@@ -584,6 +733,26 @@ var QueryRegistry = class {
|
|
|
584
733
|
}));
|
|
585
734
|
}
|
|
586
735
|
}
|
|
736
|
+
/**
|
|
737
|
+
* Send update to remote coordinator node for a distributed subscription.
|
|
738
|
+
*/
|
|
739
|
+
sendDistributedUpdate(sub, key, value, type) {
|
|
740
|
+
if (!this.clusterManager || !sub.coordinatorNodeId) return;
|
|
741
|
+
const changeType = type === "UPDATE" ? sub.previousResultKeys.has(key) ? "UPDATE" : "ENTER" : "LEAVE";
|
|
742
|
+
const payload = {
|
|
743
|
+
subscriptionId: sub.id,
|
|
744
|
+
sourceNodeId: this.nodeId || "unknown",
|
|
745
|
+
key,
|
|
746
|
+
value,
|
|
747
|
+
changeType,
|
|
748
|
+
timestamp: Date.now()
|
|
749
|
+
};
|
|
750
|
+
this.clusterManager.send(sub.coordinatorNodeId, "CLUSTER_SUB_UPDATE", payload);
|
|
751
|
+
logger.debug(
|
|
752
|
+
{ subscriptionId: sub.id, key, changeType, coordinator: sub.coordinatorNodeId },
|
|
753
|
+
"Sent distributed query update"
|
|
754
|
+
);
|
|
755
|
+
}
|
|
587
756
|
analyzeQueryFields(query) {
|
|
588
757
|
const fields = /* @__PURE__ */ new Set();
|
|
589
758
|
try {
|
|
@@ -722,7 +891,7 @@ var TopicManager = class {
|
|
|
722
891
|
};
|
|
723
892
|
|
|
724
893
|
// src/cluster/ClusterManager.ts
|
|
725
|
-
import { WebSocket, WebSocketServer } from "ws";
|
|
894
|
+
import { WebSocket as WebSocket2, WebSocketServer } from "ws";
|
|
726
895
|
import { EventEmitter as EventEmitter2 } from "events";
|
|
727
896
|
import * as dns from "dns";
|
|
728
897
|
import { readFileSync } from "fs";
|
|
@@ -1115,7 +1284,7 @@ var ClusterManager = class extends EventEmitter2 {
|
|
|
1115
1284
|
sendHeartbeatToAll() {
|
|
1116
1285
|
for (const [nodeId, member] of this.members) {
|
|
1117
1286
|
if (member.isSelf) continue;
|
|
1118
|
-
if (member.socket && member.socket.readyState ===
|
|
1287
|
+
if (member.socket && member.socket.readyState === WebSocket2.OPEN) {
|
|
1119
1288
|
this.send(nodeId, "HEARTBEAT", { timestamp: Date.now() });
|
|
1120
1289
|
}
|
|
1121
1290
|
}
|
|
@@ -1149,7 +1318,7 @@ var ClusterManager = class extends EventEmitter2 {
|
|
|
1149
1318
|
broadcastMemberList() {
|
|
1150
1319
|
for (const [nodeId, member] of this.members) {
|
|
1151
1320
|
if (member.isSelf) continue;
|
|
1152
|
-
if (member.socket && member.socket.readyState ===
|
|
1321
|
+
if (member.socket && member.socket.readyState === WebSocket2.OPEN) {
|
|
1153
1322
|
this.sendMemberList(nodeId);
|
|
1154
1323
|
}
|
|
1155
1324
|
}
|
|
@@ -1174,7 +1343,7 @@ var ClusterManager = class extends EventEmitter2 {
|
|
|
1174
1343
|
const member = this.members.get(nodeId);
|
|
1175
1344
|
if (!member) return;
|
|
1176
1345
|
logger.warn({ nodeId }, "Removing failed node from cluster");
|
|
1177
|
-
if (member.socket && member.socket.readyState !==
|
|
1346
|
+
if (member.socket && member.socket.readyState !== WebSocket2.CLOSED) {
|
|
1178
1347
|
try {
|
|
1179
1348
|
member.socket.terminate();
|
|
1180
1349
|
} catch (e) {
|
|
@@ -1248,9 +1417,9 @@ var ClusterManager = class extends EventEmitter2 {
|
|
|
1248
1417
|
if (this.config.tls.caCertPath) {
|
|
1249
1418
|
wsOptions.ca = readFileSync(this.config.tls.caCertPath);
|
|
1250
1419
|
}
|
|
1251
|
-
ws = new
|
|
1420
|
+
ws = new WebSocket2(`${protocol}${peerAddress}`, wsOptions);
|
|
1252
1421
|
} else {
|
|
1253
|
-
ws = new
|
|
1422
|
+
ws = new WebSocket2(`ws://${peerAddress}`);
|
|
1254
1423
|
}
|
|
1255
1424
|
ws.on("open", () => {
|
|
1256
1425
|
this.pendingConnections.delete(peerAddress);
|
|
@@ -1337,7 +1506,7 @@ var ClusterManager = class extends EventEmitter2 {
|
|
|
1337
1506
|
}
|
|
1338
1507
|
send(nodeId, type, payload) {
|
|
1339
1508
|
const member = this.members.get(nodeId);
|
|
1340
|
-
if (member && member.socket && member.socket.readyState ===
|
|
1509
|
+
if (member && member.socket && member.socket.readyState === WebSocket2.OPEN) {
|
|
1341
1510
|
const msg = {
|
|
1342
1511
|
type,
|
|
1343
1512
|
senderId: this.config.nodeId,
|
|
@@ -2452,7 +2621,7 @@ var SecurityManager = class {
|
|
|
2452
2621
|
};
|
|
2453
2622
|
|
|
2454
2623
|
// src/monitoring/MetricsService.ts
|
|
2455
|
-
import { Registry, Gauge, Counter, Summary, collectDefaultMetrics } from "prom-client";
|
|
2624
|
+
import { Registry, Gauge, Counter, Summary, Histogram, collectDefaultMetrics } from "prom-client";
|
|
2456
2625
|
var MetricsService = class {
|
|
2457
2626
|
constructor() {
|
|
2458
2627
|
this.registry = new Registry();
|
|
@@ -2564,6 +2733,90 @@ var MetricsService = class {
|
|
|
2564
2733
|
help: "Current connection rate per second",
|
|
2565
2734
|
registers: [this.registry]
|
|
2566
2735
|
});
|
|
2736
|
+
this.distributedSearchTotal = new Counter({
|
|
2737
|
+
name: "topgun_distributed_search_total",
|
|
2738
|
+
help: "Total number of distributed search requests",
|
|
2739
|
+
labelNames: ["map", "status"],
|
|
2740
|
+
registers: [this.registry]
|
|
2741
|
+
});
|
|
2742
|
+
this.distributedSearchDuration = new Summary({
|
|
2743
|
+
name: "topgun_distributed_search_duration_ms",
|
|
2744
|
+
help: "Distribution of distributed search execution times in milliseconds",
|
|
2745
|
+
labelNames: ["map"],
|
|
2746
|
+
percentiles: [0.5, 0.9, 0.95, 0.99],
|
|
2747
|
+
registers: [this.registry]
|
|
2748
|
+
});
|
|
2749
|
+
this.distributedSearchFailedNodes = new Counter({
|
|
2750
|
+
name: "topgun_distributed_search_failed_nodes_total",
|
|
2751
|
+
help: "Total number of node failures during distributed search",
|
|
2752
|
+
registers: [this.registry]
|
|
2753
|
+
});
|
|
2754
|
+
this.distributedSearchPartialResults = new Counter({
|
|
2755
|
+
name: "topgun_distributed_search_partial_results_total",
|
|
2756
|
+
help: "Total number of searches that returned partial results due to node failures",
|
|
2757
|
+
registers: [this.registry]
|
|
2758
|
+
});
|
|
2759
|
+
this.distributedSubTotal = new Counter({
|
|
2760
|
+
name: "topgun_distributed_sub_total",
|
|
2761
|
+
help: "Total distributed subscriptions created",
|
|
2762
|
+
labelNames: ["type", "status"],
|
|
2763
|
+
registers: [this.registry]
|
|
2764
|
+
});
|
|
2765
|
+
this.distributedSubUnsubscribeTotal = new Counter({
|
|
2766
|
+
name: "topgun_distributed_sub_unsubscribe_total",
|
|
2767
|
+
help: "Total unsubscriptions from distributed subscriptions",
|
|
2768
|
+
labelNames: ["type"],
|
|
2769
|
+
registers: [this.registry]
|
|
2770
|
+
});
|
|
2771
|
+
this.distributedSubActive = new Gauge({
|
|
2772
|
+
name: "topgun_distributed_sub_active",
|
|
2773
|
+
help: "Currently active distributed subscriptions",
|
|
2774
|
+
labelNames: ["type"],
|
|
2775
|
+
registers: [this.registry]
|
|
2776
|
+
});
|
|
2777
|
+
this.distributedSubPendingAcks = new Gauge({
|
|
2778
|
+
name: "topgun_distributed_sub_pending_acks",
|
|
2779
|
+
help: "Subscriptions waiting for ACKs from cluster nodes",
|
|
2780
|
+
registers: [this.registry]
|
|
2781
|
+
});
|
|
2782
|
+
this.distributedSubUpdates = new Counter({
|
|
2783
|
+
name: "topgun_distributed_sub_updates_total",
|
|
2784
|
+
help: "Delta updates processed for distributed subscriptions",
|
|
2785
|
+
labelNames: ["direction", "change_type"],
|
|
2786
|
+
registers: [this.registry]
|
|
2787
|
+
});
|
|
2788
|
+
this.distributedSubAckTotal = new Counter({
|
|
2789
|
+
name: "topgun_distributed_sub_ack_total",
|
|
2790
|
+
help: "Node ACK responses for distributed subscriptions",
|
|
2791
|
+
labelNames: ["status"],
|
|
2792
|
+
registers: [this.registry]
|
|
2793
|
+
});
|
|
2794
|
+
this.distributedSubNodeDisconnect = new Counter({
|
|
2795
|
+
name: "topgun_distributed_sub_node_disconnect_total",
|
|
2796
|
+
help: "Node disconnects affecting distributed subscriptions",
|
|
2797
|
+
registers: [this.registry]
|
|
2798
|
+
});
|
|
2799
|
+
this.distributedSubRegistrationDuration = new Histogram({
|
|
2800
|
+
name: "topgun_distributed_sub_registration_duration_ms",
|
|
2801
|
+
help: "Time to register subscription on all nodes",
|
|
2802
|
+
labelNames: ["type"],
|
|
2803
|
+
buckets: [10, 50, 100, 250, 500, 1e3, 2500],
|
|
2804
|
+
registers: [this.registry]
|
|
2805
|
+
});
|
|
2806
|
+
this.distributedSubUpdateLatency = new Histogram({
|
|
2807
|
+
name: "topgun_distributed_sub_update_latency_ms",
|
|
2808
|
+
help: "Latency from data change to client notification",
|
|
2809
|
+
labelNames: ["type"],
|
|
2810
|
+
buckets: [1, 5, 10, 25, 50, 100, 250],
|
|
2811
|
+
registers: [this.registry]
|
|
2812
|
+
});
|
|
2813
|
+
this.distributedSubInitialResultsCount = new Histogram({
|
|
2814
|
+
name: "topgun_distributed_sub_initial_results_count",
|
|
2815
|
+
help: "Initial result set size for distributed subscriptions",
|
|
2816
|
+
labelNames: ["type"],
|
|
2817
|
+
buckets: [0, 1, 5, 10, 25, 50, 100],
|
|
2818
|
+
registers: [this.registry]
|
|
2819
|
+
});
|
|
2567
2820
|
}
|
|
2568
2821
|
destroy() {
|
|
2569
2822
|
this.registry.clear();
|
|
@@ -2674,6 +2927,115 @@ var MetricsService = class {
|
|
|
2674
2927
|
setConnectionRatePerSecond(rate) {
|
|
2675
2928
|
this.connectionRatePerSecond.set(rate);
|
|
2676
2929
|
}
|
|
2930
|
+
// === Distributed search metric methods (Phase 14) ===
|
|
2931
|
+
/**
|
|
2932
|
+
* Record a distributed search request.
|
|
2933
|
+
* @param mapName - Name of the map being searched
|
|
2934
|
+
* @param status - 'success', 'partial', or 'error'
|
|
2935
|
+
*/
|
|
2936
|
+
incDistributedSearch(mapName, status) {
|
|
2937
|
+
this.distributedSearchTotal.inc({ map: mapName, status });
|
|
2938
|
+
}
|
|
2939
|
+
/**
|
|
2940
|
+
* Record the duration of a distributed search.
|
|
2941
|
+
* @param mapName - Name of the map being searched
|
|
2942
|
+
* @param durationMs - Duration in milliseconds
|
|
2943
|
+
*/
|
|
2944
|
+
recordDistributedSearchDuration(mapName, durationMs) {
|
|
2945
|
+
this.distributedSearchDuration.observe({ map: mapName }, durationMs);
|
|
2946
|
+
}
|
|
2947
|
+
/**
|
|
2948
|
+
* Increment counter for failed nodes during distributed search.
|
|
2949
|
+
* @param count - Number of nodes that failed (default 1)
|
|
2950
|
+
*/
|
|
2951
|
+
incDistributedSearchFailedNodes(count = 1) {
|
|
2952
|
+
this.distributedSearchFailedNodes.inc(count);
|
|
2953
|
+
}
|
|
2954
|
+
/**
|
|
2955
|
+
* Increment counter for searches returning partial results.
|
|
2956
|
+
*/
|
|
2957
|
+
incDistributedSearchPartialResults() {
|
|
2958
|
+
this.distributedSearchPartialResults.inc();
|
|
2959
|
+
}
|
|
2960
|
+
// === Distributed subscription metric methods (Phase 14.2) ===
|
|
2961
|
+
/**
|
|
2962
|
+
* Record a distributed subscription creation.
|
|
2963
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
2964
|
+
* @param status - Result status (success, failed, timeout)
|
|
2965
|
+
*/
|
|
2966
|
+
incDistributedSub(type, status) {
|
|
2967
|
+
this.distributedSubTotal.inc({ type, status });
|
|
2968
|
+
if (status === "success") {
|
|
2969
|
+
this.distributedSubActive.inc({ type });
|
|
2970
|
+
}
|
|
2971
|
+
}
|
|
2972
|
+
/**
|
|
2973
|
+
* Record a distributed subscription unsubscription.
|
|
2974
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
2975
|
+
*/
|
|
2976
|
+
incDistributedSubUnsubscribe(type) {
|
|
2977
|
+
this.distributedSubUnsubscribeTotal.inc({ type });
|
|
2978
|
+
}
|
|
2979
|
+
/**
|
|
2980
|
+
* Decrement the active distributed subscriptions gauge.
|
|
2981
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
2982
|
+
*/
|
|
2983
|
+
decDistributedSubActive(type) {
|
|
2984
|
+
this.distributedSubActive.dec({ type });
|
|
2985
|
+
}
|
|
2986
|
+
/**
|
|
2987
|
+
* Set the number of subscriptions waiting for ACKs.
|
|
2988
|
+
* @param count - Number of pending ACKs
|
|
2989
|
+
*/
|
|
2990
|
+
setDistributedSubPendingAcks(count) {
|
|
2991
|
+
this.distributedSubPendingAcks.set(count);
|
|
2992
|
+
}
|
|
2993
|
+
/**
|
|
2994
|
+
* Record a delta update for distributed subscriptions.
|
|
2995
|
+
* @param direction - Direction of update (sent or received)
|
|
2996
|
+
* @param changeType - Type of change (ENTER, UPDATE, LEAVE)
|
|
2997
|
+
*/
|
|
2998
|
+
incDistributedSubUpdates(direction, changeType) {
|
|
2999
|
+
this.distributedSubUpdates.inc({ direction, change_type: changeType });
|
|
3000
|
+
}
|
|
3001
|
+
/**
|
|
3002
|
+
* Record node ACK responses.
|
|
3003
|
+
* @param status - ACK status (success, failed, timeout)
|
|
3004
|
+
* @param count - Number of ACKs to record (default 1)
|
|
3005
|
+
*/
|
|
3006
|
+
incDistributedSubAck(status, count = 1) {
|
|
3007
|
+
this.distributedSubAckTotal.inc({ status }, count);
|
|
3008
|
+
}
|
|
3009
|
+
/**
|
|
3010
|
+
* Record a node disconnect affecting distributed subscriptions.
|
|
3011
|
+
*/
|
|
3012
|
+
incDistributedSubNodeDisconnect() {
|
|
3013
|
+
this.distributedSubNodeDisconnect.inc();
|
|
3014
|
+
}
|
|
3015
|
+
/**
|
|
3016
|
+
* Record the time to register a subscription on all nodes.
|
|
3017
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
3018
|
+
* @param durationMs - Duration in milliseconds
|
|
3019
|
+
*/
|
|
3020
|
+
recordDistributedSubRegistration(type, durationMs) {
|
|
3021
|
+
this.distributedSubRegistrationDuration.observe({ type }, durationMs);
|
|
3022
|
+
}
|
|
3023
|
+
/**
|
|
3024
|
+
* Record the latency from data change to client notification.
|
|
3025
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
3026
|
+
* @param latencyMs - Latency in milliseconds
|
|
3027
|
+
*/
|
|
3028
|
+
recordDistributedSubUpdateLatency(type, latencyMs) {
|
|
3029
|
+
this.distributedSubUpdateLatency.observe({ type }, latencyMs);
|
|
3030
|
+
}
|
|
3031
|
+
/**
|
|
3032
|
+
* Record the initial result set size for a subscription.
|
|
3033
|
+
* @param type - Subscription type (SEARCH or QUERY)
|
|
3034
|
+
* @param count - Number of initial results
|
|
3035
|
+
*/
|
|
3036
|
+
recordDistributedSubInitialResultsCount(type, count) {
|
|
3037
|
+
this.distributedSubInitialResultsCount.observe({ type }, count);
|
|
3038
|
+
}
|
|
2677
3039
|
async getMetrics() {
|
|
2678
3040
|
return this.registry.metrics();
|
|
2679
3041
|
}
|
|
@@ -3204,7 +3566,7 @@ var BackpressureRegulator = class {
|
|
|
3204
3566
|
};
|
|
3205
3567
|
|
|
3206
3568
|
// src/utils/CoalescingWriter.ts
|
|
3207
|
-
import { WebSocket as
|
|
3569
|
+
import { WebSocket as WebSocket3 } from "ws";
|
|
3208
3570
|
import { serialize as serialize3 } from "@topgunbuild/core";
|
|
3209
3571
|
|
|
3210
3572
|
// src/memory/BufferPool.ts
|
|
@@ -3774,7 +4136,7 @@ var CoalescingWriter = class {
|
|
|
3774
4136
|
}
|
|
3775
4137
|
this.state = 2 /* FLUSHING */;
|
|
3776
4138
|
try {
|
|
3777
|
-
if (this.socket.readyState !==
|
|
4139
|
+
if (this.socket.readyState !== WebSocket3.OPEN) {
|
|
3778
4140
|
this.queue = [];
|
|
3779
4141
|
this.pendingBytes = 0;
|
|
3780
4142
|
this.state = 0 /* IDLE */;
|
|
@@ -3847,7 +4209,7 @@ var CoalescingWriter = class {
|
|
|
3847
4209
|
* Send a message immediately without batching.
|
|
3848
4210
|
*/
|
|
3849
4211
|
sendImmediate(data) {
|
|
3850
|
-
if (this.socket.readyState !==
|
|
4212
|
+
if (this.socket.readyState !== WebSocket3.OPEN) {
|
|
3851
4213
|
return;
|
|
3852
4214
|
}
|
|
3853
4215
|
try {
|
|
@@ -7437,7 +7799,7 @@ var RepairScheduler = class extends EventEmitter12 {
|
|
|
7437
7799
|
this.processTimer = setInterval(() => {
|
|
7438
7800
|
this.processRepairQueue();
|
|
7439
7801
|
}, 1e3);
|
|
7440
|
-
setTimeout(() => {
|
|
7802
|
+
this.initialScanTimer = setTimeout(() => {
|
|
7441
7803
|
this.scheduleFullScan();
|
|
7442
7804
|
}, 6e4);
|
|
7443
7805
|
}
|
|
@@ -7455,9 +7817,13 @@ var RepairScheduler = class extends EventEmitter12 {
|
|
|
7455
7817
|
clearInterval(this.processTimer);
|
|
7456
7818
|
this.processTimer = void 0;
|
|
7457
7819
|
}
|
|
7820
|
+
if (this.initialScanTimer) {
|
|
7821
|
+
clearTimeout(this.initialScanTimer);
|
|
7822
|
+
this.initialScanTimer = void 0;
|
|
7823
|
+
}
|
|
7458
7824
|
this.repairQueue = [];
|
|
7459
7825
|
this.activeRepairs.clear();
|
|
7460
|
-
for (const [
|
|
7826
|
+
for (const [, req] of this.pendingRequests) {
|
|
7461
7827
|
clearTimeout(req.timer);
|
|
7462
7828
|
req.reject(new Error("Scheduler stopped"));
|
|
7463
7829
|
}
|
|
@@ -9188,11 +9554,13 @@ var EventJournalService = class extends EventJournalImpl {
|
|
|
9188
9554
|
};
|
|
9189
9555
|
|
|
9190
9556
|
// src/search/SearchCoordinator.ts
|
|
9557
|
+
import { EventEmitter as EventEmitter13 } from "events";
|
|
9191
9558
|
import {
|
|
9192
9559
|
FullTextIndex
|
|
9193
9560
|
} from "@topgunbuild/core";
|
|
9194
|
-
var SearchCoordinator = class {
|
|
9561
|
+
var SearchCoordinator = class extends EventEmitter13 {
|
|
9195
9562
|
constructor() {
|
|
9563
|
+
super();
|
|
9196
9564
|
/** Map name → FullTextIndex */
|
|
9197
9565
|
this.indexes = /* @__PURE__ */ new Map();
|
|
9198
9566
|
/** Map name → FullTextIndexConfig (for reference) */
|
|
@@ -9217,6 +9585,13 @@ var SearchCoordinator = class {
|
|
|
9217
9585
|
this.BATCH_INTERVAL = 16;
|
|
9218
9586
|
logger.debug("SearchCoordinator initialized");
|
|
9219
9587
|
}
|
|
9588
|
+
/**
|
|
9589
|
+
* Set the node ID for this server.
|
|
9590
|
+
* Required for distributed subscriptions.
|
|
9591
|
+
*/
|
|
9592
|
+
setNodeId(nodeId) {
|
|
9593
|
+
this.nodeId = nodeId;
|
|
9594
|
+
}
|
|
9220
9595
|
/**
|
|
9221
9596
|
* Set the callback for sending updates to clients.
|
|
9222
9597
|
* Called by ServerCoordinator during initialization.
|
|
@@ -9518,9 +9893,112 @@ var SearchCoordinator = class {
|
|
|
9518
9893
|
getSubscriptionCount() {
|
|
9519
9894
|
return this.subscriptions.size;
|
|
9520
9895
|
}
|
|
9896
|
+
// ============================================
|
|
9897
|
+
// Phase 14.2: Distributed Subscription Methods
|
|
9898
|
+
// ============================================
|
|
9899
|
+
/**
|
|
9900
|
+
* Register a distributed subscription from a remote coordinator.
|
|
9901
|
+
* Called when receiving CLUSTER_SUB_REGISTER message.
|
|
9902
|
+
*
|
|
9903
|
+
* @param subscriptionId - Unique subscription ID
|
|
9904
|
+
* @param mapName - Map name to search
|
|
9905
|
+
* @param query - Search query string
|
|
9906
|
+
* @param options - Search options
|
|
9907
|
+
* @param coordinatorNodeId - Node ID of the coordinator (receives updates)
|
|
9908
|
+
* @returns Initial search results from this node
|
|
9909
|
+
*/
|
|
9910
|
+
registerDistributedSubscription(subscriptionId, mapName, query, options, coordinatorNodeId) {
|
|
9911
|
+
const index = this.indexes.get(mapName);
|
|
9912
|
+
if (!index) {
|
|
9913
|
+
logger.warn({ mapName }, "Distributed subscription for map without FTS enabled");
|
|
9914
|
+
return { results: [], totalHits: 0 };
|
|
9915
|
+
}
|
|
9916
|
+
const queryTerms = index.tokenizeQuery(query);
|
|
9917
|
+
const searchResults = index.search(query, options);
|
|
9918
|
+
const currentResults = /* @__PURE__ */ new Map();
|
|
9919
|
+
const results = [];
|
|
9920
|
+
for (const result of searchResults) {
|
|
9921
|
+
const value = this.getDocumentValue ? this.getDocumentValue(mapName, result.docId) : void 0;
|
|
9922
|
+
currentResults.set(result.docId, {
|
|
9923
|
+
score: result.score,
|
|
9924
|
+
matchedTerms: result.matchedTerms || []
|
|
9925
|
+
});
|
|
9926
|
+
results.push({
|
|
9927
|
+
key: result.docId,
|
|
9928
|
+
value,
|
|
9929
|
+
score: result.score,
|
|
9930
|
+
matchedTerms: result.matchedTerms || []
|
|
9931
|
+
});
|
|
9932
|
+
}
|
|
9933
|
+
const subscription = {
|
|
9934
|
+
id: subscriptionId,
|
|
9935
|
+
clientId: `cluster:${coordinatorNodeId}`,
|
|
9936
|
+
// Virtual client ID
|
|
9937
|
+
mapName,
|
|
9938
|
+
query,
|
|
9939
|
+
queryTerms,
|
|
9940
|
+
options: options || {},
|
|
9941
|
+
currentResults,
|
|
9942
|
+
// Distributed fields
|
|
9943
|
+
coordinatorNodeId,
|
|
9944
|
+
isDistributed: true
|
|
9945
|
+
};
|
|
9946
|
+
this.subscriptions.set(subscriptionId, subscription);
|
|
9947
|
+
if (!this.subscriptionsByMap.has(mapName)) {
|
|
9948
|
+
this.subscriptionsByMap.set(mapName, /* @__PURE__ */ new Set());
|
|
9949
|
+
}
|
|
9950
|
+
this.subscriptionsByMap.get(mapName).add(subscriptionId);
|
|
9951
|
+
if (!this.subscriptionsByClient.has(subscription.clientId)) {
|
|
9952
|
+
this.subscriptionsByClient.set(subscription.clientId, /* @__PURE__ */ new Set());
|
|
9953
|
+
}
|
|
9954
|
+
this.subscriptionsByClient.get(subscription.clientId).add(subscriptionId);
|
|
9955
|
+
logger.debug(
|
|
9956
|
+
{ subscriptionId, mapName, query, coordinatorNodeId, resultCount: results.length },
|
|
9957
|
+
"Distributed search subscription registered"
|
|
9958
|
+
);
|
|
9959
|
+
return {
|
|
9960
|
+
results,
|
|
9961
|
+
totalHits: results.length
|
|
9962
|
+
};
|
|
9963
|
+
}
|
|
9964
|
+
/**
|
|
9965
|
+
* Get a distributed subscription by ID.
|
|
9966
|
+
* Returns undefined if not found or not distributed.
|
|
9967
|
+
*/
|
|
9968
|
+
getDistributedSubscription(subscriptionId) {
|
|
9969
|
+
const sub = this.subscriptions.get(subscriptionId);
|
|
9970
|
+
if (sub?.isDistributed) {
|
|
9971
|
+
return sub;
|
|
9972
|
+
}
|
|
9973
|
+
return void 0;
|
|
9974
|
+
}
|
|
9975
|
+
/**
|
|
9976
|
+
* Unsubscribe all distributed subscriptions where the given node was the coordinator.
|
|
9977
|
+
* Called when a cluster node disconnects.
|
|
9978
|
+
*
|
|
9979
|
+
* @param coordinatorNodeId - Node ID of the disconnected coordinator
|
|
9980
|
+
*/
|
|
9981
|
+
unsubscribeByCoordinator(coordinatorNodeId) {
|
|
9982
|
+
const subscriptionsToRemove = [];
|
|
9983
|
+
for (const [subId, sub] of this.subscriptions) {
|
|
9984
|
+
if (sub.isDistributed && sub.coordinatorNodeId === coordinatorNodeId) {
|
|
9985
|
+
subscriptionsToRemove.push(subId);
|
|
9986
|
+
}
|
|
9987
|
+
}
|
|
9988
|
+
for (const subId of subscriptionsToRemove) {
|
|
9989
|
+
this.unsubscribe(subId);
|
|
9990
|
+
}
|
|
9991
|
+
if (subscriptionsToRemove.length > 0) {
|
|
9992
|
+
logger.debug(
|
|
9993
|
+
{ coordinatorNodeId, count: subscriptionsToRemove.length },
|
|
9994
|
+
"Cleaned up distributed subscriptions for disconnected coordinator"
|
|
9995
|
+
);
|
|
9996
|
+
}
|
|
9997
|
+
}
|
|
9521
9998
|
/**
|
|
9522
9999
|
* Notify subscribers about a document change.
|
|
9523
10000
|
* Computes delta (ENTER/UPDATE/LEAVE) for each affected subscription.
|
|
10001
|
+
* For distributed subscriptions, emits 'distributedUpdate' event instead of sending to client.
|
|
9524
10002
|
*
|
|
9525
10003
|
* @param mapName - Name of the map that changed
|
|
9526
10004
|
* @param key - Document key that changed
|
|
@@ -9528,9 +10006,6 @@ var SearchCoordinator = class {
|
|
|
9528
10006
|
* @param changeType - Type of change
|
|
9529
10007
|
*/
|
|
9530
10008
|
notifySubscribers(mapName, key, value, changeType) {
|
|
9531
|
-
if (!this.sendUpdate) {
|
|
9532
|
-
return;
|
|
9533
|
-
}
|
|
9534
10009
|
const subscriptionIds = this.subscriptionsByMap.get(mapName);
|
|
9535
10010
|
if (!subscriptionIds || subscriptionIds.size === 0) {
|
|
9536
10011
|
return;
|
|
@@ -9571,18 +10046,50 @@ var SearchCoordinator = class {
|
|
|
9571
10046
|
}
|
|
9572
10047
|
logger.debug({ subId, key, wasInResults, isInResults, updateType, newScore }, "Update decision");
|
|
9573
10048
|
if (updateType) {
|
|
9574
|
-
|
|
9575
|
-
sub
|
|
9576
|
-
|
|
9577
|
-
|
|
9578
|
-
|
|
9579
|
-
|
|
9580
|
-
|
|
9581
|
-
|
|
9582
|
-
|
|
10049
|
+
if (sub.isDistributed && sub.coordinatorNodeId) {
|
|
10050
|
+
this.sendDistributedUpdate(sub, key, value, newScore, matchedTerms, updateType);
|
|
10051
|
+
} else if (this.sendUpdate) {
|
|
10052
|
+
this.sendUpdate(
|
|
10053
|
+
sub.clientId,
|
|
10054
|
+
subId,
|
|
10055
|
+
key,
|
|
10056
|
+
value,
|
|
10057
|
+
newScore,
|
|
10058
|
+
matchedTerms,
|
|
10059
|
+
updateType
|
|
10060
|
+
);
|
|
10061
|
+
}
|
|
9583
10062
|
}
|
|
9584
10063
|
}
|
|
9585
10064
|
}
|
|
10065
|
+
/**
|
|
10066
|
+
* Send update to remote coordinator node for a distributed subscription.
|
|
10067
|
+
* Emits 'distributedUpdate' event with ClusterSubUpdatePayload.
|
|
10068
|
+
*
|
|
10069
|
+
* @param sub - The distributed subscription
|
|
10070
|
+
* @param key - Document key
|
|
10071
|
+
* @param value - Document value
|
|
10072
|
+
* @param score - Search score
|
|
10073
|
+
* @param matchedTerms - Matched search terms
|
|
10074
|
+
* @param changeType - Change type (ENTER/UPDATE/LEAVE)
|
|
10075
|
+
*/
|
|
10076
|
+
sendDistributedUpdate(sub, key, value, score, matchedTerms, changeType) {
|
|
10077
|
+
const payload = {
|
|
10078
|
+
subscriptionId: sub.id,
|
|
10079
|
+
sourceNodeId: this.nodeId || "unknown",
|
|
10080
|
+
key,
|
|
10081
|
+
value,
|
|
10082
|
+
score,
|
|
10083
|
+
matchedTerms,
|
|
10084
|
+
changeType,
|
|
10085
|
+
timestamp: Date.now()
|
|
10086
|
+
};
|
|
10087
|
+
this.emit("distributedUpdate", payload);
|
|
10088
|
+
logger.debug(
|
|
10089
|
+
{ subscriptionId: sub.id, key, changeType, coordinator: sub.coordinatorNodeId },
|
|
10090
|
+
"Emitted distributed update"
|
|
10091
|
+
);
|
|
10092
|
+
}
|
|
9586
10093
|
/**
|
|
9587
10094
|
* Score a single document against a subscription's query.
|
|
9588
10095
|
*
|
|
@@ -9737,44 +10244,1223 @@ var SearchCoordinator = class {
|
|
|
9737
10244
|
}
|
|
9738
10245
|
};
|
|
9739
10246
|
|
|
9740
|
-
// src/
|
|
9741
|
-
|
|
9742
|
-
|
|
9743
|
-
|
|
9744
|
-
|
|
9745
|
-
|
|
9746
|
-
|
|
9747
|
-
|
|
9748
|
-
|
|
9749
|
-
|
|
9750
|
-
|
|
9751
|
-
|
|
9752
|
-
|
|
9753
|
-
|
|
9754
|
-
|
|
9755
|
-
|
|
9756
|
-
|
|
9757
|
-
|
|
9758
|
-
this.
|
|
9759
|
-
this.
|
|
9760
|
-
this.
|
|
9761
|
-
this.
|
|
9762
|
-
this.
|
|
9763
|
-
|
|
9764
|
-
|
|
9765
|
-
|
|
9766
|
-
|
|
9767
|
-
|
|
9768
|
-
|
|
9769
|
-
|
|
9770
|
-
|
|
9771
|
-
|
|
9772
|
-
|
|
9773
|
-
|
|
9774
|
-
|
|
9775
|
-
|
|
9776
|
-
|
|
9777
|
-
|
|
10247
|
+
// src/search/ClusterSearchCoordinator.ts
|
|
10248
|
+
import { EventEmitter as EventEmitter14 } from "events";
|
|
10249
|
+
import {
|
|
10250
|
+
ReciprocalRankFusion,
|
|
10251
|
+
SearchCursor,
|
|
10252
|
+
ClusterSearchReqPayloadSchema,
|
|
10253
|
+
ClusterSearchRespPayloadSchema
|
|
10254
|
+
} from "@topgunbuild/core";
|
|
10255
|
+
var DEFAULT_CONFIG6 = {
|
|
10256
|
+
rrfK: 60,
|
|
10257
|
+
defaultTimeoutMs: 5e3,
|
|
10258
|
+
defaultMinResponses: 0
|
|
10259
|
+
};
|
|
10260
|
+
var ClusterSearchCoordinator = class extends EventEmitter14 {
|
|
10261
|
+
constructor(clusterManager, partitionService, localSearchCoordinator, config, metricsService) {
|
|
10262
|
+
super();
|
|
10263
|
+
/** Pending requests awaiting responses */
|
|
10264
|
+
this.pendingRequests = /* @__PURE__ */ new Map();
|
|
10265
|
+
this.clusterManager = clusterManager;
|
|
10266
|
+
this.partitionService = partitionService;
|
|
10267
|
+
this.localSearchCoordinator = localSearchCoordinator;
|
|
10268
|
+
this.config = { ...DEFAULT_CONFIG6, ...config };
|
|
10269
|
+
this.rrf = new ReciprocalRankFusion({ k: this.config.rrfK });
|
|
10270
|
+
this.metricsService = metricsService;
|
|
10271
|
+
this.clusterManager.on("message", this.handleClusterMessage.bind(this));
|
|
10272
|
+
}
|
|
10273
|
+
/**
|
|
10274
|
+
* Execute a distributed search across the cluster.
|
|
10275
|
+
*
|
|
10276
|
+
* @param mapName - Name of the map to search
|
|
10277
|
+
* @param query - Search query text
|
|
10278
|
+
* @param options - Search options
|
|
10279
|
+
* @returns Promise resolving to merged search results
|
|
10280
|
+
*/
|
|
10281
|
+
async search(mapName, query, options = { limit: 10 }) {
|
|
10282
|
+
const startTime = performance.now();
|
|
10283
|
+
const requestId = this.generateRequestId();
|
|
10284
|
+
const timeoutMs = options.timeoutMs ?? this.config.defaultTimeoutMs;
|
|
10285
|
+
const allNodes = new Set(this.clusterManager.getMembers());
|
|
10286
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10287
|
+
if (allNodes.size === 1 && allNodes.has(myNodeId)) {
|
|
10288
|
+
return this.executeLocalSearch(mapName, query, options, startTime);
|
|
10289
|
+
}
|
|
10290
|
+
const perNodeLimit = this.calculatePerNodeLimit(options.limit, options.cursor);
|
|
10291
|
+
let cursorData = null;
|
|
10292
|
+
if (options.cursor) {
|
|
10293
|
+
cursorData = SearchCursor.decode(options.cursor);
|
|
10294
|
+
if (cursorData && !SearchCursor.isValid(cursorData, query)) {
|
|
10295
|
+
cursorData = null;
|
|
10296
|
+
logger.warn({ requestId }, "Invalid or expired cursor, ignoring");
|
|
10297
|
+
}
|
|
10298
|
+
}
|
|
10299
|
+
const promise = new Promise((resolve, reject) => {
|
|
10300
|
+
const timeoutHandle = setTimeout(() => {
|
|
10301
|
+
this.resolvePartialResults(requestId);
|
|
10302
|
+
}, timeoutMs);
|
|
10303
|
+
this.pendingRequests.set(requestId, {
|
|
10304
|
+
resolve,
|
|
10305
|
+
reject,
|
|
10306
|
+
responses: /* @__PURE__ */ new Map(),
|
|
10307
|
+
expectedNodes: allNodes,
|
|
10308
|
+
startTime,
|
|
10309
|
+
timeoutHandle,
|
|
10310
|
+
options,
|
|
10311
|
+
mapName,
|
|
10312
|
+
query
|
|
10313
|
+
});
|
|
10314
|
+
});
|
|
10315
|
+
const payload = {
|
|
10316
|
+
requestId,
|
|
10317
|
+
mapName,
|
|
10318
|
+
query,
|
|
10319
|
+
options: {
|
|
10320
|
+
limit: perNodeLimit,
|
|
10321
|
+
minScore: options.minScore,
|
|
10322
|
+
boost: options.boost,
|
|
10323
|
+
includeMatchedTerms: true,
|
|
10324
|
+
// Add cursor position if available
|
|
10325
|
+
...cursorData ? {
|
|
10326
|
+
afterScore: cursorData.nodeScores[myNodeId],
|
|
10327
|
+
afterKey: cursorData.nodeKeys[myNodeId]
|
|
10328
|
+
} : {}
|
|
10329
|
+
},
|
|
10330
|
+
timeoutMs
|
|
10331
|
+
};
|
|
10332
|
+
for (const nodeId of allNodes) {
|
|
10333
|
+
if (nodeId === myNodeId) {
|
|
10334
|
+
this.executeLocalAndRespond(requestId, mapName, query, perNodeLimit, cursorData);
|
|
10335
|
+
} else {
|
|
10336
|
+
this.clusterManager.send(nodeId, "CLUSTER_SEARCH_REQ", payload);
|
|
10337
|
+
}
|
|
10338
|
+
}
|
|
10339
|
+
return promise;
|
|
10340
|
+
}
|
|
10341
|
+
/**
|
|
10342
|
+
* Handle incoming cluster messages.
|
|
10343
|
+
*/
|
|
10344
|
+
handleClusterMessage(msg) {
|
|
10345
|
+
switch (msg.type) {
|
|
10346
|
+
case "CLUSTER_SEARCH_REQ":
|
|
10347
|
+
this.handleSearchRequest(msg.senderId, msg.payload);
|
|
10348
|
+
break;
|
|
10349
|
+
case "CLUSTER_SEARCH_RESP":
|
|
10350
|
+
this.handleSearchResponse(msg.senderId, msg.payload);
|
|
10351
|
+
break;
|
|
10352
|
+
}
|
|
10353
|
+
}
|
|
10354
|
+
/**
|
|
10355
|
+
* Handle incoming search request from another node.
|
|
10356
|
+
*/
|
|
10357
|
+
async handleSearchRequest(senderId, rawPayload) {
|
|
10358
|
+
const startTime = performance.now();
|
|
10359
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10360
|
+
const parsed = ClusterSearchReqPayloadSchema.safeParse(rawPayload);
|
|
10361
|
+
if (!parsed.success) {
|
|
10362
|
+
logger.warn(
|
|
10363
|
+
{ senderId, error: parsed.error.message },
|
|
10364
|
+
"Invalid cluster search request payload"
|
|
10365
|
+
);
|
|
10366
|
+
this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", {
|
|
10367
|
+
requestId: rawPayload?.requestId ?? "unknown",
|
|
10368
|
+
nodeId: myNodeId,
|
|
10369
|
+
results: [],
|
|
10370
|
+
totalHits: 0,
|
|
10371
|
+
executionTimeMs: performance.now() - startTime,
|
|
10372
|
+
error: "Invalid request payload"
|
|
10373
|
+
});
|
|
10374
|
+
return;
|
|
10375
|
+
}
|
|
10376
|
+
const payload = parsed.data;
|
|
10377
|
+
try {
|
|
10378
|
+
const localResult = this.localSearchCoordinator.search(
|
|
10379
|
+
payload.mapName,
|
|
10380
|
+
payload.query,
|
|
10381
|
+
{
|
|
10382
|
+
limit: payload.options.limit,
|
|
10383
|
+
minScore: payload.options.minScore,
|
|
10384
|
+
boost: payload.options.boost
|
|
10385
|
+
}
|
|
10386
|
+
);
|
|
10387
|
+
let results = localResult.results;
|
|
10388
|
+
if (payload.options.afterScore !== void 0) {
|
|
10389
|
+
results = results.filter((r) => {
|
|
10390
|
+
if (r.score < payload.options.afterScore) {
|
|
10391
|
+
return true;
|
|
10392
|
+
}
|
|
10393
|
+
if (r.score === payload.options.afterScore && payload.options.afterKey) {
|
|
10394
|
+
return r.key > payload.options.afterKey;
|
|
10395
|
+
}
|
|
10396
|
+
return false;
|
|
10397
|
+
});
|
|
10398
|
+
}
|
|
10399
|
+
const response = {
|
|
10400
|
+
requestId: payload.requestId,
|
|
10401
|
+
nodeId: myNodeId,
|
|
10402
|
+
results: results.map((r) => ({
|
|
10403
|
+
key: r.key,
|
|
10404
|
+
value: r.value,
|
|
10405
|
+
score: r.score,
|
|
10406
|
+
matchedTerms: r.matchedTerms
|
|
10407
|
+
})),
|
|
10408
|
+
totalHits: localResult.totalCount ?? results.length,
|
|
10409
|
+
executionTimeMs: performance.now() - startTime
|
|
10410
|
+
};
|
|
10411
|
+
this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", response);
|
|
10412
|
+
} catch (error) {
|
|
10413
|
+
this.clusterManager.send(senderId, "CLUSTER_SEARCH_RESP", {
|
|
10414
|
+
requestId: payload.requestId,
|
|
10415
|
+
nodeId: myNodeId,
|
|
10416
|
+
results: [],
|
|
10417
|
+
totalHits: 0,
|
|
10418
|
+
executionTimeMs: performance.now() - startTime,
|
|
10419
|
+
error: error instanceof Error ? error.message : "Unknown error"
|
|
10420
|
+
});
|
|
10421
|
+
}
|
|
10422
|
+
}
|
|
10423
|
+
/**
|
|
10424
|
+
* Handle search response from a node.
|
|
10425
|
+
*/
|
|
10426
|
+
handleSearchResponse(_senderId, rawPayload) {
|
|
10427
|
+
const parsed = ClusterSearchRespPayloadSchema.safeParse(rawPayload);
|
|
10428
|
+
if (!parsed.success) {
|
|
10429
|
+
logger.warn(
|
|
10430
|
+
{ error: parsed.error.message },
|
|
10431
|
+
"Invalid cluster search response payload"
|
|
10432
|
+
);
|
|
10433
|
+
return;
|
|
10434
|
+
}
|
|
10435
|
+
const payload = parsed.data;
|
|
10436
|
+
const pending = this.pendingRequests.get(payload.requestId);
|
|
10437
|
+
if (!pending) {
|
|
10438
|
+
logger.warn({ requestId: payload.requestId }, "Received response for unknown request");
|
|
10439
|
+
return;
|
|
10440
|
+
}
|
|
10441
|
+
pending.responses.set(payload.nodeId, payload);
|
|
10442
|
+
const minResponses = pending.options.minResponses ?? this.config.defaultMinResponses;
|
|
10443
|
+
const requiredResponses = minResponses > 0 ? minResponses : pending.expectedNodes.size;
|
|
10444
|
+
if (pending.responses.size >= requiredResponses) {
|
|
10445
|
+
clearTimeout(pending.timeoutHandle);
|
|
10446
|
+
this.mergeAndResolve(payload.requestId);
|
|
10447
|
+
}
|
|
10448
|
+
}
|
|
10449
|
+
/**
|
|
10450
|
+
* Merge results from all nodes using RRF and resolve the promise.
|
|
10451
|
+
*/
|
|
10452
|
+
mergeAndResolve(requestId) {
|
|
10453
|
+
const pending = this.pendingRequests.get(requestId);
|
|
10454
|
+
if (!pending) return;
|
|
10455
|
+
const resultSets = [];
|
|
10456
|
+
const respondedNodes = [];
|
|
10457
|
+
const failedNodes = [];
|
|
10458
|
+
let totalHits = 0;
|
|
10459
|
+
const resultsWithNodes = [];
|
|
10460
|
+
for (const [nodeId, response] of pending.responses) {
|
|
10461
|
+
if (response.error) {
|
|
10462
|
+
failedNodes.push(nodeId);
|
|
10463
|
+
logger.warn({ nodeId, error: response.error }, "Node returned error for search");
|
|
10464
|
+
} else {
|
|
10465
|
+
respondedNodes.push(nodeId);
|
|
10466
|
+
totalHits += response.totalHits;
|
|
10467
|
+
const rankedResults = response.results.map((r) => ({
|
|
10468
|
+
docId: r.key,
|
|
10469
|
+
score: r.score,
|
|
10470
|
+
source: nodeId
|
|
10471
|
+
}));
|
|
10472
|
+
resultSets.push(rankedResults);
|
|
10473
|
+
for (const r of response.results) {
|
|
10474
|
+
resultsWithNodes.push({
|
|
10475
|
+
key: r.key,
|
|
10476
|
+
score: r.score,
|
|
10477
|
+
nodeId,
|
|
10478
|
+
value: r.value,
|
|
10479
|
+
matchedTerms: r.matchedTerms || []
|
|
10480
|
+
});
|
|
10481
|
+
}
|
|
10482
|
+
}
|
|
10483
|
+
}
|
|
10484
|
+
for (const nodeId of pending.expectedNodes) {
|
|
10485
|
+
if (!pending.responses.has(nodeId)) {
|
|
10486
|
+
failedNodes.push(nodeId);
|
|
10487
|
+
}
|
|
10488
|
+
}
|
|
10489
|
+
const merged = this.rrf.merge(resultSets);
|
|
10490
|
+
const limit = pending.options.limit;
|
|
10491
|
+
const results = [];
|
|
10492
|
+
const cursorResults = [];
|
|
10493
|
+
for (const mergedResult of merged.slice(0, limit)) {
|
|
10494
|
+
const original = resultsWithNodes.find((r) => r.key === mergedResult.docId);
|
|
10495
|
+
if (original) {
|
|
10496
|
+
results.push({
|
|
10497
|
+
key: original.key,
|
|
10498
|
+
value: original.value,
|
|
10499
|
+
score: mergedResult.score,
|
|
10500
|
+
// Use RRF score
|
|
10501
|
+
matchedTerms: original.matchedTerms
|
|
10502
|
+
});
|
|
10503
|
+
cursorResults.push({
|
|
10504
|
+
key: original.key,
|
|
10505
|
+
score: original.score,
|
|
10506
|
+
// Use original score for cursor
|
|
10507
|
+
nodeId: original.nodeId
|
|
10508
|
+
});
|
|
10509
|
+
}
|
|
10510
|
+
}
|
|
10511
|
+
let nextCursor;
|
|
10512
|
+
if (merged.length > limit && cursorResults.length > 0) {
|
|
10513
|
+
nextCursor = SearchCursor.fromResults(cursorResults, pending.query);
|
|
10514
|
+
}
|
|
10515
|
+
const executionTimeMs = performance.now() - pending.startTime;
|
|
10516
|
+
if (this.metricsService) {
|
|
10517
|
+
const status = failedNodes.length === 0 ? "success" : respondedNodes.length === 0 ? "error" : "partial";
|
|
10518
|
+
this.metricsService.incDistributedSearch(pending.mapName, status);
|
|
10519
|
+
this.metricsService.recordDistributedSearchDuration(pending.mapName, executionTimeMs);
|
|
10520
|
+
if (failedNodes.length > 0) {
|
|
10521
|
+
this.metricsService.incDistributedSearchFailedNodes(failedNodes.length);
|
|
10522
|
+
if (respondedNodes.length > 0) {
|
|
10523
|
+
this.metricsService.incDistributedSearchPartialResults();
|
|
10524
|
+
}
|
|
10525
|
+
}
|
|
10526
|
+
}
|
|
10527
|
+
pending.resolve({
|
|
10528
|
+
results,
|
|
10529
|
+
totalHits,
|
|
10530
|
+
nextCursor,
|
|
10531
|
+
respondedNodes,
|
|
10532
|
+
failedNodes,
|
|
10533
|
+
executionTimeMs
|
|
10534
|
+
});
|
|
10535
|
+
this.pendingRequests.delete(requestId);
|
|
10536
|
+
logger.debug({
|
|
10537
|
+
requestId,
|
|
10538
|
+
mapName: pending.mapName,
|
|
10539
|
+
query: pending.query,
|
|
10540
|
+
resultCount: results.length,
|
|
10541
|
+
totalHits,
|
|
10542
|
+
respondedNodes: respondedNodes.length,
|
|
10543
|
+
failedNodes: failedNodes.length,
|
|
10544
|
+
executionTimeMs
|
|
10545
|
+
}, "Distributed search completed");
|
|
10546
|
+
}
|
|
10547
|
+
/**
|
|
10548
|
+
* Resolve with partial results when timeout occurs.
|
|
10549
|
+
*/
|
|
10550
|
+
resolvePartialResults(requestId) {
|
|
10551
|
+
const pending = this.pendingRequests.get(requestId);
|
|
10552
|
+
if (!pending) return;
|
|
10553
|
+
logger.warn(
|
|
10554
|
+
{
|
|
10555
|
+
requestId,
|
|
10556
|
+
received: pending.responses.size,
|
|
10557
|
+
expected: pending.expectedNodes.size
|
|
10558
|
+
},
|
|
10559
|
+
"Search request timed out, returning partial results"
|
|
10560
|
+
);
|
|
10561
|
+
this.mergeAndResolve(requestId);
|
|
10562
|
+
}
|
|
10563
|
+
/**
|
|
10564
|
+
* Execute local search and add response to pending request.
|
|
10565
|
+
*/
|
|
10566
|
+
async executeLocalAndRespond(requestId, mapName, query, limit, cursorData) {
|
|
10567
|
+
const startTime = performance.now();
|
|
10568
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10569
|
+
const pending = this.pendingRequests.get(requestId);
|
|
10570
|
+
if (!pending) return;
|
|
10571
|
+
try {
|
|
10572
|
+
const localResult = this.localSearchCoordinator.search(mapName, query, {
|
|
10573
|
+
limit,
|
|
10574
|
+
minScore: pending.options.minScore,
|
|
10575
|
+
boost: pending.options.boost
|
|
10576
|
+
});
|
|
10577
|
+
let results = localResult.results;
|
|
10578
|
+
if (cursorData) {
|
|
10579
|
+
const position = SearchCursor.getNodePosition(cursorData, myNodeId);
|
|
10580
|
+
if (position) {
|
|
10581
|
+
results = results.filter((r) => {
|
|
10582
|
+
if (r.score < position.afterScore) {
|
|
10583
|
+
return true;
|
|
10584
|
+
}
|
|
10585
|
+
if (r.score === position.afterScore) {
|
|
10586
|
+
return r.key > position.afterKey;
|
|
10587
|
+
}
|
|
10588
|
+
return false;
|
|
10589
|
+
});
|
|
10590
|
+
}
|
|
10591
|
+
}
|
|
10592
|
+
const response = {
|
|
10593
|
+
requestId,
|
|
10594
|
+
nodeId: myNodeId,
|
|
10595
|
+
results: results.map((r) => ({
|
|
10596
|
+
key: r.key,
|
|
10597
|
+
value: r.value,
|
|
10598
|
+
score: r.score,
|
|
10599
|
+
matchedTerms: r.matchedTerms
|
|
10600
|
+
})),
|
|
10601
|
+
totalHits: localResult.totalCount ?? results.length,
|
|
10602
|
+
executionTimeMs: performance.now() - startTime
|
|
10603
|
+
};
|
|
10604
|
+
this.handleSearchResponse(myNodeId, response);
|
|
10605
|
+
} catch (error) {
|
|
10606
|
+
this.handleSearchResponse(myNodeId, {
|
|
10607
|
+
requestId,
|
|
10608
|
+
nodeId: myNodeId,
|
|
10609
|
+
results: [],
|
|
10610
|
+
totalHits: 0,
|
|
10611
|
+
executionTimeMs: performance.now() - startTime,
|
|
10612
|
+
error: error instanceof Error ? error.message : "Unknown error"
|
|
10613
|
+
});
|
|
10614
|
+
}
|
|
10615
|
+
}
|
|
10616
|
+
/**
|
|
10617
|
+
* Execute search locally only (single-node optimization).
|
|
10618
|
+
*/
|
|
10619
|
+
async executeLocalSearch(mapName, query, options, startTime) {
|
|
10620
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10621
|
+
const localResult = this.localSearchCoordinator.search(mapName, query, {
|
|
10622
|
+
limit: options.limit,
|
|
10623
|
+
minScore: options.minScore,
|
|
10624
|
+
boost: options.boost
|
|
10625
|
+
});
|
|
10626
|
+
let results = localResult.results;
|
|
10627
|
+
if (options.cursor) {
|
|
10628
|
+
const cursorData = SearchCursor.decode(options.cursor);
|
|
10629
|
+
if (cursorData && SearchCursor.isValid(cursorData, query)) {
|
|
10630
|
+
const position = SearchCursor.getNodePosition(cursorData, myNodeId);
|
|
10631
|
+
if (position) {
|
|
10632
|
+
results = results.filter((r) => {
|
|
10633
|
+
if (r.score < position.afterScore) {
|
|
10634
|
+
return true;
|
|
10635
|
+
}
|
|
10636
|
+
if (r.score === position.afterScore) {
|
|
10637
|
+
return r.key > position.afterKey;
|
|
10638
|
+
}
|
|
10639
|
+
return false;
|
|
10640
|
+
});
|
|
10641
|
+
}
|
|
10642
|
+
}
|
|
10643
|
+
}
|
|
10644
|
+
results = results.slice(0, options.limit);
|
|
10645
|
+
const totalCount = localResult.totalCount ?? localResult.results.length;
|
|
10646
|
+
let nextCursor;
|
|
10647
|
+
if (totalCount > options.limit && results.length > 0) {
|
|
10648
|
+
const lastResult = results[results.length - 1];
|
|
10649
|
+
nextCursor = SearchCursor.fromResults(
|
|
10650
|
+
[{ key: lastResult.key, score: lastResult.score, nodeId: myNodeId }],
|
|
10651
|
+
query
|
|
10652
|
+
);
|
|
10653
|
+
}
|
|
10654
|
+
const executionTimeMs = performance.now() - startTime;
|
|
10655
|
+
if (this.metricsService) {
|
|
10656
|
+
this.metricsService.incDistributedSearch(mapName, "success");
|
|
10657
|
+
this.metricsService.recordDistributedSearchDuration(mapName, executionTimeMs);
|
|
10658
|
+
}
|
|
10659
|
+
return {
|
|
10660
|
+
results,
|
|
10661
|
+
totalHits: localResult.totalCount ?? results.length,
|
|
10662
|
+
nextCursor,
|
|
10663
|
+
respondedNodes: [myNodeId],
|
|
10664
|
+
failedNodes: [],
|
|
10665
|
+
executionTimeMs
|
|
10666
|
+
};
|
|
10667
|
+
}
|
|
10668
|
+
/**
|
|
10669
|
+
* Calculate per-node limit for distributed query.
|
|
10670
|
+
*
|
|
10671
|
+
* For quality RRF merge, each node should return more than the final limit.
|
|
10672
|
+
* Rule of thumb: 2x limit for good merge quality.
|
|
10673
|
+
*/
|
|
10674
|
+
calculatePerNodeLimit(limit, cursor) {
|
|
10675
|
+
if (cursor) {
|
|
10676
|
+
return limit;
|
|
10677
|
+
}
|
|
10678
|
+
return Math.min(limit * 2, 1e3);
|
|
10679
|
+
}
|
|
10680
|
+
/**
|
|
10681
|
+
* Generate unique request ID.
|
|
10682
|
+
*/
|
|
10683
|
+
generateRequestId() {
|
|
10684
|
+
return `search-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`;
|
|
10685
|
+
}
|
|
10686
|
+
/**
|
|
10687
|
+
* Get RRF constant k.
|
|
10688
|
+
*/
|
|
10689
|
+
getRrfK() {
|
|
10690
|
+
return this.config.rrfK;
|
|
10691
|
+
}
|
|
10692
|
+
/**
|
|
10693
|
+
* Clean up resources.
|
|
10694
|
+
*/
|
|
10695
|
+
destroy() {
|
|
10696
|
+
for (const [requestId, pending] of this.pendingRequests) {
|
|
10697
|
+
clearTimeout(pending.timeoutHandle);
|
|
10698
|
+
pending.reject(new Error("ClusterSearchCoordinator destroyed"));
|
|
10699
|
+
}
|
|
10700
|
+
this.pendingRequests.clear();
|
|
10701
|
+
this.removeAllListeners();
|
|
10702
|
+
}
|
|
10703
|
+
};
|
|
10704
|
+
|
|
10705
|
+
// src/subscriptions/DistributedSubscriptionCoordinator.ts
|
|
10706
|
+
import { EventEmitter as EventEmitter15 } from "events";
|
|
10707
|
+
import {
|
|
10708
|
+
ReciprocalRankFusion as ReciprocalRankFusion2,
|
|
10709
|
+
ClusterSubRegisterPayloadSchema,
|
|
10710
|
+
ClusterSubAckPayloadSchema,
|
|
10711
|
+
ClusterSubUpdatePayloadSchema,
|
|
10712
|
+
ClusterSubUnregisterPayloadSchema
|
|
10713
|
+
} from "@topgunbuild/core";
|
|
10714
|
+
import { WebSocket as WebSocket4 } from "ws";
|
|
10715
|
+
var DEFAULT_CONFIG7 = {
|
|
10716
|
+
ackTimeoutMs: 5e3,
|
|
10717
|
+
rrfK: 60
|
|
10718
|
+
};
|
|
10719
|
+
var DistributedSubscriptionCoordinator = class extends EventEmitter15 {
|
|
10720
|
+
constructor(clusterManager, queryRegistry, searchCoordinator, config, metricsService) {
|
|
10721
|
+
super();
|
|
10722
|
+
/**
|
|
10723
|
+
* Active subscriptions where this node is coordinator.
|
|
10724
|
+
* subscriptionId → SubscriptionState
|
|
10725
|
+
*/
|
|
10726
|
+
this.subscriptions = /* @__PURE__ */ new Map();
|
|
10727
|
+
/**
|
|
10728
|
+
* Track which nodes have acknowledged subscription registration.
|
|
10729
|
+
* subscriptionId → Set<nodeId>
|
|
10730
|
+
*/
|
|
10731
|
+
this.nodeAcks = /* @__PURE__ */ new Map();
|
|
10732
|
+
/**
|
|
10733
|
+
* Pending ACK promises for subscription registration.
|
|
10734
|
+
* subscriptionId → { resolve, reject, timeout, startTime }
|
|
10735
|
+
*/
|
|
10736
|
+
this.pendingAcks = /* @__PURE__ */ new Map();
|
|
10737
|
+
this.clusterManager = clusterManager;
|
|
10738
|
+
this.localQueryRegistry = queryRegistry;
|
|
10739
|
+
this.localSearchCoordinator = searchCoordinator;
|
|
10740
|
+
this.config = { ...DEFAULT_CONFIG7, ...config };
|
|
10741
|
+
this.rrf = new ReciprocalRankFusion2({ k: this.config.rrfK });
|
|
10742
|
+
this.metricsService = metricsService;
|
|
10743
|
+
this.clusterManager.on("message", this.handleClusterMessage.bind(this));
|
|
10744
|
+
this.localSearchCoordinator.on("distributedUpdate", this.handleLocalSearchUpdate.bind(this));
|
|
10745
|
+
this.clusterManager.on("memberLeft", this.handleMemberLeft.bind(this));
|
|
10746
|
+
logger.debug("DistributedSubscriptionCoordinator initialized");
|
|
10747
|
+
}
|
|
10748
|
+
/**
|
|
10749
|
+
* Create a new distributed search subscription.
|
|
10750
|
+
*
|
|
10751
|
+
* @param subscriptionId - Unique subscription ID
|
|
10752
|
+
* @param clientSocket - Client WebSocket for sending updates
|
|
10753
|
+
* @param mapName - Map name to search
|
|
10754
|
+
* @param query - Search query string
|
|
10755
|
+
* @param options - Search options
|
|
10756
|
+
* @returns Promise resolving to initial results
|
|
10757
|
+
*/
|
|
10758
|
+
async subscribeSearch(subscriptionId, clientSocket, mapName, query, options = {}) {
|
|
10759
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10760
|
+
const allNodes = new Set(this.clusterManager.getMembers());
|
|
10761
|
+
logger.debug(
|
|
10762
|
+
{ subscriptionId, mapName, query, nodes: Array.from(allNodes) },
|
|
10763
|
+
"Creating distributed search subscription"
|
|
10764
|
+
);
|
|
10765
|
+
const subscription = {
|
|
10766
|
+
id: subscriptionId,
|
|
10767
|
+
type: "SEARCH",
|
|
10768
|
+
coordinatorNodeId: myNodeId,
|
|
10769
|
+
clientSocket,
|
|
10770
|
+
mapName,
|
|
10771
|
+
searchQuery: query,
|
|
10772
|
+
searchOptions: options,
|
|
10773
|
+
registeredNodes: /* @__PURE__ */ new Set(),
|
|
10774
|
+
pendingResults: /* @__PURE__ */ new Map(),
|
|
10775
|
+
createdAt: Date.now(),
|
|
10776
|
+
currentResults: /* @__PURE__ */ new Map()
|
|
10777
|
+
};
|
|
10778
|
+
this.subscriptions.set(subscriptionId, subscription);
|
|
10779
|
+
this.nodeAcks.set(subscriptionId, /* @__PURE__ */ new Set());
|
|
10780
|
+
const registerPayload = {
|
|
10781
|
+
subscriptionId,
|
|
10782
|
+
coordinatorNodeId: myNodeId,
|
|
10783
|
+
mapName,
|
|
10784
|
+
type: "SEARCH",
|
|
10785
|
+
searchQuery: query,
|
|
10786
|
+
searchOptions: {
|
|
10787
|
+
limit: options.limit,
|
|
10788
|
+
minScore: options.minScore,
|
|
10789
|
+
boost: options.boost
|
|
10790
|
+
}
|
|
10791
|
+
};
|
|
10792
|
+
const localResult = this.registerLocalSearchSubscription(subscription);
|
|
10793
|
+
this.handleLocalAck(subscriptionId, myNodeId, localResult);
|
|
10794
|
+
for (const nodeId of allNodes) {
|
|
10795
|
+
if (nodeId !== myNodeId) {
|
|
10796
|
+
this.clusterManager.send(nodeId, "CLUSTER_SUB_REGISTER", registerPayload);
|
|
10797
|
+
}
|
|
10798
|
+
}
|
|
10799
|
+
return this.waitForAcks(subscriptionId, allNodes);
|
|
10800
|
+
}
|
|
10801
|
+
/**
|
|
10802
|
+
* Create a new distributed query subscription.
|
|
10803
|
+
*
|
|
10804
|
+
* @param subscriptionId - Unique subscription ID
|
|
10805
|
+
* @param clientSocket - Client WebSocket for sending updates
|
|
10806
|
+
* @param mapName - Map name to query
|
|
10807
|
+
* @param query - Query predicate
|
|
10808
|
+
* @returns Promise resolving to initial results
|
|
10809
|
+
*/
|
|
10810
|
+
async subscribeQuery(subscriptionId, clientSocket, mapName, query) {
|
|
10811
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10812
|
+
const allNodes = new Set(this.clusterManager.getMembers());
|
|
10813
|
+
logger.debug(
|
|
10814
|
+
{ subscriptionId, mapName, nodes: Array.from(allNodes) },
|
|
10815
|
+
"Creating distributed query subscription"
|
|
10816
|
+
);
|
|
10817
|
+
const subscription = {
|
|
10818
|
+
id: subscriptionId,
|
|
10819
|
+
type: "QUERY",
|
|
10820
|
+
coordinatorNodeId: myNodeId,
|
|
10821
|
+
clientSocket,
|
|
10822
|
+
mapName,
|
|
10823
|
+
queryPredicate: query,
|
|
10824
|
+
registeredNodes: /* @__PURE__ */ new Set(),
|
|
10825
|
+
pendingResults: /* @__PURE__ */ new Map(),
|
|
10826
|
+
createdAt: Date.now(),
|
|
10827
|
+
currentResults: /* @__PURE__ */ new Map()
|
|
10828
|
+
};
|
|
10829
|
+
this.subscriptions.set(subscriptionId, subscription);
|
|
10830
|
+
this.nodeAcks.set(subscriptionId, /* @__PURE__ */ new Set());
|
|
10831
|
+
const registerPayload = {
|
|
10832
|
+
subscriptionId,
|
|
10833
|
+
coordinatorNodeId: myNodeId,
|
|
10834
|
+
mapName,
|
|
10835
|
+
type: "QUERY",
|
|
10836
|
+
queryPredicate: query
|
|
10837
|
+
};
|
|
10838
|
+
const localResults = this.registerLocalQuerySubscription(subscription);
|
|
10839
|
+
this.handleLocalAck(subscriptionId, myNodeId, { results: localResults, totalHits: localResults.length });
|
|
10840
|
+
for (const nodeId of allNodes) {
|
|
10841
|
+
if (nodeId !== myNodeId) {
|
|
10842
|
+
this.clusterManager.send(nodeId, "CLUSTER_SUB_REGISTER", registerPayload);
|
|
10843
|
+
}
|
|
10844
|
+
}
|
|
10845
|
+
return this.waitForAcks(subscriptionId, allNodes);
|
|
10846
|
+
}
|
|
10847
|
+
/**
|
|
10848
|
+
* Unsubscribe from a distributed subscription.
|
|
10849
|
+
*
|
|
10850
|
+
* @param subscriptionId - Subscription ID to unsubscribe
|
|
10851
|
+
*/
|
|
10852
|
+
async unsubscribe(subscriptionId) {
|
|
10853
|
+
const subscription = this.subscriptions.get(subscriptionId);
|
|
10854
|
+
if (!subscription) {
|
|
10855
|
+
logger.warn({ subscriptionId }, "Attempt to unsubscribe from unknown subscription");
|
|
10856
|
+
return;
|
|
10857
|
+
}
|
|
10858
|
+
logger.debug({ subscriptionId }, "Unsubscribing from distributed subscription");
|
|
10859
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10860
|
+
const payload = { subscriptionId };
|
|
10861
|
+
for (const nodeId of subscription.registeredNodes) {
|
|
10862
|
+
if (nodeId !== myNodeId) {
|
|
10863
|
+
this.clusterManager.send(nodeId, "CLUSTER_SUB_UNREGISTER", payload);
|
|
10864
|
+
}
|
|
10865
|
+
}
|
|
10866
|
+
this.unregisterLocalSubscription(subscription);
|
|
10867
|
+
this.subscriptions.delete(subscriptionId);
|
|
10868
|
+
this.nodeAcks.delete(subscriptionId);
|
|
10869
|
+
const pendingAck = this.pendingAcks.get(subscriptionId);
|
|
10870
|
+
if (pendingAck) {
|
|
10871
|
+
clearTimeout(pendingAck.timeoutHandle);
|
|
10872
|
+
this.pendingAcks.delete(subscriptionId);
|
|
10873
|
+
}
|
|
10874
|
+
this.metricsService?.incDistributedSubUnsubscribe(subscription.type);
|
|
10875
|
+
this.metricsService?.decDistributedSubActive(subscription.type);
|
|
10876
|
+
this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
|
|
10877
|
+
}
|
|
10878
|
+
/**
|
|
10879
|
+
* Handle client disconnect - unsubscribe all their subscriptions.
|
|
10880
|
+
*/
|
|
10881
|
+
unsubscribeClient(clientSocket) {
|
|
10882
|
+
const subscriptionsToRemove = [];
|
|
10883
|
+
for (const [subId, sub] of this.subscriptions) {
|
|
10884
|
+
if (sub.clientSocket === clientSocket) {
|
|
10885
|
+
subscriptionsToRemove.push(subId);
|
|
10886
|
+
}
|
|
10887
|
+
}
|
|
10888
|
+
for (const subId of subscriptionsToRemove) {
|
|
10889
|
+
this.unsubscribe(subId);
|
|
10890
|
+
}
|
|
10891
|
+
}
|
|
10892
|
+
/**
|
|
10893
|
+
* Get active subscription count.
|
|
10894
|
+
*/
|
|
10895
|
+
getActiveSubscriptionCount() {
|
|
10896
|
+
return this.subscriptions.size;
|
|
10897
|
+
}
|
|
10898
|
+
/**
|
|
10899
|
+
* Handle incoming cluster messages with Zod validation.
|
|
10900
|
+
*/
|
|
10901
|
+
handleClusterMessage(msg) {
|
|
10902
|
+
switch (msg.type) {
|
|
10903
|
+
case "CLUSTER_SUB_REGISTER": {
|
|
10904
|
+
const parsed = ClusterSubRegisterPayloadSchema.safeParse(msg.payload);
|
|
10905
|
+
if (!parsed.success) {
|
|
10906
|
+
logger.warn(
|
|
10907
|
+
{ senderId: msg.senderId, error: parsed.error.message },
|
|
10908
|
+
"Invalid CLUSTER_SUB_REGISTER payload"
|
|
10909
|
+
);
|
|
10910
|
+
return;
|
|
10911
|
+
}
|
|
10912
|
+
this.handleSubRegister(msg.senderId, parsed.data);
|
|
10913
|
+
break;
|
|
10914
|
+
}
|
|
10915
|
+
case "CLUSTER_SUB_ACK": {
|
|
10916
|
+
const parsed = ClusterSubAckPayloadSchema.safeParse(msg.payload);
|
|
10917
|
+
if (!parsed.success) {
|
|
10918
|
+
logger.warn(
|
|
10919
|
+
{ senderId: msg.senderId, error: parsed.error.message },
|
|
10920
|
+
"Invalid CLUSTER_SUB_ACK payload"
|
|
10921
|
+
);
|
|
10922
|
+
return;
|
|
10923
|
+
}
|
|
10924
|
+
this.handleSubAck(msg.senderId, parsed.data);
|
|
10925
|
+
break;
|
|
10926
|
+
}
|
|
10927
|
+
case "CLUSTER_SUB_UPDATE": {
|
|
10928
|
+
const parsed = ClusterSubUpdatePayloadSchema.safeParse(msg.payload);
|
|
10929
|
+
if (!parsed.success) {
|
|
10930
|
+
logger.warn(
|
|
10931
|
+
{ senderId: msg.senderId, error: parsed.error.message },
|
|
10932
|
+
"Invalid CLUSTER_SUB_UPDATE payload"
|
|
10933
|
+
);
|
|
10934
|
+
return;
|
|
10935
|
+
}
|
|
10936
|
+
this.handleSubUpdate(msg.senderId, parsed.data);
|
|
10937
|
+
break;
|
|
10938
|
+
}
|
|
10939
|
+
case "CLUSTER_SUB_UNREGISTER": {
|
|
10940
|
+
const parsed = ClusterSubUnregisterPayloadSchema.safeParse(msg.payload);
|
|
10941
|
+
if (!parsed.success) {
|
|
10942
|
+
logger.warn(
|
|
10943
|
+
{ senderId: msg.senderId, error: parsed.error.message },
|
|
10944
|
+
"Invalid CLUSTER_SUB_UNREGISTER payload"
|
|
10945
|
+
);
|
|
10946
|
+
return;
|
|
10947
|
+
}
|
|
10948
|
+
this.handleSubUnregister(msg.senderId, parsed.data);
|
|
10949
|
+
break;
|
|
10950
|
+
}
|
|
10951
|
+
}
|
|
10952
|
+
}
|
|
10953
|
+
/**
|
|
10954
|
+
* Handle cluster node disconnect - cleanup subscriptions involving this node.
|
|
10955
|
+
*/
|
|
10956
|
+
handleMemberLeft(nodeId) {
|
|
10957
|
+
logger.debug({ nodeId }, "Handling member left for distributed subscriptions");
|
|
10958
|
+
const subscriptionsToRemove = [];
|
|
10959
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10960
|
+
for (const [subId, subscription] of this.subscriptions) {
|
|
10961
|
+
if (subscription.registeredNodes.has(nodeId)) {
|
|
10962
|
+
subscription.registeredNodes.delete(nodeId);
|
|
10963
|
+
for (const [key, result] of subscription.currentResults) {
|
|
10964
|
+
if (result.sourceNode === nodeId) {
|
|
10965
|
+
subscription.currentResults.delete(key);
|
|
10966
|
+
}
|
|
10967
|
+
}
|
|
10968
|
+
logger.debug(
|
|
10969
|
+
{ subscriptionId: subId, nodeId, remainingNodes: subscription.registeredNodes.size },
|
|
10970
|
+
"Removed disconnected node from subscription"
|
|
10971
|
+
);
|
|
10972
|
+
}
|
|
10973
|
+
}
|
|
10974
|
+
for (const [subId, pending] of this.pendingAcks) {
|
|
10975
|
+
const acks = this.nodeAcks.get(subId);
|
|
10976
|
+
if (acks && !acks.has(nodeId)) {
|
|
10977
|
+
acks.add(nodeId);
|
|
10978
|
+
this.checkAcksComplete(subId);
|
|
10979
|
+
}
|
|
10980
|
+
}
|
|
10981
|
+
this.localSearchCoordinator.unsubscribeByCoordinator(nodeId);
|
|
10982
|
+
this.localQueryRegistry.unregisterByCoordinator(nodeId);
|
|
10983
|
+
this.metricsService?.incDistributedSubNodeDisconnect();
|
|
10984
|
+
}
|
|
10985
|
+
/**
|
|
10986
|
+
* Handle CLUSTER_SUB_REGISTER from coordinator (we are a data node).
|
|
10987
|
+
*/
|
|
10988
|
+
handleSubRegister(senderId, payload) {
|
|
10989
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
10990
|
+
logger.debug(
|
|
10991
|
+
{ subscriptionId: payload.subscriptionId, coordinator: payload.coordinatorNodeId },
|
|
10992
|
+
"Received distributed subscription registration"
|
|
10993
|
+
);
|
|
10994
|
+
let ackPayload;
|
|
10995
|
+
try {
|
|
10996
|
+
if (payload.type === "SEARCH") {
|
|
10997
|
+
const result = this.localSearchCoordinator.registerDistributedSubscription(
|
|
10998
|
+
payload.subscriptionId,
|
|
10999
|
+
payload.mapName,
|
|
11000
|
+
payload.searchQuery,
|
|
11001
|
+
payload.searchOptions || {},
|
|
11002
|
+
payload.coordinatorNodeId
|
|
11003
|
+
);
|
|
11004
|
+
ackPayload = {
|
|
11005
|
+
subscriptionId: payload.subscriptionId,
|
|
11006
|
+
nodeId: myNodeId,
|
|
11007
|
+
success: true,
|
|
11008
|
+
initialResults: result.results.map((r) => ({
|
|
11009
|
+
key: r.key,
|
|
11010
|
+
value: r.value,
|
|
11011
|
+
score: r.score,
|
|
11012
|
+
matchedTerms: r.matchedTerms
|
|
11013
|
+
})),
|
|
11014
|
+
totalHits: result.totalHits
|
|
11015
|
+
};
|
|
11016
|
+
} else {
|
|
11017
|
+
const results = this.localQueryRegistry.registerDistributed(
|
|
11018
|
+
payload.subscriptionId,
|
|
11019
|
+
payload.mapName,
|
|
11020
|
+
payload.queryPredicate,
|
|
11021
|
+
payload.coordinatorNodeId
|
|
11022
|
+
);
|
|
11023
|
+
ackPayload = {
|
|
11024
|
+
subscriptionId: payload.subscriptionId,
|
|
11025
|
+
nodeId: myNodeId,
|
|
11026
|
+
success: true,
|
|
11027
|
+
initialResults: results.map((r) => ({
|
|
11028
|
+
key: r.key,
|
|
11029
|
+
value: r.value
|
|
11030
|
+
})),
|
|
11031
|
+
totalHits: results.length
|
|
11032
|
+
};
|
|
11033
|
+
}
|
|
11034
|
+
} catch (error) {
|
|
11035
|
+
logger.error(
|
|
11036
|
+
{ subscriptionId: payload.subscriptionId, error },
|
|
11037
|
+
"Failed to register distributed subscription locally"
|
|
11038
|
+
);
|
|
11039
|
+
ackPayload = {
|
|
11040
|
+
subscriptionId: payload.subscriptionId,
|
|
11041
|
+
nodeId: myNodeId,
|
|
11042
|
+
success: false,
|
|
11043
|
+
error: error instanceof Error ? error.message : "Unknown error"
|
|
11044
|
+
};
|
|
11045
|
+
}
|
|
11046
|
+
this.clusterManager.send(payload.coordinatorNodeId, "CLUSTER_SUB_ACK", ackPayload);
|
|
11047
|
+
}
|
|
11048
|
+
/**
|
|
11049
|
+
* Handle CLUSTER_SUB_ACK from a data node.
|
|
11050
|
+
*/
|
|
11051
|
+
handleSubAck(senderId, payload) {
|
|
11052
|
+
const subscription = this.subscriptions.get(payload.subscriptionId);
|
|
11053
|
+
if (!subscription) {
|
|
11054
|
+
logger.warn(
|
|
11055
|
+
{ subscriptionId: payload.subscriptionId, nodeId: payload.nodeId },
|
|
11056
|
+
"Received ACK for unknown subscription"
|
|
11057
|
+
);
|
|
11058
|
+
return;
|
|
11059
|
+
}
|
|
11060
|
+
const acks = this.nodeAcks.get(payload.subscriptionId);
|
|
11061
|
+
if (!acks) return;
|
|
11062
|
+
logger.debug(
|
|
11063
|
+
{ subscriptionId: payload.subscriptionId, nodeId: payload.nodeId, success: payload.success },
|
|
11064
|
+
"Received subscription ACK"
|
|
11065
|
+
);
|
|
11066
|
+
if (payload.success) {
|
|
11067
|
+
subscription.registeredNodes.add(payload.nodeId);
|
|
11068
|
+
subscription.pendingResults.set(payload.nodeId, payload);
|
|
11069
|
+
}
|
|
11070
|
+
acks.add(payload.nodeId);
|
|
11071
|
+
this.checkAcksComplete(payload.subscriptionId);
|
|
11072
|
+
}
|
|
11073
|
+
/**
|
|
11074
|
+
* Handle CLUSTER_SUB_UPDATE from a data node.
|
|
11075
|
+
*/
|
|
11076
|
+
handleSubUpdate(senderId, payload) {
|
|
11077
|
+
const subscription = this.subscriptions.get(payload.subscriptionId);
|
|
11078
|
+
if (!subscription) {
|
|
11079
|
+
logger.warn(
|
|
11080
|
+
{ subscriptionId: payload.subscriptionId },
|
|
11081
|
+
"Update for unknown subscription"
|
|
11082
|
+
);
|
|
11083
|
+
return;
|
|
11084
|
+
}
|
|
11085
|
+
logger.debug(
|
|
11086
|
+
{ subscriptionId: payload.subscriptionId, key: payload.key, changeType: payload.changeType },
|
|
11087
|
+
"Received subscription update"
|
|
11088
|
+
);
|
|
11089
|
+
if (payload.changeType === "LEAVE") {
|
|
11090
|
+
subscription.currentResults.delete(payload.key);
|
|
11091
|
+
} else {
|
|
11092
|
+
subscription.currentResults.set(payload.key, {
|
|
11093
|
+
value: payload.value,
|
|
11094
|
+
score: payload.score,
|
|
11095
|
+
sourceNode: payload.sourceNodeId
|
|
11096
|
+
});
|
|
11097
|
+
}
|
|
11098
|
+
this.forwardUpdateToClient(subscription, payload);
|
|
11099
|
+
this.metricsService?.incDistributedSubUpdates("received", payload.changeType);
|
|
11100
|
+
if (payload.timestamp) {
|
|
11101
|
+
const latencyMs = Date.now() - payload.timestamp;
|
|
11102
|
+
this.metricsService?.recordDistributedSubUpdateLatency(subscription.type, latencyMs);
|
|
11103
|
+
}
|
|
11104
|
+
}
|
|
11105
|
+
/**
|
|
11106
|
+
* Handle CLUSTER_SUB_UNREGISTER from coordinator.
|
|
11107
|
+
*/
|
|
11108
|
+
handleSubUnregister(senderId, payload) {
|
|
11109
|
+
logger.debug(
|
|
11110
|
+
{ subscriptionId: payload.subscriptionId },
|
|
11111
|
+
"Received subscription unregister request"
|
|
11112
|
+
);
|
|
11113
|
+
this.localSearchCoordinator.unsubscribe(payload.subscriptionId);
|
|
11114
|
+
this.localQueryRegistry.unregister(payload.subscriptionId);
|
|
11115
|
+
}
|
|
11116
|
+
/**
|
|
11117
|
+
* Handle local search update (emitted by SearchCoordinator for distributed subscriptions).
|
|
11118
|
+
*/
|
|
11119
|
+
handleLocalSearchUpdate(payload) {
|
|
11120
|
+
const coordinatorNodeId = this.getCoordinatorForSubscription(payload.subscriptionId);
|
|
11121
|
+
if (!coordinatorNodeId) return;
|
|
11122
|
+
const myNodeId = this.clusterManager.config.nodeId;
|
|
11123
|
+
if (coordinatorNodeId === myNodeId) {
|
|
11124
|
+
this.handleSubUpdate(myNodeId, payload);
|
|
11125
|
+
} else {
|
|
11126
|
+
this.clusterManager.send(coordinatorNodeId, "CLUSTER_SUB_UPDATE", payload);
|
|
11127
|
+
}
|
|
11128
|
+
this.metricsService?.incDistributedSubUpdates("sent", payload.changeType);
|
|
11129
|
+
}
|
|
11130
|
+
/**
|
|
11131
|
+
* Register a local search subscription for a distributed coordinator.
|
|
11132
|
+
*/
|
|
11133
|
+
registerLocalSearchSubscription(subscription) {
|
|
11134
|
+
return this.localSearchCoordinator.registerDistributedSubscription(
|
|
11135
|
+
subscription.id,
|
|
11136
|
+
subscription.mapName,
|
|
11137
|
+
subscription.searchQuery,
|
|
11138
|
+
subscription.searchOptions || {},
|
|
11139
|
+
subscription.coordinatorNodeId
|
|
11140
|
+
);
|
|
11141
|
+
}
|
|
11142
|
+
/**
|
|
11143
|
+
* Register a local query subscription for a distributed coordinator.
|
|
11144
|
+
*/
|
|
11145
|
+
registerLocalQuerySubscription(subscription) {
|
|
11146
|
+
return this.localQueryRegistry.registerDistributed(
|
|
11147
|
+
subscription.id,
|
|
11148
|
+
subscription.mapName,
|
|
11149
|
+
subscription.queryPredicate,
|
|
11150
|
+
subscription.coordinatorNodeId
|
|
11151
|
+
);
|
|
11152
|
+
}
|
|
11153
|
+
/**
|
|
11154
|
+
* Unregister local subscription.
|
|
11155
|
+
*/
|
|
11156
|
+
unregisterLocalSubscription(subscription) {
|
|
11157
|
+
if (subscription.type === "SEARCH") {
|
|
11158
|
+
this.localSearchCoordinator.unsubscribe(subscription.id);
|
|
11159
|
+
} else {
|
|
11160
|
+
this.localQueryRegistry.unregister(subscription.id);
|
|
11161
|
+
}
|
|
11162
|
+
}
|
|
11163
|
+
/**
|
|
11164
|
+
* Handle local ACK (from this node's registration).
|
|
11165
|
+
*/
|
|
11166
|
+
handleLocalAck(subscriptionId, nodeId, result) {
|
|
11167
|
+
const subscription = this.subscriptions.get(subscriptionId);
|
|
11168
|
+
if (!subscription) return;
|
|
11169
|
+
const acks = this.nodeAcks.get(subscriptionId);
|
|
11170
|
+
if (!acks) return;
|
|
11171
|
+
subscription.registeredNodes.add(nodeId);
|
|
11172
|
+
subscription.pendingResults.set(nodeId, {
|
|
11173
|
+
subscriptionId,
|
|
11174
|
+
nodeId,
|
|
11175
|
+
success: true,
|
|
11176
|
+
initialResults: (result.results || []).map((r) => ({
|
|
11177
|
+
key: r.key,
|
|
11178
|
+
value: r.value,
|
|
11179
|
+
score: r.score,
|
|
11180
|
+
matchedTerms: r.matchedTerms
|
|
11181
|
+
})),
|
|
11182
|
+
totalHits: result.totalHits || (result.results?.length ?? 0)
|
|
11183
|
+
});
|
|
11184
|
+
acks.add(nodeId);
|
|
11185
|
+
this.checkAcksComplete(subscriptionId);
|
|
11186
|
+
}
|
|
11187
|
+
/**
|
|
11188
|
+
* Wait for all node ACKs with timeout.
|
|
11189
|
+
*/
|
|
11190
|
+
waitForAcks(subscriptionId, expectedNodes) {
|
|
11191
|
+
return new Promise((resolve, reject) => {
|
|
11192
|
+
const startTime = performance.now();
|
|
11193
|
+
const timeoutHandle = setTimeout(() => {
|
|
11194
|
+
this.resolveWithPartialAcks(subscriptionId);
|
|
11195
|
+
}, this.config.ackTimeoutMs);
|
|
11196
|
+
this.pendingAcks.set(subscriptionId, { resolve, reject, timeoutHandle, startTime });
|
|
11197
|
+
this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
|
|
11198
|
+
this.checkAcksComplete(subscriptionId);
|
|
11199
|
+
});
|
|
11200
|
+
}
|
|
11201
|
+
/**
|
|
11202
|
+
* Check if all ACKs have been received.
|
|
11203
|
+
*/
|
|
11204
|
+
checkAcksComplete(subscriptionId) {
|
|
11205
|
+
const subscription = this.subscriptions.get(subscriptionId);
|
|
11206
|
+
const acks = this.nodeAcks.get(subscriptionId);
|
|
11207
|
+
const pendingAck = this.pendingAcks.get(subscriptionId);
|
|
11208
|
+
if (!subscription || !acks || !pendingAck) return;
|
|
11209
|
+
const allNodes = new Set(this.clusterManager.getMembers());
|
|
11210
|
+
if (acks.size >= allNodes.size) {
|
|
11211
|
+
clearTimeout(pendingAck.timeoutHandle);
|
|
11212
|
+
this.pendingAcks.delete(subscriptionId);
|
|
11213
|
+
const duration = performance.now() - pendingAck.startTime;
|
|
11214
|
+
const result = this.mergeInitialResults(subscription);
|
|
11215
|
+
const hasFailures = result.failedNodes.length > 0;
|
|
11216
|
+
this.recordCompletionMetrics(
|
|
11217
|
+
subscription,
|
|
11218
|
+
result,
|
|
11219
|
+
duration,
|
|
11220
|
+
hasFailures ? "timeout" : "success"
|
|
11221
|
+
);
|
|
11222
|
+
pendingAck.resolve(result);
|
|
11223
|
+
}
|
|
11224
|
+
}
|
|
11225
|
+
/**
|
|
11226
|
+
* Resolve with partial ACKs (on timeout).
|
|
11227
|
+
*/
|
|
11228
|
+
resolveWithPartialAcks(subscriptionId) {
|
|
11229
|
+
const subscription = this.subscriptions.get(subscriptionId);
|
|
11230
|
+
const pendingAck = this.pendingAcks.get(subscriptionId);
|
|
11231
|
+
if (!subscription || !pendingAck) return;
|
|
11232
|
+
this.pendingAcks.delete(subscriptionId);
|
|
11233
|
+
logger.warn(
|
|
11234
|
+
{ subscriptionId, registeredNodes: Array.from(subscription.registeredNodes) },
|
|
11235
|
+
"Subscription ACK timeout, resolving with partial results"
|
|
11236
|
+
);
|
|
11237
|
+
const duration = performance.now() - pendingAck.startTime;
|
|
11238
|
+
const result = this.mergeInitialResults(subscription);
|
|
11239
|
+
this.recordCompletionMetrics(subscription, result, duration, "timeout");
|
|
11240
|
+
pendingAck.resolve(result);
|
|
11241
|
+
}
|
|
11242
|
+
/**
|
|
11243
|
+
* Record metrics when subscription registration completes.
|
|
11244
|
+
*/
|
|
11245
|
+
recordCompletionMetrics(subscription, result, durationMs, status) {
|
|
11246
|
+
this.metricsService?.incDistributedSub(subscription.type, status);
|
|
11247
|
+
this.metricsService?.recordDistributedSubRegistration(subscription.type, durationMs);
|
|
11248
|
+
this.metricsService?.recordDistributedSubInitialResultsCount(subscription.type, result.results.length);
|
|
11249
|
+
this.metricsService?.setDistributedSubPendingAcks(this.pendingAcks.size);
|
|
11250
|
+
this.metricsService?.incDistributedSubAck("success", subscription.registeredNodes.size);
|
|
11251
|
+
this.metricsService?.incDistributedSubAck("timeout", result.failedNodes.length);
|
|
11252
|
+
}
|
|
11253
|
+
/**
|
|
11254
|
+
* Merge initial results from all nodes.
|
|
11255
|
+
*/
|
|
11256
|
+
mergeInitialResults(subscription) {
|
|
11257
|
+
const allNodes = new Set(this.clusterManager.getMembers());
|
|
11258
|
+
const failedNodes = Array.from(allNodes).filter((n) => !subscription.registeredNodes.has(n));
|
|
11259
|
+
if (subscription.type === "SEARCH") {
|
|
11260
|
+
return this.mergeSearchResults(subscription, failedNodes);
|
|
11261
|
+
} else {
|
|
11262
|
+
return this.mergeQueryResults(subscription, failedNodes);
|
|
11263
|
+
}
|
|
11264
|
+
}
|
|
11265
|
+
/**
|
|
11266
|
+
* Merge search results using RRF.
|
|
11267
|
+
*/
|
|
11268
|
+
mergeSearchResults(subscription, failedNodes) {
|
|
11269
|
+
const resultSets = [];
|
|
11270
|
+
const resultDataMap = /* @__PURE__ */ new Map();
|
|
11271
|
+
for (const [nodeId, ack] of subscription.pendingResults) {
|
|
11272
|
+
if (ack.success && ack.initialResults) {
|
|
11273
|
+
const rankedResults = [];
|
|
11274
|
+
for (const r of ack.initialResults) {
|
|
11275
|
+
rankedResults.push({
|
|
11276
|
+
docId: r.key,
|
|
11277
|
+
score: r.score ?? 0,
|
|
11278
|
+
source: nodeId
|
|
11279
|
+
});
|
|
11280
|
+
if (!resultDataMap.has(r.key)) {
|
|
11281
|
+
resultDataMap.set(r.key, {
|
|
11282
|
+
key: r.key,
|
|
11283
|
+
value: r.value,
|
|
11284
|
+
score: r.score ?? 0,
|
|
11285
|
+
matchedTerms: r.matchedTerms,
|
|
11286
|
+
nodeId
|
|
11287
|
+
});
|
|
11288
|
+
}
|
|
11289
|
+
}
|
|
11290
|
+
if (rankedResults.length > 0) {
|
|
11291
|
+
resultSets.push(rankedResults);
|
|
11292
|
+
}
|
|
11293
|
+
}
|
|
11294
|
+
}
|
|
11295
|
+
const mergedResults = this.rrf.merge(resultSets);
|
|
11296
|
+
const limit = subscription.searchOptions?.limit ?? 10;
|
|
11297
|
+
const results = [];
|
|
11298
|
+
for (const merged of mergedResults.slice(0, limit)) {
|
|
11299
|
+
const original = resultDataMap.get(merged.docId);
|
|
11300
|
+
if (original) {
|
|
11301
|
+
results.push({
|
|
11302
|
+
key: original.key,
|
|
11303
|
+
value: original.value,
|
|
11304
|
+
score: merged.score,
|
|
11305
|
+
// Use RRF score
|
|
11306
|
+
matchedTerms: original.matchedTerms
|
|
11307
|
+
});
|
|
11308
|
+
subscription.currentResults.set(original.key, {
|
|
11309
|
+
value: original.value,
|
|
11310
|
+
score: merged.score,
|
|
11311
|
+
sourceNode: original.nodeId
|
|
11312
|
+
});
|
|
11313
|
+
}
|
|
11314
|
+
}
|
|
11315
|
+
let totalHits = 0;
|
|
11316
|
+
for (const ack of subscription.pendingResults.values()) {
|
|
11317
|
+
totalHits += ack.totalHits ?? 0;
|
|
11318
|
+
}
|
|
11319
|
+
return {
|
|
11320
|
+
subscriptionId: subscription.id,
|
|
11321
|
+
results,
|
|
11322
|
+
totalHits,
|
|
11323
|
+
registeredNodes: Array.from(subscription.registeredNodes),
|
|
11324
|
+
failedNodes
|
|
11325
|
+
};
|
|
11326
|
+
}
|
|
11327
|
+
/**
|
|
11328
|
+
* Merge query results (simple dedupe by key).
|
|
11329
|
+
*/
|
|
11330
|
+
mergeQueryResults(subscription, failedNodes) {
|
|
11331
|
+
const resultMap = /* @__PURE__ */ new Map();
|
|
11332
|
+
for (const [nodeId, ack] of subscription.pendingResults) {
|
|
11333
|
+
if (ack.success && ack.initialResults) {
|
|
11334
|
+
for (const result of ack.initialResults) {
|
|
11335
|
+
if (!resultMap.has(result.key)) {
|
|
11336
|
+
resultMap.set(result.key, { key: result.key, value: result.value });
|
|
11337
|
+
subscription.currentResults.set(result.key, {
|
|
11338
|
+
value: result.value,
|
|
11339
|
+
sourceNode: nodeId
|
|
11340
|
+
});
|
|
11341
|
+
}
|
|
11342
|
+
}
|
|
11343
|
+
}
|
|
11344
|
+
}
|
|
11345
|
+
const results = Array.from(resultMap.values());
|
|
11346
|
+
return {
|
|
11347
|
+
subscriptionId: subscription.id,
|
|
11348
|
+
results,
|
|
11349
|
+
totalHits: results.length,
|
|
11350
|
+
registeredNodes: Array.from(subscription.registeredNodes),
|
|
11351
|
+
failedNodes
|
|
11352
|
+
};
|
|
11353
|
+
}
|
|
11354
|
+
/**
|
|
11355
|
+
* Forward update to client WebSocket.
|
|
11356
|
+
*/
|
|
11357
|
+
forwardUpdateToClient(subscription, payload) {
|
|
11358
|
+
if (subscription.clientSocket.readyState !== WebSocket4.OPEN) {
|
|
11359
|
+
logger.warn(
|
|
11360
|
+
{ subscriptionId: subscription.id },
|
|
11361
|
+
"Cannot forward update, client socket not open"
|
|
11362
|
+
);
|
|
11363
|
+
return;
|
|
11364
|
+
}
|
|
11365
|
+
const message = subscription.type === "SEARCH" ? {
|
|
11366
|
+
type: "SEARCH_UPDATE",
|
|
11367
|
+
payload: {
|
|
11368
|
+
subscriptionId: payload.subscriptionId,
|
|
11369
|
+
key: payload.key,
|
|
11370
|
+
value: payload.value,
|
|
11371
|
+
score: payload.score,
|
|
11372
|
+
matchedTerms: payload.matchedTerms,
|
|
11373
|
+
type: payload.changeType
|
|
11374
|
+
}
|
|
11375
|
+
} : {
|
|
11376
|
+
type: "QUERY_UPDATE",
|
|
11377
|
+
payload: {
|
|
11378
|
+
queryId: payload.subscriptionId,
|
|
11379
|
+
key: payload.key,
|
|
11380
|
+
value: payload.value,
|
|
11381
|
+
type: payload.changeType
|
|
11382
|
+
}
|
|
11383
|
+
};
|
|
11384
|
+
try {
|
|
11385
|
+
subscription.clientSocket.send(JSON.stringify(message));
|
|
11386
|
+
} catch (error) {
|
|
11387
|
+
logger.error(
|
|
11388
|
+
{ subscriptionId: subscription.id, error },
|
|
11389
|
+
"Failed to send update to client"
|
|
11390
|
+
);
|
|
11391
|
+
}
|
|
11392
|
+
}
|
|
11393
|
+
/**
|
|
11394
|
+
* Get coordinator node for a subscription.
|
|
11395
|
+
* For remote subscriptions (where we are a data node), this returns the coordinator.
|
|
11396
|
+
*/
|
|
11397
|
+
getCoordinatorForSubscription(subscriptionId) {
|
|
11398
|
+
if (this.subscriptions.has(subscriptionId)) {
|
|
11399
|
+
return this.clusterManager.config.nodeId;
|
|
11400
|
+
}
|
|
11401
|
+
const searchSub = this.localSearchCoordinator.getDistributedSubscription(subscriptionId);
|
|
11402
|
+
if (searchSub?.coordinatorNodeId) {
|
|
11403
|
+
return searchSub.coordinatorNodeId;
|
|
11404
|
+
}
|
|
11405
|
+
const querySub = this.localQueryRegistry.getDistributedSubscription(subscriptionId);
|
|
11406
|
+
if (querySub?.coordinatorNodeId) {
|
|
11407
|
+
return querySub.coordinatorNodeId;
|
|
11408
|
+
}
|
|
11409
|
+
return null;
|
|
11410
|
+
}
|
|
11411
|
+
/**
|
|
11412
|
+
* Cleanup on destroy.
|
|
11413
|
+
*/
|
|
11414
|
+
destroy() {
|
|
11415
|
+
for (const subscriptionId of this.subscriptions.keys()) {
|
|
11416
|
+
this.unsubscribe(subscriptionId);
|
|
11417
|
+
}
|
|
11418
|
+
for (const pending of this.pendingAcks.values()) {
|
|
11419
|
+
clearTimeout(pending.timeoutHandle);
|
|
11420
|
+
}
|
|
11421
|
+
this.pendingAcks.clear();
|
|
11422
|
+
this.removeAllListeners();
|
|
11423
|
+
}
|
|
11424
|
+
};
|
|
11425
|
+
|
|
11426
|
+
// src/ServerCoordinator.ts
|
|
11427
|
+
var GC_INTERVAL_MS = 60 * 60 * 1e3;
|
|
11428
|
+
var GC_AGE_MS = 30 * 24 * 60 * 60 * 1e3;
|
|
11429
|
+
var CLIENT_HEARTBEAT_TIMEOUT_MS = 2e4;
|
|
11430
|
+
var CLIENT_HEARTBEAT_CHECK_INTERVAL_MS = 5e3;
|
|
11431
|
+
var ServerCoordinator = class {
|
|
11432
|
+
constructor(config) {
|
|
11433
|
+
this.clients = /* @__PURE__ */ new Map();
|
|
11434
|
+
// Interceptors
|
|
11435
|
+
this.interceptors = [];
|
|
11436
|
+
// In-memory storage (partitioned later)
|
|
11437
|
+
this.maps = /* @__PURE__ */ new Map();
|
|
11438
|
+
this.pendingClusterQueries = /* @__PURE__ */ new Map();
|
|
11439
|
+
// GC Consensus State
|
|
11440
|
+
this.gcReports = /* @__PURE__ */ new Map();
|
|
11441
|
+
// Track map loading state to avoid returning empty results during async load
|
|
11442
|
+
this.mapLoadingPromises = /* @__PURE__ */ new Map();
|
|
11443
|
+
// Track pending batch operations for testing purposes
|
|
11444
|
+
this.pendingBatchOperations = /* @__PURE__ */ new Set();
|
|
11445
|
+
this.journalSubscriptions = /* @__PURE__ */ new Map();
|
|
11446
|
+
this._actualPort = 0;
|
|
11447
|
+
this._actualClusterPort = 0;
|
|
11448
|
+
this._readyPromise = new Promise((resolve) => {
|
|
11449
|
+
this._readyResolve = resolve;
|
|
11450
|
+
});
|
|
11451
|
+
this._nodeId = config.nodeId;
|
|
11452
|
+
this.hlc = new HLC2(config.nodeId);
|
|
11453
|
+
this.storage = config.storage;
|
|
11454
|
+
const rawSecret = config.jwtSecret || process.env.JWT_SECRET || "topgun-secret-dev";
|
|
11455
|
+
this.jwtSecret = rawSecret.replace(/\\n/g, "\n");
|
|
11456
|
+
this.queryRegistry = new QueryRegistry();
|
|
11457
|
+
this.securityManager = new SecurityManager(config.securityPolicies || []);
|
|
11458
|
+
this.interceptors = config.interceptors || [];
|
|
11459
|
+
this.metricsService = new MetricsService();
|
|
11460
|
+
this.eventExecutor = new StripedEventExecutor({
|
|
11461
|
+
stripeCount: config.eventStripeCount ?? 4,
|
|
11462
|
+
queueCapacity: config.eventQueueCapacity ?? 1e4,
|
|
11463
|
+
name: `${config.nodeId}-event-executor`,
|
|
9778
11464
|
onReject: (task) => {
|
|
9779
11465
|
logger.warn({ nodeId: config.nodeId, key: task.key }, "Event task rejected due to queue capacity");
|
|
9780
11466
|
this.metricsService.incEventQueueRejected();
|
|
@@ -9922,7 +11608,7 @@ var ServerCoordinator = class {
|
|
|
9922
11608
|
cluster: this.cluster,
|
|
9923
11609
|
sendToClient: (clientId, message) => {
|
|
9924
11610
|
const client = this.clients.get(clientId);
|
|
9925
|
-
if (client && client.socket.readyState ===
|
|
11611
|
+
if (client && client.socket.readyState === WebSocket5.OPEN) {
|
|
9926
11612
|
client.writer.write(message);
|
|
9927
11613
|
}
|
|
9928
11614
|
}
|
|
@@ -10022,6 +11708,26 @@ var ServerCoordinator = class {
|
|
|
10022
11708
|
logger.info({ mapName, fields: ftsConfig.fields }, "FTS enabled for map");
|
|
10023
11709
|
}
|
|
10024
11710
|
}
|
|
11711
|
+
this.clusterSearchCoordinator = new ClusterSearchCoordinator(
|
|
11712
|
+
this.cluster,
|
|
11713
|
+
this.partitionService,
|
|
11714
|
+
this.searchCoordinator,
|
|
11715
|
+
config.distributedSearch,
|
|
11716
|
+
this.metricsService
|
|
11717
|
+
);
|
|
11718
|
+
logger.info("ClusterSearchCoordinator initialized for distributed search");
|
|
11719
|
+
this.distributedSubCoordinator = new DistributedSubscriptionCoordinator(
|
|
11720
|
+
this.cluster,
|
|
11721
|
+
this.queryRegistry,
|
|
11722
|
+
this.searchCoordinator,
|
|
11723
|
+
void 0,
|
|
11724
|
+
// Use default config
|
|
11725
|
+
this.metricsService
|
|
11726
|
+
);
|
|
11727
|
+
logger.info("DistributedSubscriptionCoordinator initialized for distributed live subscriptions");
|
|
11728
|
+
this.searchCoordinator.setNodeId(config.nodeId);
|
|
11729
|
+
this.queryRegistry.setClusterManager(this.cluster, config.nodeId);
|
|
11730
|
+
this.queryRegistry.setMapGetter((name) => this.getMap(name));
|
|
10025
11731
|
this.systemManager = new SystemManager(
|
|
10026
11732
|
this.cluster,
|
|
10027
11733
|
this.metricsService,
|
|
@@ -10290,7 +11996,7 @@ var ServerCoordinator = class {
|
|
|
10290
11996
|
const shutdownMsg = serialize4({ type: "SHUTDOWN_PENDING", retryAfter: 5e3 });
|
|
10291
11997
|
for (const client of this.clients.values()) {
|
|
10292
11998
|
try {
|
|
10293
|
-
if (client.socket.readyState ===
|
|
11999
|
+
if (client.socket.readyState === WebSocket5.OPEN) {
|
|
10294
12000
|
client.socket.send(shutdownMsg);
|
|
10295
12001
|
if (client.writer) {
|
|
10296
12002
|
client.writer.close();
|
|
@@ -10353,6 +12059,12 @@ var ServerCoordinator = class {
|
|
|
10353
12059
|
if (this.eventJournalService) {
|
|
10354
12060
|
this.eventJournalService.dispose();
|
|
10355
12061
|
}
|
|
12062
|
+
if (this.clusterSearchCoordinator) {
|
|
12063
|
+
this.clusterSearchCoordinator.destroy();
|
|
12064
|
+
}
|
|
12065
|
+
if (this.distributedSubCoordinator) {
|
|
12066
|
+
this.distributedSubCoordinator.destroy();
|
|
12067
|
+
}
|
|
10356
12068
|
logger.info("Server Coordinator shutdown complete.");
|
|
10357
12069
|
}
|
|
10358
12070
|
async handleConnection(ws) {
|
|
@@ -10461,6 +12173,9 @@ var ServerCoordinator = class {
|
|
|
10461
12173
|
this.topicManager.unsubscribeAll(clientId);
|
|
10462
12174
|
this.counterHandler.unsubscribeAll(clientId);
|
|
10463
12175
|
this.searchCoordinator.unsubscribeClient(clientId);
|
|
12176
|
+
if (this.distributedSubCoordinator && connection) {
|
|
12177
|
+
this.distributedSubCoordinator.unsubscribeClient(connection.socket);
|
|
12178
|
+
}
|
|
10464
12179
|
const members = this.cluster.getMembers();
|
|
10465
12180
|
for (const memberId of members) {
|
|
10466
12181
|
if (!this.cluster.isLocal(memberId)) {
|
|
@@ -10535,70 +12250,115 @@ var ServerCoordinator = class {
|
|
|
10535
12250
|
}
|
|
10536
12251
|
logger.info({ clientId: client.id, mapName, query }, "Client subscribed");
|
|
10537
12252
|
this.metricsService.incOp("SUBSCRIBE", mapName);
|
|
10538
|
-
|
|
10539
|
-
|
|
10540
|
-
|
|
10541
|
-
|
|
10542
|
-
|
|
10543
|
-
|
|
12253
|
+
if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
|
|
12254
|
+
this.distributedSubCoordinator.subscribeQuery(
|
|
12255
|
+
queryId,
|
|
12256
|
+
client.socket,
|
|
12257
|
+
mapName,
|
|
12258
|
+
query
|
|
12259
|
+
).then((result) => {
|
|
12260
|
+
const filteredResults = result.results.map((res) => {
|
|
12261
|
+
const filteredValue = this.securityManager.filterObject(res.value, client.principal, mapName);
|
|
12262
|
+
return { ...res, value: filteredValue };
|
|
12263
|
+
});
|
|
12264
|
+
client.writer.write({
|
|
12265
|
+
type: "QUERY_RESP",
|
|
12266
|
+
payload: {
|
|
12267
|
+
queryId,
|
|
12268
|
+
results: filteredResults
|
|
12269
|
+
}
|
|
12270
|
+
});
|
|
12271
|
+
client.subscriptions.add(queryId);
|
|
12272
|
+
logger.debug({
|
|
12273
|
+
clientId: client.id,
|
|
12274
|
+
queryId,
|
|
10544
12275
|
mapName,
|
|
10545
|
-
|
|
10546
|
-
|
|
10547
|
-
|
|
10548
|
-
|
|
10549
|
-
|
|
12276
|
+
resultCount: result.results.length,
|
|
12277
|
+
totalHits: result.totalHits,
|
|
12278
|
+
nodes: result.registeredNodes
|
|
12279
|
+
}, "Distributed query subscription created");
|
|
12280
|
+
}).catch((err) => {
|
|
12281
|
+
logger.error({ err, queryId }, "Distributed query subscription failed");
|
|
12282
|
+
client.writer.write({
|
|
12283
|
+
type: "QUERY_RESP",
|
|
12284
|
+
payload: {
|
|
12285
|
+
queryId,
|
|
12286
|
+
results: [],
|
|
12287
|
+
error: "Failed to create distributed subscription"
|
|
10550
12288
|
}
|
|
10551
12289
|
});
|
|
10552
|
-
|
|
10553
|
-
|
|
10554
|
-
|
|
10555
|
-
|
|
10556
|
-
|
|
10557
|
-
|
|
10558
|
-
|
|
12290
|
+
});
|
|
12291
|
+
} else {
|
|
12292
|
+
const allMembers = this.cluster.getMembers();
|
|
12293
|
+
let remoteMembers = allMembers.filter((id) => !this.cluster.isLocal(id));
|
|
12294
|
+
const queryKey = query._id || query.where?._id;
|
|
12295
|
+
if (queryKey && typeof queryKey === "string" && this.readReplicaHandler) {
|
|
12296
|
+
try {
|
|
12297
|
+
const targetNode = this.readReplicaHandler.selectReadNode({
|
|
12298
|
+
mapName,
|
|
12299
|
+
key: queryKey,
|
|
12300
|
+
options: {
|
|
12301
|
+
// Default to EVENTUAL for read scaling unless specified otherwise
|
|
12302
|
+
// In future, we could extract consistency from query options if available
|
|
12303
|
+
consistency: ConsistencyLevel3.EVENTUAL
|
|
12304
|
+
}
|
|
12305
|
+
});
|
|
12306
|
+
if (targetNode) {
|
|
12307
|
+
if (this.cluster.isLocal(targetNode)) {
|
|
12308
|
+
remoteMembers = [];
|
|
12309
|
+
logger.debug({ clientId: client.id, mapName, key: queryKey }, "Read optimization: Serving locally");
|
|
12310
|
+
} else if (remoteMembers.includes(targetNode)) {
|
|
12311
|
+
remoteMembers = [targetNode];
|
|
12312
|
+
logger.debug({ clientId: client.id, mapName, key: queryKey, targetNode }, "Read optimization: Routing to replica");
|
|
12313
|
+
}
|
|
10559
12314
|
}
|
|
12315
|
+
} catch (e) {
|
|
12316
|
+
logger.warn({ err: e }, "Error in ReadReplicaHandler selection");
|
|
10560
12317
|
}
|
|
10561
|
-
} catch (e) {
|
|
10562
|
-
logger.warn({ err: e }, "Error in ReadReplicaHandler selection");
|
|
10563
12318
|
}
|
|
10564
|
-
|
|
10565
|
-
|
|
10566
|
-
|
|
10567
|
-
|
|
10568
|
-
|
|
10569
|
-
|
|
10570
|
-
|
|
10571
|
-
|
|
10572
|
-
|
|
10573
|
-
|
|
10574
|
-
|
|
10575
|
-
|
|
10576
|
-
|
|
10577
|
-
|
|
10578
|
-
|
|
10579
|
-
|
|
10580
|
-
|
|
10581
|
-
|
|
10582
|
-
|
|
10583
|
-
|
|
10584
|
-
|
|
10585
|
-
|
|
10586
|
-
|
|
10587
|
-
|
|
10588
|
-
|
|
10589
|
-
}
|
|
12319
|
+
const requestId = crypto.randomUUID();
|
|
12320
|
+
const pending = {
|
|
12321
|
+
requestId,
|
|
12322
|
+
client,
|
|
12323
|
+
queryId,
|
|
12324
|
+
mapName,
|
|
12325
|
+
query,
|
|
12326
|
+
results: [],
|
|
12327
|
+
// Will populate with local results first
|
|
12328
|
+
expectedNodes: new Set(remoteMembers),
|
|
12329
|
+
respondedNodes: /* @__PURE__ */ new Set(),
|
|
12330
|
+
timer: setTimeout(() => this.finalizeClusterQuery(requestId, true), 5e3)
|
|
12331
|
+
// 5s timeout
|
|
12332
|
+
};
|
|
12333
|
+
this.pendingClusterQueries.set(requestId, pending);
|
|
12334
|
+
try {
|
|
12335
|
+
const localResults = await this.executeLocalQuery(mapName, query);
|
|
12336
|
+
pending.results.push(...localResults);
|
|
12337
|
+
if (remoteMembers.length > 0) {
|
|
12338
|
+
for (const nodeId of remoteMembers) {
|
|
12339
|
+
this.cluster.send(nodeId, "CLUSTER_QUERY_EXEC", {
|
|
12340
|
+
requestId,
|
|
12341
|
+
mapName,
|
|
12342
|
+
query
|
|
12343
|
+
});
|
|
12344
|
+
}
|
|
12345
|
+
} else {
|
|
12346
|
+
this.finalizeClusterQuery(requestId);
|
|
10590
12347
|
}
|
|
10591
|
-
}
|
|
12348
|
+
} catch (err) {
|
|
12349
|
+
logger.error({ err, mapName }, "Failed to execute local query");
|
|
10592
12350
|
this.finalizeClusterQuery(requestId);
|
|
10593
12351
|
}
|
|
10594
|
-
} catch (err) {
|
|
10595
|
-
logger.error({ err, mapName }, "Failed to execute local query");
|
|
10596
|
-
this.finalizeClusterQuery(requestId);
|
|
10597
12352
|
}
|
|
10598
12353
|
break;
|
|
10599
12354
|
}
|
|
10600
12355
|
case "QUERY_UNSUB": {
|
|
10601
12356
|
const { queryId: unsubId } = message.payload;
|
|
12357
|
+
if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
|
|
12358
|
+
this.distributedSubCoordinator.unsubscribe(unsubId).catch((err) => {
|
|
12359
|
+
logger.warn({ err, queryId: unsubId }, "Failed to unsubscribe from distributed coordinator");
|
|
12360
|
+
});
|
|
12361
|
+
}
|
|
10602
12362
|
this.queryRegistry.unregister(unsubId);
|
|
10603
12363
|
client.subscriptions.delete(unsubId);
|
|
10604
12364
|
break;
|
|
@@ -10922,7 +12682,7 @@ var ServerCoordinator = class {
|
|
|
10922
12682
|
client.writer.write(result.response);
|
|
10923
12683
|
for (const targetClientId of result.broadcastTo) {
|
|
10924
12684
|
const targetClient = this.clients.get(targetClientId);
|
|
10925
|
-
if (targetClient && targetClient.socket.readyState ===
|
|
12685
|
+
if (targetClient && targetClient.socket.readyState === WebSocket5.OPEN) {
|
|
10926
12686
|
targetClient.writer.write(result.broadcastMessage);
|
|
10927
12687
|
}
|
|
10928
12688
|
}
|
|
@@ -11402,7 +13162,7 @@ var ServerCoordinator = class {
|
|
|
11402
13162
|
});
|
|
11403
13163
|
break;
|
|
11404
13164
|
}
|
|
11405
|
-
// Phase 11.1: Full-Text Search
|
|
13165
|
+
// Phase 11.1: Full-Text Search (Phase 14: Distributed Search)
|
|
11406
13166
|
case "SEARCH": {
|
|
11407
13167
|
const { requestId: searchReqId, mapName: searchMapName, query: searchQuery, options: searchOptions } = message.payload;
|
|
11408
13168
|
if (!this.securityManager.checkPermission(client.principal, searchMapName, "READ")) {
|
|
@@ -11430,18 +13190,58 @@ var ServerCoordinator = class {
|
|
|
11430
13190
|
});
|
|
11431
13191
|
break;
|
|
11432
13192
|
}
|
|
11433
|
-
|
|
11434
|
-
|
|
11435
|
-
|
|
11436
|
-
|
|
11437
|
-
|
|
11438
|
-
|
|
11439
|
-
|
|
11440
|
-
|
|
11441
|
-
|
|
11442
|
-
|
|
11443
|
-
|
|
11444
|
-
|
|
13193
|
+
if (this.clusterSearchCoordinator && this.cluster.getMembers().length > 1) {
|
|
13194
|
+
this.clusterSearchCoordinator.search(searchMapName, searchQuery, {
|
|
13195
|
+
limit: searchOptions?.limit ?? 10,
|
|
13196
|
+
minScore: searchOptions?.minScore,
|
|
13197
|
+
boost: searchOptions?.boost
|
|
13198
|
+
}).then((distributedResult) => {
|
|
13199
|
+
logger.debug({
|
|
13200
|
+
clientId: client.id,
|
|
13201
|
+
mapName: searchMapName,
|
|
13202
|
+
query: searchQuery,
|
|
13203
|
+
resultCount: distributedResult.results.length,
|
|
13204
|
+
totalHits: distributedResult.totalHits,
|
|
13205
|
+
respondedNodes: distributedResult.respondedNodes.length,
|
|
13206
|
+
failedNodes: distributedResult.failedNodes.length,
|
|
13207
|
+
executionTimeMs: distributedResult.executionTimeMs
|
|
13208
|
+
}, "Distributed search executed");
|
|
13209
|
+
client.writer.write({
|
|
13210
|
+
type: "SEARCH_RESP",
|
|
13211
|
+
payload: {
|
|
13212
|
+
requestId: searchReqId,
|
|
13213
|
+
results: distributedResult.results,
|
|
13214
|
+
totalCount: distributedResult.totalHits,
|
|
13215
|
+
// Include cursor for pagination if available
|
|
13216
|
+
nextCursor: distributedResult.nextCursor
|
|
13217
|
+
}
|
|
13218
|
+
});
|
|
13219
|
+
}).catch((err) => {
|
|
13220
|
+
logger.error({ err, mapName: searchMapName, query: searchQuery }, "Distributed search failed");
|
|
13221
|
+
client.writer.write({
|
|
13222
|
+
type: "SEARCH_RESP",
|
|
13223
|
+
payload: {
|
|
13224
|
+
requestId: searchReqId,
|
|
13225
|
+
results: [],
|
|
13226
|
+
totalCount: 0,
|
|
13227
|
+
error: `Distributed search failed: ${err.message}`
|
|
13228
|
+
}
|
|
13229
|
+
});
|
|
13230
|
+
});
|
|
13231
|
+
} else {
|
|
13232
|
+
const searchResult = this.searchCoordinator.search(searchMapName, searchQuery, searchOptions);
|
|
13233
|
+
searchResult.requestId = searchReqId;
|
|
13234
|
+
logger.debug({
|
|
13235
|
+
clientId: client.id,
|
|
13236
|
+
mapName: searchMapName,
|
|
13237
|
+
query: searchQuery,
|
|
13238
|
+
resultCount: searchResult.results.length
|
|
13239
|
+
}, "Local search executed");
|
|
13240
|
+
client.writer.write({
|
|
13241
|
+
type: "SEARCH_RESP",
|
|
13242
|
+
payload: searchResult
|
|
13243
|
+
});
|
|
13244
|
+
}
|
|
11445
13245
|
break;
|
|
11446
13246
|
}
|
|
11447
13247
|
// Phase 11.1b: Live Search Subscriptions
|
|
@@ -11472,33 +13272,75 @@ var ServerCoordinator = class {
|
|
|
11472
13272
|
});
|
|
11473
13273
|
break;
|
|
11474
13274
|
}
|
|
11475
|
-
|
|
11476
|
-
|
|
11477
|
-
|
|
11478
|
-
|
|
11479
|
-
|
|
11480
|
-
|
|
11481
|
-
|
|
11482
|
-
|
|
11483
|
-
|
|
11484
|
-
|
|
11485
|
-
|
|
11486
|
-
|
|
11487
|
-
|
|
11488
|
-
|
|
11489
|
-
|
|
11490
|
-
|
|
11491
|
-
|
|
11492
|
-
|
|
11493
|
-
|
|
11494
|
-
|
|
11495
|
-
|
|
11496
|
-
|
|
13275
|
+
if (this.distributedSubCoordinator && this.cluster && this.cluster.getMembers().length > 1) {
|
|
13276
|
+
this.distributedSubCoordinator.subscribeSearch(
|
|
13277
|
+
subscriptionId,
|
|
13278
|
+
client.socket,
|
|
13279
|
+
subMapName,
|
|
13280
|
+
subQuery,
|
|
13281
|
+
subOptions || {}
|
|
13282
|
+
).then((result) => {
|
|
13283
|
+
client.writer.write({
|
|
13284
|
+
type: "SEARCH_RESP",
|
|
13285
|
+
payload: {
|
|
13286
|
+
requestId: subscriptionId,
|
|
13287
|
+
results: result.results,
|
|
13288
|
+
totalCount: result.totalHits
|
|
13289
|
+
}
|
|
13290
|
+
});
|
|
13291
|
+
logger.debug({
|
|
13292
|
+
clientId: client.id,
|
|
13293
|
+
subscriptionId,
|
|
13294
|
+
mapName: subMapName,
|
|
13295
|
+
query: subQuery,
|
|
13296
|
+
resultCount: result.results.length,
|
|
13297
|
+
totalHits: result.totalHits,
|
|
13298
|
+
nodes: result.registeredNodes
|
|
13299
|
+
}, "Distributed search subscription created");
|
|
13300
|
+
}).catch((err) => {
|
|
13301
|
+
logger.error({ err, subscriptionId }, "Distributed search subscription failed");
|
|
13302
|
+
client.writer.write({
|
|
13303
|
+
type: "SEARCH_RESP",
|
|
13304
|
+
payload: {
|
|
13305
|
+
requestId: subscriptionId,
|
|
13306
|
+
results: [],
|
|
13307
|
+
totalCount: 0,
|
|
13308
|
+
error: "Failed to create distributed subscription"
|
|
13309
|
+
}
|
|
13310
|
+
});
|
|
13311
|
+
});
|
|
13312
|
+
} else {
|
|
13313
|
+
const initialResults = this.searchCoordinator.subscribe(
|
|
13314
|
+
client.id,
|
|
13315
|
+
subscriptionId,
|
|
13316
|
+
subMapName,
|
|
13317
|
+
subQuery,
|
|
13318
|
+
subOptions
|
|
13319
|
+
);
|
|
13320
|
+
logger.debug({
|
|
13321
|
+
clientId: client.id,
|
|
13322
|
+
subscriptionId,
|
|
13323
|
+
mapName: subMapName,
|
|
13324
|
+
query: subQuery,
|
|
13325
|
+
resultCount: initialResults.length
|
|
13326
|
+
}, "Search subscription created (local)");
|
|
13327
|
+
client.writer.write({
|
|
13328
|
+
type: "SEARCH_RESP",
|
|
13329
|
+
payload: {
|
|
13330
|
+
requestId: subscriptionId,
|
|
13331
|
+
results: initialResults,
|
|
13332
|
+
totalCount: initialResults.length
|
|
13333
|
+
}
|
|
13334
|
+
});
|
|
13335
|
+
}
|
|
11497
13336
|
break;
|
|
11498
13337
|
}
|
|
11499
13338
|
case "SEARCH_UNSUB": {
|
|
11500
13339
|
const { subscriptionId: unsubId } = message.payload;
|
|
11501
13340
|
this.searchCoordinator.unsubscribe(unsubId);
|
|
13341
|
+
if (this.distributedSubCoordinator) {
|
|
13342
|
+
this.distributedSubCoordinator.unsubscribe(unsubId);
|
|
13343
|
+
}
|
|
11502
13344
|
logger.debug({ clientId: client.id, subscriptionId: unsubId }, "Search unsubscription");
|
|
11503
13345
|
break;
|
|
11504
13346
|
}
|
|
@@ -11539,7 +13381,7 @@ var ServerCoordinator = class {
|
|
|
11539
13381
|
};
|
|
11540
13382
|
let broadcastCount = 0;
|
|
11541
13383
|
for (const client of this.clients.values()) {
|
|
11542
|
-
if (client.isAuthenticated && client.socket.readyState ===
|
|
13384
|
+
if (client.isAuthenticated && client.socket.readyState === WebSocket5.OPEN && client.writer) {
|
|
11543
13385
|
client.writer.write(message);
|
|
11544
13386
|
broadcastCount++;
|
|
11545
13387
|
}
|
|
@@ -11971,7 +13813,7 @@ var ServerCoordinator = class {
|
|
|
11971
13813
|
async executeLocalQuery(mapName, query) {
|
|
11972
13814
|
const map = await this.getMapAsync(mapName);
|
|
11973
13815
|
const localQuery = { ...query };
|
|
11974
|
-
delete localQuery.
|
|
13816
|
+
delete localQuery.cursor;
|
|
11975
13817
|
delete localQuery.limit;
|
|
11976
13818
|
if (map instanceof IndexedLWWMap2) {
|
|
11977
13819
|
const coreQuery = this.convertToCoreQuery(localQuery);
|
|
@@ -12090,7 +13932,7 @@ var ServerCoordinator = class {
|
|
|
12090
13932
|
};
|
|
12091
13933
|
return mapping[op] || null;
|
|
12092
13934
|
}
|
|
12093
|
-
finalizeClusterQuery(requestId, timeout = false) {
|
|
13935
|
+
async finalizeClusterQuery(requestId, timeout = false) {
|
|
12094
13936
|
const pending = this.pendingClusterQueries.get(requestId);
|
|
12095
13937
|
if (!pending) return;
|
|
12096
13938
|
if (timeout) {
|
|
@@ -12115,7 +13957,49 @@ var ServerCoordinator = class {
|
|
|
12115
13957
|
return 0;
|
|
12116
13958
|
});
|
|
12117
13959
|
}
|
|
12118
|
-
|
|
13960
|
+
let slicedResults = finalResults;
|
|
13961
|
+
let nextCursor;
|
|
13962
|
+
let hasMore = false;
|
|
13963
|
+
let cursorStatus = "none";
|
|
13964
|
+
if (query.cursor || query.limit) {
|
|
13965
|
+
const sort = query.sort || {};
|
|
13966
|
+
const sortEntries = Object.entries(sort);
|
|
13967
|
+
const sortField = sortEntries.length > 0 ? sortEntries[0][0] : "_key";
|
|
13968
|
+
if (query.cursor) {
|
|
13969
|
+
const cursorData = QueryCursor2.decode(query.cursor);
|
|
13970
|
+
if (!cursorData) {
|
|
13971
|
+
cursorStatus = "invalid";
|
|
13972
|
+
} else if (!QueryCursor2.isValid(cursorData, query.predicate ?? query.where, sort)) {
|
|
13973
|
+
if (Date.now() - cursorData.timestamp > DEFAULT_QUERY_CURSOR_MAX_AGE_MS2) {
|
|
13974
|
+
cursorStatus = "expired";
|
|
13975
|
+
} else {
|
|
13976
|
+
cursorStatus = "invalid";
|
|
13977
|
+
}
|
|
13978
|
+
} else {
|
|
13979
|
+
cursorStatus = "valid";
|
|
13980
|
+
slicedResults = finalResults.filter((r) => {
|
|
13981
|
+
const sortValue = r.value[sortField];
|
|
13982
|
+
return QueryCursor2.isAfterCursor(
|
|
13983
|
+
{ key: r.key, sortValue },
|
|
13984
|
+
cursorData
|
|
13985
|
+
);
|
|
13986
|
+
});
|
|
13987
|
+
}
|
|
13988
|
+
}
|
|
13989
|
+
if (query.limit) {
|
|
13990
|
+
hasMore = slicedResults.length > query.limit;
|
|
13991
|
+
slicedResults = slicedResults.slice(0, query.limit);
|
|
13992
|
+
if (hasMore && slicedResults.length > 0) {
|
|
13993
|
+
const lastResult = slicedResults[slicedResults.length - 1];
|
|
13994
|
+
const sortValue = lastResult.value[sortField];
|
|
13995
|
+
nextCursor = QueryCursor2.fromLastResult(
|
|
13996
|
+
{ key: lastResult.key, sortValue },
|
|
13997
|
+
sort,
|
|
13998
|
+
query.predicate ?? query.where
|
|
13999
|
+
);
|
|
14000
|
+
}
|
|
14001
|
+
}
|
|
14002
|
+
}
|
|
12119
14003
|
const resultKeys = new Set(slicedResults.map((r) => r.key));
|
|
12120
14004
|
const sub = {
|
|
12121
14005
|
id: queryId,
|
|
@@ -12134,7 +14018,7 @@ var ServerCoordinator = class {
|
|
|
12134
14018
|
});
|
|
12135
14019
|
client.writer.write({
|
|
12136
14020
|
type: "QUERY_RESP",
|
|
12137
|
-
payload: { queryId, results: filteredResults }
|
|
14021
|
+
payload: { queryId, results: filteredResults, nextCursor, hasMore, cursorStatus }
|
|
12138
14022
|
});
|
|
12139
14023
|
}
|
|
12140
14024
|
/**
|
|
@@ -12246,17 +14130,6 @@ var ServerCoordinator = class {
|
|
|
12246
14130
|
}
|
|
12247
14131
|
return { eventPayload, oldRecord };
|
|
12248
14132
|
}
|
|
12249
|
-
/**
|
|
12250
|
-
* Broadcast event to cluster members (excluding self).
|
|
12251
|
-
*/
|
|
12252
|
-
broadcastToCluster(eventPayload) {
|
|
12253
|
-
const members = this.cluster.getMembers();
|
|
12254
|
-
for (const memberId of members) {
|
|
12255
|
-
if (!this.cluster.isLocal(memberId)) {
|
|
12256
|
-
this.cluster.send(memberId, "CLUSTER_EVENT", eventPayload);
|
|
12257
|
-
}
|
|
12258
|
-
}
|
|
12259
|
-
}
|
|
12260
14133
|
/**
|
|
12261
14134
|
* Apply replicated operation from another node (callback for ReplicationPipeline)
|
|
12262
14135
|
* This is called when we receive a replicated operation as a backup node
|
|
@@ -12382,7 +14255,6 @@ var ServerCoordinator = class {
|
|
|
12382
14255
|
payload: eventPayload,
|
|
12383
14256
|
timestamp: this.hlc.now()
|
|
12384
14257
|
}, originalSenderId);
|
|
12385
|
-
this.broadcastToCluster(eventPayload);
|
|
12386
14258
|
this.runAfterInterceptors(op, context);
|
|
12387
14259
|
}
|
|
12388
14260
|
/**
|
|
@@ -12509,7 +14381,6 @@ var ServerCoordinator = class {
|
|
|
12509
14381
|
});
|
|
12510
14382
|
}
|
|
12511
14383
|
batchedEvents.push(eventPayload);
|
|
12512
|
-
this.broadcastToCluster(eventPayload);
|
|
12513
14384
|
this.runAfterInterceptors(op, context);
|
|
12514
14385
|
}
|
|
12515
14386
|
handleClusterEvent(payload) {
|
|
@@ -12754,7 +14625,7 @@ var ServerCoordinator = class {
|
|
|
12754
14625
|
idleTime: now - client.lastPingReceived,
|
|
12755
14626
|
timeoutMs: CLIENT_HEARTBEAT_TIMEOUT_MS
|
|
12756
14627
|
}, "Evicting dead client (heartbeat timeout)");
|
|
12757
|
-
if (client.socket.readyState ===
|
|
14628
|
+
if (client.socket.readyState === WebSocket5.OPEN) {
|
|
12758
14629
|
client.socket.close(4002, "Heartbeat timeout");
|
|
12759
14630
|
}
|
|
12760
14631
|
}
|
|
@@ -12852,11 +14723,18 @@ var ServerCoordinator = class {
|
|
|
12852
14723
|
payload: eventPayload,
|
|
12853
14724
|
timestamp: this.hlc.now()
|
|
12854
14725
|
});
|
|
12855
|
-
|
|
12856
|
-
|
|
12857
|
-
|
|
12858
|
-
|
|
12859
|
-
|
|
14726
|
+
this.queryRegistry.processChange(name, map, key, tombstone, record);
|
|
14727
|
+
if (this.replicationPipeline) {
|
|
14728
|
+
const op = {
|
|
14729
|
+
opType: "set",
|
|
14730
|
+
mapName: name,
|
|
14731
|
+
key,
|
|
14732
|
+
record: tombstone
|
|
14733
|
+
};
|
|
14734
|
+
const opId = `ttl:${name}:${key}:${Date.now()}`;
|
|
14735
|
+
this.replicationPipeline.replicate(op, opId, key).catch((err) => {
|
|
14736
|
+
logger.warn({ opId, key, err }, "TTL expiration replication failed (non-fatal)");
|
|
14737
|
+
});
|
|
12860
14738
|
}
|
|
12861
14739
|
}
|
|
12862
14740
|
}
|
|
@@ -12889,6 +14767,7 @@ var ServerCoordinator = class {
|
|
|
12889
14767
|
}
|
|
12890
14768
|
for (const { key, tag } of tagsToExpire) {
|
|
12891
14769
|
logger.info({ mapName: name, key, tag }, "ORMap Record expired (TTL). Removing.");
|
|
14770
|
+
const oldRecords = map.getRecords(key);
|
|
12892
14771
|
map.applyTombstone(tag);
|
|
12893
14772
|
if (this.storage) {
|
|
12894
14773
|
const records = map.getRecords(key);
|
|
@@ -12914,11 +14793,19 @@ var ServerCoordinator = class {
|
|
|
12914
14793
|
payload: eventPayload,
|
|
12915
14794
|
timestamp: this.hlc.now()
|
|
12916
14795
|
});
|
|
12917
|
-
const
|
|
12918
|
-
|
|
12919
|
-
|
|
12920
|
-
|
|
12921
|
-
|
|
14796
|
+
const newRecords = map.getRecords(key);
|
|
14797
|
+
this.queryRegistry.processChange(name, map, key, newRecords, oldRecords);
|
|
14798
|
+
if (this.replicationPipeline) {
|
|
14799
|
+
const op = {
|
|
14800
|
+
opType: "OR_REMOVE",
|
|
14801
|
+
mapName: name,
|
|
14802
|
+
key,
|
|
14803
|
+
orTag: tag
|
|
14804
|
+
};
|
|
14805
|
+
const opId = `ttl:${name}:${key}:${tag}:${Date.now()}`;
|
|
14806
|
+
this.replicationPipeline.replicate(op, opId, key).catch((err) => {
|
|
14807
|
+
logger.warn({ opId, key, err }, "ORMap TTL expiration replication failed (non-fatal)");
|
|
14808
|
+
});
|
|
12922
14809
|
}
|
|
12923
14810
|
}
|
|
12924
14811
|
const removedTags = map.prune(olderThan);
|
|
@@ -13512,7 +15399,7 @@ function logNativeStatus() {
|
|
|
13512
15399
|
}
|
|
13513
15400
|
|
|
13514
15401
|
// src/cluster/ClusterCoordinator.ts
|
|
13515
|
-
import { EventEmitter as
|
|
15402
|
+
import { EventEmitter as EventEmitter16 } from "events";
|
|
13516
15403
|
import {
|
|
13517
15404
|
DEFAULT_MIGRATION_CONFIG as DEFAULT_MIGRATION_CONFIG3,
|
|
13518
15405
|
DEFAULT_REPLICATION_CONFIG as DEFAULT_REPLICATION_CONFIG3
|
|
@@ -13523,7 +15410,7 @@ var DEFAULT_CLUSTER_COORDINATOR_CONFIG = {
|
|
|
13523
15410
|
replication: DEFAULT_REPLICATION_CONFIG3,
|
|
13524
15411
|
replicationEnabled: true
|
|
13525
15412
|
};
|
|
13526
|
-
var ClusterCoordinator = class extends
|
|
15413
|
+
var ClusterCoordinator = class extends EventEmitter16 {
|
|
13527
15414
|
constructor(config) {
|
|
13528
15415
|
super();
|
|
13529
15416
|
this.replicationPipeline = null;
|