@delali/narsil 0.2.3-next.33 → 0.2.3-next.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{chunk-C6VVLMRX.mjs → chunk-36ER4WCG.mjs} +1 -0
- package/dist/{chunk-MFFAJ2NR.mjs → chunk-GODECMDT.mjs} +1 -1
- package/dist/{chunk-TEAJO7CU.mjs → chunk-OOA7OBLA.mjs} +29 -3
- package/dist/{client-DQHB_swQ.d.ts → client-DAJ_UHOa.d.ts} +1 -1
- package/dist/client.d.ts +2 -2
- package/dist/{config-ChZ7lfqc.d.ts → config-CzTJJili.d.ts} +1 -1
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +94 -15
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/coordinator/in-memory.mjs +10 -3
- package/dist/distribution/transport/grpc.mjs +2 -2
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +30 -6
- package/dist/distribution.mjs +1910 -1042
- package/dist/{engine-DScGlT2Z.d.ts → engine-DEtRLYpB.d.ts} +1 -1
- package/dist/index.browser.mjs +19 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +2 -2
- package/dist/react.d.ts +3 -3
- package/dist/{search-CQ-9bKxo.d.ts → search-Dt4dkzpf.d.ts} +24 -1
- package/dist/server.d.ts +2 -2
- package/dist/{types-Bfhhnh3G.d.ts → types-BhdINMtE.d.ts} +2 -0
- package/package.json +1 -1
package/dist/distribution.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, MAX_PARTITION_COUNT, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, MAX_INDEX_NAME_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum,
|
|
2
|
-
import { ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-
|
|
1
|
+
import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, MAX_PARTITION_COUNT, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, MAX_INDEX_NAME_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-OOA7OBLA.mjs';
|
|
2
|
+
import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-36ER4WCG.mjs';
|
|
3
3
|
import { deserializePayloadV2, createPartitionIndex, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, getLanguage, DEFAULT_PAGE_SIZE, requireWithinResultWindow, fnv1a, FILTER_EXPRESSION_KEYS, MAX_CURSOR_LENGTH, mergePartitionStats, FIELD_FILTER_OPERATORS, MAX_NESTING_DEPTH } from './chunk-RO4EFWE6.mjs';
|
|
4
4
|
import './chunk-TGP5M5R7.mjs';
|
|
5
5
|
import './chunk-OBXS7OZM.mjs';
|
|
@@ -60,61 +60,6 @@ function startRenewalInterval(state, coordinator, nodeId, ttlMs, onLeaseLost) {
|
|
|
60
60
|
function scheduleStandbyRetry(state, retryMs, onRetry) {
|
|
61
61
|
state.standbyTimer = setTimeout(onRetry, retryMs);
|
|
62
62
|
}
|
|
63
|
-
async function requestInsyncRemoval(indexName, partitionId, replicaNodeId, primaryTerm, controllerNodeId, transport, sourceNodeId) {
|
|
64
|
-
const payload = {
|
|
65
|
-
indexName,
|
|
66
|
-
partitionId,
|
|
67
|
-
replicaNodeId,
|
|
68
|
-
primaryTerm
|
|
69
|
-
};
|
|
70
|
-
const message = createInsyncRemoveMessage(payload, sourceNodeId);
|
|
71
|
-
const response = await transport.send(controllerNodeId, message);
|
|
72
|
-
const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
|
|
73
|
-
return { accepted: confirmPayload.accepted };
|
|
74
|
-
}
|
|
75
|
-
async function handleInsyncRemoval(payload, coordinator) {
|
|
76
|
-
const table = await coordinator.getAllocation(payload.indexName);
|
|
77
|
-
if (table === null) {
|
|
78
|
-
return {
|
|
79
|
-
indexName: payload.indexName,
|
|
80
|
-
partitionId: payload.partitionId,
|
|
81
|
-
accepted: false
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
const assignment = table.assignments.get(payload.partitionId);
|
|
85
|
-
if (assignment === void 0) {
|
|
86
|
-
return {
|
|
87
|
-
indexName: payload.indexName,
|
|
88
|
-
partitionId: payload.partitionId,
|
|
89
|
-
accepted: false
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
if (payload.primaryTerm !== assignment.primaryTerm) {
|
|
93
|
-
return {
|
|
94
|
-
indexName: payload.indexName,
|
|
95
|
-
partitionId: payload.partitionId,
|
|
96
|
-
accepted: false
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
|
-
const updatedInSyncSet = assignment.inSyncSet.filter((nodeId) => nodeId !== payload.replicaNodeId);
|
|
100
|
-
const updatedAssignment = {
|
|
101
|
-
...assignment,
|
|
102
|
-
inSyncSet: updatedInSyncSet
|
|
103
|
-
};
|
|
104
|
-
const updatedAssignments = new Map(table.assignments);
|
|
105
|
-
updatedAssignments.set(payload.partitionId, updatedAssignment);
|
|
106
|
-
const updatedTable = {
|
|
107
|
-
...table,
|
|
108
|
-
version: table.version + 1,
|
|
109
|
-
assignments: updatedAssignments
|
|
110
|
-
};
|
|
111
|
-
const written = await coordinator.putAllocation(payload.indexName, updatedTable, table.version);
|
|
112
|
-
return {
|
|
113
|
-
indexName: payload.indexName,
|
|
114
|
-
partitionId: payload.partitionId,
|
|
115
|
-
accepted: written
|
|
116
|
-
};
|
|
117
|
-
}
|
|
118
63
|
|
|
119
64
|
// src/distribution/cluster/allocator/deciders/balance.ts
|
|
120
65
|
var balanceDecider = {
|
|
@@ -292,8 +237,18 @@ function countNodeAssignments(assignments) {
|
|
|
292
237
|
}
|
|
293
238
|
return counts;
|
|
294
239
|
}
|
|
240
|
+
function countPrimaryAssignments(assignments) {
|
|
241
|
+
const counts = /* @__PURE__ */ new Map();
|
|
242
|
+
for (const assignment of assignments.values()) {
|
|
243
|
+
if (assignment.primary !== null) {
|
|
244
|
+
counts.set(assignment.primary, (counts.get(assignment.primary) ?? 0) + 1);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return counts;
|
|
248
|
+
}
|
|
295
249
|
function computeNodeWeights(nodes, assignments) {
|
|
296
250
|
const assignmentCounts = countNodeAssignments(assignments);
|
|
251
|
+
const primaryCounts = countPrimaryAssignments(assignments);
|
|
297
252
|
let totalCapacity = 0;
|
|
298
253
|
for (const node of nodes) {
|
|
299
254
|
totalCapacity += node.capacity.memoryBytes;
|
|
@@ -302,12 +257,14 @@ function computeNodeWeights(nodes, assignments) {
|
|
|
302
257
|
const weights = [];
|
|
303
258
|
for (const node of nodes) {
|
|
304
259
|
const partitionCount = assignmentCounts.get(node.nodeId) ?? 0;
|
|
260
|
+
const primaryCount = primaryCounts.get(node.nodeId) ?? 0;
|
|
305
261
|
const normalizedCapacity = averageCapacity > 0 ? node.capacity.memoryBytes / averageCapacity : 1;
|
|
306
|
-
const weight = partitionCount / normalizedCapacity;
|
|
307
262
|
weights.push({
|
|
308
263
|
nodeId: node.nodeId,
|
|
309
|
-
weight,
|
|
264
|
+
weight: partitionCount / normalizedCapacity,
|
|
310
265
|
partitionCount,
|
|
266
|
+
primaryWeight: primaryCount / normalizedCapacity,
|
|
267
|
+
primaryCount,
|
|
311
268
|
capacity: node.capacity.memoryBytes
|
|
312
269
|
});
|
|
313
270
|
}
|
|
@@ -326,6 +283,11 @@ function findBestNode(candidates, weights, deciders, context) {
|
|
|
326
283
|
const sorted = [...candidates].sort((a, b) => {
|
|
327
284
|
const wA = weightByNodeId.get(a);
|
|
328
285
|
const wB = weightByNodeId.get(b);
|
|
286
|
+
if (context.role === "primary") {
|
|
287
|
+
const primaryA = wA?.primaryWeight ?? 0;
|
|
288
|
+
const primaryB = wB?.primaryWeight ?? 0;
|
|
289
|
+
if (primaryA !== primaryB) return primaryA - primaryB;
|
|
290
|
+
}
|
|
329
291
|
const weightA = wA?.weight ?? 0;
|
|
330
292
|
const weightB = wB?.weight ?? 0;
|
|
331
293
|
if (weightA !== weightB) return weightA - weightB;
|
|
@@ -392,6 +354,7 @@ function initialAllocate(nodes, indexName, partitionCount, replicationFactor, co
|
|
|
392
354
|
primary: primaryNodeId,
|
|
393
355
|
replicas,
|
|
394
356
|
inSyncSet: [],
|
|
357
|
+
commitPoint: 0,
|
|
395
358
|
state: "INITIALISING",
|
|
396
359
|
primaryTerm: 1
|
|
397
360
|
};
|
|
@@ -434,6 +397,74 @@ function initialAllocate(nodes, indexName, partitionCount, replicationFactor, co
|
|
|
434
397
|
};
|
|
435
398
|
}
|
|
436
399
|
|
|
400
|
+
// src/distribution/cluster/allocator/leadership.ts
|
|
401
|
+
var LEADERSHIP_IMBALANCE_THRESHOLD = 2;
|
|
402
|
+
function loadOf(nodeId, counts, shares) {
|
|
403
|
+
const share = shares.get(nodeId);
|
|
404
|
+
return (counts.get(nodeId) ?? 0) / (share === void 0 || share <= 0 ? 1 : share);
|
|
405
|
+
}
|
|
406
|
+
function heaviestLeader(shares, counts) {
|
|
407
|
+
let heaviest;
|
|
408
|
+
for (const nodeId of shares.keys()) {
|
|
409
|
+
if (heaviest === void 0 || loadOf(nodeId, counts, shares) > loadOf(heaviest, counts, shares)) {
|
|
410
|
+
heaviest = nodeId;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
return heaviest;
|
|
414
|
+
}
|
|
415
|
+
function findSwap(assignments, heaviest, counts, shares) {
|
|
416
|
+
const heaviestLoad = loadOf(heaviest, counts, shares);
|
|
417
|
+
let best = null;
|
|
418
|
+
for (const assignment of assignments.values()) {
|
|
419
|
+
if (assignment.state !== "ACTIVE" || assignment.primary !== heaviest) {
|
|
420
|
+
continue;
|
|
421
|
+
}
|
|
422
|
+
for (const candidateNodeId of assignment.inSyncSet) {
|
|
423
|
+
const candidateLoad = loadOf(candidateNodeId, counts, shares);
|
|
424
|
+
if (heaviestLoad - candidateLoad < LEADERSHIP_IMBALANCE_THRESHOLD) {
|
|
425
|
+
continue;
|
|
426
|
+
}
|
|
427
|
+
if (best === null || candidateLoad < best.candidateLoad) {
|
|
428
|
+
best = { assignment, candidateNodeId, candidateLoad };
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
return best;
|
|
433
|
+
}
|
|
434
|
+
function applySwap(swap, previousPrimary) {
|
|
435
|
+
const { assignment, candidateNodeId } = swap;
|
|
436
|
+
assignment.primary = candidateNodeId;
|
|
437
|
+
assignment.replicas = [...assignment.replicas.filter((nodeId) => nodeId !== candidateNodeId), previousPrimary];
|
|
438
|
+
assignment.inSyncSet = assignment.inSyncSet.filter((nodeId) => nodeId !== candidateNodeId);
|
|
439
|
+
assignment.primaryTerm += 1;
|
|
440
|
+
}
|
|
441
|
+
function rebalanceLeadership(assignments, shares) {
|
|
442
|
+
for (let swaps = 0; swaps < assignments.size; swaps++) {
|
|
443
|
+
const counts = countPrimaryAssignments(assignments);
|
|
444
|
+
const heaviest = heaviestLeader(shares, counts);
|
|
445
|
+
if (heaviest === void 0) {
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
const swap = findSwap(assignments, heaviest, counts, shares);
|
|
449
|
+
if (swap === null) {
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
applySwap(swap, heaviest);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
function capacityShares(nodes) {
|
|
456
|
+
const shares = /* @__PURE__ */ new Map();
|
|
457
|
+
let totalCapacity = 0;
|
|
458
|
+
for (const node of nodes) {
|
|
459
|
+
totalCapacity += node.capacity.memoryBytes;
|
|
460
|
+
}
|
|
461
|
+
const averageCapacity = nodes.length > 0 ? totalCapacity / nodes.length : 0;
|
|
462
|
+
for (const node of nodes) {
|
|
463
|
+
shares.set(node.nodeId, averageCapacity > 0 ? node.capacity.memoryBytes / averageCapacity : 1);
|
|
464
|
+
}
|
|
465
|
+
return shares;
|
|
466
|
+
}
|
|
467
|
+
|
|
437
468
|
// src/distribution/cluster/allocator/rebalance.ts
|
|
438
469
|
function cloneAssignments(assignments) {
|
|
439
470
|
const cloned = /* @__PURE__ */ new Map();
|
|
@@ -442,6 +473,7 @@ function cloneAssignments(assignments) {
|
|
|
442
473
|
primary: assignment.primary,
|
|
443
474
|
replicas: [...assignment.replicas],
|
|
444
475
|
inSyncSet: [...assignment.inSyncSet],
|
|
476
|
+
commitPoint: assignment.commitPoint,
|
|
445
477
|
state: assignment.state,
|
|
446
478
|
primaryTerm: assignment.primaryTerm
|
|
447
479
|
});
|
|
@@ -454,7 +486,27 @@ function markUnassigned(assignment) {
|
|
|
454
486
|
assignment.inSyncSet = [];
|
|
455
487
|
assignment.state = "UNASSIGNED";
|
|
456
488
|
}
|
|
457
|
-
function
|
|
489
|
+
function primaryLoadOf(nodeId, primaryCounts, shares) {
|
|
490
|
+
const share = shares.get(nodeId);
|
|
491
|
+
return (primaryCounts.get(nodeId) ?? 0) / (share === void 0 || share <= 0 ? 1 : share);
|
|
492
|
+
}
|
|
493
|
+
function leastLoadedCandidate(candidates, primaryCounts, shares) {
|
|
494
|
+
let best;
|
|
495
|
+
for (const candidate of candidates) {
|
|
496
|
+
if (best === void 0) {
|
|
497
|
+
best = candidate;
|
|
498
|
+
continue;
|
|
499
|
+
}
|
|
500
|
+
const bestLoad = primaryLoadOf(best, primaryCounts, shares);
|
|
501
|
+
const candidateLoad = primaryLoadOf(candidate, primaryCounts, shares);
|
|
502
|
+
if (candidateLoad < bestLoad || candidateLoad === bestLoad && candidate < best) {
|
|
503
|
+
best = candidate;
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
return best;
|
|
507
|
+
}
|
|
508
|
+
function handleLostNodes(assignments, activeNodeIds, shares) {
|
|
509
|
+
const primaryCounts = countPrimaryAssignments(assignments);
|
|
458
510
|
for (const assignment of assignments.values()) {
|
|
459
511
|
const primaryWasLost = assignment.primary !== null && !activeNodeIds.has(assignment.primary);
|
|
460
512
|
if (primaryWasLost) {
|
|
@@ -473,7 +525,12 @@ function handleLostNodes(assignments, activeNodeIds) {
|
|
|
473
525
|
markUnassigned(assignment);
|
|
474
526
|
continue;
|
|
475
527
|
}
|
|
476
|
-
const promoted = inSyncCandidates
|
|
528
|
+
const promoted = leastLoadedCandidate(inSyncCandidates, primaryCounts, shares);
|
|
529
|
+
if (promoted === void 0) {
|
|
530
|
+
markUnassigned(assignment);
|
|
531
|
+
continue;
|
|
532
|
+
}
|
|
533
|
+
primaryCounts.set(promoted, (primaryCounts.get(promoted) ?? 0) + 1);
|
|
477
534
|
assignment.primary = promoted;
|
|
478
535
|
assignment.replicas = assignment.replicas.filter((id) => id !== promoted);
|
|
479
536
|
assignment.inSyncSet = assignment.inSyncSet.filter((id) => id !== promoted);
|
|
@@ -506,7 +563,16 @@ function fillReplicaSlots(assignments, sortedNodes, nodeMap, replicationFactor,
|
|
|
506
563
|
}
|
|
507
564
|
}
|
|
508
565
|
}
|
|
509
|
-
function
|
|
566
|
+
function slotWeightOf(nodeId, shares) {
|
|
567
|
+
const share = shares.get(nodeId);
|
|
568
|
+
return 1 / (share === void 0 || share <= 0 ? 1 : share);
|
|
569
|
+
}
|
|
570
|
+
function moveNarrowsGap(mostLoaded, leastLoaded, shares) {
|
|
571
|
+
const gap = mostLoaded.weight - leastLoaded.weight;
|
|
572
|
+
const afterMove = Math.abs(gap - slotWeightOf(mostLoaded.nodeId, shares) - slotWeightOf(leastLoaded.nodeId, shares));
|
|
573
|
+
return afterMove < gap;
|
|
574
|
+
}
|
|
575
|
+
function rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, deciders, shares) {
|
|
510
576
|
let totalSlots = 0;
|
|
511
577
|
for (const assignment of assignments.values()) {
|
|
512
578
|
if (assignment.primary !== null) totalSlots++;
|
|
@@ -522,35 +588,60 @@ function rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, dec
|
|
|
522
588
|
if (mostLoaded.weight - leastLoaded.weight < REBALANCE_THRESHOLD) {
|
|
523
589
|
break;
|
|
524
590
|
}
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
if (moved) break;
|
|
528
|
-
const replicaIndex = assignment.replicas.indexOf(mostLoaded.nodeId);
|
|
529
|
-
if (replicaIndex >= 0) {
|
|
530
|
-
const removedReplica = assignment.replicas[replicaIndex];
|
|
531
|
-
assignment.replicas.splice(replicaIndex, 1);
|
|
532
|
-
const nodeAssignmentCounts = countNodeAssignments(assignments);
|
|
533
|
-
const moveContext = {
|
|
534
|
-
partitionId,
|
|
535
|
-
role: "replica",
|
|
536
|
-
currentAssignment: assignment,
|
|
537
|
-
allAssignments: assignments,
|
|
538
|
-
nodeAssignmentCounts,
|
|
539
|
-
nodes: nodeMap,
|
|
540
|
-
constraints
|
|
541
|
-
};
|
|
542
|
-
const target = findBestNode([leastLoaded.nodeId], weights, deciders, moveContext);
|
|
543
|
-
if (target !== null) {
|
|
544
|
-
assignment.replicas.push(target);
|
|
545
|
-
moved = true;
|
|
546
|
-
} else {
|
|
547
|
-
assignment.replicas.splice(replicaIndex, 0, removedReplica);
|
|
548
|
-
}
|
|
549
|
-
}
|
|
591
|
+
if (!moveNarrowsGap(mostLoaded, leastLoaded, shares)) {
|
|
592
|
+
break;
|
|
550
593
|
}
|
|
594
|
+
const move = {
|
|
595
|
+
fromNodeId: mostLoaded.nodeId,
|
|
596
|
+
toNodeId: leastLoaded.nodeId,
|
|
597
|
+
weights,
|
|
598
|
+
nodeMap,
|
|
599
|
+
constraints,
|
|
600
|
+
deciders
|
|
601
|
+
};
|
|
602
|
+
const laggingFirst = moveOneReplica(assignments, move, true);
|
|
603
|
+
const moved = laggingFirst || moveOneReplica(assignments, move, false);
|
|
551
604
|
if (!moved) break;
|
|
552
605
|
}
|
|
553
606
|
}
|
|
607
|
+
function moveOneReplica(assignments, move, laggingOnly) {
|
|
608
|
+
for (const [partitionId, assignment] of assignments) {
|
|
609
|
+
const replicaIndex = assignment.replicas.indexOf(move.fromNodeId);
|
|
610
|
+
if (replicaIndex < 0) {
|
|
611
|
+
continue;
|
|
612
|
+
}
|
|
613
|
+
if (laggingOnly && assignment.inSyncSet.includes(move.fromNodeId)) {
|
|
614
|
+
continue;
|
|
615
|
+
}
|
|
616
|
+
const removedReplica = assignment.replicas[replicaIndex];
|
|
617
|
+
assignment.replicas.splice(replicaIndex, 1);
|
|
618
|
+
const moveContext = {
|
|
619
|
+
partitionId,
|
|
620
|
+
role: "replica",
|
|
621
|
+
currentAssignment: assignment,
|
|
622
|
+
allAssignments: assignments,
|
|
623
|
+
nodeAssignmentCounts: countNodeAssignments(assignments),
|
|
624
|
+
nodes: move.nodeMap,
|
|
625
|
+
constraints: move.constraints
|
|
626
|
+
};
|
|
627
|
+
const target = findBestNode([move.toNodeId], move.weights, move.deciders, moveContext);
|
|
628
|
+
if (target === null) {
|
|
629
|
+
assignment.replicas.splice(replicaIndex, 0, removedReplica);
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
assignment.replicas.push(target);
|
|
633
|
+
return true;
|
|
634
|
+
}
|
|
635
|
+
return false;
|
|
636
|
+
}
|
|
637
|
+
function pruneInSyncSets(assignments) {
|
|
638
|
+
for (const assignment of assignments.values()) {
|
|
639
|
+
const stale = assignment.inSyncSet.some((nodeId) => !assignment.replicas.includes(nodeId));
|
|
640
|
+
if (stale) {
|
|
641
|
+
assignment.inSyncSet = assignment.inSyncSet.filter((nodeId) => assignment.replicas.includes(nodeId));
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
}
|
|
554
645
|
function rebalanceAllocate(nodes, currentTable, constraints, deciders) {
|
|
555
646
|
const sortedNodes = [...nodes].sort((a, b) => {
|
|
556
647
|
if (a.nodeId < b.nodeId) return -1;
|
|
@@ -563,9 +654,12 @@ function rebalanceAllocate(nodes, currentTable, constraints, deciders) {
|
|
|
563
654
|
nodeMap.set(node.nodeId, node);
|
|
564
655
|
}
|
|
565
656
|
const assignments = cloneAssignments(currentTable.assignments);
|
|
566
|
-
|
|
657
|
+
const shares = capacityShares(sortedNodes);
|
|
658
|
+
handleLostNodes(assignments, activeNodeIds, shares);
|
|
567
659
|
fillReplicaSlots(assignments, sortedNodes, nodeMap, currentTable.replicationFactor, constraints, deciders);
|
|
568
|
-
rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, deciders);
|
|
660
|
+
rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, deciders, shares);
|
|
661
|
+
rebalanceLeadership(assignments, shares);
|
|
662
|
+
pruneInSyncSets(assignments);
|
|
569
663
|
const warnings = collectReplicationWarnings(assignments, currentTable.replicationFactor);
|
|
570
664
|
return {
|
|
571
665
|
table: {
|
|
@@ -613,203 +707,14 @@ function allocate(nodes, currentTable, indexName, partitionCount, replicationFac
|
|
|
613
707
|
}
|
|
614
708
|
return rebalanceAllocate(dataNodes, currentTable, constraints, deciders);
|
|
615
709
|
}
|
|
616
|
-
function isRecord(value) {
|
|
617
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
618
|
-
}
|
|
619
|
-
function isValidInteger(value) {
|
|
620
|
-
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
621
|
-
}
|
|
622
|
-
function validateBootstrapCompletePayload(decoded) {
|
|
623
|
-
if (!isRecord(decoded)) {
|
|
624
|
-
return null;
|
|
625
|
-
}
|
|
626
|
-
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
627
|
-
return null;
|
|
628
|
-
}
|
|
629
|
-
if (!isValidInteger(decoded.partitionId) || decoded.partitionId < 0) {
|
|
630
|
-
return null;
|
|
631
|
-
}
|
|
632
|
-
if (typeof decoded.nodeId !== "string") {
|
|
633
|
-
return null;
|
|
634
|
-
}
|
|
635
|
-
if (!isValidInteger(decoded.primaryTerm)) {
|
|
636
|
-
return null;
|
|
637
|
-
}
|
|
638
|
-
return {
|
|
639
|
-
indexName: decoded.indexName,
|
|
640
|
-
partitionId: decoded.partitionId,
|
|
641
|
-
nodeId: decoded.nodeId,
|
|
642
|
-
primaryTerm: decoded.primaryTerm
|
|
643
|
-
};
|
|
644
|
-
}
|
|
645
|
-
async function sendRejection(respond, controllerNodeId, requestId, indexName, partitionId) {
|
|
646
|
-
const resultPayload = {
|
|
647
|
-
indexName,
|
|
648
|
-
partitionId,
|
|
649
|
-
accepted: false
|
|
650
|
-
};
|
|
651
|
-
const response = {
|
|
652
|
-
type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
|
|
653
|
-
sourceId: controllerNodeId,
|
|
654
|
-
requestId,
|
|
655
|
-
payload: encode(resultPayload)
|
|
656
|
-
};
|
|
657
|
-
await respond(response);
|
|
658
|
-
}
|
|
659
|
-
function handleBootstrapCompleteMessage(message, respond, coordinator, controllerNodeId) {
|
|
660
|
-
let decoded;
|
|
661
|
-
try {
|
|
662
|
-
decoded = decode(message.payload);
|
|
663
|
-
} catch (_) {
|
|
664
|
-
void sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
665
|
-
return;
|
|
666
|
-
}
|
|
667
|
-
const payload = validateBootstrapCompletePayload(decoded);
|
|
668
|
-
if (payload === null) {
|
|
669
|
-
void sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
670
|
-
return;
|
|
671
|
-
}
|
|
672
|
-
if (message.sourceId !== payload.nodeId) {
|
|
673
|
-
void sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
|
|
674
|
-
return;
|
|
675
|
-
}
|
|
676
|
-
processBootstrapComplete(payload, coordinator).then(async (accepted) => {
|
|
677
|
-
const resultPayload = {
|
|
678
|
-
indexName: payload.indexName,
|
|
679
|
-
partitionId: payload.partitionId,
|
|
680
|
-
accepted
|
|
681
|
-
};
|
|
682
|
-
const response = {
|
|
683
|
-
type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
|
|
684
|
-
sourceId: controllerNodeId,
|
|
685
|
-
requestId: message.requestId,
|
|
686
|
-
payload: encode(resultPayload)
|
|
687
|
-
};
|
|
688
|
-
await respond(response);
|
|
689
|
-
}).catch(() => sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId));
|
|
690
|
-
}
|
|
691
|
-
function ensurePrimaryInSyncSet(assignment, inSyncSet) {
|
|
692
|
-
if (assignment.primary === null) {
|
|
693
|
-
return inSyncSet;
|
|
694
|
-
}
|
|
695
|
-
if (inSyncSet.includes(assignment.primary)) {
|
|
696
|
-
return inSyncSet;
|
|
697
|
-
}
|
|
698
|
-
return [assignment.primary, ...inSyncSet];
|
|
699
|
-
}
|
|
700
|
-
async function processBootstrapComplete(payload, coordinator) {
|
|
701
|
-
const table = await coordinator.getAllocation(payload.indexName);
|
|
702
|
-
if (table === null) {
|
|
703
|
-
return false;
|
|
704
|
-
}
|
|
705
|
-
const assignment = table.assignments.get(payload.partitionId);
|
|
706
|
-
if (assignment === void 0) {
|
|
707
|
-
return false;
|
|
708
|
-
}
|
|
709
|
-
if (payload.primaryTerm !== assignment.primaryTerm) {
|
|
710
|
-
return false;
|
|
711
|
-
}
|
|
712
|
-
const isAssigned = assignment.primary === payload.nodeId || assignment.replicas.includes(payload.nodeId);
|
|
713
|
-
if (!isAssigned) {
|
|
714
|
-
return false;
|
|
715
|
-
}
|
|
716
|
-
if (assignment.state === "ACTIVE") {
|
|
717
|
-
if (assignment.inSyncSet.includes(payload.nodeId)) {
|
|
718
|
-
return true;
|
|
719
|
-
}
|
|
720
|
-
const updatedInSyncSet2 = [...assignment.inSyncSet, payload.nodeId];
|
|
721
|
-
const updatedAssignment2 = {
|
|
722
|
-
...assignment,
|
|
723
|
-
inSyncSet: updatedInSyncSet2
|
|
724
|
-
};
|
|
725
|
-
const updatedAssignments2 = new Map(table.assignments);
|
|
726
|
-
updatedAssignments2.set(payload.partitionId, updatedAssignment2);
|
|
727
|
-
const updatedTable2 = {
|
|
728
|
-
...table,
|
|
729
|
-
version: table.version + 1,
|
|
730
|
-
assignments: updatedAssignments2
|
|
731
|
-
};
|
|
732
|
-
return coordinator.putAllocation(payload.indexName, updatedTable2, table.version);
|
|
733
|
-
}
|
|
734
|
-
if (assignment.state !== "INITIALISING") {
|
|
735
|
-
return false;
|
|
736
|
-
}
|
|
737
|
-
const baseInSyncSet = assignment.inSyncSet.includes(payload.nodeId) ? assignment.inSyncSet : [...assignment.inSyncSet, payload.nodeId];
|
|
738
|
-
const updatedInSyncSet = ensurePrimaryInSyncSet(assignment, baseInSyncSet);
|
|
739
|
-
const updatedAssignment = {
|
|
740
|
-
...assignment,
|
|
741
|
-
state: "ACTIVE",
|
|
742
|
-
inSyncSet: updatedInSyncSet
|
|
743
|
-
};
|
|
744
|
-
const updatedAssignments = new Map(table.assignments);
|
|
745
|
-
updatedAssignments.set(payload.partitionId, updatedAssignment);
|
|
746
|
-
const updatedTable = {
|
|
747
|
-
...table,
|
|
748
|
-
version: table.version + 1,
|
|
749
|
-
assignments: updatedAssignments
|
|
750
|
-
};
|
|
751
|
-
return coordinator.putAllocation(payload.indexName, updatedTable, table.version);
|
|
752
|
-
}
|
|
753
710
|
|
|
754
|
-
// src/distribution/cluster/controller/event-loop.ts
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
debounceTimer: null,
|
|
762
|
-
insyncQueue: Promise.resolve()
|
|
763
|
-
};
|
|
764
|
-
}
|
|
765
|
-
function clearEventLoopWatchers(state) {
|
|
766
|
-
if (state.unwatchNodes !== null) {
|
|
767
|
-
state.unwatchNodes();
|
|
768
|
-
state.unwatchNodes = null;
|
|
769
|
-
}
|
|
770
|
-
if (state.unwatchSchemas !== null) {
|
|
771
|
-
state.unwatchSchemas();
|
|
772
|
-
state.unwatchSchemas = null;
|
|
773
|
-
}
|
|
774
|
-
if (state.unwatchTransport !== null) {
|
|
775
|
-
state.unwatchTransport();
|
|
776
|
-
state.unwatchTransport = null;
|
|
777
|
-
}
|
|
778
|
-
if (state.debounceTimer !== null) {
|
|
779
|
-
clearTimeout(state.debounceTimer);
|
|
780
|
-
state.debounceTimer = null;
|
|
781
|
-
}
|
|
782
|
-
}
|
|
783
|
-
function isRecord2(value) {
|
|
784
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
785
|
-
}
|
|
786
|
-
function isValidInteger2(value) {
|
|
787
|
-
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
788
|
-
}
|
|
789
|
-
function validateInsyncRemovePayload(decoded) {
|
|
790
|
-
if (!isRecord2(decoded)) {
|
|
791
|
-
return null;
|
|
792
|
-
}
|
|
793
|
-
if (typeof decoded.indexName !== "string") {
|
|
794
|
-
return null;
|
|
795
|
-
}
|
|
796
|
-
if (!isValidInteger2(decoded.partitionId)) {
|
|
797
|
-
return null;
|
|
798
|
-
}
|
|
799
|
-
if (typeof decoded.replicaNodeId !== "string") {
|
|
800
|
-
return null;
|
|
801
|
-
}
|
|
802
|
-
if (!isValidInteger2(decoded.primaryTerm)) {
|
|
803
|
-
return null;
|
|
804
|
-
}
|
|
805
|
-
return {
|
|
806
|
-
indexName: decoded.indexName,
|
|
807
|
-
partitionId: decoded.partitionId,
|
|
808
|
-
replicaNodeId: decoded.replicaNodeId,
|
|
809
|
-
primaryTerm: decoded.primaryTerm
|
|
810
|
-
};
|
|
811
|
-
}
|
|
812
|
-
async function runAllocatorForIndex(coordinator, indexName, nodes, isActive) {
|
|
711
|
+
// src/distribution/cluster/controller/event-loop/allocation.ts
|
|
712
|
+
var ALLOCATION_CAS_ATTEMPTS = 5;
|
|
713
|
+
var ALLOCATION_RETRY_DELAY_MS = 1e3;
|
|
714
|
+
var TEARDOWN_CAS_ATTEMPTS = 5;
|
|
715
|
+
var TEARDOWN_RETRY_ROUNDS = 3;
|
|
716
|
+
var TEARDOWN_RETRY_DELAY_MS = 500;
|
|
717
|
+
async function runAllocatorForIndex(coordinator, indexName, nodes, isActive, attempt = 0) {
|
|
813
718
|
if (!isActive()) {
|
|
814
719
|
return;
|
|
815
720
|
}
|
|
@@ -832,9 +737,20 @@ async function runAllocatorForIndex(coordinator, indexName, nodes, isActive) {
|
|
|
832
737
|
if (!isActive()) {
|
|
833
738
|
return;
|
|
834
739
|
}
|
|
835
|
-
await coordinator.putAllocation(indexName, result.table, currentTable.version)
|
|
740
|
+
if (await coordinator.putAllocation(indexName, result.table, currentTable.version)) {
|
|
741
|
+
return;
|
|
742
|
+
}
|
|
743
|
+
if (attempt + 1 < ALLOCATION_CAS_ATTEMPTS) {
|
|
744
|
+
await runAllocatorForIndex(coordinator, indexName, nodes, isActive, attempt + 1);
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
747
|
+
throw new NarsilError(
|
|
748
|
+
ErrorCodes.ALLOCATION_FAILED,
|
|
749
|
+
`Allocation for index '${indexName}' lost ${ALLOCATION_CAS_ATTEMPTS} compare-and-set attempts`,
|
|
750
|
+
{ indexName, attempts: ALLOCATION_CAS_ATTEMPTS }
|
|
751
|
+
);
|
|
836
752
|
}
|
|
837
|
-
async function runAllocatorForAllIndexes(
|
|
753
|
+
async function runAllocatorForAllIndexes(state, coordinator, isActive, onError) {
|
|
838
754
|
if (!isActive()) {
|
|
839
755
|
return;
|
|
840
756
|
}
|
|
@@ -842,18 +758,23 @@ async function runAllocatorForAllIndexes(coordinator, knownIndexes, isActive, on
|
|
|
842
758
|
if (nodes.length === 0 || !isActive()) {
|
|
843
759
|
return;
|
|
844
760
|
}
|
|
845
|
-
|
|
761
|
+
let anyIndexFailed = false;
|
|
762
|
+
for (const indexName of state.knownIndexes) {
|
|
846
763
|
if (!isActive()) {
|
|
847
764
|
return;
|
|
848
765
|
}
|
|
849
766
|
try {
|
|
850
767
|
await runAllocatorForIndex(coordinator, indexName, nodes, isActive);
|
|
851
768
|
} catch (error) {
|
|
769
|
+
anyIndexFailed = true;
|
|
852
770
|
if (onError !== void 0) {
|
|
853
771
|
onError(indexName, error);
|
|
854
772
|
}
|
|
855
773
|
}
|
|
856
774
|
}
|
|
775
|
+
if (anyIndexFailed && isActive()) {
|
|
776
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError, ALLOCATION_RETRY_DELAY_MS);
|
|
777
|
+
}
|
|
857
778
|
}
|
|
858
779
|
function scheduleDebouncedAllocation(state, coordinator, isActive, onError, debounceMs = 500) {
|
|
859
780
|
if (state.debounceTimer !== null) {
|
|
@@ -861,9 +782,13 @@ function scheduleDebouncedAllocation(state, coordinator, isActive, onError, debo
|
|
|
861
782
|
}
|
|
862
783
|
state.debounceTimer = setTimeout(() => {
|
|
863
784
|
state.debounceTimer = null;
|
|
864
|
-
runAllocatorForAllIndexes(
|
|
785
|
+
runAllocatorForAllIndexes(state, coordinator, isActive, onError).catch(() => {
|
|
786
|
+
if (isActive()) {
|
|
787
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError, ALLOCATION_RETRY_DELAY_MS);
|
|
788
|
+
}
|
|
865
789
|
});
|
|
866
790
|
}, debounceMs);
|
|
791
|
+
state.debounceTimer.unref?.();
|
|
867
792
|
}
|
|
868
793
|
async function handleSchemaCreated(indexName, coordinator, knownIndexes, isActive) {
|
|
869
794
|
knownIndexes.add(indexName);
|
|
@@ -895,14 +820,31 @@ async function handleSchemaCreated(indexName, coordinator, knownIndexes, isActiv
|
|
|
895
820
|
return;
|
|
896
821
|
}
|
|
897
822
|
const expectedVersion = storedTable !== null ? storedTable.version : null;
|
|
898
|
-
await coordinator.putAllocation(indexName, result.table, expectedVersion)
|
|
823
|
+
if (await coordinator.putAllocation(indexName, result.table, expectedVersion)) {
|
|
824
|
+
return;
|
|
825
|
+
}
|
|
826
|
+
await runAllocatorForIndex(coordinator, indexName, nodes, isActive);
|
|
899
827
|
}
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
828
|
+
function scheduleTeardownRetry(indexName, coordinator, state, isActive, round, onError) {
|
|
829
|
+
const existing = state.teardownTimers.get(indexName);
|
|
830
|
+
if (existing !== void 0) {
|
|
831
|
+
clearTimeout(existing);
|
|
832
|
+
}
|
|
833
|
+
const timer = setTimeout(() => {
|
|
834
|
+
state.teardownTimers.delete(indexName);
|
|
835
|
+
tearDownAllocation(indexName, coordinator, state, isActive, round, onError).catch((error) => {
|
|
836
|
+
if (onError !== void 0) {
|
|
837
|
+
onError(indexName, error);
|
|
838
|
+
}
|
|
839
|
+
});
|
|
840
|
+
}, TEARDOWN_RETRY_DELAY_MS);
|
|
841
|
+
timer.unref?.();
|
|
842
|
+
state.teardownTimers.set(indexName, timer);
|
|
843
|
+
}
|
|
844
|
+
async function tearDownAllocation(indexName, coordinator, state, isActive, round, onError) {
|
|
845
|
+
for (let attempt = 0; attempt < TEARDOWN_CAS_ATTEMPTS; attempt++) {
|
|
846
|
+
if (!isActive()) {
|
|
847
|
+
return;
|
|
906
848
|
}
|
|
907
849
|
const currentTable = await coordinator.getAllocation(indexName);
|
|
908
850
|
if (currentTable === null) {
|
|
@@ -929,438 +871,858 @@ async function handleSchemaDropped(indexName, coordinator, knownIndexes, isActiv
|
|
|
929
871
|
await coordinator.deleteAllocation(indexName);
|
|
930
872
|
return;
|
|
931
873
|
}
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
await handleSchemaCreated(event.indexName, coordinator, knownIndexes, isActive);
|
|
936
|
-
} else if (event.type === "schema_dropped") {
|
|
937
|
-
await handleSchemaDropped(event.indexName, coordinator, knownIndexes, isActive);
|
|
874
|
+
if (round + 1 < TEARDOWN_RETRY_ROUNDS && isActive()) {
|
|
875
|
+
scheduleTeardownRetry(indexName, coordinator, state, isActive, round + 1, onError);
|
|
876
|
+
return;
|
|
938
877
|
}
|
|
878
|
+
throw new NarsilError(
|
|
879
|
+
ErrorCodes.ALLOCATION_FAILED,
|
|
880
|
+
`Teardown of the allocation for dropped index '${indexName}' lost every compare-and-set attempt`,
|
|
881
|
+
{ indexName, attempts: TEARDOWN_CAS_ATTEMPTS * TEARDOWN_RETRY_ROUNDS }
|
|
882
|
+
);
|
|
939
883
|
}
|
|
940
|
-
async function
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
944
|
-
});
|
|
945
|
-
state.unwatchNodes = unwatchNodes;
|
|
946
|
-
const unwatchSchemas = await coordinator.watchSchemas((event) => {
|
|
947
|
-
handleSchemaEvent(event, coordinator, state.knownIndexes, isActive).catch(() => {
|
|
948
|
-
});
|
|
949
|
-
});
|
|
950
|
-
state.unwatchSchemas = unwatchSchemas;
|
|
951
|
-
const unwatchTransport = await transport.listen((message, respond) => {
|
|
952
|
-
if (!isActive()) {
|
|
953
|
-
return;
|
|
954
|
-
}
|
|
955
|
-
if (message.type === ReplicationMessageTypes.INSYNC_REMOVE) {
|
|
956
|
-
handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId);
|
|
957
|
-
} else if (message.type === ClusterMessageTypes.BOOTSTRAP_COMPLETE) {
|
|
958
|
-
handleBootstrapCompleteMessage(message, respond, coordinator, nodeId);
|
|
959
|
-
}
|
|
960
|
-
});
|
|
961
|
-
state.unwatchTransport = unwatchTransport;
|
|
962
|
-
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
963
|
-
}
|
|
964
|
-
function handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId) {
|
|
965
|
-
let decoded;
|
|
966
|
-
try {
|
|
967
|
-
decoded = decode(message.payload);
|
|
968
|
-
} catch (_) {
|
|
884
|
+
async function handleSchemaEvent(event, coordinator, state, isActive, onError) {
|
|
885
|
+
if (event.type === "schema_created") {
|
|
886
|
+
await handleSchemaCreated(event.indexName, coordinator, state.knownIndexes, isActive);
|
|
969
887
|
return;
|
|
970
888
|
}
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
889
|
+
if (event.type === "schema_dropped") {
|
|
890
|
+
state.knownIndexes.delete(event.indexName);
|
|
891
|
+
await tearDownAllocation(event.indexName, coordinator, state, isActive, 0, onError);
|
|
974
892
|
}
|
|
975
|
-
|
|
976
|
-
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
// src/distribution/payload-guards.ts
|
|
896
|
+
function isRecord(value) {
|
|
897
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
898
|
+
}
|
|
899
|
+
function isValidInteger(value) {
|
|
900
|
+
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
901
|
+
}
|
|
902
|
+
var INSYNC_CAS_ATTEMPTS = 5;
|
|
903
|
+
var REFUSE = { outcome: "refuse" };
|
|
904
|
+
var ACCEPT = { outcome: "accept" };
|
|
905
|
+
async function updateInsyncSet(request, coordinator, decide) {
|
|
906
|
+
const refused = {
|
|
907
|
+
indexName: request.indexName,
|
|
908
|
+
partitionId: request.partitionId,
|
|
909
|
+
accepted: false
|
|
910
|
+
};
|
|
911
|
+
for (let attempt = 0; attempt < INSYNC_CAS_ATTEMPTS; attempt++) {
|
|
912
|
+
const table = await coordinator.getAllocation(request.indexName);
|
|
977
913
|
if (table === null) {
|
|
978
|
-
return;
|
|
914
|
+
return refused;
|
|
979
915
|
}
|
|
980
|
-
const assignment = table.assignments.get(
|
|
916
|
+
const assignment = table.assignments.get(request.partitionId);
|
|
981
917
|
if (assignment === void 0) {
|
|
982
|
-
return;
|
|
918
|
+
return refused;
|
|
983
919
|
}
|
|
984
|
-
if (
|
|
985
|
-
return;
|
|
920
|
+
if (request.primaryTerm !== assignment.primaryTerm) {
|
|
921
|
+
return refused;
|
|
986
922
|
}
|
|
987
|
-
const
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
}).catch(() => {
|
|
991
|
-
});
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
// src/distribution/cluster/controller/index.ts
|
|
995
|
-
function createController(config) {
|
|
996
|
-
const { nodeId, coordinator, transport, leaseTtlMs, standbyRetryMs, knownIndexNames, onError } = config;
|
|
997
|
-
let electionState = createElectionState();
|
|
998
|
-
let eventLoopState = createEventLoopState(knownIndexNames);
|
|
999
|
-
let stopped = false;
|
|
1000
|
-
function isActive() {
|
|
1001
|
-
return electionState.active;
|
|
1002
|
-
}
|
|
1003
|
-
function stepDown() {
|
|
1004
|
-
if (!electionState.active) {
|
|
1005
|
-
return;
|
|
923
|
+
const decision = decide(assignment);
|
|
924
|
+
if (decision.outcome === "refuse") {
|
|
925
|
+
return refused;
|
|
1006
926
|
}
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
}
|
|
927
|
+
if (decision.outcome === "accept") {
|
|
928
|
+
return { ...refused, accepted: true };
|
|
929
|
+
}
|
|
930
|
+
const updatedAssignments = new Map(table.assignments);
|
|
931
|
+
updatedAssignments.set(request.partitionId, decision.assignment);
|
|
932
|
+
const written = await coordinator.putAllocation(
|
|
933
|
+
request.indexName,
|
|
934
|
+
{ ...table, version: table.version + 1, assignments: updatedAssignments },
|
|
935
|
+
table.version
|
|
936
|
+
);
|
|
937
|
+
if (written) {
|
|
938
|
+
return { ...refused, accepted: true };
|
|
1015
939
|
}
|
|
1016
940
|
}
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
941
|
+
return refused;
|
|
942
|
+
}
|
|
943
|
+
async function requestInsyncRemoval(indexName, partitionId, replicaNodeId, primaryTerm, controllerNodeId, transport, sourceNodeId) {
|
|
944
|
+
const payload = {
|
|
945
|
+
indexName,
|
|
946
|
+
partitionId,
|
|
947
|
+
replicaNodeId,
|
|
948
|
+
primaryTerm
|
|
949
|
+
};
|
|
950
|
+
const message = createInsyncRemoveMessage(payload, sourceNodeId);
|
|
951
|
+
const response = await transport.send(controllerNodeId, message);
|
|
952
|
+
const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
|
|
953
|
+
return { accepted: confirmPayload.accepted };
|
|
954
|
+
}
|
|
955
|
+
function handleInsyncRemoval(payload, coordinator) {
|
|
956
|
+
return updateInsyncSet(payload, coordinator, (assignment) => {
|
|
957
|
+
if (!assignment.inSyncSet.includes(payload.replicaNodeId)) {
|
|
958
|
+
return ACCEPT;
|
|
1020
959
|
}
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
960
|
+
return {
|
|
961
|
+
outcome: "write",
|
|
962
|
+
assignment: {
|
|
963
|
+
...assignment,
|
|
964
|
+
inSyncSet: assignment.inSyncSet.filter((nodeId) => nodeId !== payload.replicaNodeId)
|
|
1026
965
|
}
|
|
1027
|
-
}
|
|
966
|
+
};
|
|
967
|
+
});
|
|
968
|
+
}
|
|
969
|
+
async function requestInsyncAdmission(payload, controllerNodeId, transport, sourceNodeId) {
|
|
970
|
+
const message = createInsyncAddMessage(payload, sourceNodeId);
|
|
971
|
+
const response = await transport.send(controllerNodeId, message);
|
|
972
|
+
const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
|
|
973
|
+
return { accepted: confirmPayload.accepted };
|
|
974
|
+
}
|
|
975
|
+
function handleInsyncAdmission(payload, coordinator) {
|
|
976
|
+
return updateInsyncSet(payload, coordinator, (assignment) => {
|
|
977
|
+
if (assignment.state !== "ACTIVE") {
|
|
978
|
+
return REFUSE;
|
|
1028
979
|
}
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
async function tryElection() {
|
|
1032
|
-
if (stopped) {
|
|
1033
|
-
return;
|
|
980
|
+
if (!assignment.replicas.includes(payload.replicaNodeId)) {
|
|
981
|
+
return REFUSE;
|
|
1034
982
|
}
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
await becomeActive();
|
|
1038
|
-
} else {
|
|
1039
|
-
scheduleStandbyRetry(electionState, standbyRetryMs, () => {
|
|
1040
|
-
tryElection().catch(() => {
|
|
1041
|
-
});
|
|
1042
|
-
});
|
|
983
|
+
if (payload.appliedSeqNo < assignment.commitPoint || payload.appliedSeqNo < payload.commitPoint) {
|
|
984
|
+
return REFUSE;
|
|
1043
985
|
}
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
return
|
|
1048
|
-
},
|
|
1049
|
-
async start() {
|
|
1050
|
-
stopped = false;
|
|
1051
|
-
await tryElection();
|
|
1052
|
-
},
|
|
1053
|
-
async stop() {
|
|
1054
|
-
stopped = true;
|
|
1055
|
-
clearElectionTimers(electionState);
|
|
1056
|
-
clearEventLoopWatchers(eventLoopState);
|
|
1057
|
-
if (electionState.active) {
|
|
1058
|
-
electionState.active = false;
|
|
1059
|
-
try {
|
|
1060
|
-
await releaseLease(coordinator);
|
|
1061
|
-
} catch (_) {
|
|
1062
|
-
}
|
|
1063
|
-
}
|
|
1064
|
-
},
|
|
1065
|
-
async shutdown() {
|
|
1066
|
-
await controller.stop();
|
|
1067
|
-
eventLoopState = createEventLoopState([]);
|
|
1068
|
-
electionState = createElectionState();
|
|
986
|
+
const alreadyInSync = assignment.inSyncSet.includes(payload.replicaNodeId);
|
|
987
|
+
const commitPoint = Math.max(assignment.commitPoint, payload.commitPoint);
|
|
988
|
+
if (alreadyInSync && commitPoint === assignment.commitPoint) {
|
|
989
|
+
return ACCEPT;
|
|
1069
990
|
}
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
991
|
+
return {
|
|
992
|
+
outcome: "write",
|
|
993
|
+
assignment: {
|
|
994
|
+
...assignment,
|
|
995
|
+
inSyncSet: alreadyInSync ? assignment.inSyncSet : [...assignment.inSyncSet, payload.replicaNodeId],
|
|
996
|
+
commitPoint
|
|
997
|
+
}
|
|
998
|
+
};
|
|
999
|
+
});
|
|
1078
1000
|
}
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
if (table === null) {
|
|
1001
|
+
function validateBootstrapCompletePayload(decoded) {
|
|
1002
|
+
if (!isRecord(decoded)) {
|
|
1082
1003
|
return null;
|
|
1083
1004
|
}
|
|
1084
|
-
|
|
1085
|
-
if (assignment === void 0) {
|
|
1005
|
+
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
1086
1006
|
return null;
|
|
1087
1007
|
}
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
async function resolveControllerTargets(coordinator, controllerNodeId) {
|
|
1091
|
-
const targets = [controllerNodeId];
|
|
1092
|
-
try {
|
|
1093
|
-
const nodes = await coordinator.listNodes();
|
|
1094
|
-
const registration = nodes.find((entry) => entry.nodeId === controllerNodeId);
|
|
1095
|
-
if (registration !== void 0 && registration.address.length > 0 && registration.address !== controllerNodeId) {
|
|
1096
|
-
targets.push(registration.address);
|
|
1097
|
-
}
|
|
1098
|
-
} catch (_) {
|
|
1099
|
-
return targets;
|
|
1008
|
+
if (!isValidInteger(decoded.partitionId) || decoded.partitionId < 0) {
|
|
1009
|
+
return null;
|
|
1100
1010
|
}
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
async function reportBootstrapComplete(indexName, partitionId, nodeId, coordinator, transport) {
|
|
1104
|
-
const controllerNodeId = await coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
1105
|
-
if (controllerNodeId === null) {
|
|
1106
|
-
return false;
|
|
1011
|
+
if (typeof decoded.nodeId !== "string") {
|
|
1012
|
+
return null;
|
|
1107
1013
|
}
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
return false;
|
|
1014
|
+
if (!isValidInteger(decoded.primaryTerm)) {
|
|
1015
|
+
return null;
|
|
1111
1016
|
}
|
|
1112
|
-
|
|
1017
|
+
return {
|
|
1018
|
+
indexName: decoded.indexName,
|
|
1019
|
+
partitionId: decoded.partitionId,
|
|
1020
|
+
nodeId: decoded.nodeId,
|
|
1021
|
+
primaryTerm: decoded.primaryTerm
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
async function sendRejection(respond, controllerNodeId, requestId, indexName, partitionId) {
|
|
1025
|
+
const resultPayload = {
|
|
1113
1026
|
indexName,
|
|
1114
1027
|
partitionId,
|
|
1115
|
-
|
|
1116
|
-
primaryTerm
|
|
1028
|
+
accepted: false
|
|
1117
1029
|
};
|
|
1118
|
-
const
|
|
1030
|
+
const response = {
|
|
1119
1031
|
type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
|
|
1120
|
-
sourceId:
|
|
1121
|
-
requestId
|
|
1122
|
-
payload: encode(
|
|
1032
|
+
sourceId: controllerNodeId,
|
|
1033
|
+
requestId,
|
|
1034
|
+
payload: encode(resultPayload)
|
|
1123
1035
|
};
|
|
1124
|
-
|
|
1125
|
-
try {
|
|
1126
|
-
const response = await transport.send(target, message);
|
|
1127
|
-
return isAcceptedResponse(response);
|
|
1128
|
-
} catch (_) {
|
|
1129
|
-
}
|
|
1130
|
-
}
|
|
1131
|
-
return false;
|
|
1036
|
+
await respond(response);
|
|
1132
1037
|
}
|
|
1133
|
-
function
|
|
1038
|
+
async function rejectBootstrapComplete(message, respond, controllerNodeId) {
|
|
1039
|
+
let decoded;
|
|
1134
1040
|
try {
|
|
1135
|
-
|
|
1136
|
-
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
1137
|
-
return false;
|
|
1138
|
-
}
|
|
1139
|
-
return decoded.accepted === true;
|
|
1041
|
+
decoded = decode(message.payload);
|
|
1140
1042
|
} catch (_) {
|
|
1141
|
-
|
|
1043
|
+
await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
1044
|
+
return;
|
|
1045
|
+
}
|
|
1046
|
+
const payload = validateBootstrapCompletePayload(decoded);
|
|
1047
|
+
if (payload === null) {
|
|
1048
|
+
await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
1049
|
+
return;
|
|
1142
1050
|
}
|
|
1051
|
+
await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
|
|
1143
1052
|
}
|
|
1144
|
-
async function
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
return false;
|
|
1152
|
-
}
|
|
1153
|
-
if (synced) {
|
|
1154
|
-
return retryReportBootstrapComplete(
|
|
1155
|
-
state,
|
|
1156
|
-
coordinator,
|
|
1157
|
-
transport,
|
|
1158
|
-
nodeId,
|
|
1159
|
-
bootstrapRetryBaseMs,
|
|
1160
|
-
bootstrapRetryMaxMs,
|
|
1161
|
-
bootstrapMaxRetries,
|
|
1162
|
-
onError
|
|
1163
|
-
);
|
|
1164
|
-
}
|
|
1165
|
-
if (attempt < bootstrapMaxRetries) {
|
|
1166
|
-
const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
|
|
1167
|
-
await waitWithAbort(state, backoffMs);
|
|
1168
|
-
}
|
|
1053
|
+
async function handleBootstrapCompleteMessage(message, respond, coordinator, controllerNodeId) {
|
|
1054
|
+
let decoded;
|
|
1055
|
+
try {
|
|
1056
|
+
decoded = decode(message.payload);
|
|
1057
|
+
} catch (_) {
|
|
1058
|
+
await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
1059
|
+
return;
|
|
1169
1060
|
}
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1061
|
+
const payload = validateBootstrapCompletePayload(decoded);
|
|
1062
|
+
if (payload === null) {
|
|
1063
|
+
await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1066
|
+
if (message.sourceId !== payload.nodeId) {
|
|
1067
|
+
await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
|
|
1068
|
+
return;
|
|
1069
|
+
}
|
|
1070
|
+
let accepted;
|
|
1173
1071
|
try {
|
|
1174
|
-
|
|
1175
|
-
} catch (
|
|
1176
|
-
|
|
1072
|
+
accepted = await processBootstrapComplete(payload, coordinator);
|
|
1073
|
+
} catch (_) {
|
|
1074
|
+
await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
|
|
1075
|
+
return;
|
|
1076
|
+
}
|
|
1077
|
+
const resultPayload = {
|
|
1078
|
+
indexName: payload.indexName,
|
|
1079
|
+
partitionId: payload.partitionId,
|
|
1080
|
+
accepted
|
|
1081
|
+
};
|
|
1082
|
+
await respond({
|
|
1083
|
+
type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
|
|
1084
|
+
sourceId: controllerNodeId,
|
|
1085
|
+
requestId: message.requestId,
|
|
1086
|
+
payload: encode(resultPayload)
|
|
1087
|
+
});
|
|
1088
|
+
}
|
|
1089
|
+
var BOOTSTRAP_CAS_ATTEMPTS = 5;
|
|
1090
|
+
async function processBootstrapComplete(payload, coordinator) {
|
|
1091
|
+
for (let attempt = 0; attempt < BOOTSTRAP_CAS_ATTEMPTS; attempt++) {
|
|
1092
|
+
const table = await coordinator.getAllocation(payload.indexName);
|
|
1093
|
+
if (table === null) {
|
|
1177
1094
|
return false;
|
|
1178
1095
|
}
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
new NarsilError(
|
|
1183
|
-
ErrorCodes.NODE_BOOTSTRAP_FAILED,
|
|
1184
|
-
`Bootstrap sync failed for ${state.indexName}:${state.partitionId}`,
|
|
1185
|
-
{ indexName: state.indexName, partitionId: state.partitionId, cause }
|
|
1186
|
-
)
|
|
1187
|
-
);
|
|
1096
|
+
const assignment = table.assignments.get(payload.partitionId);
|
|
1097
|
+
if (assignment === void 0) {
|
|
1098
|
+
return false;
|
|
1188
1099
|
}
|
|
1189
|
-
|
|
1190
|
-
}
|
|
1191
|
-
}
|
|
1192
|
-
async function retryReportBootstrapComplete(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onError) {
|
|
1193
|
-
for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
|
|
1194
|
-
if (state.aborted) {
|
|
1100
|
+
if (payload.primaryTerm !== assignment.primaryTerm) {
|
|
1195
1101
|
return false;
|
|
1196
1102
|
}
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
return true;
|
|
1103
|
+
if (assignment.primary !== payload.nodeId && !assignment.replicas.includes(payload.nodeId)) {
|
|
1104
|
+
return false;
|
|
1200
1105
|
}
|
|
1201
|
-
if (
|
|
1202
|
-
|
|
1203
|
-
await waitWithAbort(state, backoffMs);
|
|
1106
|
+
if (assignment.state !== "INITIALISING") {
|
|
1107
|
+
return false;
|
|
1204
1108
|
}
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
{ indexName: state.indexName, partitionId: state.partitionId }
|
|
1212
|
-
)
|
|
1109
|
+
const updatedAssignments = new Map(table.assignments);
|
|
1110
|
+
updatedAssignments.set(payload.partitionId, { ...assignment, state: "ACTIVE" });
|
|
1111
|
+
const written = await coordinator.putAllocation(
|
|
1112
|
+
payload.indexName,
|
|
1113
|
+
{ ...table, version: table.version + 1, assignments: updatedAssignments },
|
|
1114
|
+
table.version
|
|
1213
1115
|
);
|
|
1116
|
+
if (written) {
|
|
1117
|
+
return true;
|
|
1118
|
+
}
|
|
1214
1119
|
}
|
|
1215
1120
|
return false;
|
|
1216
1121
|
}
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
state.abortResolve = resolve;
|
|
1224
|
-
state.retryTimer = setTimeout(() => {
|
|
1225
|
-
state.retryTimer = null;
|
|
1226
|
-
state.abortResolve = null;
|
|
1227
|
-
resolve();
|
|
1228
|
-
}, ms);
|
|
1229
|
-
});
|
|
1230
|
-
}
|
|
1231
|
-
function abortBootstrapState(state) {
|
|
1232
|
-
state.aborted = true;
|
|
1233
|
-
if (state.retryTimer !== null) {
|
|
1234
|
-
clearTimeout(state.retryTimer);
|
|
1235
|
-
state.retryTimer = null;
|
|
1122
|
+
|
|
1123
|
+
// src/distribution/cluster/controller/event-loop/insync-messages.ts
|
|
1124
|
+
var UNKNOWN_PARTITION_ID = -1;
|
|
1125
|
+
function validateInsyncRemovePayload(decoded) {
|
|
1126
|
+
if (!isRecord(decoded)) {
|
|
1127
|
+
return null;
|
|
1236
1128
|
}
|
|
1237
|
-
if (
|
|
1238
|
-
|
|
1239
|
-
|
|
1129
|
+
if (typeof decoded.indexName !== "string") {
|
|
1130
|
+
return null;
|
|
1131
|
+
}
|
|
1132
|
+
if (!isValidInteger(decoded.partitionId)) {
|
|
1133
|
+
return null;
|
|
1134
|
+
}
|
|
1135
|
+
if (typeof decoded.replicaNodeId !== "string") {
|
|
1136
|
+
return null;
|
|
1137
|
+
}
|
|
1138
|
+
if (!isValidInteger(decoded.primaryTerm)) {
|
|
1139
|
+
return null;
|
|
1240
1140
|
}
|
|
1241
|
-
}
|
|
1242
|
-
function createBootstrapState(indexName, partitionId, primaryNodeId) {
|
|
1243
1141
|
return {
|
|
1244
|
-
indexName,
|
|
1245
|
-
partitionId,
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
retryTimer: null,
|
|
1249
|
-
aborted: false,
|
|
1250
|
-
abortResolve: null
|
|
1142
|
+
indexName: decoded.indexName,
|
|
1143
|
+
partitionId: decoded.partitionId,
|
|
1144
|
+
replicaNodeId: decoded.replicaNodeId,
|
|
1145
|
+
primaryTerm: decoded.primaryTerm
|
|
1251
1146
|
};
|
|
1252
1147
|
}
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1148
|
+
function validateInsyncAddPayload(decoded) {
|
|
1149
|
+
const base = validateInsyncRemovePayload(decoded);
|
|
1150
|
+
if (base === null || !isRecord(decoded)) {
|
|
1151
|
+
return null;
|
|
1152
|
+
}
|
|
1153
|
+
if (!isValidInteger(decoded.appliedSeqNo) || decoded.appliedSeqNo < 0) {
|
|
1154
|
+
return null;
|
|
1155
|
+
}
|
|
1156
|
+
if (!isValidInteger(decoded.commitPoint) || decoded.commitPoint < 0) {
|
|
1157
|
+
return null;
|
|
1158
|
+
}
|
|
1159
|
+
return { ...base, appliedSeqNo: decoded.appliedSeqNo, commitPoint: decoded.commitPoint };
|
|
1257
1160
|
}
|
|
1258
|
-
function
|
|
1161
|
+
function decodeMessage(message) {
|
|
1162
|
+
try {
|
|
1163
|
+
return decode(message.payload);
|
|
1164
|
+
} catch (_) {
|
|
1165
|
+
return null;
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1168
|
+
function describeRequest(decoded) {
|
|
1169
|
+
if (!isRecord(decoded)) {
|
|
1170
|
+
return { indexName: "", partitionId: UNKNOWN_PARTITION_ID };
|
|
1171
|
+
}
|
|
1259
1172
|
return {
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
activeBootstraps: /* @__PURE__ */ new Map(),
|
|
1263
|
-
trackedPartitions: /* @__PURE__ */ new Map(),
|
|
1264
|
-
pendingTables: /* @__PURE__ */ new Map(),
|
|
1265
|
-
stopped: false
|
|
1173
|
+
indexName: typeof decoded.indexName === "string" ? decoded.indexName : "",
|
|
1174
|
+
partitionId: isValidInteger(decoded.partitionId) ? decoded.partitionId : UNKNOWN_PARTITION_ID
|
|
1266
1175
|
};
|
|
1267
1176
|
}
|
|
1268
|
-
async function
|
|
1269
|
-
const
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1177
|
+
async function assertSenderLeadsPartition(coordinator, indexName, partitionId, sourceId) {
|
|
1178
|
+
const table = await coordinator.getAllocation(indexName);
|
|
1179
|
+
const assignment = table?.assignments.get(partitionId);
|
|
1180
|
+
if (assignment === void 0) {
|
|
1181
|
+
return false;
|
|
1182
|
+
}
|
|
1183
|
+
return assignment.primary === sourceId;
|
|
1184
|
+
}
|
|
1185
|
+
function enqueue(state, task) {
|
|
1186
|
+
state.insyncQueue = state.insyncQueue.then(task).catch(() => {
|
|
1274
1187
|
});
|
|
1275
|
-
state.unwatchAllocation = unwatchAllocation;
|
|
1276
1188
|
}
|
|
1277
|
-
function
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1189
|
+
async function respondRefusal(respond, request, nodeId, requestId) {
|
|
1190
|
+
const payload = {
|
|
1191
|
+
indexName: request.indexName,
|
|
1192
|
+
partitionId: request.partitionId,
|
|
1193
|
+
accepted: false
|
|
1194
|
+
};
|
|
1195
|
+
try {
|
|
1196
|
+
await respond(createInsyncConfirmMessage(payload, nodeId, requestId));
|
|
1197
|
+
} catch (_) {
|
|
1281
1198
|
}
|
|
1282
|
-
state.debounceTimer = setTimeout(() => {
|
|
1283
|
-
state.debounceTimer = null;
|
|
1284
|
-
const tablesToProcess = Array.from(state.pendingTables.values());
|
|
1285
|
-
state.pendingTables.clear();
|
|
1286
|
-
for (const pending of tablesToProcess) {
|
|
1287
|
-
processAllocationChange(state, config, pending);
|
|
1288
|
-
}
|
|
1289
|
-
}, config.allocationDebounceMs);
|
|
1290
1199
|
}
|
|
1291
|
-
function
|
|
1292
|
-
|
|
1200
|
+
function handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId, isActive) {
|
|
1201
|
+
const decoded = decodeMessage(message);
|
|
1202
|
+
const payload = validateInsyncRemovePayload(decoded);
|
|
1203
|
+
if (payload === null) {
|
|
1204
|
+
void respondRefusal(respond, describeRequest(decoded), nodeId, message.requestId);
|
|
1293
1205
|
return;
|
|
1294
1206
|
}
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
if (existing === void 0) {
|
|
1306
|
-
handleNewAssignment(state, config, table.indexName, partitionId, assignment, nodeId);
|
|
1307
|
-
} else {
|
|
1308
|
-
handleExistingAssignment(state, config, existing, assignment, nodeId);
|
|
1207
|
+
enqueue(state, async () => {
|
|
1208
|
+
const leadsPartition = await assertSenderLeadsPartition(
|
|
1209
|
+
coordinator,
|
|
1210
|
+
payload.indexName,
|
|
1211
|
+
payload.partitionId,
|
|
1212
|
+
message.sourceId
|
|
1213
|
+
);
|
|
1214
|
+
if (!leadsPartition || !isActive()) {
|
|
1215
|
+
await respondRefusal(respond, payload, nodeId, message.requestId);
|
|
1216
|
+
return;
|
|
1309
1217
|
}
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
isPrimary,
|
|
1314
|
-
primaryNodeId: assignment.primary,
|
|
1315
|
-
primaryTerm: assignment.primaryTerm
|
|
1316
|
-
});
|
|
1317
|
-
}
|
|
1318
|
-
removeUnassignedPartitions(state, config, table.indexName, currentKeys);
|
|
1319
|
-
}
|
|
1320
|
-
function isNodeAssigned(assignment, nodeId) {
|
|
1321
|
-
if (assignment.primary === nodeId) {
|
|
1322
|
-
return true;
|
|
1323
|
-
}
|
|
1324
|
-
return assignment.replicas.includes(nodeId);
|
|
1218
|
+
const confirmPayload = await handleInsyncRemoval(payload, coordinator);
|
|
1219
|
+
await respond(createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId));
|
|
1220
|
+
});
|
|
1325
1221
|
}
|
|
1326
|
-
function
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1222
|
+
function handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive, onError) {
|
|
1223
|
+
const decoded = decodeMessage(message);
|
|
1224
|
+
const payload = validateInsyncAddPayload(decoded);
|
|
1225
|
+
if (payload === null) {
|
|
1226
|
+
void respondRefusal(respond, describeRequest(decoded), nodeId, message.requestId);
|
|
1331
1227
|
return;
|
|
1332
1228
|
}
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1229
|
+
enqueue(state, async () => {
|
|
1230
|
+
const leadsPartition = await assertSenderLeadsPartition(
|
|
1231
|
+
coordinator,
|
|
1232
|
+
payload.indexName,
|
|
1233
|
+
payload.partitionId,
|
|
1234
|
+
message.sourceId
|
|
1235
|
+
);
|
|
1236
|
+
if (!leadsPartition || !isActive()) {
|
|
1237
|
+
await respondRefusal(respond, payload, nodeId, message.requestId);
|
|
1238
|
+
return;
|
|
1336
1239
|
}
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1240
|
+
const confirmPayload = await handleInsyncAdmission(payload, coordinator);
|
|
1241
|
+
await respond(createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId));
|
|
1242
|
+
if (confirmPayload.accepted && isActive()) {
|
|
1243
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
1244
|
+
}
|
|
1245
|
+
});
|
|
1340
1246
|
}
|
|
1341
|
-
function
|
|
1342
|
-
|
|
1343
|
-
if (
|
|
1344
|
-
|
|
1345
|
-
config.onPrimaryDemotion(existing.indexName, existing.partitionId, assignment.primary);
|
|
1346
|
-
}
|
|
1347
|
-
startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
|
|
1247
|
+
function handleQueuedBootstrapComplete(state, message, respond, coordinator, nodeId, isActive) {
|
|
1248
|
+
enqueue(state, () => {
|
|
1249
|
+
if (!isActive()) {
|
|
1250
|
+
return rejectBootstrapComplete(message, respond, nodeId);
|
|
1348
1251
|
}
|
|
1349
|
-
return;
|
|
1350
|
-
}
|
|
1351
|
-
if (assignment.primary !== null && replicaNeedsResync(state, existing, assignment, nodeId)) {
|
|
1352
|
-
startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
|
|
1353
|
-
}
|
|
1252
|
+
return handleBootstrapCompleteMessage(message, respond, coordinator, nodeId);
|
|
1253
|
+
});
|
|
1354
1254
|
}
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1255
|
+
|
|
1256
|
+
// src/distribution/cluster/controller/event-loop/state.ts
|
|
1257
|
+
function createEventLoopState(initialIndexNames) {
|
|
1258
|
+
return {
|
|
1259
|
+
unwatchNodes: null,
|
|
1260
|
+
unwatchSchemas: null,
|
|
1261
|
+
unwatchTransport: null,
|
|
1262
|
+
knownIndexes: new Set(initialIndexNames),
|
|
1263
|
+
debounceTimer: null,
|
|
1264
|
+
teardownTimers: /* @__PURE__ */ new Map(),
|
|
1265
|
+
insyncQueue: Promise.resolve()
|
|
1266
|
+
};
|
|
1267
|
+
}
|
|
1268
|
+
function clearEventLoopWatchers(state) {
|
|
1269
|
+
if (state.unwatchNodes !== null) {
|
|
1270
|
+
state.unwatchNodes();
|
|
1271
|
+
state.unwatchNodes = null;
|
|
1272
|
+
}
|
|
1273
|
+
if (state.unwatchSchemas !== null) {
|
|
1274
|
+
state.unwatchSchemas();
|
|
1275
|
+
state.unwatchSchemas = null;
|
|
1276
|
+
}
|
|
1277
|
+
if (state.unwatchTransport !== null) {
|
|
1278
|
+
state.unwatchTransport();
|
|
1279
|
+
state.unwatchTransport = null;
|
|
1280
|
+
}
|
|
1281
|
+
if (state.debounceTimer !== null) {
|
|
1282
|
+
clearTimeout(state.debounceTimer);
|
|
1283
|
+
state.debounceTimer = null;
|
|
1284
|
+
}
|
|
1285
|
+
for (const timer of state.teardownTimers.values()) {
|
|
1286
|
+
clearTimeout(timer);
|
|
1287
|
+
}
|
|
1288
|
+
state.teardownTimers.clear();
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
// src/distribution/cluster/controller/event-loop/index.ts
|
|
1292
|
+
async function startEventLoop(state, coordinator, transport, nodeId, isActive, onError) {
|
|
1293
|
+
clearEventLoopWatchers(state);
|
|
1294
|
+
try {
|
|
1295
|
+
state.unwatchNodes = await coordinator.watchNodes((_event) => {
|
|
1296
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
1297
|
+
});
|
|
1298
|
+
state.unwatchSchemas = await coordinator.watchSchemas((event) => {
|
|
1299
|
+
handleSchemaEvent(event, coordinator, state, isActive, onError).catch((error) => {
|
|
1300
|
+
if (onError !== void 0) {
|
|
1301
|
+
onError(event.indexName, error);
|
|
1302
|
+
}
|
|
1303
|
+
});
|
|
1304
|
+
});
|
|
1305
|
+
state.unwatchTransport = await transport.listen((message, respond) => {
|
|
1306
|
+
if (!isActive()) {
|
|
1307
|
+
return;
|
|
1308
|
+
}
|
|
1309
|
+
if (message.type === ReplicationMessageTypes.INSYNC_REMOVE) {
|
|
1310
|
+
handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId, isActive);
|
|
1311
|
+
} else if (message.type === ReplicationMessageTypes.INSYNC_ADD) {
|
|
1312
|
+
handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive, onError);
|
|
1313
|
+
} else if (message.type === ClusterMessageTypes.BOOTSTRAP_COMPLETE) {
|
|
1314
|
+
handleQueuedBootstrapComplete(state, message, respond, coordinator, nodeId, isActive);
|
|
1315
|
+
}
|
|
1316
|
+
});
|
|
1317
|
+
} catch (error) {
|
|
1318
|
+
clearEventLoopWatchers(state);
|
|
1319
|
+
throw error;
|
|
1320
|
+
}
|
|
1321
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
// src/distribution/cluster/controller/index.ts
|
|
1325
|
+
function createController(config) {
|
|
1326
|
+
const { nodeId, coordinator, transport, leaseTtlMs, standbyRetryMs, knownIndexNames, onError } = config;
|
|
1327
|
+
let electionState = createElectionState();
|
|
1328
|
+
let eventLoopState = createEventLoopState(knownIndexNames);
|
|
1329
|
+
let stopped = false;
|
|
1330
|
+
function isActive() {
|
|
1331
|
+
return electionState.active;
|
|
1332
|
+
}
|
|
1333
|
+
function stepDown() {
|
|
1334
|
+
if (!electionState.active) {
|
|
1335
|
+
return;
|
|
1336
|
+
}
|
|
1337
|
+
electionState.active = false;
|
|
1338
|
+
clearElectionTimers(electionState);
|
|
1339
|
+
clearEventLoopWatchers(eventLoopState);
|
|
1340
|
+
if (!stopped) {
|
|
1341
|
+
scheduleStandbyRetry(electionState, standbyRetryMs, () => {
|
|
1342
|
+
tryElection().catch(() => {
|
|
1343
|
+
});
|
|
1344
|
+
});
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
async function becomeActive() {
|
|
1348
|
+
if (electionState.active) {
|
|
1349
|
+
return;
|
|
1350
|
+
}
|
|
1351
|
+
electionState.active = true;
|
|
1352
|
+
startRenewalInterval(electionState, coordinator, nodeId, leaseTtlMs, stepDown);
|
|
1353
|
+
try {
|
|
1354
|
+
for (const indexName of await coordinator.listSchemas()) {
|
|
1355
|
+
eventLoopState.knownIndexes.add(indexName);
|
|
1356
|
+
}
|
|
1357
|
+
} catch (_) {
|
|
1358
|
+
}
|
|
1359
|
+
try {
|
|
1360
|
+
await startEventLoop(eventLoopState, coordinator, transport, nodeId, isActive, onError);
|
|
1361
|
+
} catch (error) {
|
|
1362
|
+
electionState.active = false;
|
|
1363
|
+
clearElectionTimers(electionState);
|
|
1364
|
+
try {
|
|
1365
|
+
await releaseLease(coordinator);
|
|
1366
|
+
} catch (_) {
|
|
1367
|
+
}
|
|
1368
|
+
throw error;
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
async function tryElection() {
|
|
1372
|
+
if (stopped) {
|
|
1373
|
+
return;
|
|
1374
|
+
}
|
|
1375
|
+
const acquired = await tryAcquireLease(coordinator, nodeId, leaseTtlMs);
|
|
1376
|
+
if (acquired) {
|
|
1377
|
+
await becomeActive();
|
|
1378
|
+
} else {
|
|
1379
|
+
scheduleStandbyRetry(electionState, standbyRetryMs, () => {
|
|
1380
|
+
tryElection().catch(() => {
|
|
1381
|
+
});
|
|
1382
|
+
});
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
const controller = {
|
|
1386
|
+
get isActive() {
|
|
1387
|
+
return electionState.active;
|
|
1388
|
+
},
|
|
1389
|
+
async start() {
|
|
1390
|
+
stopped = false;
|
|
1391
|
+
await tryElection();
|
|
1392
|
+
},
|
|
1393
|
+
async stop() {
|
|
1394
|
+
stopped = true;
|
|
1395
|
+
clearElectionTimers(electionState);
|
|
1396
|
+
clearEventLoopWatchers(eventLoopState);
|
|
1397
|
+
if (electionState.active) {
|
|
1398
|
+
electionState.active = false;
|
|
1399
|
+
try {
|
|
1400
|
+
await releaseLease(coordinator);
|
|
1401
|
+
} catch (_) {
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
},
|
|
1405
|
+
async shutdown() {
|
|
1406
|
+
await controller.stop();
|
|
1407
|
+
eventLoopState = createEventLoopState([]);
|
|
1408
|
+
electionState = createElectionState();
|
|
1409
|
+
}
|
|
1410
|
+
};
|
|
1411
|
+
return controller;
|
|
1412
|
+
}
|
|
1413
|
+
function computeBackoffMs(baseMs, maxMs, retryCount) {
|
|
1414
|
+
const exponential = baseMs * 2 ** retryCount;
|
|
1415
|
+
const capped = Math.min(exponential, maxMs);
|
|
1416
|
+
const jitter = capped * (0.5 + Math.random() * 0.5);
|
|
1417
|
+
return Math.floor(jitter);
|
|
1418
|
+
}
|
|
1419
|
+
async function resolvePrimaryTerm(indexName, partitionId, coordinator) {
|
|
1420
|
+
const table = await coordinator.getAllocation(indexName);
|
|
1421
|
+
if (table === null) {
|
|
1422
|
+
return null;
|
|
1423
|
+
}
|
|
1424
|
+
const assignment = table.assignments.get(partitionId);
|
|
1425
|
+
if (assignment === void 0) {
|
|
1426
|
+
return null;
|
|
1427
|
+
}
|
|
1428
|
+
return assignment.primaryTerm;
|
|
1429
|
+
}
|
|
1430
|
+
async function resolveControllerTargets(coordinator, controllerNodeId) {
|
|
1431
|
+
const targets = [controllerNodeId];
|
|
1432
|
+
try {
|
|
1433
|
+
const nodes = await coordinator.listNodes();
|
|
1434
|
+
const registration = nodes.find((entry) => entry.nodeId === controllerNodeId);
|
|
1435
|
+
if (registration !== void 0 && registration.address.length > 0 && registration.address !== controllerNodeId) {
|
|
1436
|
+
targets.push(registration.address);
|
|
1437
|
+
}
|
|
1438
|
+
} catch (_) {
|
|
1439
|
+
return targets;
|
|
1440
|
+
}
|
|
1441
|
+
return targets;
|
|
1442
|
+
}
|
|
1443
|
+
async function reportBootstrapComplete(indexName, partitionId, nodeId, coordinator, transport) {
|
|
1444
|
+
const controllerNodeId = await coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
1445
|
+
if (controllerNodeId === null) {
|
|
1446
|
+
return false;
|
|
1447
|
+
}
|
|
1448
|
+
const primaryTerm = await resolvePrimaryTerm(indexName, partitionId, coordinator);
|
|
1449
|
+
if (primaryTerm === null) {
|
|
1450
|
+
return false;
|
|
1451
|
+
}
|
|
1452
|
+
const payload = {
|
|
1453
|
+
indexName,
|
|
1454
|
+
partitionId,
|
|
1455
|
+
nodeId,
|
|
1456
|
+
primaryTerm
|
|
1457
|
+
};
|
|
1458
|
+
const message = {
|
|
1459
|
+
type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
|
|
1460
|
+
sourceId: nodeId,
|
|
1461
|
+
requestId: generateId(),
|
|
1462
|
+
payload: encode(payload)
|
|
1463
|
+
};
|
|
1464
|
+
for (const target of await resolveControllerTargets(coordinator, controllerNodeId)) {
|
|
1465
|
+
try {
|
|
1466
|
+
const response = await transport.send(target, message);
|
|
1467
|
+
return isAcceptedResponse(response);
|
|
1468
|
+
} catch (_) {
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
return false;
|
|
1472
|
+
}
|
|
1473
|
+
function isAcceptedResponse(response) {
|
|
1474
|
+
try {
|
|
1475
|
+
const decoded = decode(response.payload);
|
|
1476
|
+
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
1477
|
+
return false;
|
|
1478
|
+
}
|
|
1479
|
+
return decoded.accepted === true;
|
|
1480
|
+
} catch (_) {
|
|
1481
|
+
return false;
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
async function partitionAwaitsBootstrapReport(state, coordinator) {
|
|
1485
|
+
try {
|
|
1486
|
+
const table = await coordinator.getAllocation(state.indexName);
|
|
1487
|
+
const assignment = table?.assignments.get(state.partitionId);
|
|
1488
|
+
if (assignment === void 0) {
|
|
1489
|
+
return true;
|
|
1490
|
+
}
|
|
1491
|
+
return assignment.state === "INITIALISING";
|
|
1492
|
+
} catch (_) {
|
|
1493
|
+
return true;
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
async function bootstrapPartition(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onBootstrapPartition, onError) {
|
|
1497
|
+
for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
|
|
1498
|
+
if (state.aborted) {
|
|
1499
|
+
return false;
|
|
1500
|
+
}
|
|
1501
|
+
const synced = await runBootstrapSyncAttempt(state, onBootstrapPartition, onError);
|
|
1502
|
+
if (state.aborted) {
|
|
1503
|
+
return false;
|
|
1504
|
+
}
|
|
1505
|
+
if (synced) {
|
|
1506
|
+
if (!await partitionAwaitsBootstrapReport(state, coordinator)) {
|
|
1507
|
+
return true;
|
|
1508
|
+
}
|
|
1509
|
+
return retryReportBootstrapComplete(
|
|
1510
|
+
state,
|
|
1511
|
+
coordinator,
|
|
1512
|
+
transport,
|
|
1513
|
+
nodeId,
|
|
1514
|
+
bootstrapRetryBaseMs,
|
|
1515
|
+
bootstrapRetryMaxMs,
|
|
1516
|
+
bootstrapMaxRetries,
|
|
1517
|
+
onError
|
|
1518
|
+
);
|
|
1519
|
+
}
|
|
1520
|
+
if (attempt < bootstrapMaxRetries) {
|
|
1521
|
+
const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
|
|
1522
|
+
await waitWithAbort(state, backoffMs);
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
return false;
|
|
1526
|
+
}
|
|
1527
|
+
async function runBootstrapSyncAttempt(state, onBootstrapPartition, onError) {
|
|
1528
|
+
try {
|
|
1529
|
+
return await onBootstrapPartition(state.indexName, state.partitionId, state.primaryNodeId);
|
|
1530
|
+
} catch (error) {
|
|
1531
|
+
if (state.aborted) {
|
|
1532
|
+
return false;
|
|
1533
|
+
}
|
|
1534
|
+
if (onError !== void 0) {
|
|
1535
|
+
const cause = error instanceof Error ? error.message : String(error);
|
|
1536
|
+
onError(
|
|
1537
|
+
new NarsilError(
|
|
1538
|
+
ErrorCodes.NODE_BOOTSTRAP_FAILED,
|
|
1539
|
+
`Bootstrap sync failed for ${state.indexName}:${state.partitionId}`,
|
|
1540
|
+
{ indexName: state.indexName, partitionId: state.partitionId, cause }
|
|
1541
|
+
)
|
|
1542
|
+
);
|
|
1543
|
+
}
|
|
1544
|
+
return false;
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
async function retryReportBootstrapComplete(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onError) {
|
|
1548
|
+
for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
|
|
1549
|
+
if (state.aborted) {
|
|
1550
|
+
return false;
|
|
1551
|
+
}
|
|
1552
|
+
if (attempt > 0 && !await partitionAwaitsBootstrapReport(state, coordinator)) {
|
|
1553
|
+
return true;
|
|
1554
|
+
}
|
|
1555
|
+
const accepted = await reportBootstrapComplete(state.indexName, state.partitionId, nodeId, coordinator, transport);
|
|
1556
|
+
if (accepted) {
|
|
1557
|
+
return true;
|
|
1558
|
+
}
|
|
1559
|
+
if (!await partitionAwaitsBootstrapReport(state, coordinator)) {
|
|
1560
|
+
return true;
|
|
1561
|
+
}
|
|
1562
|
+
if (attempt < bootstrapMaxRetries && !state.aborted) {
|
|
1563
|
+
const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
|
|
1564
|
+
await waitWithAbort(state, backoffMs);
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
if (onError !== void 0) {
|
|
1568
|
+
onError(
|
|
1569
|
+
new NarsilError(
|
|
1570
|
+
ErrorCodes.NODE_BOOTSTRAP_FAILED,
|
|
1571
|
+
`Failed to report bootstrap completion for ${state.indexName}:${state.partitionId} after ${bootstrapMaxRetries + 1} attempts`,
|
|
1572
|
+
{ indexName: state.indexName, partitionId: state.partitionId }
|
|
1573
|
+
)
|
|
1574
|
+
);
|
|
1575
|
+
}
|
|
1576
|
+
return false;
|
|
1577
|
+
}
|
|
1578
|
+
function waitWithAbort(state, ms) {
|
|
1579
|
+
return new Promise((resolve) => {
|
|
1580
|
+
if (state.aborted) {
|
|
1581
|
+
resolve();
|
|
1582
|
+
return;
|
|
1583
|
+
}
|
|
1584
|
+
state.abortResolve = resolve;
|
|
1585
|
+
state.retryTimer = setTimeout(() => {
|
|
1586
|
+
state.retryTimer = null;
|
|
1587
|
+
state.abortResolve = null;
|
|
1588
|
+
resolve();
|
|
1589
|
+
}, ms);
|
|
1590
|
+
});
|
|
1591
|
+
}
|
|
1592
|
+
function abortBootstrapState(state) {
|
|
1593
|
+
state.aborted = true;
|
|
1594
|
+
if (state.retryTimer !== null) {
|
|
1595
|
+
clearTimeout(state.retryTimer);
|
|
1596
|
+
state.retryTimer = null;
|
|
1597
|
+
}
|
|
1598
|
+
if (state.abortResolve !== null) {
|
|
1599
|
+
state.abortResolve();
|
|
1600
|
+
state.abortResolve = null;
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
function createBootstrapState(indexName, partitionId, primaryNodeId) {
|
|
1604
|
+
return {
|
|
1605
|
+
indexName,
|
|
1606
|
+
partitionId,
|
|
1607
|
+
primaryNodeId,
|
|
1608
|
+
retryCount: 0,
|
|
1609
|
+
retryTimer: null,
|
|
1610
|
+
aborted: false,
|
|
1611
|
+
abortResolve: null
|
|
1612
|
+
};
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
// src/distribution/cluster/node-lifecycle/allocation-watcher.ts
|
|
1616
|
+
function partitionKey(indexName, partitionId) {
|
|
1617
|
+
return `${indexName}:${partitionId}`;
|
|
1618
|
+
}
|
|
1619
|
+
function createAllocationWatcherState() {
|
|
1620
|
+
return {
|
|
1621
|
+
unwatchAllocation: null,
|
|
1622
|
+
debounceTimer: null,
|
|
1623
|
+
activeBootstraps: /* @__PURE__ */ new Map(),
|
|
1624
|
+
trackedPartitions: /* @__PURE__ */ new Map(),
|
|
1625
|
+
pendingTables: /* @__PURE__ */ new Map(),
|
|
1626
|
+
restartWaiters: /* @__PURE__ */ new Map(),
|
|
1627
|
+
stopped: false
|
|
1628
|
+
};
|
|
1629
|
+
}
|
|
1630
|
+
async function startAllocationWatcher(state, config) {
|
|
1631
|
+
const unwatchAllocation = await config.coordinator.watchAllocation((event) => {
|
|
1632
|
+
if (state.stopped) {
|
|
1633
|
+
return;
|
|
1634
|
+
}
|
|
1635
|
+
scheduleProcessAllocation(state, config, event.table);
|
|
1636
|
+
});
|
|
1637
|
+
state.unwatchAllocation = unwatchAllocation;
|
|
1638
|
+
}
|
|
1639
|
+
function scheduleProcessAllocation(state, config, table) {
|
|
1640
|
+
state.pendingTables.set(table.indexName, table);
|
|
1641
|
+
if (state.debounceTimer !== null) {
|
|
1642
|
+
clearTimeout(state.debounceTimer);
|
|
1643
|
+
}
|
|
1644
|
+
state.debounceTimer = setTimeout(() => {
|
|
1645
|
+
state.debounceTimer = null;
|
|
1646
|
+
const tablesToProcess = Array.from(state.pendingTables.values());
|
|
1647
|
+
state.pendingTables.clear();
|
|
1648
|
+
for (const pending of tablesToProcess) {
|
|
1649
|
+
processAllocationChange(state, config, pending);
|
|
1650
|
+
}
|
|
1651
|
+
}, config.allocationDebounceMs);
|
|
1652
|
+
}
|
|
1653
|
+
function processAllocationChange(state, config, table) {
|
|
1654
|
+
if (state.stopped) {
|
|
1655
|
+
return;
|
|
1656
|
+
}
|
|
1657
|
+
const nodeId = config.registration.nodeId;
|
|
1658
|
+
const currentKeys = /* @__PURE__ */ new Set();
|
|
1659
|
+
for (const [partitionId, assignment] of table.assignments) {
|
|
1660
|
+
if (!isNodeAssigned(assignment, nodeId)) {
|
|
1661
|
+
continue;
|
|
1662
|
+
}
|
|
1663
|
+
const key = partitionKey(table.indexName, partitionId);
|
|
1664
|
+
currentKeys.add(key);
|
|
1665
|
+
const existing = state.trackedPartitions.get(key);
|
|
1666
|
+
const isPrimary = assignment.primary === nodeId;
|
|
1667
|
+
if (existing === void 0) {
|
|
1668
|
+
handleNewAssignment(state, config, table.indexName, partitionId, assignment, nodeId);
|
|
1669
|
+
} else {
|
|
1670
|
+
handleExistingAssignment(state, config, existing, assignment, nodeId);
|
|
1671
|
+
}
|
|
1672
|
+
state.trackedPartitions.set(key, {
|
|
1673
|
+
indexName: table.indexName,
|
|
1674
|
+
partitionId,
|
|
1675
|
+
isPrimary,
|
|
1676
|
+
primaryNodeId: assignment.primary,
|
|
1677
|
+
primaryTerm: assignment.primaryTerm
|
|
1678
|
+
});
|
|
1679
|
+
}
|
|
1680
|
+
removeUnassignedPartitions(state, config, table.indexName, currentKeys);
|
|
1681
|
+
}
|
|
1682
|
+
function isNodeAssigned(assignment, nodeId) {
|
|
1683
|
+
if (assignment.primary === nodeId) {
|
|
1684
|
+
return true;
|
|
1685
|
+
}
|
|
1686
|
+
return assignment.replicas.includes(nodeId);
|
|
1687
|
+
}
|
|
1688
|
+
function handleNewAssignment(state, config, indexName, partitionId, assignment, nodeId) {
|
|
1689
|
+
if (assignment.state === "ACTIVE" && assignment.primary === nodeId) {
|
|
1690
|
+
return;
|
|
1691
|
+
}
|
|
1692
|
+
if (assignment.primary === null) {
|
|
1693
|
+
return;
|
|
1694
|
+
}
|
|
1695
|
+
if (assignment.primary === nodeId) {
|
|
1696
|
+
if (assignment.state === "INITIALISING") {
|
|
1697
|
+
startBootstrap(state, config, indexName, partitionId, nodeId);
|
|
1698
|
+
}
|
|
1699
|
+
return;
|
|
1700
|
+
}
|
|
1701
|
+
startBootstrap(state, config, indexName, partitionId, assignment.primary);
|
|
1702
|
+
}
|
|
1703
|
+
function handleExistingAssignment(state, config, existing, assignment, nodeId) {
|
|
1704
|
+
if (existing.isPrimary && assignment.primary !== nodeId && assignment.primaryTerm > existing.primaryTerm) {
|
|
1705
|
+
if (assignment.primary !== null) {
|
|
1706
|
+
if (config.onPrimaryDemotion !== void 0) {
|
|
1707
|
+
config.onPrimaryDemotion(existing.indexName, existing.partitionId, assignment.primary);
|
|
1708
|
+
}
|
|
1709
|
+
startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
|
|
1710
|
+
}
|
|
1711
|
+
return;
|
|
1712
|
+
}
|
|
1713
|
+
if (assignment.primary !== null && replicaNeedsResync(state, existing, assignment, nodeId)) {
|
|
1714
|
+
startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
|
|
1715
|
+
}
|
|
1716
|
+
}
|
|
1717
|
+
function replicaNeedsResync(state, existing, assignment, nodeId) {
|
|
1718
|
+
if (assignment.state !== "ACTIVE") {
|
|
1719
|
+
return false;
|
|
1720
|
+
}
|
|
1721
|
+
if (assignment.primary === null || assignment.primary === nodeId) {
|
|
1722
|
+
return false;
|
|
1723
|
+
}
|
|
1724
|
+
if (!assignment.replicas.includes(nodeId)) {
|
|
1725
|
+
return false;
|
|
1364
1726
|
}
|
|
1365
1727
|
if (assignment.inSyncSet.includes(nodeId)) {
|
|
1366
1728
|
return false;
|
|
@@ -1391,12 +1753,14 @@ function startBootstrap(state, config, indexName, partitionId, primaryNodeId) {
|
|
|
1391
1753
|
}
|
|
1392
1754
|
if (!succeeded) {
|
|
1393
1755
|
state.trackedPartitions.delete(key);
|
|
1756
|
+
void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
|
|
1394
1757
|
}
|
|
1395
1758
|
}).catch(() => {
|
|
1396
1759
|
if (state.activeBootstraps.get(key) === bootstrapState) {
|
|
1397
1760
|
state.activeBootstraps.delete(key);
|
|
1398
1761
|
}
|
|
1399
1762
|
state.trackedPartitions.delete(key);
|
|
1763
|
+
void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
|
|
1400
1764
|
});
|
|
1401
1765
|
}
|
|
1402
1766
|
function removeUnassignedPartitions(state, config, indexName, currentKeys) {
|
|
@@ -1428,6 +1792,11 @@ function stopAllocationWatcher(state) {
|
|
|
1428
1792
|
clearTimeout(state.debounceTimer);
|
|
1429
1793
|
state.debounceTimer = null;
|
|
1430
1794
|
}
|
|
1795
|
+
for (const [timer, resolve] of state.restartWaiters) {
|
|
1796
|
+
clearTimeout(timer);
|
|
1797
|
+
resolve();
|
|
1798
|
+
}
|
|
1799
|
+
state.restartWaiters.clear();
|
|
1431
1800
|
for (const bootstrapState of state.activeBootstraps.values()) {
|
|
1432
1801
|
abortBootstrapState(bootstrapState);
|
|
1433
1802
|
}
|
|
@@ -1440,10 +1809,90 @@ function processInitialAllocations(state, config, tables) {
|
|
|
1440
1809
|
processAllocationChange(state, config, table);
|
|
1441
1810
|
}
|
|
1442
1811
|
}
|
|
1812
|
+
function waitBeforeBootstrapRestart(state, config) {
|
|
1813
|
+
const delayMs = Math.min(config.bootstrapRetryMaxMs, config.bootstrapRetryBaseMs);
|
|
1814
|
+
if (delayMs <= 0) {
|
|
1815
|
+
return Promise.resolve();
|
|
1816
|
+
}
|
|
1817
|
+
return new Promise((resolve) => {
|
|
1818
|
+
const timer = setTimeout(() => {
|
|
1819
|
+
state.restartWaiters.delete(timer);
|
|
1820
|
+
resolve();
|
|
1821
|
+
}, delayMs);
|
|
1822
|
+
timer.unref?.();
|
|
1823
|
+
state.restartWaiters.set(timer, resolve);
|
|
1824
|
+
});
|
|
1825
|
+
}
|
|
1826
|
+
async function restartBootstrapWhileOutOfSync(state, config, indexName, partitionId) {
|
|
1827
|
+
if (state.stopped) {
|
|
1828
|
+
return;
|
|
1829
|
+
}
|
|
1830
|
+
await waitBeforeBootstrapRestart(state, config);
|
|
1831
|
+
if (state.stopped) {
|
|
1832
|
+
return;
|
|
1833
|
+
}
|
|
1834
|
+
const nodeId = config.registration.nodeId;
|
|
1835
|
+
let table;
|
|
1836
|
+
try {
|
|
1837
|
+
table = await config.coordinator.getAllocation(indexName);
|
|
1838
|
+
} catch (error) {
|
|
1839
|
+
if (config.onError !== void 0) {
|
|
1840
|
+
config.onError(error);
|
|
1841
|
+
}
|
|
1842
|
+
if (!state.stopped) {
|
|
1843
|
+
void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
|
|
1844
|
+
}
|
|
1845
|
+
return;
|
|
1846
|
+
}
|
|
1847
|
+
if (state.stopped || table === null) {
|
|
1848
|
+
return;
|
|
1849
|
+
}
|
|
1850
|
+
const assignment = table.assignments.get(partitionId);
|
|
1851
|
+
if (assignment === void 0 || assignment.primary === null || assignment.primary === nodeId) {
|
|
1852
|
+
return;
|
|
1853
|
+
}
|
|
1854
|
+
if (!assignment.replicas.includes(nodeId) || assignment.inSyncSet.includes(nodeId)) {
|
|
1855
|
+
return;
|
|
1856
|
+
}
|
|
1857
|
+
if (assignment.state !== "ACTIVE" && assignment.state !== "INITIALISING" && assignment.state !== "MIGRATING") {
|
|
1858
|
+
return;
|
|
1859
|
+
}
|
|
1860
|
+
if (state.activeBootstraps.has(partitionKey(indexName, partitionId))) {
|
|
1861
|
+
return;
|
|
1862
|
+
}
|
|
1863
|
+
startBootstrap(state, config, indexName, partitionId, assignment.primary);
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1866
|
+
// src/distribution/cluster/node-lifecycle/heartbeat.ts
|
|
1867
|
+
function createRegistrationHeartbeatState() {
|
|
1868
|
+
return { timer: null, inFlight: Promise.resolve() };
|
|
1869
|
+
}
|
|
1870
|
+
function clearHeartbeatTimer(state) {
|
|
1871
|
+
if (state.timer !== null) {
|
|
1872
|
+
clearInterval(state.timer);
|
|
1873
|
+
state.timer = null;
|
|
1874
|
+
}
|
|
1875
|
+
}
|
|
1876
|
+
function startRegistrationHeartbeat(state, config) {
|
|
1877
|
+
clearHeartbeatTimer(state);
|
|
1878
|
+
state.timer = setInterval(() => {
|
|
1879
|
+
state.inFlight = state.inFlight.then(() => config.coordinator.registerNode(config.registration)).catch((error) => {
|
|
1880
|
+
if (config.onError !== void 0) {
|
|
1881
|
+
config.onError(error);
|
|
1882
|
+
}
|
|
1883
|
+
});
|
|
1884
|
+
}, config.nodeHeartbeatIntervalMs);
|
|
1885
|
+
state.timer.unref?.();
|
|
1886
|
+
}
|
|
1887
|
+
async function stopRegistrationHeartbeat(state) {
|
|
1888
|
+
clearHeartbeatTimer(state);
|
|
1889
|
+
await state.inFlight;
|
|
1890
|
+
}
|
|
1443
1891
|
|
|
1444
1892
|
// src/distribution/cluster/node-lifecycle/join.ts
|
|
1445
|
-
async function joinCluster(config, watcherState) {
|
|
1893
|
+
async function joinCluster(config, watcherState, heartbeatState) {
|
|
1446
1894
|
await config.coordinator.registerNode(config.registration);
|
|
1895
|
+
startRegistrationHeartbeat(heartbeatState, config);
|
|
1447
1896
|
const initialTables = await loadInitialAllocations(config);
|
|
1448
1897
|
processInitialAllocations(watcherState, config, initialTables);
|
|
1449
1898
|
await startAllocationWatcher(watcherState, config);
|
|
@@ -1465,15 +1914,17 @@ async function leaveCluster(config) {
|
|
|
1465
1914
|
// src/distribution/cluster/node-lifecycle/types.ts
|
|
1466
1915
|
var DEFAULT_NODE_LIFECYCLE_CONFIG = {
|
|
1467
1916
|
bootstrapRetryBaseMs: 1e3,
|
|
1468
|
-
bootstrapRetryMaxMs:
|
|
1917
|
+
bootstrapRetryMaxMs: 5e3,
|
|
1469
1918
|
bootstrapMaxRetries: 10,
|
|
1470
|
-
allocationDebounceMs: 250
|
|
1919
|
+
allocationDebounceMs: 250,
|
|
1920
|
+
nodeHeartbeatIntervalMs: 1e4
|
|
1471
1921
|
};
|
|
1472
1922
|
|
|
1473
1923
|
// src/distribution/cluster/node-lifecycle/index.ts
|
|
1474
1924
|
function createDataNodeLifecycle(config) {
|
|
1475
1925
|
let status = "stopped";
|
|
1476
1926
|
let watcherState = createAllocationWatcherState();
|
|
1927
|
+
const heartbeatState = createRegistrationHeartbeatState();
|
|
1477
1928
|
let operationLock = Promise.resolve();
|
|
1478
1929
|
function withLock(fn) {
|
|
1479
1930
|
const prev = operationLock;
|
|
@@ -1516,9 +1967,10 @@ function createDataNodeLifecycle(config) {
|
|
|
1516
1967
|
}
|
|
1517
1968
|
status = "joining";
|
|
1518
1969
|
try {
|
|
1519
|
-
await joinCluster(config, watcherState);
|
|
1970
|
+
await joinCluster(config, watcherState, heartbeatState);
|
|
1520
1971
|
status = "active";
|
|
1521
1972
|
} catch (error) {
|
|
1973
|
+
await stopRegistrationHeartbeat(heartbeatState);
|
|
1522
1974
|
status = "stopped";
|
|
1523
1975
|
throw error;
|
|
1524
1976
|
}
|
|
@@ -1530,6 +1982,7 @@ function createDataNodeLifecycle(config) {
|
|
|
1530
1982
|
return;
|
|
1531
1983
|
}
|
|
1532
1984
|
status = "leaving";
|
|
1985
|
+
await stopRegistrationHeartbeat(heartbeatState);
|
|
1533
1986
|
stopAllocationWatcher(watcherState);
|
|
1534
1987
|
try {
|
|
1535
1988
|
await leaveCluster(config);
|
|
@@ -1544,6 +1997,7 @@ function createDataNodeLifecycle(config) {
|
|
|
1544
1997
|
if (status === "shutdown") {
|
|
1545
1998
|
return;
|
|
1546
1999
|
}
|
|
2000
|
+
await stopRegistrationHeartbeat(heartbeatState);
|
|
1547
2001
|
stopAllocationWatcher(watcherState);
|
|
1548
2002
|
if (status === "active" || status === "joining") {
|
|
1549
2003
|
try {
|
|
@@ -2818,11 +3272,47 @@ function validateLocalPartitionCount(engine, indexName, primaryNodeId, partition
|
|
|
2818
3272
|
// src/distribution/cluster-node/bootstrap-sync/targets.ts
|
|
2819
3273
|
function readLocalLogState(indexName, partitionId, deps) {
|
|
2820
3274
|
const log = deps.getReplicationLog(indexName, partitionId);
|
|
2821
|
-
const lastSeqNo = log.
|
|
2822
|
-
const lastPrimaryTerm = log.
|
|
3275
|
+
const lastSeqNo = log.localLogEnd;
|
|
3276
|
+
const lastPrimaryTerm = log.localLogEndPrimaryTerm;
|
|
2823
3277
|
const newestEntry = log.getEntry(lastSeqNo);
|
|
2824
3278
|
return { lastSeqNo, lastPrimaryTerm: newestEntry?.primaryTerm ?? lastPrimaryTerm };
|
|
2825
3279
|
}
|
|
3280
|
+
var SYNCED_POSITION_REPORT_TIMEOUT_MS = 5e3;
|
|
3281
|
+
async function reportSyncedPosition(indexName, partitionId, target, knownSeqNo, deadline, deps) {
|
|
3282
|
+
const logState = readLocalLogState(indexName, partitionId, deps);
|
|
3283
|
+
if (logState.lastSeqNo <= knownSeqNo) {
|
|
3284
|
+
return;
|
|
3285
|
+
}
|
|
3286
|
+
const budgetMs = Math.min(SYNCED_POSITION_REPORT_TIMEOUT_MS, deadline - Date.now());
|
|
3287
|
+
if (budgetMs <= 0) {
|
|
3288
|
+
return;
|
|
3289
|
+
}
|
|
3290
|
+
const payload = {
|
|
3291
|
+
indexName,
|
|
3292
|
+
partitionId,
|
|
3293
|
+
lastSeqNo: logState.lastSeqNo,
|
|
3294
|
+
lastPrimaryTerm: logState.lastPrimaryTerm
|
|
3295
|
+
};
|
|
3296
|
+
const request = {
|
|
3297
|
+
type: ReplicationMessageTypes.SYNC_REQUEST,
|
|
3298
|
+
sourceId: deps.sourceNodeId,
|
|
3299
|
+
requestId: generateId(),
|
|
3300
|
+
payload: encode(payload)
|
|
3301
|
+
};
|
|
3302
|
+
try {
|
|
3303
|
+
await withDeadline(
|
|
3304
|
+
deps.transport.stream(target, request, () => {
|
|
3305
|
+
}),
|
|
3306
|
+
budgetMs,
|
|
3307
|
+
indexName,
|
|
3308
|
+
"synced-position-report"
|
|
3309
|
+
);
|
|
3310
|
+
} catch (error) {
|
|
3311
|
+
if (deps.onError !== void 0) {
|
|
3312
|
+
deps.onError(error);
|
|
3313
|
+
}
|
|
3314
|
+
}
|
|
3315
|
+
}
|
|
2826
3316
|
async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNodeId, targets, coordinatorSchema, partitionCount, deadline, deps, abortCheck) {
|
|
2827
3317
|
let lastError = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "no live sync attempts were made", {
|
|
2828
3318
|
indexName,
|
|
@@ -2856,6 +3346,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
|
|
|
2856
3346
|
)
|
|
2857
3347
|
};
|
|
2858
3348
|
}
|
|
3349
|
+
const positionBeforeAttempt = readLocalLogState(indexName, partitionId, deps).lastSeqNo;
|
|
2859
3350
|
const attempt = await syncFromTarget(
|
|
2860
3351
|
state,
|
|
2861
3352
|
entry,
|
|
@@ -2869,6 +3360,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
|
|
|
2869
3360
|
abortCheck
|
|
2870
3361
|
);
|
|
2871
3362
|
if (attempt.ok) {
|
|
3363
|
+
await reportSyncedPosition(indexName, partitionId, target, positionBeforeAttempt, deadline, deps);
|
|
2872
3364
|
return attempt;
|
|
2873
3365
|
}
|
|
2874
3366
|
lastError = new NarsilError(attempt.error.code, attempt.error.message, {
|
|
@@ -3156,171 +3648,586 @@ async function applyRestore(state, entry, indexName, primaryNodeId, bytes, coord
|
|
|
3156
3648
|
if (!anotherBootstrapOwnsKey(state, key, entry)) {
|
|
3157
3649
|
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
3158
3650
|
}
|
|
3159
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3160
|
-
return false;
|
|
3651
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3652
|
+
return false;
|
|
3653
|
+
}
|
|
3654
|
+
const schemaError = validateRestoredSchema(deps.engine, indexName, primaryNodeId, coordinatorSchema);
|
|
3655
|
+
if (schemaError !== null) {
|
|
3656
|
+
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
3657
|
+
surfaceError(deps, indexName, primaryNodeId, schemaError);
|
|
3658
|
+
return false;
|
|
3659
|
+
}
|
|
3660
|
+
return true;
|
|
3661
|
+
}
|
|
3662
|
+
|
|
3663
|
+
// src/distribution/cluster-node/bootstrap-sync/index.ts
|
|
3664
|
+
async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
|
|
3665
|
+
const validationError = validateArguments(indexName, partitionId, primaryNodeId);
|
|
3666
|
+
if (validationError !== null) {
|
|
3667
|
+
surfaceError(deps, indexName, primaryNodeId, validationError);
|
|
3668
|
+
return false;
|
|
3669
|
+
}
|
|
3670
|
+
const key = entryKey(indexName, partitionId);
|
|
3671
|
+
if (state.completed.has(key)) {
|
|
3672
|
+
if (!await replicaResyncRequired(indexName, partitionId, deps)) {
|
|
3673
|
+
return true;
|
|
3674
|
+
}
|
|
3675
|
+
clearBootstrapSyncIndex(state, indexName, partitionId);
|
|
3676
|
+
}
|
|
3677
|
+
const existing = state.inFlight.get(key);
|
|
3678
|
+
if (existing !== void 0) {
|
|
3679
|
+
const result = await Promise.race([existing.promise, existing.abortPromise]);
|
|
3680
|
+
if (result === ABORT_SENTINEL) {
|
|
3681
|
+
return false;
|
|
3682
|
+
}
|
|
3683
|
+
return result;
|
|
3684
|
+
}
|
|
3685
|
+
const entry = createEntry(state, indexName, partitionId, key);
|
|
3686
|
+
entry.promise = executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps).finally(() => {
|
|
3687
|
+
const current = state.inFlight.get(key);
|
|
3688
|
+
if (current === entry) {
|
|
3689
|
+
state.inFlight.delete(key);
|
|
3690
|
+
}
|
|
3691
|
+
if (!state.completed.has(key) && !state.inFlight.has(key)) {
|
|
3692
|
+
const currentGen = state.generations.get(key);
|
|
3693
|
+
if (currentGen !== void 0 && currentGen !== entry.generation) {
|
|
3694
|
+
state.generations.delete(key);
|
|
3695
|
+
}
|
|
3696
|
+
}
|
|
3697
|
+
});
|
|
3698
|
+
state.inFlight.set(key, entry);
|
|
3699
|
+
return entry.promise;
|
|
3700
|
+
}
|
|
3701
|
+
async function replicaResyncRequired(indexName, partitionId, deps) {
|
|
3702
|
+
const allocation = await deps.coordinator.getAllocation(indexName);
|
|
3703
|
+
const assignment = allocation?.assignments.get(partitionId);
|
|
3704
|
+
if (assignment === void 0) {
|
|
3705
|
+
return false;
|
|
3706
|
+
}
|
|
3707
|
+
return assignment.state === "ACTIVE" && assignment.replicas.includes(deps.sourceNodeId) && !assignment.inSyncSet.includes(deps.sourceNodeId);
|
|
3708
|
+
}
|
|
3709
|
+
async function executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps) {
|
|
3710
|
+
const deadlineMs = deps.deadlineMs ?? DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS;
|
|
3711
|
+
const deadline = Date.now() + deadlineMs;
|
|
3712
|
+
const key = entryKey(indexName, partitionId);
|
|
3713
|
+
const abortCheck = () => {
|
|
3714
|
+
const currentGeneration = state.generations.get(key) ?? 0;
|
|
3715
|
+
if (currentGeneration !== entry.generation) {
|
|
3716
|
+
entry.aborted = true;
|
|
3717
|
+
}
|
|
3718
|
+
return entry.aborted;
|
|
3719
|
+
};
|
|
3720
|
+
if (hasLiveBootstrapSyncDeps(deps)) {
|
|
3721
|
+
return executeLiveBootstrapSync(
|
|
3722
|
+
state,
|
|
3723
|
+
entry,
|
|
3724
|
+
indexName,
|
|
3725
|
+
partitionId,
|
|
3726
|
+
primaryNodeId,
|
|
3727
|
+
deps,
|
|
3728
|
+
deadlineMs,
|
|
3729
|
+
deadline,
|
|
3730
|
+
abortCheck
|
|
3731
|
+
);
|
|
3732
|
+
}
|
|
3733
|
+
const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
|
|
3734
|
+
if (schemaResult === "aborted") {
|
|
3735
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3736
|
+
return false;
|
|
3737
|
+
}
|
|
3738
|
+
if (schemaResult === null) {
|
|
3739
|
+
return false;
|
|
3740
|
+
}
|
|
3741
|
+
if (abortCheck()) {
|
|
3742
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3743
|
+
return false;
|
|
3744
|
+
}
|
|
3745
|
+
if (Date.now() >= deadline) {
|
|
3746
|
+
surfaceError(
|
|
3747
|
+
deps,
|
|
3748
|
+
indexName,
|
|
3749
|
+
primaryNodeId,
|
|
3750
|
+
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
|
|
3751
|
+
indexName,
|
|
3752
|
+
deadlineMs
|
|
3753
|
+
})
|
|
3754
|
+
);
|
|
3755
|
+
return false;
|
|
3756
|
+
}
|
|
3757
|
+
const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
|
|
3758
|
+
if (targetsResult instanceof NarsilError) {
|
|
3759
|
+
surfaceError(deps, indexName, primaryNodeId, targetsResult);
|
|
3760
|
+
return false;
|
|
3761
|
+
}
|
|
3762
|
+
const fetchDeps = { transport: deps.transport, sourceNodeId: deps.sourceNodeId };
|
|
3763
|
+
const fetchResult = await fetchSnapshotFromAnyTarget(
|
|
3764
|
+
indexName,
|
|
3765
|
+
primaryNodeId,
|
|
3766
|
+
targetsResult,
|
|
3767
|
+
deadline,
|
|
3768
|
+
fetchDeps,
|
|
3769
|
+
abortCheck,
|
|
3770
|
+
partitionId
|
|
3771
|
+
);
|
|
3772
|
+
if (!fetchResult.ok) {
|
|
3773
|
+
if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
|
|
3774
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3775
|
+
return false;
|
|
3776
|
+
}
|
|
3777
|
+
surfaceError(
|
|
3778
|
+
deps,
|
|
3779
|
+
indexName,
|
|
3780
|
+
primaryNodeId,
|
|
3781
|
+
new NarsilError(fetchResult.code, fetchResult.message, {
|
|
3782
|
+
indexName,
|
|
3783
|
+
primaryNodeId,
|
|
3784
|
+
...fetchResult.details
|
|
3785
|
+
})
|
|
3786
|
+
);
|
|
3787
|
+
return false;
|
|
3788
|
+
}
|
|
3789
|
+
if (abortCheck()) {
|
|
3790
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3791
|
+
return false;
|
|
3792
|
+
}
|
|
3793
|
+
const restoreSucceeded = await applyRestore(
|
|
3794
|
+
state,
|
|
3795
|
+
entry,
|
|
3796
|
+
indexName,
|
|
3797
|
+
primaryNodeId,
|
|
3798
|
+
fetchResult.bytes,
|
|
3799
|
+
schemaResult,
|
|
3800
|
+
deadline,
|
|
3801
|
+
deps
|
|
3802
|
+
);
|
|
3803
|
+
if (!restoreSucceeded) {
|
|
3804
|
+
return false;
|
|
3805
|
+
}
|
|
3806
|
+
if (abortCheck()) {
|
|
3807
|
+
if (!anotherBootstrapOwnsKey(state, key, entry)) {
|
|
3808
|
+
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
3809
|
+
}
|
|
3810
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3811
|
+
return false;
|
|
3812
|
+
}
|
|
3813
|
+
deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
|
|
3814
|
+
state.completed.add(key);
|
|
3815
|
+
return true;
|
|
3816
|
+
}
|
|
3817
|
+
|
|
3818
|
+
// src/distribution/cluster-node/write-routing/assignment.ts
|
|
3819
|
+
function resolvePartitionId(docId, partitionCount) {
|
|
3820
|
+
return fnv1a(docId) % partitionCount;
|
|
3821
|
+
}
|
|
3822
|
+
async function resolveNodeTargets(nodeId, deps) {
|
|
3823
|
+
if (deps.resolveNodeTargets === void 0) {
|
|
3824
|
+
return [nodeId];
|
|
3825
|
+
}
|
|
3826
|
+
const targets = await deps.resolveNodeTargets(nodeId);
|
|
3827
|
+
return targets.length > 0 ? targets : [nodeId];
|
|
3828
|
+
}
|
|
3829
|
+
function requireAssignedPrimary(assignment, indexName, partitionId) {
|
|
3830
|
+
if (assignment === void 0 || assignment.primary === null) {
|
|
3831
|
+
throw new NarsilError(
|
|
3832
|
+
ErrorCodes.PARTITION_UNASSIGNED,
|
|
3833
|
+
`No primary assigned for partition ${partitionId} of index '${indexName}'`,
|
|
3834
|
+
{ indexName, partitionId }
|
|
3835
|
+
);
|
|
3836
|
+
}
|
|
3837
|
+
return assignment;
|
|
3838
|
+
}
|
|
3839
|
+
async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrimary) {
|
|
3840
|
+
const table = await deps.coordinator.getAllocation(indexName);
|
|
3841
|
+
if (table === null || table.assignments.size === 0) {
|
|
3842
|
+
if (requireLocalPrimary) {
|
|
3843
|
+
throw new NarsilError(
|
|
3844
|
+
ErrorCodes.QUERY_ROUTING_FAILED,
|
|
3845
|
+
`No allocation table is available for forwarded write to index '${indexName}'`,
|
|
3846
|
+
{ indexName }
|
|
3847
|
+
);
|
|
3848
|
+
}
|
|
3849
|
+
return null;
|
|
3850
|
+
}
|
|
3851
|
+
const partitionCount = table.assignments.size;
|
|
3852
|
+
const partitionId = resolvePartitionId(docId, partitionCount);
|
|
3853
|
+
const assignment = requireAssignedPrimary(table.assignments.get(partitionId), indexName, partitionId);
|
|
3854
|
+
if (requireLocalPrimary && assignment.primary !== deps.nodeId) {
|
|
3855
|
+
throw new NarsilError(
|
|
3856
|
+
ErrorCodes.PARTITION_NOT_PRIMARY,
|
|
3857
|
+
`Node '${deps.nodeId}' is not primary for partition ${partitionId} of index '${indexName}'`,
|
|
3858
|
+
{ indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
|
|
3859
|
+
);
|
|
3860
|
+
}
|
|
3861
|
+
return { partitionId, assignment };
|
|
3862
|
+
}
|
|
3863
|
+
function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
|
|
3864
|
+
const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
|
|
3865
|
+
if (requiredActiveCopies <= 1) {
|
|
3866
|
+
return;
|
|
3867
|
+
}
|
|
3868
|
+
const activeCopies = 1 + getInSyncReplicaTargets(assignment, deps.nodeId).length;
|
|
3869
|
+
if (activeCopies < requiredActiveCopies) {
|
|
3870
|
+
throw new NarsilError(
|
|
3871
|
+
ErrorCodes.INSUFFICIENT_REPLICAS,
|
|
3872
|
+
`Write to partition ${partitionId} of index '${indexName}' requires ${requiredActiveCopies} active copies but only ${activeCopies} are available`,
|
|
3873
|
+
{ indexName, partitionId, requiredActiveCopies, activeCopies }
|
|
3874
|
+
);
|
|
3875
|
+
}
|
|
3876
|
+
}
|
|
3877
|
+
function getInSyncReplicaTargets(assignment, localNodeId, pendingAdmissions = []) {
|
|
3878
|
+
const configuredReplicas = new Set(assignment.replicas);
|
|
3879
|
+
const targets = [];
|
|
3880
|
+
for (const nodeId of [...assignment.inSyncSet, ...pendingAdmissions]) {
|
|
3881
|
+
if (nodeId !== localNodeId && configuredReplicas.has(nodeId) && !targets.includes(nodeId)) {
|
|
3882
|
+
targets.push(nodeId);
|
|
3883
|
+
}
|
|
3161
3884
|
}
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3885
|
+
return targets;
|
|
3886
|
+
}
|
|
3887
|
+
|
|
3888
|
+
// src/distribution/cluster-node/catch-up/state.ts
|
|
3889
|
+
var CATCH_UP_TICK_MS = 1e3;
|
|
3890
|
+
var CATCH_UP_IN_FLIGHT_BYTE_CEILING = 67108864;
|
|
3891
|
+
function partitionKey2(indexName, partitionId) {
|
|
3892
|
+
return `${indexName}:${partitionId}`;
|
|
3893
|
+
}
|
|
3894
|
+
function createCatchUpState() {
|
|
3895
|
+
return {
|
|
3896
|
+
cursors: /* @__PURE__ */ new Map(),
|
|
3897
|
+
pendingAdmissions: /* @__PURE__ */ new Map(),
|
|
3898
|
+
inFlightBytes: 0,
|
|
3899
|
+
timer: null,
|
|
3900
|
+
activeTick: null,
|
|
3901
|
+
stopped: false
|
|
3902
|
+
};
|
|
3903
|
+
}
|
|
3904
|
+
function recordReplicaPosition(state, indexName, partitionId, replicaNodeId, appliedSeqNo) {
|
|
3905
|
+
const key = partitionKey2(indexName, partitionId);
|
|
3906
|
+
const partition = state.cursors.get(key) ?? /* @__PURE__ */ new Map();
|
|
3907
|
+
const existing = partition.get(replicaNodeId);
|
|
3908
|
+
if (existing === void 0) {
|
|
3909
|
+
partition.set(replicaNodeId, { appliedSeqNo, sending: false, syncEpoch: 0 });
|
|
3910
|
+
} else if (appliedSeqNo > existing.appliedSeqNo) {
|
|
3911
|
+
existing.appliedSeqNo = appliedSeqNo;
|
|
3912
|
+
} else if (appliedSeqNo < existing.appliedSeqNo) {
|
|
3913
|
+
existing.appliedSeqNo = appliedSeqNo;
|
|
3914
|
+
existing.syncEpoch += 1;
|
|
3915
|
+
}
|
|
3916
|
+
state.cursors.set(key, partition);
|
|
3917
|
+
}
|
|
3918
|
+
function forgetReplica(state, indexName, partitionId, replicaNodeId) {
|
|
3919
|
+
const key = partitionKey2(indexName, partitionId);
|
|
3920
|
+
const partition = state.cursors.get(key);
|
|
3921
|
+
if (partition !== void 0) {
|
|
3922
|
+
partition.delete(replicaNodeId);
|
|
3923
|
+
if (partition.size === 0) {
|
|
3924
|
+
state.cursors.delete(key);
|
|
3925
|
+
}
|
|
3926
|
+
}
|
|
3927
|
+
clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
|
|
3928
|
+
}
|
|
3929
|
+
function getPendingAdmissions(state, indexName, partitionId) {
|
|
3930
|
+
const pending = state.pendingAdmissions.get(partitionKey2(indexName, partitionId));
|
|
3931
|
+
return pending === void 0 ? [] : [...pending];
|
|
3932
|
+
}
|
|
3933
|
+
function markPendingAdmission(state, indexName, partitionId, replicaNodeId) {
|
|
3934
|
+
const key = partitionKey2(indexName, partitionId);
|
|
3935
|
+
const pending = state.pendingAdmissions.get(key) ?? /* @__PURE__ */ new Set();
|
|
3936
|
+
if (pending.has(replicaNodeId)) {
|
|
3166
3937
|
return false;
|
|
3167
3938
|
}
|
|
3939
|
+
pending.add(replicaNodeId);
|
|
3940
|
+
state.pendingAdmissions.set(key, pending);
|
|
3168
3941
|
return true;
|
|
3169
3942
|
}
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
surfaceError(deps, indexName, primaryNodeId, validationError);
|
|
3176
|
-
return false;
|
|
3943
|
+
function clearPendingAdmission(state, indexName, partitionId, replicaNodeId) {
|
|
3944
|
+
const key = partitionKey2(indexName, partitionId);
|
|
3945
|
+
const pending = state.pendingAdmissions.get(key);
|
|
3946
|
+
if (pending === void 0) {
|
|
3947
|
+
return;
|
|
3177
3948
|
}
|
|
3178
|
-
|
|
3179
|
-
if (
|
|
3180
|
-
|
|
3181
|
-
return true;
|
|
3182
|
-
}
|
|
3183
|
-
clearBootstrapSyncIndex(state, indexName, partitionId);
|
|
3949
|
+
pending.delete(replicaNodeId);
|
|
3950
|
+
if (pending.size === 0) {
|
|
3951
|
+
state.pendingAdmissions.delete(key);
|
|
3184
3952
|
}
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3953
|
+
}
|
|
3954
|
+
|
|
3955
|
+
// src/distribution/cluster-node/catch-up/admission.ts
|
|
3956
|
+
var ADMISSION_TIMEOUT_MS = 1e4;
|
|
3957
|
+
async function sendToController(payload, controllerNodeId, deps) {
|
|
3958
|
+
const targets = await resolveNodeTargets(controllerNodeId, deps);
|
|
3959
|
+
for (const target of targets) {
|
|
3960
|
+
try {
|
|
3961
|
+
const result = await requestInsyncAdmission(payload, target, deps.transport, deps.nodeId);
|
|
3962
|
+
return result.accepted;
|
|
3963
|
+
} catch (_) {
|
|
3190
3964
|
}
|
|
3191
|
-
return result;
|
|
3192
3965
|
}
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3966
|
+
return false;
|
|
3967
|
+
}
|
|
3968
|
+
function withTimeout(work, timeoutMs, fallback) {
|
|
3969
|
+
return new Promise((resolve) => {
|
|
3970
|
+
const timer = setTimeout(() => resolve(fallback), timeoutMs);
|
|
3971
|
+
timer.unref?.();
|
|
3972
|
+
work.then((value) => {
|
|
3973
|
+
clearTimeout(timer);
|
|
3974
|
+
resolve(value);
|
|
3975
|
+
}).catch(() => {
|
|
3976
|
+
clearTimeout(timer);
|
|
3977
|
+
resolve(fallback);
|
|
3978
|
+
});
|
|
3205
3979
|
});
|
|
3206
|
-
state.inFlight.set(key, entry);
|
|
3207
|
-
return entry.promise;
|
|
3208
3980
|
}
|
|
3209
|
-
async function
|
|
3210
|
-
const
|
|
3211
|
-
const
|
|
3212
|
-
|
|
3213
|
-
|
|
3981
|
+
async function withdrawAdmission(payload, controllerNodeId, deps) {
|
|
3982
|
+
const targets = await resolveNodeTargets(controllerNodeId, deps);
|
|
3983
|
+
for (const target of targets) {
|
|
3984
|
+
try {
|
|
3985
|
+
const result = await requestInsyncRemoval(
|
|
3986
|
+
payload.indexName,
|
|
3987
|
+
payload.partitionId,
|
|
3988
|
+
payload.replicaNodeId,
|
|
3989
|
+
payload.primaryTerm,
|
|
3990
|
+
target,
|
|
3991
|
+
deps.transport,
|
|
3992
|
+
deps.nodeId
|
|
3993
|
+
);
|
|
3994
|
+
if (result.accepted) {
|
|
3995
|
+
return;
|
|
3996
|
+
}
|
|
3997
|
+
} catch (_) {
|
|
3998
|
+
}
|
|
3214
3999
|
}
|
|
3215
|
-
return assignment.state === "ACTIVE" && assignment.replicas.includes(deps.sourceNodeId) && !assignment.inSyncSet.includes(deps.sourceNodeId);
|
|
3216
4000
|
}
|
|
3217
|
-
async function
|
|
3218
|
-
const
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
4001
|
+
async function proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
|
|
4002
|
+
const log = deps.getReplicationLog(indexName, partitionId);
|
|
4003
|
+
if (cursor.appliedSeqNo < log.commitPoint) {
|
|
4004
|
+
return;
|
|
4005
|
+
}
|
|
4006
|
+
if (!markPendingAdmission(state, indexName, partitionId, replicaNodeId)) {
|
|
4007
|
+
return;
|
|
4008
|
+
}
|
|
4009
|
+
const proposedEpoch = cursor.syncEpoch;
|
|
4010
|
+
try {
|
|
4011
|
+
const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
4012
|
+
if (controllerNodeId === null) {
|
|
4013
|
+
return;
|
|
3225
4014
|
}
|
|
3226
|
-
|
|
3227
|
-
};
|
|
3228
|
-
if (hasLiveBootstrapSyncDeps(deps)) {
|
|
3229
|
-
return executeLiveBootstrapSync(
|
|
3230
|
-
state,
|
|
3231
|
-
entry,
|
|
4015
|
+
const payload = {
|
|
3232
4016
|
indexName,
|
|
3233
4017
|
partitionId,
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
);
|
|
4018
|
+
replicaNodeId,
|
|
4019
|
+
primaryTerm: assignment.primaryTerm,
|
|
4020
|
+
appliedSeqNo: cursor.appliedSeqNo,
|
|
4021
|
+
commitPoint: log.commitPoint
|
|
4022
|
+
};
|
|
4023
|
+
const accepted = await withTimeout(sendToController(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
|
|
4024
|
+
if (accepted && cursor.syncEpoch !== proposedEpoch) {
|
|
4025
|
+
await withTimeout(withdrawAdmission(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, void 0);
|
|
4026
|
+
}
|
|
4027
|
+
} finally {
|
|
4028
|
+
clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
|
|
3240
4029
|
}
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
4030
|
+
}
|
|
4031
|
+
async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
|
|
4032
|
+
const message = createEntryMessage(entry, sourceNodeId);
|
|
4033
|
+
return sendAndCollectAcks(message, entry, inSyncReplicas, transport, resolveNodeTargets3);
|
|
4034
|
+
}
|
|
4035
|
+
async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
|
|
4036
|
+
if (entries.length === 0) {
|
|
4037
|
+
return { acknowledged: [], failed: [] };
|
|
3245
4038
|
}
|
|
3246
|
-
if (
|
|
3247
|
-
return
|
|
4039
|
+
if (entries.length === 1) {
|
|
4040
|
+
return replicateToReplicas(entries[0], inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3);
|
|
3248
4041
|
}
|
|
3249
|
-
|
|
3250
|
-
|
|
4042
|
+
const message = createEntryBatchMessage(entries, sourceNodeId);
|
|
4043
|
+
const lastEntry = entries[entries.length - 1];
|
|
4044
|
+
return sendAndCollectAcks(message, lastEntry, inSyncReplicas, transport, resolveNodeTargets3);
|
|
4045
|
+
}
|
|
4046
|
+
async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport, resolveNodeTargets3) {
|
|
4047
|
+
const uniqueReplicas = [...new Set(inSyncReplicas)];
|
|
4048
|
+
if (uniqueReplicas.length === 0) {
|
|
4049
|
+
return { acknowledged: [], failed: [] };
|
|
4050
|
+
}
|
|
4051
|
+
const sendResults = await Promise.allSettled(
|
|
4052
|
+
uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
|
|
4053
|
+
);
|
|
4054
|
+
const acknowledged = [];
|
|
4055
|
+
const failed = [];
|
|
4056
|
+
for (let i = 0; i < sendResults.length; i++) {
|
|
4057
|
+
const result = sendResults[i];
|
|
4058
|
+
const replicaNodeId = uniqueReplicas[i];
|
|
4059
|
+
if (result.status === "rejected") {
|
|
4060
|
+
if (result.reason instanceof TransportError) {
|
|
4061
|
+
failed.push(replicaNodeId);
|
|
4062
|
+
continue;
|
|
4063
|
+
}
|
|
4064
|
+
throw result.reason;
|
|
4065
|
+
}
|
|
4066
|
+
const response = result.value;
|
|
4067
|
+
if (isMatchingAck(response, ackEntry)) {
|
|
4068
|
+
acknowledged.push(replicaNodeId);
|
|
4069
|
+
} else {
|
|
4070
|
+
failed.push(replicaNodeId);
|
|
4071
|
+
}
|
|
4072
|
+
}
|
|
4073
|
+
return { acknowledged, failed };
|
|
4074
|
+
}
|
|
4075
|
+
async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
|
|
4076
|
+
const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
|
|
4077
|
+
const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
|
|
4078
|
+
let lastError;
|
|
4079
|
+
for (const target of resolvedTargets) {
|
|
4080
|
+
try {
|
|
4081
|
+
return await transport.send(target, message);
|
|
4082
|
+
} catch (error) {
|
|
4083
|
+
lastError = error;
|
|
4084
|
+
}
|
|
4085
|
+
}
|
|
4086
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
4087
|
+
}
|
|
4088
|
+
function isMatchingAck(response, entry) {
|
|
4089
|
+
if (response.type !== ReplicationMessageTypes.ACK) {
|
|
3251
4090
|
return false;
|
|
3252
4091
|
}
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
primaryNodeId,
|
|
3258
|
-
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
|
|
3259
|
-
indexName,
|
|
3260
|
-
deadlineMs
|
|
3261
|
-
})
|
|
3262
|
-
);
|
|
4092
|
+
try {
|
|
4093
|
+
const payload = validateAckPayload(decode(response.payload));
|
|
4094
|
+
return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
|
|
4095
|
+
} catch {
|
|
3263
4096
|
return false;
|
|
3264
4097
|
}
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
4098
|
+
}
|
|
4099
|
+
|
|
4100
|
+
// src/distribution/cluster-node/catch-up/pump.ts
|
|
4101
|
+
var ENTRY_FIXED_OVERHEAD_BYTES = 40;
|
|
4102
|
+
function entryBytes(entry) {
|
|
4103
|
+
return ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
4104
|
+
}
|
|
4105
|
+
function batchBytes(entries) {
|
|
4106
|
+
let total = 0;
|
|
4107
|
+
for (const entry of entries) {
|
|
4108
|
+
total += entryBytes(entry);
|
|
4109
|
+
}
|
|
4110
|
+
return total;
|
|
4111
|
+
}
|
|
4112
|
+
function replicaHasFallenOutOfRetention(log, cursor) {
|
|
4113
|
+
const oldest = log.oldestSeqNo;
|
|
4114
|
+
if (oldest === void 0) {
|
|
3268
4115
|
return false;
|
|
3269
4116
|
}
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
)
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
4117
|
+
return cursor.appliedSeqNo + 1 < oldest;
|
|
4118
|
+
}
|
|
4119
|
+
function nextWireBatch(state, pending) {
|
|
4120
|
+
const chunks = chunkByBudget(pending, {
|
|
4121
|
+
...WIRE_BATCH_BUDGET,
|
|
4122
|
+
payloadBytesOf: (entry) => entry.document?.byteLength ?? 0,
|
|
4123
|
+
breaksRun: (entry, previous) => entry.seqNo !== previous.seqNo + 1
|
|
4124
|
+
});
|
|
4125
|
+
const batch = chunks[0] ?? [];
|
|
4126
|
+
if (batch.length === 0) {
|
|
4127
|
+
return [];
|
|
4128
|
+
}
|
|
4129
|
+
const available = CATCH_UP_IN_FLIGHT_BYTE_CEILING - state.inFlightBytes;
|
|
4130
|
+
return batchBytes(batch) > available ? [] : batch;
|
|
4131
|
+
}
|
|
4132
|
+
async function pumpReplica(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
|
|
4133
|
+
if (cursor.sending) {
|
|
4134
|
+
return;
|
|
4135
|
+
}
|
|
4136
|
+
const log = deps.getReplicationLog(indexName, partitionId);
|
|
4137
|
+
if (replicaHasFallenOutOfRetention(log, cursor)) {
|
|
4138
|
+
forgetReplica(state, indexName, partitionId, replicaNodeId);
|
|
4139
|
+
return;
|
|
4140
|
+
}
|
|
4141
|
+
if (cursor.appliedSeqNo >= log.localLogEnd) {
|
|
4142
|
+
await proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
|
|
4143
|
+
return;
|
|
4144
|
+
}
|
|
4145
|
+
const batch = nextWireBatch(state, log.getEntriesFrom(cursor.appliedSeqNo + 1));
|
|
4146
|
+
if (batch.length === 0) {
|
|
4147
|
+
return;
|
|
4148
|
+
}
|
|
4149
|
+
const reservedBytes = batchBytes(batch);
|
|
4150
|
+
const sendingEpoch = cursor.syncEpoch;
|
|
4151
|
+
cursor.sending = true;
|
|
4152
|
+
state.inFlightBytes += reservedBytes;
|
|
4153
|
+
try {
|
|
4154
|
+
const result = await replicateBatchToReplicas(
|
|
4155
|
+
batch,
|
|
4156
|
+
[replicaNodeId],
|
|
4157
|
+
deps.transport,
|
|
4158
|
+
deps.nodeId,
|
|
4159
|
+
deps.resolveNodeTargets
|
|
3294
4160
|
);
|
|
3295
|
-
|
|
4161
|
+
if (result.acknowledged.includes(replicaNodeId) && cursor.syncEpoch === sendingEpoch) {
|
|
4162
|
+
cursor.appliedSeqNo = batch[batch.length - 1].seqNo;
|
|
4163
|
+
}
|
|
4164
|
+
} finally {
|
|
4165
|
+
state.inFlightBytes -= reservedBytes;
|
|
4166
|
+
cursor.sending = false;
|
|
4167
|
+
}
|
|
4168
|
+
}
|
|
4169
|
+
async function pumpPartition(state, deps, key, replicas) {
|
|
4170
|
+
const separator = key.lastIndexOf(":");
|
|
4171
|
+
const indexName = key.slice(0, separator);
|
|
4172
|
+
const partitionId = Number(key.slice(separator + 1));
|
|
4173
|
+
const table = await deps.coordinator.getAllocation(indexName);
|
|
4174
|
+
const assignment = table?.assignments.get(partitionId);
|
|
4175
|
+
if (assignment === void 0 || assignment.primary !== deps.nodeId) {
|
|
4176
|
+
state.cursors.delete(key);
|
|
4177
|
+
state.pendingAdmissions.delete(key);
|
|
4178
|
+
return;
|
|
3296
4179
|
}
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
4180
|
+
for (const [replicaNodeId, cursor] of [...replicas]) {
|
|
4181
|
+
if (state.stopped) {
|
|
4182
|
+
return;
|
|
4183
|
+
}
|
|
4184
|
+
if (!assignment.replicas.includes(replicaNodeId) || assignment.inSyncSet.includes(replicaNodeId)) {
|
|
4185
|
+
forgetReplica(state, indexName, partitionId, replicaNodeId);
|
|
4186
|
+
continue;
|
|
4187
|
+
}
|
|
4188
|
+
try {
|
|
4189
|
+
await pumpReplica(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
|
|
4190
|
+
} catch (_) {
|
|
4191
|
+
}
|
|
3300
4192
|
}
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
primaryNodeId,
|
|
3306
|
-
fetchResult.bytes,
|
|
3307
|
-
schemaResult,
|
|
3308
|
-
deadline,
|
|
3309
|
-
deps
|
|
3310
|
-
);
|
|
3311
|
-
if (!restoreSucceeded) {
|
|
3312
|
-
return false;
|
|
4193
|
+
}
|
|
4194
|
+
function runCatchUpTick(state, deps) {
|
|
4195
|
+
if (state.stopped || state.activeTick !== null || state.cursors.size === 0) {
|
|
4196
|
+
return state.activeTick ?? Promise.resolve();
|
|
3313
4197
|
}
|
|
3314
|
-
|
|
3315
|
-
|
|
3316
|
-
|
|
4198
|
+
const tick = (async () => {
|
|
4199
|
+
for (const [key, replicas] of [...state.cursors]) {
|
|
4200
|
+
if (state.stopped) {
|
|
4201
|
+
return;
|
|
4202
|
+
}
|
|
4203
|
+
try {
|
|
4204
|
+
await pumpPartition(state, deps, key, replicas);
|
|
4205
|
+
} catch (_) {
|
|
4206
|
+
}
|
|
3317
4207
|
}
|
|
3318
|
-
|
|
3319
|
-
|
|
4208
|
+
})().finally(() => {
|
|
4209
|
+
state.activeTick = null;
|
|
4210
|
+
});
|
|
4211
|
+
state.activeTick = tick;
|
|
4212
|
+
return tick;
|
|
4213
|
+
}
|
|
4214
|
+
function startCatchUpPump(state, deps) {
|
|
4215
|
+
void stopCatchUpPump(state);
|
|
4216
|
+
state.stopped = false;
|
|
4217
|
+
state.timer = setInterval(() => {
|
|
4218
|
+
void runCatchUpTick(state, deps);
|
|
4219
|
+
}, CATCH_UP_TICK_MS);
|
|
4220
|
+
state.timer.unref?.();
|
|
4221
|
+
}
|
|
4222
|
+
async function stopCatchUpPump(state) {
|
|
4223
|
+
state.stopped = true;
|
|
4224
|
+
if (state.timer !== null) {
|
|
4225
|
+
clearInterval(state.timer);
|
|
4226
|
+
state.timer = null;
|
|
4227
|
+
}
|
|
4228
|
+
if (state.activeTick !== null) {
|
|
4229
|
+
await state.activeTick;
|
|
3320
4230
|
}
|
|
3321
|
-
deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
|
|
3322
|
-
state.completed.add(key);
|
|
3323
|
-
return true;
|
|
3324
4231
|
}
|
|
3325
4232
|
|
|
3326
4233
|
// src/distribution/cluster-node/local-engine.ts
|
|
@@ -3663,21 +4570,14 @@ async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
|
|
|
3663
4570
|
}
|
|
3664
4571
|
let sawAssignment = false;
|
|
3665
4572
|
for (const assignment of allocation.assignments.values()) {
|
|
3666
|
-
const
|
|
3667
|
-
|
|
3668
|
-
if (!isPrimary && !isReplica) {
|
|
4573
|
+
const isAssigned = assignment.primary === sourceId || assignment.replicas.includes(sourceId);
|
|
4574
|
+
if (!isAssigned) {
|
|
3669
4575
|
continue;
|
|
3670
4576
|
}
|
|
3671
4577
|
sawAssignment = true;
|
|
3672
4578
|
if (!BOOTSTRAPPABLE_STATES.has(assignment.state)) {
|
|
3673
4579
|
continue;
|
|
3674
4580
|
}
|
|
3675
|
-
if (isPrimary) {
|
|
3676
|
-
return { outcome: "authorized" };
|
|
3677
|
-
}
|
|
3678
|
-
if (!assignmentAllowsReplicaBootstrap(assignment, sourceId)) {
|
|
3679
|
-
continue;
|
|
3680
|
-
}
|
|
3681
4581
|
return { outcome: "authorized" };
|
|
3682
4582
|
}
|
|
3683
4583
|
if (sawAssignment) {
|
|
@@ -3693,12 +4593,6 @@ async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
|
|
|
3693
4593
|
reason: `node '${sourceId}' is not an assigned replica for index '${indexName}'`
|
|
3694
4594
|
};
|
|
3695
4595
|
}
|
|
3696
|
-
function assignmentAllowsReplicaBootstrap(assignment, sourceId) {
|
|
3697
|
-
if (assignment.state === "INITIALISING" || assignment.state === "MIGRATING") {
|
|
3698
|
-
return true;
|
|
3699
|
-
}
|
|
3700
|
-
return assignment.inSyncSet.includes(sourceId);
|
|
3701
|
-
}
|
|
3702
4596
|
|
|
3703
4597
|
// src/distribution/cluster-node/snapshot-cache.ts
|
|
3704
4598
|
var DEFAULT_MAX_CONCURRENT_SNAPSHOTS = 2;
|
|
@@ -4217,7 +5111,7 @@ var NULL_BYTE = String.fromCharCode(0);
|
|
|
4217
5111
|
function throwInvalid(code, message, details) {
|
|
4218
5112
|
throw new NarsilError(code, message, details);
|
|
4219
5113
|
}
|
|
4220
|
-
function
|
|
5114
|
+
function isRecord2(value) {
|
|
4221
5115
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
4222
5116
|
}
|
|
4223
5117
|
function isInteger(value) {
|
|
@@ -4335,14 +5229,14 @@ function validateIndexNameField(value, fieldLabel) {
|
|
|
4335
5229
|
return value;
|
|
4336
5230
|
}
|
|
4337
5231
|
function validateDocumentId(value, fieldLabel) {
|
|
4338
|
-
if (!
|
|
5232
|
+
if (!isRecord2(value)) {
|
|
4339
5233
|
throwInvalid(CONFIG_INVALID, `Invalid FetchPayload: "${fieldLabel}" must be an object`);
|
|
4340
5234
|
}
|
|
4341
5235
|
validateStringField(value.docId, `${fieldLabel}.docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
|
|
4342
5236
|
validatePartitionId(value.partitionId, `${fieldLabel}.partitionId`, CONFIG_INVALID);
|
|
4343
5237
|
}
|
|
4344
5238
|
function validateHighlightConfig(value) {
|
|
4345
|
-
if (!
|
|
5239
|
+
if (!isRecord2(value)) {
|
|
4346
5240
|
throwInvalid(CONFIG_INVALID, 'Invalid FetchPayload: "highlight" must be an object or null');
|
|
4347
5241
|
}
|
|
4348
5242
|
if (value.fields !== null) {
|
|
@@ -4368,7 +5262,7 @@ function validateHighlightConfig(value) {
|
|
|
4368
5262
|
}
|
|
4369
5263
|
}
|
|
4370
5264
|
function validateFetchPayload(decoded) {
|
|
4371
|
-
if (!
|
|
5265
|
+
if (!isRecord2(decoded)) {
|
|
4372
5266
|
throwInvalid(CONFIG_INVALID, "Invalid FetchPayload: expected an object");
|
|
4373
5267
|
}
|
|
4374
5268
|
validateIndexNameField(decoded.indexName, "indexName");
|
|
@@ -4394,7 +5288,7 @@ function validateFetchPayload(decoded) {
|
|
|
4394
5288
|
return decoded;
|
|
4395
5289
|
}
|
|
4396
5290
|
function validateFetchResultPayload(decoded) {
|
|
4397
|
-
if (!
|
|
5291
|
+
if (!isRecord2(decoded)) {
|
|
4398
5292
|
throwInvalid(CONFIG_INVALID, "Invalid FetchResultPayload: expected an object");
|
|
4399
5293
|
}
|
|
4400
5294
|
if (!Array.isArray(decoded.documents)) {
|
|
@@ -4402,14 +5296,14 @@ function validateFetchResultPayload(decoded) {
|
|
|
4402
5296
|
}
|
|
4403
5297
|
for (let i = 0; i < decoded.documents.length; i++) {
|
|
4404
5298
|
const doc = decoded.documents[i];
|
|
4405
|
-
if (!
|
|
5299
|
+
if (!isRecord2(doc)) {
|
|
4406
5300
|
throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}]" must be an object`);
|
|
4407
5301
|
}
|
|
4408
5302
|
validateStringField(doc.docId, `documents[${i}].docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
|
|
4409
|
-
if (!
|
|
5303
|
+
if (!isRecord2(doc.document)) {
|
|
4410
5304
|
throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}].document" must be an object`);
|
|
4411
5305
|
}
|
|
4412
|
-
if (doc.highlights !== null && !
|
|
5306
|
+
if (doc.highlights !== null && !isRecord2(doc.highlights)) {
|
|
4413
5307
|
throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}].highlights" must be an object or null`);
|
|
4414
5308
|
}
|
|
4415
5309
|
}
|
|
@@ -4506,7 +5400,7 @@ function isKnownLeafKey(key) {
|
|
|
4506
5400
|
return FIELD_FILTER_OPERATORS.includes(key);
|
|
4507
5401
|
}
|
|
4508
5402
|
function validateGeoRadius(value, fieldLabel, errorCode) {
|
|
4509
|
-
if (!
|
|
5403
|
+
if (!isRecord2(value)) {
|
|
4510
5404
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
4511
5405
|
}
|
|
4512
5406
|
validateNumericOperand(value.lat, `${fieldLabel}.lat`, errorCode);
|
|
@@ -4526,7 +5420,7 @@ function validateGeoRadius(value, fieldLabel, errorCode) {
|
|
|
4526
5420
|
}
|
|
4527
5421
|
}
|
|
4528
5422
|
function validateGeoPolygon(value, fieldLabel, errorCode) {
|
|
4529
|
-
if (!
|
|
5423
|
+
if (!isRecord2(value)) {
|
|
4530
5424
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
4531
5425
|
}
|
|
4532
5426
|
if (!Array.isArray(value.points)) {
|
|
@@ -4542,7 +5436,7 @@ function validateGeoPolygon(value, fieldLabel, errorCode) {
|
|
|
4542
5436
|
}
|
|
4543
5437
|
for (let i = 0; i < points.length; i++) {
|
|
4544
5438
|
const point = points[i];
|
|
4545
|
-
if (!
|
|
5439
|
+
if (!isRecord2(point)) {
|
|
4546
5440
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}.points[${i}]" must be an object`);
|
|
4547
5441
|
}
|
|
4548
5442
|
validateNumericOperand(point.lat, `${fieldLabel}.points[${i}].lat`, errorCode);
|
|
@@ -4583,7 +5477,7 @@ function validateLeafOperator(opKey, opValue, leafLabel, errorCode) {
|
|
|
4583
5477
|
return;
|
|
4584
5478
|
}
|
|
4585
5479
|
if (opKey === "size") {
|
|
4586
|
-
if (!
|
|
5480
|
+
if (!isRecord2(opValue)) {
|
|
4587
5481
|
throwInvalid(errorCode, `Invalid payload: "${opLabel}" must be an object`);
|
|
4588
5482
|
}
|
|
4589
5483
|
for (const [innerKey, innerValue] of Object.entries(opValue)) {
|
|
@@ -4605,7 +5499,7 @@ function validateLeafOperator(opKey, opValue, leafLabel, errorCode) {
|
|
|
4605
5499
|
throwInvalid(errorCode, `Invalid payload: "${leafLabel}" contains unsupported operator "${opKey}"`);
|
|
4606
5500
|
}
|
|
4607
5501
|
function validateFieldFilter(value, fieldLabel, errorCode) {
|
|
4608
|
-
if (!
|
|
5502
|
+
if (!isRecord2(value)) {
|
|
4609
5503
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
4610
5504
|
}
|
|
4611
5505
|
const entries = Object.entries(value);
|
|
@@ -4623,7 +5517,7 @@ function validateFieldFilter(value, fieldLabel, errorCode) {
|
|
|
4623
5517
|
}
|
|
4624
5518
|
}
|
|
4625
5519
|
function validateFieldsRecord(value, fieldLabel, errorCode) {
|
|
4626
|
-
if (!
|
|
5520
|
+
if (!isRecord2(value)) {
|
|
4627
5521
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
4628
5522
|
}
|
|
4629
5523
|
const entries = Object.entries(value);
|
|
@@ -4666,7 +5560,7 @@ function validateFilterExpression(value, fieldLabel, errorCode = SEARCH_INVALID_
|
|
|
4666
5560
|
limit: MAX_FILTER_DEPTH
|
|
4667
5561
|
});
|
|
4668
5562
|
}
|
|
4669
|
-
if (!
|
|
5563
|
+
if (!isRecord2(value)) {
|
|
4670
5564
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
4671
5565
|
}
|
|
4672
5566
|
const entries = Object.entries(value);
|
|
@@ -4704,7 +5598,7 @@ function validateIndexNameField2(value, fieldLabel) {
|
|
|
4704
5598
|
return value;
|
|
4705
5599
|
}
|
|
4706
5600
|
function validateVectorParams(value) {
|
|
4707
|
-
if (!
|
|
5601
|
+
if (!isRecord2(value)) {
|
|
4708
5602
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.vector" must be an object');
|
|
4709
5603
|
}
|
|
4710
5604
|
validateFieldName(value.field, "params.vector.field", SEARCH_INVALID_FIELD);
|
|
@@ -4749,7 +5643,7 @@ function validateVectorParams(value) {
|
|
|
4749
5643
|
}
|
|
4750
5644
|
}
|
|
4751
5645
|
function validateHybridParams(value) {
|
|
4752
|
-
if (!
|
|
5646
|
+
if (!isRecord2(value)) {
|
|
4753
5647
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.hybrid" must be an object');
|
|
4754
5648
|
}
|
|
4755
5649
|
if (!ALLOWED_HYBRID_STRATEGIES.includes(value.strategy)) {
|
|
@@ -4769,7 +5663,7 @@ function validateHybridParams(value) {
|
|
|
4769
5663
|
}
|
|
4770
5664
|
}
|
|
4771
5665
|
function validateGroupParams(value) {
|
|
4772
|
-
if (!
|
|
5666
|
+
if (!isRecord2(value)) {
|
|
4773
5667
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.group" must be an object');
|
|
4774
5668
|
}
|
|
4775
5669
|
validateFieldName(value.field, "params.group.field", SEARCH_INVALID_FIELD);
|
|
@@ -4796,7 +5690,7 @@ function validateSortParams(value) {
|
|
|
4796
5690
|
}
|
|
4797
5691
|
for (let i = 0; i < value.length; i++) {
|
|
4798
5692
|
const entry = value[i];
|
|
4799
|
-
if (!
|
|
5693
|
+
if (!isRecord2(entry)) {
|
|
4800
5694
|
throwInvalid(CONFIG_INVALID, `Invalid SearchPayload: "params.sort[${i}]" must be an object`);
|
|
4801
5695
|
}
|
|
4802
5696
|
validateFieldName(entry.field, `params.sort[${i}].field`, SEARCH_INVALID_FIELD);
|
|
@@ -4809,7 +5703,7 @@ function validateSortParams(value) {
|
|
|
4809
5703
|
}
|
|
4810
5704
|
}
|
|
4811
5705
|
function validateBoostParams(value) {
|
|
4812
|
-
if (!
|
|
5706
|
+
if (!isRecord2(value)) {
|
|
4813
5707
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.boost" must be an object or null');
|
|
4814
5708
|
}
|
|
4815
5709
|
const entries = Object.entries(value);
|
|
@@ -4828,16 +5722,16 @@ function validateBoostParams(value) {
|
|
|
4828
5722
|
}
|
|
4829
5723
|
}
|
|
4830
5724
|
function validateGlobalStatsShape(value) {
|
|
4831
|
-
if (!
|
|
5725
|
+
if (!isRecord2(value)) {
|
|
4832
5726
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats" must be an object or null');
|
|
4833
5727
|
}
|
|
4834
5728
|
if (!isFiniteNumber(value.totalDocuments) || value.totalDocuments < 0) {
|
|
4835
5729
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.totalDocuments" must be a non-negative number');
|
|
4836
5730
|
}
|
|
4837
|
-
if (!
|
|
5731
|
+
if (!isRecord2(value.docFrequencies)) {
|
|
4838
5732
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.docFrequencies" must be an object');
|
|
4839
5733
|
}
|
|
4840
|
-
if (!
|
|
5734
|
+
if (!isRecord2(value.totalFieldLengths)) {
|
|
4841
5735
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.totalFieldLengths" must be an object');
|
|
4842
5736
|
}
|
|
4843
5737
|
}
|
|
@@ -4927,18 +5821,18 @@ function validateParams(params) {
|
|
|
4927
5821
|
}
|
|
4928
5822
|
}
|
|
4929
5823
|
function validateWireParams(params) {
|
|
4930
|
-
if (!
|
|
5824
|
+
if (!isRecord2(params)) {
|
|
4931
5825
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params" must be an object');
|
|
4932
5826
|
}
|
|
4933
5827
|
validateParams(params);
|
|
4934
5828
|
}
|
|
4935
5829
|
function validateSearchPayload(decoded) {
|
|
4936
|
-
if (!
|
|
5830
|
+
if (!isRecord2(decoded)) {
|
|
4937
5831
|
throwInvalid(CONFIG_INVALID, "Invalid SearchPayload: expected an object");
|
|
4938
5832
|
}
|
|
4939
5833
|
validateIndexNameField2(decoded.indexName, "indexName");
|
|
4940
5834
|
validatePartitionIdsArray(decoded.partitionIds, "partitionIds", CONFIG_INVALID);
|
|
4941
|
-
if (!
|
|
5835
|
+
if (!isRecord2(decoded.params)) {
|
|
4942
5836
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params" must be an object');
|
|
4943
5837
|
}
|
|
4944
5838
|
validateParams(decoded.params);
|
|
@@ -4974,7 +5868,7 @@ function validateIndexNameField3(decoded, payloadLabel) {
|
|
|
4974
5868
|
validateStringField(decoded.indexName, `${payloadLabel}.indexName`, 255, CONFIG_INVALID);
|
|
4975
5869
|
}
|
|
4976
5870
|
function validateCountPayload(decoded) {
|
|
4977
|
-
if (!
|
|
5871
|
+
if (!isRecord2(decoded)) {
|
|
4978
5872
|
throwInvalid(CONFIG_INVALID, "Invalid CountPayload: expected an object");
|
|
4979
5873
|
}
|
|
4980
5874
|
validateIndexNameField3(decoded, "CountPayload");
|
|
@@ -4982,7 +5876,7 @@ function validateCountPayload(decoded) {
|
|
|
4982
5876
|
return decoded;
|
|
4983
5877
|
}
|
|
4984
5878
|
function validateCountResultPayload(decoded) {
|
|
4985
|
-
if (!
|
|
5879
|
+
if (!isRecord2(decoded)) {
|
|
4986
5880
|
throwInvalid(CONFIG_INVALID, "Invalid CountResultPayload: expected an object");
|
|
4987
5881
|
}
|
|
4988
5882
|
if (!Array.isArray(decoded.partitions)) {
|
|
@@ -4990,7 +5884,7 @@ function validateCountResultPayload(decoded) {
|
|
|
4990
5884
|
}
|
|
4991
5885
|
for (let i = 0; i < decoded.partitions.length; i++) {
|
|
4992
5886
|
const entry = decoded.partitions[i];
|
|
4993
|
-
if (!
|
|
5887
|
+
if (!isRecord2(entry)) {
|
|
4994
5888
|
throwInvalid(CONFIG_INVALID, `Invalid CountResultPayload: "partitions[${i}]" must be an object`);
|
|
4995
5889
|
}
|
|
4996
5890
|
validateNonNegativeInteger(entry.partitionId, `partitions[${i}].partitionId`, MAX_COUNT_VALUE, CONFIG_INVALID);
|
|
@@ -5017,7 +5911,7 @@ function validateListSort(value) {
|
|
|
5017
5911
|
}
|
|
5018
5912
|
for (let i = 0; i < value.length; i++) {
|
|
5019
5913
|
const entry = value[i];
|
|
5020
|
-
if (!
|
|
5914
|
+
if (!isRecord2(entry)) {
|
|
5021
5915
|
throwInvalid(CONFIG_INVALID, `Invalid ListPayload: "sort[${i}]" must be an object`);
|
|
5022
5916
|
}
|
|
5023
5917
|
validateFieldName(entry.field, `sort[${i}].field`, SEARCH_INVALID_FIELD);
|
|
@@ -5027,7 +5921,7 @@ function validateListSort(value) {
|
|
|
5027
5921
|
}
|
|
5028
5922
|
}
|
|
5029
5923
|
function validateListPayload(decoded) {
|
|
5030
|
-
if (!
|
|
5924
|
+
if (!isRecord2(decoded)) {
|
|
5031
5925
|
throwInvalid(CONFIG_INVALID, "Invalid ListPayload: expected an object");
|
|
5032
5926
|
}
|
|
5033
5927
|
validateIndexNameField3(decoded, "ListPayload");
|
|
@@ -5072,7 +5966,7 @@ function validateSortValues(value, fieldLabel) {
|
|
|
5072
5966
|
}
|
|
5073
5967
|
}
|
|
5074
5968
|
function validateListResultPayload(decoded) {
|
|
5075
|
-
if (!
|
|
5969
|
+
if (!isRecord2(decoded)) {
|
|
5076
5970
|
throwInvalid(CONFIG_INVALID, "Invalid ListResultPayload: expected an object");
|
|
5077
5971
|
}
|
|
5078
5972
|
if (!Array.isArray(decoded.entries)) {
|
|
@@ -5083,11 +5977,11 @@ function validateListResultPayload(decoded) {
|
|
|
5083
5977
|
}
|
|
5084
5978
|
for (let i = 0; i < decoded.entries.length; i++) {
|
|
5085
5979
|
const entry = decoded.entries[i];
|
|
5086
|
-
if (!
|
|
5980
|
+
if (!isRecord2(entry)) {
|
|
5087
5981
|
throwInvalid(CONFIG_INVALID, `Invalid ListResultPayload: "entries[${i}]" must be an object`);
|
|
5088
5982
|
}
|
|
5089
5983
|
validateStringField(entry.docId, `entries[${i}].docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
|
|
5090
|
-
if (!
|
|
5984
|
+
if (!isRecord2(entry.document)) {
|
|
5091
5985
|
throwInvalid(CONFIG_INVALID, `Invalid ListResultPayload: "entries[${i}].document" must be an object`);
|
|
5092
5986
|
}
|
|
5093
5987
|
if (entry.sortValues !== null) {
|
|
@@ -5101,7 +5995,7 @@ function validateListResultPayload(decoded) {
|
|
|
5101
5995
|
return decoded;
|
|
5102
5996
|
}
|
|
5103
5997
|
function validateSuggestPayload(decoded) {
|
|
5104
|
-
if (!
|
|
5998
|
+
if (!isRecord2(decoded)) {
|
|
5105
5999
|
throwInvalid(CONFIG_INVALID, "Invalid SuggestPayload: expected an object");
|
|
5106
6000
|
}
|
|
5107
6001
|
validateIndexNameField3(decoded, "SuggestPayload");
|
|
@@ -5111,7 +6005,7 @@ function validateSuggestPayload(decoded) {
|
|
|
5111
6005
|
return decoded;
|
|
5112
6006
|
}
|
|
5113
6007
|
function validateSuggestResultPayload(decoded) {
|
|
5114
|
-
if (!
|
|
6008
|
+
if (!isRecord2(decoded)) {
|
|
5115
6009
|
throwInvalid(CONFIG_INVALID, "Invalid SuggestResultPayload: expected an object");
|
|
5116
6010
|
}
|
|
5117
6011
|
if (!Array.isArray(decoded.terms)) {
|
|
@@ -5125,7 +6019,7 @@ function validateSuggestResultPayload(decoded) {
|
|
|
5125
6019
|
}
|
|
5126
6020
|
for (let i = 0; i < decoded.terms.length; i++) {
|
|
5127
6021
|
const entry = decoded.terms[i];
|
|
5128
|
-
if (!
|
|
6022
|
+
if (!isRecord2(entry)) {
|
|
5129
6023
|
throwInvalid(CONFIG_INVALID, `Invalid SuggestResultPayload: "terms[${i}]" must be an object`);
|
|
5130
6024
|
}
|
|
5131
6025
|
validateStringField(entry.term, `terms[${i}].term`, MAX_TERM_LENGTH, CONFIG_INVALID);
|
|
@@ -5142,7 +6036,7 @@ function validateSuggestResultPayload(decoded) {
|
|
|
5142
6036
|
return decoded;
|
|
5143
6037
|
}
|
|
5144
6038
|
function validatePreflightPayload(decoded) {
|
|
5145
|
-
if (!
|
|
6039
|
+
if (!isRecord2(decoded)) {
|
|
5146
6040
|
throwInvalid(CONFIG_INVALID, "Invalid PreflightPayload: expected an object");
|
|
5147
6041
|
}
|
|
5148
6042
|
validateIndexNameField3(decoded, "PreflightPayload");
|
|
@@ -5151,7 +6045,7 @@ function validatePreflightPayload(decoded) {
|
|
|
5151
6045
|
return decoded;
|
|
5152
6046
|
}
|
|
5153
6047
|
function validatePreflightResultPayload(decoded) {
|
|
5154
|
-
if (!
|
|
6048
|
+
if (!isRecord2(decoded)) {
|
|
5155
6049
|
throwInvalid(CONFIG_INVALID, "Invalid PreflightResultPayload: expected an object");
|
|
5156
6050
|
}
|
|
5157
6051
|
if (!isInteger(decoded.count) || decoded.count < 0) {
|
|
@@ -5170,7 +6064,7 @@ var MAX_FACET_FIELDS = 64;
|
|
|
5170
6064
|
var MAX_FACET_BUCKETS = 1e4;
|
|
5171
6065
|
var MAX_FACET_VALUE_LENGTH = 1024;
|
|
5172
6066
|
function validateScoredEntry(value, fieldLabel) {
|
|
5173
|
-
if (!
|
|
6067
|
+
if (!isRecord2(value)) {
|
|
5174
6068
|
throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
|
|
5175
6069
|
}
|
|
5176
6070
|
if (typeof value.docId !== "string") {
|
|
@@ -5216,7 +6110,7 @@ function validateScoredEntry(value, fieldLabel) {
|
|
|
5216
6110
|
}
|
|
5217
6111
|
}
|
|
5218
6112
|
function validatePartitionSearchResult(value, fieldLabel) {
|
|
5219
|
-
if (!
|
|
6113
|
+
if (!isRecord2(value)) {
|
|
5220
6114
|
throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
|
|
5221
6115
|
}
|
|
5222
6116
|
validatePartitionId(value.partitionId, `${fieldLabel}.partitionId`, CONFIG_INVALID);
|
|
@@ -5241,7 +6135,7 @@ function validatePartitionSearchResult(value, fieldLabel) {
|
|
|
5241
6135
|
}
|
|
5242
6136
|
}
|
|
5243
6137
|
function validateFacetBucket(value, fieldLabel) {
|
|
5244
|
-
if (!
|
|
6138
|
+
if (!isRecord2(value)) {
|
|
5245
6139
|
throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
|
|
5246
6140
|
}
|
|
5247
6141
|
if (typeof value.value !== "string") {
|
|
@@ -5260,7 +6154,7 @@ function validateFacetBucket(value, fieldLabel) {
|
|
|
5260
6154
|
return { value: value.value, count: value.count };
|
|
5261
6155
|
}
|
|
5262
6156
|
function validateFacets(value) {
|
|
5263
|
-
if (!
|
|
6157
|
+
if (!isRecord2(value)) {
|
|
5264
6158
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchResultPayload: "facets" must be an object or null');
|
|
5265
6159
|
}
|
|
5266
6160
|
const entries = Object.entries(value);
|
|
@@ -5291,7 +6185,7 @@ function validateFacets(value) {
|
|
|
5291
6185
|
}
|
|
5292
6186
|
}
|
|
5293
6187
|
function validateFacetErrorBounds(value) {
|
|
5294
|
-
if (!
|
|
6188
|
+
if (!isRecord2(value)) {
|
|
5295
6189
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchResultPayload: "facetErrorBounds" must be an object or null');
|
|
5296
6190
|
}
|
|
5297
6191
|
const entries = Object.entries(value);
|
|
@@ -5312,7 +6206,7 @@ function validateFacetErrorBounds(value) {
|
|
|
5312
6206
|
}
|
|
5313
6207
|
}
|
|
5314
6208
|
function validateSearchResultPayload(decoded) {
|
|
5315
|
-
if (!
|
|
6209
|
+
if (!isRecord2(decoded)) {
|
|
5316
6210
|
throwInvalid(CONFIG_INVALID, "Invalid SearchResultPayload: expected an object");
|
|
5317
6211
|
}
|
|
5318
6212
|
if (!Array.isArray(decoded.results)) {
|
|
@@ -5339,7 +6233,7 @@ function validateIndexNameField4(value, fieldLabel) {
|
|
|
5339
6233
|
return value;
|
|
5340
6234
|
}
|
|
5341
6235
|
function validateNumericRecord(value, fieldLabel) {
|
|
5342
|
-
if (!
|
|
6236
|
+
if (!isRecord2(value)) {
|
|
5343
6237
|
throwInvalid(CONFIG_INVALID, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
5344
6238
|
}
|
|
5345
6239
|
for (const [key, entry] of Object.entries(value)) {
|
|
@@ -5352,7 +6246,7 @@ function validateNumericRecord(value, fieldLabel) {
|
|
|
5352
6246
|
}
|
|
5353
6247
|
}
|
|
5354
6248
|
function validateStatsPayload(decoded) {
|
|
5355
|
-
if (!
|
|
6249
|
+
if (!isRecord2(decoded)) {
|
|
5356
6250
|
throwInvalid(CONFIG_INVALID, "Invalid StatsPayload: expected an object");
|
|
5357
6251
|
}
|
|
5358
6252
|
validateIndexNameField4(decoded.indexName, "indexName");
|
|
@@ -5361,7 +6255,7 @@ function validateStatsPayload(decoded) {
|
|
|
5361
6255
|
return decoded;
|
|
5362
6256
|
}
|
|
5363
6257
|
function validateStatsResultPayload(decoded) {
|
|
5364
|
-
if (!
|
|
6258
|
+
if (!isRecord2(decoded)) {
|
|
5365
6259
|
throwInvalid(CONFIG_INVALID, "Invalid StatsResultPayload: expected an object");
|
|
5366
6260
|
}
|
|
5367
6261
|
if (!isInteger(decoded.totalDocuments) || decoded.totalDocuments < 0) {
|
|
@@ -5535,7 +6429,8 @@ function distributedResultToLocal(result, documents = /* @__PURE__ */ new Map())
|
|
|
5535
6429
|
count: result.totalHits,
|
|
5536
6430
|
elapsed: 0,
|
|
5537
6431
|
cursor: result.cursor ?? void 0,
|
|
5538
|
-
facets: result.facets !== null ? convertWireFacetsToLocal(result.facets, result.facetErrorBounds) : void 0
|
|
6432
|
+
facets: result.facets !== null ? convertWireFacetsToLocal(result.facets, result.facetErrorBounds) : void 0,
|
|
6433
|
+
coverage: result.coverage
|
|
5539
6434
|
};
|
|
5540
6435
|
}
|
|
5541
6436
|
function convertLocalSortToWire(sort) {
|
|
@@ -5761,10 +6656,10 @@ function toWireSortValue2(value) {
|
|
|
5761
6656
|
function decideSyncTier(log, replicaLastSeqNo) {
|
|
5762
6657
|
const oldest = log.oldestSeqNo;
|
|
5763
6658
|
if (oldest === void 0) {
|
|
5764
|
-
if (log.
|
|
6659
|
+
if (log.localLogEnd === 0) {
|
|
5765
6660
|
return replicaLastSeqNo === 0 ? "incremental" : "snapshot";
|
|
5766
6661
|
}
|
|
5767
|
-
return replicaLastSeqNo >= log.
|
|
6662
|
+
return replicaLastSeqNo >= log.localLogEnd ? "incremental" : "snapshot";
|
|
5768
6663
|
}
|
|
5769
6664
|
if (oldest <= replicaLastSeqNo + 1) {
|
|
5770
6665
|
return "incremental";
|
|
@@ -5843,13 +6738,22 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
5843
6738
|
}
|
|
5844
6739
|
);
|
|
5845
6740
|
}
|
|
6741
|
+
if (message.sourceId !== deps.nodeId) {
|
|
6742
|
+
recordReplicaPosition(
|
|
6743
|
+
deps.writeDeps.catchUp,
|
|
6744
|
+
request.indexName,
|
|
6745
|
+
request.partitionId,
|
|
6746
|
+
message.sourceId,
|
|
6747
|
+
request.lastSeqNo
|
|
6748
|
+
);
|
|
6749
|
+
}
|
|
5846
6750
|
const log = deps.writeDeps.getReplicationLog(request.indexName, request.partitionId);
|
|
5847
6751
|
const tier = decideSyncTier(log, request.lastSeqNo);
|
|
5848
6752
|
if (tier === "incremental") {
|
|
5849
6753
|
await sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1));
|
|
5850
6754
|
return;
|
|
5851
6755
|
}
|
|
5852
|
-
const snapshotSeqNo = log.
|
|
6756
|
+
const snapshotSeqNo = log.localLogEnd;
|
|
5853
6757
|
const sink = createSingleResponseSink(respond);
|
|
5854
6758
|
await streamValidatedSnapshotRequest(
|
|
5855
6759
|
message,
|
|
@@ -5902,76 +6806,6 @@ var DEFAULT_CAPACITY = {
|
|
|
5902
6806
|
cpuCores: 4,
|
|
5903
6807
|
diskBytes: null
|
|
5904
6808
|
};
|
|
5905
|
-
|
|
5906
|
-
// src/distribution/cluster-node/write-routing/assignment.ts
|
|
5907
|
-
function resolvePartitionId(docId, partitionCount) {
|
|
5908
|
-
return fnv1a(docId) % partitionCount;
|
|
5909
|
-
}
|
|
5910
|
-
async function resolveNodeTargets(nodeId, deps) {
|
|
5911
|
-
if (deps.resolveNodeTargets === void 0) {
|
|
5912
|
-
return [nodeId];
|
|
5913
|
-
}
|
|
5914
|
-
const targets = await deps.resolveNodeTargets(nodeId);
|
|
5915
|
-
return targets.length > 0 ? targets : [nodeId];
|
|
5916
|
-
}
|
|
5917
|
-
function requireAssignedPrimary(assignment, indexName, partitionId) {
|
|
5918
|
-
if (assignment === void 0 || assignment.primary === null) {
|
|
5919
|
-
throw new NarsilError(
|
|
5920
|
-
ErrorCodes.PARTITION_UNASSIGNED,
|
|
5921
|
-
`No primary assigned for partition ${partitionId} of index '${indexName}'`,
|
|
5922
|
-
{ indexName, partitionId }
|
|
5923
|
-
);
|
|
5924
|
-
}
|
|
5925
|
-
return assignment;
|
|
5926
|
-
}
|
|
5927
|
-
async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrimary) {
|
|
5928
|
-
const table = await deps.coordinator.getAllocation(indexName);
|
|
5929
|
-
if (table === null || table.assignments.size === 0) {
|
|
5930
|
-
if (requireLocalPrimary) {
|
|
5931
|
-
throw new NarsilError(
|
|
5932
|
-
ErrorCodes.QUERY_ROUTING_FAILED,
|
|
5933
|
-
`No allocation table is available for forwarded write to index '${indexName}'`,
|
|
5934
|
-
{ indexName }
|
|
5935
|
-
);
|
|
5936
|
-
}
|
|
5937
|
-
return null;
|
|
5938
|
-
}
|
|
5939
|
-
const partitionCount = table.assignments.size;
|
|
5940
|
-
const partitionId = resolvePartitionId(docId, partitionCount);
|
|
5941
|
-
const assignment = requireAssignedPrimary(table.assignments.get(partitionId), indexName, partitionId);
|
|
5942
|
-
if (requireLocalPrimary && assignment.primary !== deps.nodeId) {
|
|
5943
|
-
throw new NarsilError(
|
|
5944
|
-
ErrorCodes.PARTITION_NOT_PRIMARY,
|
|
5945
|
-
`Node '${deps.nodeId}' is not primary for partition ${partitionId} of index '${indexName}'`,
|
|
5946
|
-
{ indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
|
|
5947
|
-
);
|
|
5948
|
-
}
|
|
5949
|
-
return { partitionId, assignment };
|
|
5950
|
-
}
|
|
5951
|
-
function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
|
|
5952
|
-
const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
|
|
5953
|
-
if (requiredActiveCopies <= 1) {
|
|
5954
|
-
return;
|
|
5955
|
-
}
|
|
5956
|
-
const activeCopies = 1 + getInSyncReplicaTargets(assignment, deps.nodeId).length;
|
|
5957
|
-
if (activeCopies < requiredActiveCopies) {
|
|
5958
|
-
throw new NarsilError(
|
|
5959
|
-
ErrorCodes.INSUFFICIENT_REPLICAS,
|
|
5960
|
-
`Write to partition ${partitionId} of index '${indexName}' requires ${requiredActiveCopies} active copies but only ${activeCopies} are available`,
|
|
5961
|
-
{ indexName, partitionId, requiredActiveCopies, activeCopies }
|
|
5962
|
-
);
|
|
5963
|
-
}
|
|
5964
|
-
}
|
|
5965
|
-
function getInSyncReplicaTargets(assignment, localNodeId) {
|
|
5966
|
-
const configuredReplicas = new Set(assignment.replicas);
|
|
5967
|
-
const targets = [];
|
|
5968
|
-
for (const nodeId of assignment.inSyncSet) {
|
|
5969
|
-
if (nodeId !== localNodeId && configuredReplicas.has(nodeId)) {
|
|
5970
|
-
targets.push(nodeId);
|
|
5971
|
-
}
|
|
5972
|
-
}
|
|
5973
|
-
return targets;
|
|
5974
|
-
}
|
|
5975
6809
|
function assertForwardResponse(response, indexName, primaryNodeId) {
|
|
5976
6810
|
const decoded = decode(response.payload);
|
|
5977
6811
|
if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
|
|
@@ -6039,76 +6873,6 @@ async function sendToNode(nodeId, message, deps) {
|
|
|
6039
6873
|
}
|
|
6040
6874
|
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
6041
6875
|
}
|
|
6042
|
-
async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
|
|
6043
|
-
const message = createEntryMessage(entry, sourceNodeId);
|
|
6044
|
-
return sendAndCollectAcks(message, entry, inSyncReplicas, transport, resolveNodeTargets3);
|
|
6045
|
-
}
|
|
6046
|
-
async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
|
|
6047
|
-
if (entries.length === 0) {
|
|
6048
|
-
return { acknowledged: [], failed: [] };
|
|
6049
|
-
}
|
|
6050
|
-
if (entries.length === 1) {
|
|
6051
|
-
return replicateToReplicas(entries[0], inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3);
|
|
6052
|
-
}
|
|
6053
|
-
const message = createEntryBatchMessage(entries, sourceNodeId);
|
|
6054
|
-
const lastEntry = entries[entries.length - 1];
|
|
6055
|
-
return sendAndCollectAcks(message, lastEntry, inSyncReplicas, transport, resolveNodeTargets3);
|
|
6056
|
-
}
|
|
6057
|
-
async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport, resolveNodeTargets3) {
|
|
6058
|
-
const uniqueReplicas = [...new Set(inSyncReplicas)];
|
|
6059
|
-
if (uniqueReplicas.length === 0) {
|
|
6060
|
-
return { acknowledged: [], failed: [] };
|
|
6061
|
-
}
|
|
6062
|
-
const sendResults = await Promise.allSettled(
|
|
6063
|
-
uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
|
|
6064
|
-
);
|
|
6065
|
-
const acknowledged = [];
|
|
6066
|
-
const failed = [];
|
|
6067
|
-
for (let i = 0; i < sendResults.length; i++) {
|
|
6068
|
-
const result = sendResults[i];
|
|
6069
|
-
const replicaNodeId = uniqueReplicas[i];
|
|
6070
|
-
if (result.status === "rejected") {
|
|
6071
|
-
if (result.reason instanceof TransportError) {
|
|
6072
|
-
failed.push(replicaNodeId);
|
|
6073
|
-
continue;
|
|
6074
|
-
}
|
|
6075
|
-
throw result.reason;
|
|
6076
|
-
}
|
|
6077
|
-
const response = result.value;
|
|
6078
|
-
if (isMatchingAck(response, ackEntry)) {
|
|
6079
|
-
acknowledged.push(replicaNodeId);
|
|
6080
|
-
} else {
|
|
6081
|
-
failed.push(replicaNodeId);
|
|
6082
|
-
}
|
|
6083
|
-
}
|
|
6084
|
-
return { acknowledged, failed };
|
|
6085
|
-
}
|
|
6086
|
-
async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
|
|
6087
|
-
const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
|
|
6088
|
-
const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
|
|
6089
|
-
let lastError;
|
|
6090
|
-
for (const target of resolvedTargets) {
|
|
6091
|
-
try {
|
|
6092
|
-
return await transport.send(target, message);
|
|
6093
|
-
} catch (error) {
|
|
6094
|
-
lastError = error;
|
|
6095
|
-
}
|
|
6096
|
-
}
|
|
6097
|
-
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
6098
|
-
}
|
|
6099
|
-
function isMatchingAck(response, entry) {
|
|
6100
|
-
if (response.type !== ReplicationMessageTypes.ACK) {
|
|
6101
|
-
return false;
|
|
6102
|
-
}
|
|
6103
|
-
try {
|
|
6104
|
-
const payload = validateAckPayload(decode(response.payload));
|
|
6105
|
-
return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
|
|
6106
|
-
} catch {
|
|
6107
|
-
return false;
|
|
6108
|
-
}
|
|
6109
|
-
}
|
|
6110
|
-
|
|
6111
|
-
// src/distribution/cluster-node/write-routing/replication.ts
|
|
6112
6876
|
async function assertPrimaryWriteAuthority(entry, deps) {
|
|
6113
6877
|
const table = await deps.coordinator.getAllocation(entry.indexName);
|
|
6114
6878
|
const currentAssignment = table?.assignments.get(entry.partitionId);
|
|
@@ -6204,17 +6968,32 @@ async function requestInsyncRemovalWithTargets(indexName, partitionId, replicaNo
|
|
|
6204
6968
|
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
6205
6969
|
}
|
|
6206
6970
|
async function replicateEntry(entry, assignment, deps) {
|
|
6207
|
-
const
|
|
6971
|
+
const pending = getPendingAdmissions(deps.catchUp, entry.indexName, entry.partitionId);
|
|
6972
|
+
const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId, pending);
|
|
6208
6973
|
const result = await replicateToReplicas(entry, replicaTargets, deps.transport, deps.nodeId, deps.resolveNodeTargets);
|
|
6209
|
-
|
|
6974
|
+
const inSyncFailures = cancelAdmissionsForFailures(entry, result.failed, assignment, deps);
|
|
6975
|
+
await removeFailedReplicasFromInsync(entry, inSyncFailures, deps);
|
|
6210
6976
|
await assertPrimaryWriteAuthority(entry, deps);
|
|
6977
|
+
deps.getReplicationLog(entry.indexName, entry.partitionId).advanceCommitPoint(entry.seqNo);
|
|
6978
|
+
}
|
|
6979
|
+
function cancelAdmissionsForFailures(entry, failedReplicas, assignment, deps) {
|
|
6980
|
+
const inSyncFailures = [];
|
|
6981
|
+
for (const replicaNodeId of failedReplicas) {
|
|
6982
|
+
if (assignment.inSyncSet.includes(replicaNodeId)) {
|
|
6983
|
+
inSyncFailures.push(replicaNodeId);
|
|
6984
|
+
continue;
|
|
6985
|
+
}
|
|
6986
|
+
clearPendingAdmission(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId);
|
|
6987
|
+
}
|
|
6988
|
+
return inSyncFailures;
|
|
6211
6989
|
}
|
|
6212
6990
|
async function replicateEntryBatch(entries, assignment, deps) {
|
|
6213
6991
|
if (entries.length === 0) {
|
|
6214
6992
|
return;
|
|
6215
6993
|
}
|
|
6216
6994
|
const lastEntry = entries[entries.length - 1];
|
|
6217
|
-
const
|
|
6995
|
+
const pending = getPendingAdmissions(deps.catchUp, lastEntry.indexName, lastEntry.partitionId);
|
|
6996
|
+
const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId, pending);
|
|
6218
6997
|
const result = await replicateBatchToReplicas(
|
|
6219
6998
|
entries,
|
|
6220
6999
|
replicaTargets,
|
|
@@ -6222,8 +7001,10 @@ async function replicateEntryBatch(entries, assignment, deps) {
|
|
|
6222
7001
|
deps.nodeId,
|
|
6223
7002
|
deps.resolveNodeTargets
|
|
6224
7003
|
);
|
|
6225
|
-
|
|
7004
|
+
const inSyncFailures = cancelAdmissionsForFailures(lastEntry, result.failed, assignment, deps);
|
|
7005
|
+
await removeFailedReplicasFromInsync(lastEntry, inSyncFailures, deps);
|
|
6226
7006
|
await assertPrimaryWriteAuthority(lastEntry, deps);
|
|
7007
|
+
deps.getReplicationLog(lastEntry.indexName, lastEntry.partitionId).advanceCommitPoint(lastEntry.seqNo);
|
|
6227
7008
|
}
|
|
6228
7009
|
function chunkReplicationEntries(items) {
|
|
6229
7010
|
return chunkByBudget(items, {
|
|
@@ -7174,17 +7955,17 @@ async function handleForwardBatch(message, respond, deps) {
|
|
|
7174
7955
|
async function handleReplicationEntry(message, respond, deps) {
|
|
7175
7956
|
const payload = validateEntryPayload(decode(message.payload));
|
|
7176
7957
|
const { entry } = payload;
|
|
7177
|
-
|
|
7178
|
-
await applyEntryToLog(entry,
|
|
7958
|
+
await resolveValidatedAssignment(entry, message.sourceId, deps);
|
|
7959
|
+
await applyEntryToLog(entry, deps);
|
|
7179
7960
|
await respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
|
|
7180
7961
|
}
|
|
7181
7962
|
async function handleReplicationEntryBatch(message, respond, deps) {
|
|
7182
7963
|
const payload = validateEntryBatchPayload(decode(message.payload));
|
|
7183
7964
|
const firstEntry = payload.entries[0];
|
|
7184
7965
|
const lastEntry = payload.entries[payload.entries.length - 1];
|
|
7185
|
-
|
|
7966
|
+
await resolveValidatedAssignment(firstEntry, message.sourceId, deps);
|
|
7186
7967
|
for (const entry of payload.entries) {
|
|
7187
|
-
await applyEntryToLog(entry,
|
|
7968
|
+
await applyEntryToLog(entry, deps);
|
|
7188
7969
|
}
|
|
7189
7970
|
await respond(
|
|
7190
7971
|
createAckMessage(lastEntry.seqNo, lastEntry.partitionId, lastEntry.indexName, deps.nodeId, message.requestId)
|
|
@@ -7219,23 +8000,23 @@ async function resolveValidatedAssignment(entry, sourceNodeId, deps) {
|
|
|
7219
8000
|
}
|
|
7220
8001
|
);
|
|
7221
8002
|
}
|
|
7222
|
-
if (
|
|
8003
|
+
if (entry.primaryTerm > assignment.primaryTerm) {
|
|
7223
8004
|
throw new NarsilError(
|
|
7224
8005
|
ErrorCodes.REPLICATION_TERM_MISMATCH,
|
|
7225
|
-
`Replication entry term ${entry.primaryTerm}
|
|
8006
|
+
`Replication entry term ${entry.primaryTerm} is newer than allocation term ${assignment.primaryTerm}`,
|
|
7226
8007
|
{ indexName: entry.indexName, partitionId: entry.partitionId }
|
|
7227
8008
|
);
|
|
7228
8009
|
}
|
|
7229
|
-
if (!assignment.replicas.includes(deps.nodeId)
|
|
8010
|
+
if (!assignment.replicas.includes(deps.nodeId)) {
|
|
7230
8011
|
throw new NarsilError(
|
|
7231
8012
|
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
7232
|
-
`Node '${deps.nodeId}' is not an
|
|
8013
|
+
`Node '${deps.nodeId}' is not an assigned replica for index '${entry.indexName}' partition ${entry.partitionId}`,
|
|
7233
8014
|
{ indexName: entry.indexName, partitionId: entry.partitionId, nodeId: deps.nodeId }
|
|
7234
8015
|
);
|
|
7235
8016
|
}
|
|
7236
8017
|
return assignment;
|
|
7237
8018
|
}
|
|
7238
|
-
async function applyEntryToLog(entry,
|
|
8019
|
+
async function applyEntryToLog(entry, deps) {
|
|
7239
8020
|
let log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
|
|
7240
8021
|
const existing = log.getEntry(entry.seqNo);
|
|
7241
8022
|
if (existing !== void 0) {
|
|
@@ -7261,7 +8042,7 @@ async function applyEntryToLog(entry, assignment, deps) {
|
|
|
7261
8042
|
deps.writeDeps.resetReplicationLog(entry.indexName, entry.partitionId, entry.seqNo, entry.primaryTerm);
|
|
7262
8043
|
log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
|
|
7263
8044
|
}
|
|
7264
|
-
const validation = validateReplicationEntry(entry,
|
|
8045
|
+
const validation = validateReplicationEntry(entry, log.localLogEndPrimaryTerm, log);
|
|
7265
8046
|
if (!validation.valid) {
|
|
7266
8047
|
throw new NarsilError(
|
|
7267
8048
|
validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
@@ -7950,9 +8731,11 @@ async function distributedQuery(indexName, params, deps, config, selector) {
|
|
|
7950
8731
|
const routing = selectReplicasForQuery(allocationTable, randomSelector);
|
|
7951
8732
|
const totalPartitions = allocationTable.assignments.size;
|
|
7952
8733
|
if (routing.unavailablePartitions.length > 0 && !resolvedConfig.allowPartialResults) {
|
|
7953
|
-
throw new NarsilError(
|
|
7954
|
-
|
|
7955
|
-
|
|
8734
|
+
throw new NarsilError(
|
|
8735
|
+
ErrorCodes.QUERY_PARTIAL_FAILURE,
|
|
8736
|
+
"No active replica serves one or more partitions, and this node refuses partial results",
|
|
8737
|
+
{ unavailablePartitions: routing.unavailablePartitions }
|
|
8738
|
+
);
|
|
7956
8739
|
}
|
|
7957
8740
|
const isHybrid = params.hybrid !== null && params.term !== null && params.vector !== null;
|
|
7958
8741
|
if (isHybrid && params.searchAfter !== null) {
|
|
@@ -8163,17 +8946,20 @@ function worstCaseCoverage(a, b) {
|
|
|
8163
8946
|
failedPartitions: failed
|
|
8164
8947
|
};
|
|
8165
8948
|
}
|
|
8166
|
-
|
|
8167
|
-
const allocation = await deps.config.coordinator.getAllocation(indexName);
|
|
8949
|
+
function servesAnyPartition(allocation) {
|
|
8168
8950
|
if (allocation === null || allocation.assignments.size === 0) {
|
|
8169
|
-
return
|
|
8951
|
+
return false;
|
|
8170
8952
|
}
|
|
8171
8953
|
for (const [, assignment] of allocation.assignments) {
|
|
8172
8954
|
if (assignment.state === "ACTIVE") {
|
|
8173
|
-
return
|
|
8955
|
+
return true;
|
|
8174
8956
|
}
|
|
8175
8957
|
}
|
|
8176
|
-
return
|
|
8958
|
+
return false;
|
|
8959
|
+
}
|
|
8960
|
+
async function activeAllocation(deps, indexName) {
|
|
8961
|
+
const allocation = await deps.config.coordinator.getAllocation(indexName);
|
|
8962
|
+
return servesAnyPartition(allocation) ? allocation : null;
|
|
8177
8963
|
}
|
|
8178
8964
|
function strictScatterGroups(allocation, indexName) {
|
|
8179
8965
|
const routing = selectReplicasForQuery(allocation, randomSelector);
|
|
@@ -8186,8 +8972,30 @@ function strictScatterGroups(allocation, indexName) {
|
|
|
8186
8972
|
}
|
|
8187
8973
|
return Array.from(routing.nodeToPartitions.entries(), ([nodeId, partitionIds]) => ({ nodeId, partitionIds }));
|
|
8188
8974
|
}
|
|
8975
|
+
var READ_FAILURE_BY_TRANSPORT_CODE = {
|
|
8976
|
+
[TransportErrorCodes.CONNECT_FAILED]: ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
8977
|
+
[TransportErrorCodes.PEER_UNAVAILABLE]: ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
|
|
8978
|
+
[TransportErrorCodes.TIMEOUT]: ErrorCodes.QUERY_NODE_TIMEOUT,
|
|
8979
|
+
[TransportErrorCodes.MESSAGE_TOO_LARGE]: ErrorCodes.QUERY_ROUTING_FAILED,
|
|
8980
|
+
[TransportErrorCodes.DECODE_FAILED]: ErrorCodes.QUERY_ROUTING_FAILED
|
|
8981
|
+
};
|
|
8982
|
+
function readFailure(error, targetNodeId, indexName) {
|
|
8983
|
+
if (error instanceof NarsilError) {
|
|
8984
|
+
return error;
|
|
8985
|
+
}
|
|
8986
|
+
if (error instanceof TransportError) {
|
|
8987
|
+
return new NarsilError(
|
|
8988
|
+
READ_FAILURE_BY_TRANSPORT_CODE[error.code],
|
|
8989
|
+
`Node '${targetNodeId}' did not answer a read request for index '${indexName}'`,
|
|
8990
|
+
{ indexName, targetNodeId, transportCode: error.code }
|
|
8991
|
+
);
|
|
8992
|
+
}
|
|
8993
|
+
return error;
|
|
8994
|
+
}
|
|
8189
8995
|
async function sendReadRequest(deps, targetNodeId, message, indexName, validate) {
|
|
8190
|
-
const response = await sendToNode2(deps.config, targetNodeId, message)
|
|
8996
|
+
const response = await sendToNode2(deps.config, targetNodeId, message).catch((error) => {
|
|
8997
|
+
throw readFailure(error, targetNodeId, indexName);
|
|
8998
|
+
});
|
|
8191
8999
|
const decoded = decode(response.payload);
|
|
8192
9000
|
if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
|
|
8193
9001
|
throw new NarsilError(
|
|
@@ -8510,11 +9318,36 @@ async function readClusterDocuments(deps, indexName, docIds) {
|
|
|
8510
9318
|
}
|
|
8511
9319
|
return readDistributedDocuments(deps.config, deps.nodeId, deps.engine, indexName, docIds, allocation);
|
|
8512
9320
|
}
|
|
9321
|
+
function routingConfig(query) {
|
|
9322
|
+
if (query === void 0) {
|
|
9323
|
+
return void 0;
|
|
9324
|
+
}
|
|
9325
|
+
const config = {};
|
|
9326
|
+
if (query.allowPartialResults !== void 0) {
|
|
9327
|
+
config.allowPartialResults = query.allowPartialResults;
|
|
9328
|
+
}
|
|
9329
|
+
if (query.partitionTimeout !== void 0) {
|
|
9330
|
+
config.partitionTimeout = query.partitionTimeout;
|
|
9331
|
+
}
|
|
9332
|
+
return config;
|
|
9333
|
+
}
|
|
9334
|
+
async function queryLocalCopy(deps, indexName, params, allocation) {
|
|
9335
|
+
const result = await deps.engine.query(indexName, params);
|
|
9336
|
+
if (allocation === null || allocation.assignments.size === 0) {
|
|
9337
|
+
return result;
|
|
9338
|
+
}
|
|
9339
|
+
const totalPartitions = allocation.assignments.size;
|
|
9340
|
+
return {
|
|
9341
|
+
...result,
|
|
9342
|
+
coverage: { totalPartitions, queriedPartitions: 0, timedOutPartitions: 0, failedPartitions: totalPartitions }
|
|
9343
|
+
};
|
|
9344
|
+
}
|
|
8513
9345
|
async function queryCluster(deps, indexName, params) {
|
|
8514
|
-
const
|
|
8515
|
-
if (
|
|
8516
|
-
return deps
|
|
9346
|
+
const table = await deps.config.coordinator.getAllocation(indexName);
|
|
9347
|
+
if (!servesAnyPartition(table)) {
|
|
9348
|
+
return queryLocalCopy(deps, indexName, params, table);
|
|
8517
9349
|
}
|
|
9350
|
+
const allocation = table;
|
|
8518
9351
|
const wireParams = localParamsToWire(params);
|
|
8519
9352
|
const queryDeps = {
|
|
8520
9353
|
transport: deps.config.transport,
|
|
@@ -8522,7 +9355,7 @@ async function queryCluster(deps, indexName, params) {
|
|
|
8522
9355
|
getAllocation: (idx) => deps.config.coordinator.getAllocation(idx),
|
|
8523
9356
|
resolveNodeTargets: deps.resolveNodeTargets
|
|
8524
9357
|
};
|
|
8525
|
-
const distributed = await distributedQuery(indexName, wireParams, queryDeps);
|
|
9358
|
+
const distributed = await distributedQuery(indexName, wireParams, queryDeps, routingConfig(deps.config.query));
|
|
8526
9359
|
const documents = await fetchDistributedDocuments(
|
|
8527
9360
|
deps.config,
|
|
8528
9361
|
deps.nodeId,
|
|
@@ -8652,16 +9485,17 @@ function createClusterNodeOperations(deps) {
|
|
|
8652
9485
|
var DEFAULT_LOG_RETENTION_BYTES = 268435456;
|
|
8653
9486
|
|
|
8654
9487
|
// src/distribution/replication/log.ts
|
|
8655
|
-
var
|
|
9488
|
+
var ENTRY_FIXED_OVERHEAD_BYTES2 = 40;
|
|
8656
9489
|
function estimateEntrySize(entry) {
|
|
8657
|
-
return
|
|
9490
|
+
return ENTRY_FIXED_OVERHEAD_BYTES2 + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
8658
9491
|
}
|
|
8659
9492
|
var computeChecksum = computeEntryChecksum;
|
|
8660
9493
|
function createReplicationLog(partitionId, config) {
|
|
8661
9494
|
const retentionBytes = config?.logRetentionBytes ?? DEFAULT_LOG_RETENTION_BYTES;
|
|
8662
9495
|
let nextSeqNo = config?.startSeqNo ?? 1;
|
|
8663
|
-
let
|
|
8664
|
-
let
|
|
9496
|
+
let localLogEnd = nextSeqNo - 1;
|
|
9497
|
+
let localLogEndPrimaryTerm = config?.lastPrimaryTerm ?? 0;
|
|
9498
|
+
let commitPoint = nextSeqNo - 1;
|
|
8665
9499
|
let entries = [];
|
|
8666
9500
|
let headIndex = 0;
|
|
8667
9501
|
let totalSizeBytes = 0;
|
|
@@ -8704,7 +9538,7 @@ function createReplicationLog(partitionId, config) {
|
|
|
8704
9538
|
}
|
|
8705
9539
|
}
|
|
8706
9540
|
function assertEntryFitsRetention(entry) {
|
|
8707
|
-
const entrySize =
|
|
9541
|
+
const entrySize = ENTRY_FIXED_OVERHEAD_BYTES2 + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
8708
9542
|
if (entrySize > retentionBytes) {
|
|
8709
9543
|
throw new NarsilError(
|
|
8710
9544
|
ErrorCodes.REPLICATION_LOG_FULL,
|
|
@@ -8715,9 +9549,9 @@ function createReplicationLog(partitionId, config) {
|
|
|
8715
9549
|
}
|
|
8716
9550
|
function storeEntry(entry) {
|
|
8717
9551
|
entries.push(entry);
|
|
8718
|
-
if (entry.seqNo >=
|
|
8719
|
-
|
|
8720
|
-
|
|
9552
|
+
if (entry.seqNo >= localLogEnd) {
|
|
9553
|
+
localLogEnd = entry.seqNo;
|
|
9554
|
+
localLogEndPrimaryTerm = entry.primaryTerm;
|
|
8721
9555
|
}
|
|
8722
9556
|
totalSizeBytes += estimateEntrySize(entry);
|
|
8723
9557
|
evictOldEntries();
|
|
@@ -8820,11 +9654,20 @@ function createReplicationLog(partitionId, config) {
|
|
|
8820
9654
|
verifyChecksum(entry) {
|
|
8821
9655
|
return verifyEntryChecksum(entry);
|
|
8822
9656
|
},
|
|
8823
|
-
get
|
|
8824
|
-
return
|
|
9657
|
+
get localLogEnd() {
|
|
9658
|
+
return localLogEnd;
|
|
9659
|
+
},
|
|
9660
|
+
get localLogEndPrimaryTerm() {
|
|
9661
|
+
return localLogEndPrimaryTerm;
|
|
9662
|
+
},
|
|
9663
|
+
get commitPoint() {
|
|
9664
|
+
return commitPoint;
|
|
8825
9665
|
},
|
|
8826
|
-
|
|
8827
|
-
|
|
9666
|
+
advanceCommitPoint(seqNo) {
|
|
9667
|
+
const bounded = Math.min(seqNo, localLogEnd);
|
|
9668
|
+
if (bounded > commitPoint) {
|
|
9669
|
+
commitPoint = bounded;
|
|
9670
|
+
}
|
|
8828
9671
|
},
|
|
8829
9672
|
get oldestSeqNo() {
|
|
8830
9673
|
if (liveCount() === 0) return void 0;
|
|
@@ -8844,7 +9687,8 @@ function createReplicationLog(partitionId, config) {
|
|
|
8844
9687
|
entries = [];
|
|
8845
9688
|
headIndex = 0;
|
|
8846
9689
|
totalSizeBytes = 0;
|
|
8847
|
-
|
|
9690
|
+
localLogEnd = nextSeqNo - 1;
|
|
9691
|
+
commitPoint = nextSeqNo - 1;
|
|
8848
9692
|
}
|
|
8849
9693
|
};
|
|
8850
9694
|
}
|
|
@@ -8875,6 +9719,7 @@ function seedReplicationLog(replicationLogs, indexName, partitionId, startSeqNo,
|
|
|
8875
9719
|
|
|
8876
9720
|
// src/distribution/cluster-node/transport-listener.ts
|
|
8877
9721
|
var CONTROLLER_MESSAGE_TYPES = /* @__PURE__ */ new Set([
|
|
9722
|
+
ReplicationMessageTypes.INSYNC_ADD,
|
|
8878
9723
|
ReplicationMessageTypes.INSYNC_REMOVE,
|
|
8879
9724
|
ClusterMessageTypes.BOOTSTRAP_COMPLETE
|
|
8880
9725
|
]);
|
|
@@ -8971,6 +9816,25 @@ function validateClusterNodeConfig(config) {
|
|
|
8971
9816
|
);
|
|
8972
9817
|
}
|
|
8973
9818
|
}
|
|
9819
|
+
if (config.query !== void 0) {
|
|
9820
|
+
const { allowPartialResults, partitionTimeout } = config.query;
|
|
9821
|
+
if (allowPartialResults !== void 0 && typeof allowPartialResults !== "boolean") {
|
|
9822
|
+
throw new NarsilError(
|
|
9823
|
+
ErrorCodes.CONFIG_INVALID,
|
|
9824
|
+
"ClusterNodeConfig.query.allowPartialResults must be a boolean",
|
|
9825
|
+
{
|
|
9826
|
+
allowPartialResults
|
|
9827
|
+
}
|
|
9828
|
+
);
|
|
9829
|
+
}
|
|
9830
|
+
if (partitionTimeout !== void 0 && (!Number.isInteger(partitionTimeout) || partitionTimeout <= 0)) {
|
|
9831
|
+
throw new NarsilError(
|
|
9832
|
+
ErrorCodes.CONFIG_INVALID,
|
|
9833
|
+
"ClusterNodeConfig.query.partitionTimeout must be a positive integer",
|
|
9834
|
+
{ partitionTimeout }
|
|
9835
|
+
);
|
|
9836
|
+
}
|
|
9837
|
+
}
|
|
8974
9838
|
}
|
|
8975
9839
|
|
|
8976
9840
|
// src/distribution/cluster-node/index.ts
|
|
@@ -9022,6 +9886,7 @@ async function createClusterNode(config) {
|
|
|
9022
9886
|
getReplicationLog: getReplicationLog2,
|
|
9023
9887
|
resetReplicationLog: seedReplicationLog2,
|
|
9024
9888
|
partitionWriteQueues: createPartitionWriteQueues(),
|
|
9889
|
+
catchUp: createCatchUpState(),
|
|
9025
9890
|
resolveNodeTargets: resolveNodeTargets3,
|
|
9026
9891
|
waitForActiveReplicas: config.replication?.waitForActiveReplicas
|
|
9027
9892
|
};
|
|
@@ -9092,6 +9957,7 @@ async function createClusterNode(config) {
|
|
|
9092
9957
|
bootstrapRetryMaxMs: DEFAULT_NODE_LIFECYCLE_CONFIG.bootstrapRetryMaxMs,
|
|
9093
9958
|
bootstrapMaxRetries: DEFAULT_NODE_LIFECYCLE_CONFIG.bootstrapMaxRetries,
|
|
9094
9959
|
allocationDebounceMs: DEFAULT_NODE_LIFECYCLE_CONFIG.allocationDebounceMs,
|
|
9960
|
+
nodeHeartbeatIntervalMs: DEFAULT_NODE_LIFECYCLE_CONFIG.nodeHeartbeatIntervalMs,
|
|
9095
9961
|
onBootstrapPartition: bootstrapPartition2,
|
|
9096
9962
|
onRemovePartition: (indexName, partitionId) => {
|
|
9097
9963
|
clearBootstrapSyncIndex(bootstrapSyncState, indexName, partitionId);
|
|
@@ -9203,9 +10069,11 @@ async function createClusterNode(config) {
|
|
|
9203
10069
|
});
|
|
9204
10070
|
const listener = controllerTransport !== null ? controllerTransport.createHandler(dataHandler) : dataHandler;
|
|
9205
10071
|
unregisterHandler = await config.transport.listen(listener);
|
|
10072
|
+
startCatchUpPump(writeDeps.catchUp, writeDeps);
|
|
9206
10073
|
}
|
|
9207
10074
|
},
|
|
9208
10075
|
async shutdown() {
|
|
10076
|
+
await stopCatchUpPump(writeDeps.catchUp);
|
|
9209
10077
|
if (isShutdown) {
|
|
9210
10078
|
return;
|
|
9211
10079
|
}
|