@delali/narsil 0.2.3-next.33 → 0.2.3-next.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
- import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, MAX_PARTITION_COUNT, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, MAX_INDEX_NAME_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum, createInsyncConfirmMessage, createEntryBatchMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload } from './chunk-TEAJO7CU.mjs';
2
- import { ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-C6VVLMRX.mjs';
1
+ import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, MAX_PARTITION_COUNT, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, MAX_INDEX_NAME_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-OOA7OBLA.mjs';
2
+ import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-36ER4WCG.mjs';
3
3
  import { deserializePayloadV2, createPartitionIndex, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, getLanguage, DEFAULT_PAGE_SIZE, requireWithinResultWindow, fnv1a, FILTER_EXPRESSION_KEYS, MAX_CURSOR_LENGTH, mergePartitionStats, FIELD_FILTER_OPERATORS, MAX_NESTING_DEPTH } from './chunk-RO4EFWE6.mjs';
4
4
  import './chunk-TGP5M5R7.mjs';
5
5
  import './chunk-OBXS7OZM.mjs';
@@ -60,61 +60,6 @@ function startRenewalInterval(state, coordinator, nodeId, ttlMs, onLeaseLost) {
60
60
  function scheduleStandbyRetry(state, retryMs, onRetry) {
61
61
  state.standbyTimer = setTimeout(onRetry, retryMs);
62
62
  }
63
- async function requestInsyncRemoval(indexName, partitionId, replicaNodeId, primaryTerm, controllerNodeId, transport, sourceNodeId) {
64
- const payload = {
65
- indexName,
66
- partitionId,
67
- replicaNodeId,
68
- primaryTerm
69
- };
70
- const message = createInsyncRemoveMessage(payload, sourceNodeId);
71
- const response = await transport.send(controllerNodeId, message);
72
- const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
73
- return { accepted: confirmPayload.accepted };
74
- }
75
- async function handleInsyncRemoval(payload, coordinator) {
76
- const table = await coordinator.getAllocation(payload.indexName);
77
- if (table === null) {
78
- return {
79
- indexName: payload.indexName,
80
- partitionId: payload.partitionId,
81
- accepted: false
82
- };
83
- }
84
- const assignment = table.assignments.get(payload.partitionId);
85
- if (assignment === void 0) {
86
- return {
87
- indexName: payload.indexName,
88
- partitionId: payload.partitionId,
89
- accepted: false
90
- };
91
- }
92
- if (payload.primaryTerm !== assignment.primaryTerm) {
93
- return {
94
- indexName: payload.indexName,
95
- partitionId: payload.partitionId,
96
- accepted: false
97
- };
98
- }
99
- const updatedInSyncSet = assignment.inSyncSet.filter((nodeId) => nodeId !== payload.replicaNodeId);
100
- const updatedAssignment = {
101
- ...assignment,
102
- inSyncSet: updatedInSyncSet
103
- };
104
- const updatedAssignments = new Map(table.assignments);
105
- updatedAssignments.set(payload.partitionId, updatedAssignment);
106
- const updatedTable = {
107
- ...table,
108
- version: table.version + 1,
109
- assignments: updatedAssignments
110
- };
111
- const written = await coordinator.putAllocation(payload.indexName, updatedTable, table.version);
112
- return {
113
- indexName: payload.indexName,
114
- partitionId: payload.partitionId,
115
- accepted: written
116
- };
117
- }
118
63
 
119
64
  // src/distribution/cluster/allocator/deciders/balance.ts
120
65
  var balanceDecider = {
@@ -292,8 +237,18 @@ function countNodeAssignments(assignments) {
292
237
  }
293
238
  return counts;
294
239
  }
240
+ function countPrimaryAssignments(assignments) {
241
+ const counts = /* @__PURE__ */ new Map();
242
+ for (const assignment of assignments.values()) {
243
+ if (assignment.primary !== null) {
244
+ counts.set(assignment.primary, (counts.get(assignment.primary) ?? 0) + 1);
245
+ }
246
+ }
247
+ return counts;
248
+ }
295
249
  function computeNodeWeights(nodes, assignments) {
296
250
  const assignmentCounts = countNodeAssignments(assignments);
251
+ const primaryCounts = countPrimaryAssignments(assignments);
297
252
  let totalCapacity = 0;
298
253
  for (const node of nodes) {
299
254
  totalCapacity += node.capacity.memoryBytes;
@@ -302,12 +257,14 @@ function computeNodeWeights(nodes, assignments) {
302
257
  const weights = [];
303
258
  for (const node of nodes) {
304
259
  const partitionCount = assignmentCounts.get(node.nodeId) ?? 0;
260
+ const primaryCount = primaryCounts.get(node.nodeId) ?? 0;
305
261
  const normalizedCapacity = averageCapacity > 0 ? node.capacity.memoryBytes / averageCapacity : 1;
306
- const weight = partitionCount / normalizedCapacity;
307
262
  weights.push({
308
263
  nodeId: node.nodeId,
309
- weight,
264
+ weight: partitionCount / normalizedCapacity,
310
265
  partitionCount,
266
+ primaryWeight: primaryCount / normalizedCapacity,
267
+ primaryCount,
311
268
  capacity: node.capacity.memoryBytes
312
269
  });
313
270
  }
@@ -326,6 +283,11 @@ function findBestNode(candidates, weights, deciders, context) {
326
283
  const sorted = [...candidates].sort((a, b) => {
327
284
  const wA = weightByNodeId.get(a);
328
285
  const wB = weightByNodeId.get(b);
286
+ if (context.role === "primary") {
287
+ const primaryA = wA?.primaryWeight ?? 0;
288
+ const primaryB = wB?.primaryWeight ?? 0;
289
+ if (primaryA !== primaryB) return primaryA - primaryB;
290
+ }
329
291
  const weightA = wA?.weight ?? 0;
330
292
  const weightB = wB?.weight ?? 0;
331
293
  if (weightA !== weightB) return weightA - weightB;
@@ -392,6 +354,7 @@ function initialAllocate(nodes, indexName, partitionCount, replicationFactor, co
392
354
  primary: primaryNodeId,
393
355
  replicas,
394
356
  inSyncSet: [],
357
+ commitPoint: 0,
395
358
  state: "INITIALISING",
396
359
  primaryTerm: 1
397
360
  };
@@ -434,6 +397,74 @@ function initialAllocate(nodes, indexName, partitionCount, replicationFactor, co
434
397
  };
435
398
  }
436
399
 
400
+ // src/distribution/cluster/allocator/leadership.ts
401
+ var LEADERSHIP_IMBALANCE_THRESHOLD = 2;
402
+ function loadOf(nodeId, counts, shares) {
403
+ const share = shares.get(nodeId);
404
+ return (counts.get(nodeId) ?? 0) / (share === void 0 || share <= 0 ? 1 : share);
405
+ }
406
+ function heaviestLeader(shares, counts) {
407
+ let heaviest;
408
+ for (const nodeId of shares.keys()) {
409
+ if (heaviest === void 0 || loadOf(nodeId, counts, shares) > loadOf(heaviest, counts, shares)) {
410
+ heaviest = nodeId;
411
+ }
412
+ }
413
+ return heaviest;
414
+ }
415
+ function findSwap(assignments, heaviest, counts, shares) {
416
+ const heaviestLoad = loadOf(heaviest, counts, shares);
417
+ let best = null;
418
+ for (const assignment of assignments.values()) {
419
+ if (assignment.state !== "ACTIVE" || assignment.primary !== heaviest) {
420
+ continue;
421
+ }
422
+ for (const candidateNodeId of assignment.inSyncSet) {
423
+ const candidateLoad = loadOf(candidateNodeId, counts, shares);
424
+ if (heaviestLoad - candidateLoad < LEADERSHIP_IMBALANCE_THRESHOLD) {
425
+ continue;
426
+ }
427
+ if (best === null || candidateLoad < best.candidateLoad) {
428
+ best = { assignment, candidateNodeId, candidateLoad };
429
+ }
430
+ }
431
+ }
432
+ return best;
433
+ }
434
+ function applySwap(swap, previousPrimary) {
435
+ const { assignment, candidateNodeId } = swap;
436
+ assignment.primary = candidateNodeId;
437
+ assignment.replicas = [...assignment.replicas.filter((nodeId) => nodeId !== candidateNodeId), previousPrimary];
438
+ assignment.inSyncSet = assignment.inSyncSet.filter((nodeId) => nodeId !== candidateNodeId);
439
+ assignment.primaryTerm += 1;
440
+ }
441
+ function rebalanceLeadership(assignments, shares) {
442
+ for (let swaps = 0; swaps < assignments.size; swaps++) {
443
+ const counts = countPrimaryAssignments(assignments);
444
+ const heaviest = heaviestLeader(shares, counts);
445
+ if (heaviest === void 0) {
446
+ return;
447
+ }
448
+ const swap = findSwap(assignments, heaviest, counts, shares);
449
+ if (swap === null) {
450
+ return;
451
+ }
452
+ applySwap(swap, heaviest);
453
+ }
454
+ }
455
+ function capacityShares(nodes) {
456
+ const shares = /* @__PURE__ */ new Map();
457
+ let totalCapacity = 0;
458
+ for (const node of nodes) {
459
+ totalCapacity += node.capacity.memoryBytes;
460
+ }
461
+ const averageCapacity = nodes.length > 0 ? totalCapacity / nodes.length : 0;
462
+ for (const node of nodes) {
463
+ shares.set(node.nodeId, averageCapacity > 0 ? node.capacity.memoryBytes / averageCapacity : 1);
464
+ }
465
+ return shares;
466
+ }
467
+
437
468
  // src/distribution/cluster/allocator/rebalance.ts
438
469
  function cloneAssignments(assignments) {
439
470
  const cloned = /* @__PURE__ */ new Map();
@@ -442,6 +473,7 @@ function cloneAssignments(assignments) {
442
473
  primary: assignment.primary,
443
474
  replicas: [...assignment.replicas],
444
475
  inSyncSet: [...assignment.inSyncSet],
476
+ commitPoint: assignment.commitPoint,
445
477
  state: assignment.state,
446
478
  primaryTerm: assignment.primaryTerm
447
479
  });
@@ -454,7 +486,27 @@ function markUnassigned(assignment) {
454
486
  assignment.inSyncSet = [];
455
487
  assignment.state = "UNASSIGNED";
456
488
  }
457
- function handleLostNodes(assignments, activeNodeIds) {
489
+ function primaryLoadOf(nodeId, primaryCounts, shares) {
490
+ const share = shares.get(nodeId);
491
+ return (primaryCounts.get(nodeId) ?? 0) / (share === void 0 || share <= 0 ? 1 : share);
492
+ }
493
+ function leastLoadedCandidate(candidates, primaryCounts, shares) {
494
+ let best;
495
+ for (const candidate of candidates) {
496
+ if (best === void 0) {
497
+ best = candidate;
498
+ continue;
499
+ }
500
+ const bestLoad = primaryLoadOf(best, primaryCounts, shares);
501
+ const candidateLoad = primaryLoadOf(candidate, primaryCounts, shares);
502
+ if (candidateLoad < bestLoad || candidateLoad === bestLoad && candidate < best) {
503
+ best = candidate;
504
+ }
505
+ }
506
+ return best;
507
+ }
508
+ function handleLostNodes(assignments, activeNodeIds, shares) {
509
+ const primaryCounts = countPrimaryAssignments(assignments);
458
510
  for (const assignment of assignments.values()) {
459
511
  const primaryWasLost = assignment.primary !== null && !activeNodeIds.has(assignment.primary);
460
512
  if (primaryWasLost) {
@@ -473,7 +525,12 @@ function handleLostNodes(assignments, activeNodeIds) {
473
525
  markUnassigned(assignment);
474
526
  continue;
475
527
  }
476
- const promoted = inSyncCandidates[0];
528
+ const promoted = leastLoadedCandidate(inSyncCandidates, primaryCounts, shares);
529
+ if (promoted === void 0) {
530
+ markUnassigned(assignment);
531
+ continue;
532
+ }
533
+ primaryCounts.set(promoted, (primaryCounts.get(promoted) ?? 0) + 1);
477
534
  assignment.primary = promoted;
478
535
  assignment.replicas = assignment.replicas.filter((id) => id !== promoted);
479
536
  assignment.inSyncSet = assignment.inSyncSet.filter((id) => id !== promoted);
@@ -506,7 +563,16 @@ function fillReplicaSlots(assignments, sortedNodes, nodeMap, replicationFactor,
506
563
  }
507
564
  }
508
565
  }
509
- function rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, deciders) {
566
+ function slotWeightOf(nodeId, shares) {
567
+ const share = shares.get(nodeId);
568
+ return 1 / (share === void 0 || share <= 0 ? 1 : share);
569
+ }
570
+ function moveNarrowsGap(mostLoaded, leastLoaded, shares) {
571
+ const gap = mostLoaded.weight - leastLoaded.weight;
572
+ const afterMove = Math.abs(gap - slotWeightOf(mostLoaded.nodeId, shares) - slotWeightOf(leastLoaded.nodeId, shares));
573
+ return afterMove < gap;
574
+ }
575
+ function rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, deciders, shares) {
510
576
  let totalSlots = 0;
511
577
  for (const assignment of assignments.values()) {
512
578
  if (assignment.primary !== null) totalSlots++;
@@ -522,35 +588,60 @@ function rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, dec
522
588
  if (mostLoaded.weight - leastLoaded.weight < REBALANCE_THRESHOLD) {
523
589
  break;
524
590
  }
525
- let moved = false;
526
- for (const [partitionId, assignment] of assignments) {
527
- if (moved) break;
528
- const replicaIndex = assignment.replicas.indexOf(mostLoaded.nodeId);
529
- if (replicaIndex >= 0) {
530
- const removedReplica = assignment.replicas[replicaIndex];
531
- assignment.replicas.splice(replicaIndex, 1);
532
- const nodeAssignmentCounts = countNodeAssignments(assignments);
533
- const moveContext = {
534
- partitionId,
535
- role: "replica",
536
- currentAssignment: assignment,
537
- allAssignments: assignments,
538
- nodeAssignmentCounts,
539
- nodes: nodeMap,
540
- constraints
541
- };
542
- const target = findBestNode([leastLoaded.nodeId], weights, deciders, moveContext);
543
- if (target !== null) {
544
- assignment.replicas.push(target);
545
- moved = true;
546
- } else {
547
- assignment.replicas.splice(replicaIndex, 0, removedReplica);
548
- }
549
- }
591
+ if (!moveNarrowsGap(mostLoaded, leastLoaded, shares)) {
592
+ break;
550
593
  }
594
+ const move = {
595
+ fromNodeId: mostLoaded.nodeId,
596
+ toNodeId: leastLoaded.nodeId,
597
+ weights,
598
+ nodeMap,
599
+ constraints,
600
+ deciders
601
+ };
602
+ const laggingFirst = moveOneReplica(assignments, move, true);
603
+ const moved = laggingFirst || moveOneReplica(assignments, move, false);
551
604
  if (!moved) break;
552
605
  }
553
606
  }
607
+ function moveOneReplica(assignments, move, laggingOnly) {
608
+ for (const [partitionId, assignment] of assignments) {
609
+ const replicaIndex = assignment.replicas.indexOf(move.fromNodeId);
610
+ if (replicaIndex < 0) {
611
+ continue;
612
+ }
613
+ if (laggingOnly && assignment.inSyncSet.includes(move.fromNodeId)) {
614
+ continue;
615
+ }
616
+ const removedReplica = assignment.replicas[replicaIndex];
617
+ assignment.replicas.splice(replicaIndex, 1);
618
+ const moveContext = {
619
+ partitionId,
620
+ role: "replica",
621
+ currentAssignment: assignment,
622
+ allAssignments: assignments,
623
+ nodeAssignmentCounts: countNodeAssignments(assignments),
624
+ nodes: move.nodeMap,
625
+ constraints: move.constraints
626
+ };
627
+ const target = findBestNode([move.toNodeId], move.weights, move.deciders, moveContext);
628
+ if (target === null) {
629
+ assignment.replicas.splice(replicaIndex, 0, removedReplica);
630
+ continue;
631
+ }
632
+ assignment.replicas.push(target);
633
+ return true;
634
+ }
635
+ return false;
636
+ }
637
+ function pruneInSyncSets(assignments) {
638
+ for (const assignment of assignments.values()) {
639
+ const stale = assignment.inSyncSet.some((nodeId) => !assignment.replicas.includes(nodeId));
640
+ if (stale) {
641
+ assignment.inSyncSet = assignment.inSyncSet.filter((nodeId) => assignment.replicas.includes(nodeId));
642
+ }
643
+ }
644
+ }
554
645
  function rebalanceAllocate(nodes, currentTable, constraints, deciders) {
555
646
  const sortedNodes = [...nodes].sort((a, b) => {
556
647
  if (a.nodeId < b.nodeId) return -1;
@@ -563,9 +654,12 @@ function rebalanceAllocate(nodes, currentTable, constraints, deciders) {
563
654
  nodeMap.set(node.nodeId, node);
564
655
  }
565
656
  const assignments = cloneAssignments(currentTable.assignments);
566
- handleLostNodes(assignments, activeNodeIds);
657
+ const shares = capacityShares(sortedNodes);
658
+ handleLostNodes(assignments, activeNodeIds, shares);
567
659
  fillReplicaSlots(assignments, sortedNodes, nodeMap, currentTable.replicationFactor, constraints, deciders);
568
- rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, deciders);
660
+ rebalanceForBalance(assignments, sortedNodes, nodeMap, constraints, deciders, shares);
661
+ rebalanceLeadership(assignments, shares);
662
+ pruneInSyncSets(assignments);
569
663
  const warnings = collectReplicationWarnings(assignments, currentTable.replicationFactor);
570
664
  return {
571
665
  table: {
@@ -613,203 +707,14 @@ function allocate(nodes, currentTable, indexName, partitionCount, replicationFac
613
707
  }
614
708
  return rebalanceAllocate(dataNodes, currentTable, constraints, deciders);
615
709
  }
616
- function isRecord(value) {
617
- return value !== null && typeof value === "object" && !Array.isArray(value);
618
- }
619
- function isValidInteger(value) {
620
- return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
621
- }
622
- function validateBootstrapCompletePayload(decoded) {
623
- if (!isRecord(decoded)) {
624
- return null;
625
- }
626
- if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
627
- return null;
628
- }
629
- if (!isValidInteger(decoded.partitionId) || decoded.partitionId < 0) {
630
- return null;
631
- }
632
- if (typeof decoded.nodeId !== "string") {
633
- return null;
634
- }
635
- if (!isValidInteger(decoded.primaryTerm)) {
636
- return null;
637
- }
638
- return {
639
- indexName: decoded.indexName,
640
- partitionId: decoded.partitionId,
641
- nodeId: decoded.nodeId,
642
- primaryTerm: decoded.primaryTerm
643
- };
644
- }
645
- async function sendRejection(respond, controllerNodeId, requestId, indexName, partitionId) {
646
- const resultPayload = {
647
- indexName,
648
- partitionId,
649
- accepted: false
650
- };
651
- const response = {
652
- type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
653
- sourceId: controllerNodeId,
654
- requestId,
655
- payload: encode(resultPayload)
656
- };
657
- await respond(response);
658
- }
659
- function handleBootstrapCompleteMessage(message, respond, coordinator, controllerNodeId) {
660
- let decoded;
661
- try {
662
- decoded = decode(message.payload);
663
- } catch (_) {
664
- void sendRejection(respond, controllerNodeId, message.requestId, "", -1);
665
- return;
666
- }
667
- const payload = validateBootstrapCompletePayload(decoded);
668
- if (payload === null) {
669
- void sendRejection(respond, controllerNodeId, message.requestId, "", -1);
670
- return;
671
- }
672
- if (message.sourceId !== payload.nodeId) {
673
- void sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
674
- return;
675
- }
676
- processBootstrapComplete(payload, coordinator).then(async (accepted) => {
677
- const resultPayload = {
678
- indexName: payload.indexName,
679
- partitionId: payload.partitionId,
680
- accepted
681
- };
682
- const response = {
683
- type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
684
- sourceId: controllerNodeId,
685
- requestId: message.requestId,
686
- payload: encode(resultPayload)
687
- };
688
- await respond(response);
689
- }).catch(() => sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId));
690
- }
691
- function ensurePrimaryInSyncSet(assignment, inSyncSet) {
692
- if (assignment.primary === null) {
693
- return inSyncSet;
694
- }
695
- if (inSyncSet.includes(assignment.primary)) {
696
- return inSyncSet;
697
- }
698
- return [assignment.primary, ...inSyncSet];
699
- }
700
- async function processBootstrapComplete(payload, coordinator) {
701
- const table = await coordinator.getAllocation(payload.indexName);
702
- if (table === null) {
703
- return false;
704
- }
705
- const assignment = table.assignments.get(payload.partitionId);
706
- if (assignment === void 0) {
707
- return false;
708
- }
709
- if (payload.primaryTerm !== assignment.primaryTerm) {
710
- return false;
711
- }
712
- const isAssigned = assignment.primary === payload.nodeId || assignment.replicas.includes(payload.nodeId);
713
- if (!isAssigned) {
714
- return false;
715
- }
716
- if (assignment.state === "ACTIVE") {
717
- if (assignment.inSyncSet.includes(payload.nodeId)) {
718
- return true;
719
- }
720
- const updatedInSyncSet2 = [...assignment.inSyncSet, payload.nodeId];
721
- const updatedAssignment2 = {
722
- ...assignment,
723
- inSyncSet: updatedInSyncSet2
724
- };
725
- const updatedAssignments2 = new Map(table.assignments);
726
- updatedAssignments2.set(payload.partitionId, updatedAssignment2);
727
- const updatedTable2 = {
728
- ...table,
729
- version: table.version + 1,
730
- assignments: updatedAssignments2
731
- };
732
- return coordinator.putAllocation(payload.indexName, updatedTable2, table.version);
733
- }
734
- if (assignment.state !== "INITIALISING") {
735
- return false;
736
- }
737
- const baseInSyncSet = assignment.inSyncSet.includes(payload.nodeId) ? assignment.inSyncSet : [...assignment.inSyncSet, payload.nodeId];
738
- const updatedInSyncSet = ensurePrimaryInSyncSet(assignment, baseInSyncSet);
739
- const updatedAssignment = {
740
- ...assignment,
741
- state: "ACTIVE",
742
- inSyncSet: updatedInSyncSet
743
- };
744
- const updatedAssignments = new Map(table.assignments);
745
- updatedAssignments.set(payload.partitionId, updatedAssignment);
746
- const updatedTable = {
747
- ...table,
748
- version: table.version + 1,
749
- assignments: updatedAssignments
750
- };
751
- return coordinator.putAllocation(payload.indexName, updatedTable, table.version);
752
- }
753
710
 
754
- // src/distribution/cluster/controller/event-loop.ts
755
- function createEventLoopState(initialIndexNames) {
756
- return {
757
- unwatchNodes: null,
758
- unwatchSchemas: null,
759
- unwatchTransport: null,
760
- knownIndexes: new Set(initialIndexNames),
761
- debounceTimer: null,
762
- insyncQueue: Promise.resolve()
763
- };
764
- }
765
- function clearEventLoopWatchers(state) {
766
- if (state.unwatchNodes !== null) {
767
- state.unwatchNodes();
768
- state.unwatchNodes = null;
769
- }
770
- if (state.unwatchSchemas !== null) {
771
- state.unwatchSchemas();
772
- state.unwatchSchemas = null;
773
- }
774
- if (state.unwatchTransport !== null) {
775
- state.unwatchTransport();
776
- state.unwatchTransport = null;
777
- }
778
- if (state.debounceTimer !== null) {
779
- clearTimeout(state.debounceTimer);
780
- state.debounceTimer = null;
781
- }
782
- }
783
- function isRecord2(value) {
784
- return value !== null && typeof value === "object" && !Array.isArray(value);
785
- }
786
- function isValidInteger2(value) {
787
- return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
788
- }
789
- function validateInsyncRemovePayload(decoded) {
790
- if (!isRecord2(decoded)) {
791
- return null;
792
- }
793
- if (typeof decoded.indexName !== "string") {
794
- return null;
795
- }
796
- if (!isValidInteger2(decoded.partitionId)) {
797
- return null;
798
- }
799
- if (typeof decoded.replicaNodeId !== "string") {
800
- return null;
801
- }
802
- if (!isValidInteger2(decoded.primaryTerm)) {
803
- return null;
804
- }
805
- return {
806
- indexName: decoded.indexName,
807
- partitionId: decoded.partitionId,
808
- replicaNodeId: decoded.replicaNodeId,
809
- primaryTerm: decoded.primaryTerm
810
- };
811
- }
812
- async function runAllocatorForIndex(coordinator, indexName, nodes, isActive) {
711
+ // src/distribution/cluster/controller/event-loop/allocation.ts
712
+ var ALLOCATION_CAS_ATTEMPTS = 5;
713
+ var ALLOCATION_RETRY_DELAY_MS = 1e3;
714
+ var TEARDOWN_CAS_ATTEMPTS = 5;
715
+ var TEARDOWN_RETRY_ROUNDS = 3;
716
+ var TEARDOWN_RETRY_DELAY_MS = 500;
717
+ async function runAllocatorForIndex(coordinator, indexName, nodes, isActive, attempt = 0) {
813
718
  if (!isActive()) {
814
719
  return;
815
720
  }
@@ -832,9 +737,20 @@ async function runAllocatorForIndex(coordinator, indexName, nodes, isActive) {
832
737
  if (!isActive()) {
833
738
  return;
834
739
  }
835
- await coordinator.putAllocation(indexName, result.table, currentTable.version);
740
+ if (await coordinator.putAllocation(indexName, result.table, currentTable.version)) {
741
+ return;
742
+ }
743
+ if (attempt + 1 < ALLOCATION_CAS_ATTEMPTS) {
744
+ await runAllocatorForIndex(coordinator, indexName, nodes, isActive, attempt + 1);
745
+ return;
746
+ }
747
+ throw new NarsilError(
748
+ ErrorCodes.ALLOCATION_FAILED,
749
+ `Allocation for index '${indexName}' lost ${ALLOCATION_CAS_ATTEMPTS} compare-and-set attempts`,
750
+ { indexName, attempts: ALLOCATION_CAS_ATTEMPTS }
751
+ );
836
752
  }
837
- async function runAllocatorForAllIndexes(coordinator, knownIndexes, isActive, onError) {
753
+ async function runAllocatorForAllIndexes(state, coordinator, isActive, onError) {
838
754
  if (!isActive()) {
839
755
  return;
840
756
  }
@@ -842,18 +758,23 @@ async function runAllocatorForAllIndexes(coordinator, knownIndexes, isActive, on
842
758
  if (nodes.length === 0 || !isActive()) {
843
759
  return;
844
760
  }
845
- for (const indexName of knownIndexes) {
761
+ let anyIndexFailed = false;
762
+ for (const indexName of state.knownIndexes) {
846
763
  if (!isActive()) {
847
764
  return;
848
765
  }
849
766
  try {
850
767
  await runAllocatorForIndex(coordinator, indexName, nodes, isActive);
851
768
  } catch (error) {
769
+ anyIndexFailed = true;
852
770
  if (onError !== void 0) {
853
771
  onError(indexName, error);
854
772
  }
855
773
  }
856
774
  }
775
+ if (anyIndexFailed && isActive()) {
776
+ scheduleDebouncedAllocation(state, coordinator, isActive, onError, ALLOCATION_RETRY_DELAY_MS);
777
+ }
857
778
  }
858
779
  function scheduleDebouncedAllocation(state, coordinator, isActive, onError, debounceMs = 500) {
859
780
  if (state.debounceTimer !== null) {
@@ -861,9 +782,13 @@ function scheduleDebouncedAllocation(state, coordinator, isActive, onError, debo
861
782
  }
862
783
  state.debounceTimer = setTimeout(() => {
863
784
  state.debounceTimer = null;
864
- runAllocatorForAllIndexes(coordinator, state.knownIndexes, isActive, onError).catch(() => {
785
+ runAllocatorForAllIndexes(state, coordinator, isActive, onError).catch(() => {
786
+ if (isActive()) {
787
+ scheduleDebouncedAllocation(state, coordinator, isActive, onError, ALLOCATION_RETRY_DELAY_MS);
788
+ }
865
789
  });
866
790
  }, debounceMs);
791
+ state.debounceTimer.unref?.();
867
792
  }
868
793
  async function handleSchemaCreated(indexName, coordinator, knownIndexes, isActive) {
869
794
  knownIndexes.add(indexName);
@@ -895,14 +820,31 @@ async function handleSchemaCreated(indexName, coordinator, knownIndexes, isActiv
895
820
  return;
896
821
  }
897
822
  const expectedVersion = storedTable !== null ? storedTable.version : null;
898
- await coordinator.putAllocation(indexName, result.table, expectedVersion);
823
+ if (await coordinator.putAllocation(indexName, result.table, expectedVersion)) {
824
+ return;
825
+ }
826
+ await runAllocatorForIndex(coordinator, indexName, nodes, isActive);
899
827
  }
900
- var TEARDOWN_CAS_ATTEMPTS = 5;
901
- async function handleSchemaDropped(indexName, coordinator, knownIndexes, isActive) {
902
- knownIndexes.delete(indexName);
903
- for (let attempt = 0; attempt < TEARDOWN_CAS_ATTEMPTS; attempt++) {
904
- if (!isActive()) {
905
- return;
828
+ function scheduleTeardownRetry(indexName, coordinator, state, isActive, round, onError) {
829
+ const existing = state.teardownTimers.get(indexName);
830
+ if (existing !== void 0) {
831
+ clearTimeout(existing);
832
+ }
833
+ const timer = setTimeout(() => {
834
+ state.teardownTimers.delete(indexName);
835
+ tearDownAllocation(indexName, coordinator, state, isActive, round, onError).catch((error) => {
836
+ if (onError !== void 0) {
837
+ onError(indexName, error);
838
+ }
839
+ });
840
+ }, TEARDOWN_RETRY_DELAY_MS);
841
+ timer.unref?.();
842
+ state.teardownTimers.set(indexName, timer);
843
+ }
844
+ async function tearDownAllocation(indexName, coordinator, state, isActive, round, onError) {
845
+ for (let attempt = 0; attempt < TEARDOWN_CAS_ATTEMPTS; attempt++) {
846
+ if (!isActive()) {
847
+ return;
906
848
  }
907
849
  const currentTable = await coordinator.getAllocation(indexName);
908
850
  if (currentTable === null) {
@@ -929,438 +871,858 @@ async function handleSchemaDropped(indexName, coordinator, knownIndexes, isActiv
929
871
  await coordinator.deleteAllocation(indexName);
930
872
  return;
931
873
  }
932
- }
933
- async function handleSchemaEvent(event, coordinator, knownIndexes, isActive) {
934
- if (event.type === "schema_created") {
935
- await handleSchemaCreated(event.indexName, coordinator, knownIndexes, isActive);
936
- } else if (event.type === "schema_dropped") {
937
- await handleSchemaDropped(event.indexName, coordinator, knownIndexes, isActive);
874
+ if (round + 1 < TEARDOWN_RETRY_ROUNDS && isActive()) {
875
+ scheduleTeardownRetry(indexName, coordinator, state, isActive, round + 1, onError);
876
+ return;
938
877
  }
878
+ throw new NarsilError(
879
+ ErrorCodes.ALLOCATION_FAILED,
880
+ `Teardown of the allocation for dropped index '${indexName}' lost every compare-and-set attempt`,
881
+ { indexName, attempts: TEARDOWN_CAS_ATTEMPTS * TEARDOWN_RETRY_ROUNDS }
882
+ );
939
883
  }
940
- async function startEventLoop(state, coordinator, transport, nodeId, isActive, onError) {
941
- clearEventLoopWatchers(state);
942
- const unwatchNodes = await coordinator.watchNodes((_event) => {
943
- scheduleDebouncedAllocation(state, coordinator, isActive, onError);
944
- });
945
- state.unwatchNodes = unwatchNodes;
946
- const unwatchSchemas = await coordinator.watchSchemas((event) => {
947
- handleSchemaEvent(event, coordinator, state.knownIndexes, isActive).catch(() => {
948
- });
949
- });
950
- state.unwatchSchemas = unwatchSchemas;
951
- const unwatchTransport = await transport.listen((message, respond) => {
952
- if (!isActive()) {
953
- return;
954
- }
955
- if (message.type === ReplicationMessageTypes.INSYNC_REMOVE) {
956
- handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId);
957
- } else if (message.type === ClusterMessageTypes.BOOTSTRAP_COMPLETE) {
958
- handleBootstrapCompleteMessage(message, respond, coordinator, nodeId);
959
- }
960
- });
961
- state.unwatchTransport = unwatchTransport;
962
- scheduleDebouncedAllocation(state, coordinator, isActive, onError);
963
- }
964
- function handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId) {
965
- let decoded;
966
- try {
967
- decoded = decode(message.payload);
968
- } catch (_) {
884
+ async function handleSchemaEvent(event, coordinator, state, isActive, onError) {
885
+ if (event.type === "schema_created") {
886
+ await handleSchemaCreated(event.indexName, coordinator, state.knownIndexes, isActive);
969
887
  return;
970
888
  }
971
- const payload = validateInsyncRemovePayload(decoded);
972
- if (payload === null) {
973
- return;
889
+ if (event.type === "schema_dropped") {
890
+ state.knownIndexes.delete(event.indexName);
891
+ await tearDownAllocation(event.indexName, coordinator, state, isActive, 0, onError);
974
892
  }
975
- state.insyncQueue = state.insyncQueue.then(async () => {
976
- const table = await coordinator.getAllocation(payload.indexName);
893
+ }
894
+
895
+ // src/distribution/payload-guards.ts
896
+ function isRecord(value) {
897
+ return value !== null && typeof value === "object" && !Array.isArray(value);
898
+ }
899
+ function isValidInteger(value) {
900
+ return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
901
+ }
902
+ var INSYNC_CAS_ATTEMPTS = 5;
903
+ var REFUSE = { outcome: "refuse" };
904
+ var ACCEPT = { outcome: "accept" };
905
+ async function updateInsyncSet(request, coordinator, decide) {
906
+ const refused = {
907
+ indexName: request.indexName,
908
+ partitionId: request.partitionId,
909
+ accepted: false
910
+ };
911
+ for (let attempt = 0; attempt < INSYNC_CAS_ATTEMPTS; attempt++) {
912
+ const table = await coordinator.getAllocation(request.indexName);
977
913
  if (table === null) {
978
- return;
914
+ return refused;
979
915
  }
980
- const assignment = table.assignments.get(payload.partitionId);
916
+ const assignment = table.assignments.get(request.partitionId);
981
917
  if (assignment === void 0) {
982
- return;
918
+ return refused;
983
919
  }
984
- if (assignment.primary !== message.sourceId) {
985
- return;
920
+ if (request.primaryTerm !== assignment.primaryTerm) {
921
+ return refused;
986
922
  }
987
- const confirmPayload = await handleInsyncRemoval(payload, coordinator);
988
- const response = createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId);
989
- await respond(response);
990
- }).catch(() => {
991
- });
992
- }
993
-
994
- // src/distribution/cluster/controller/index.ts
995
- function createController(config) {
996
- const { nodeId, coordinator, transport, leaseTtlMs, standbyRetryMs, knownIndexNames, onError } = config;
997
- let electionState = createElectionState();
998
- let eventLoopState = createEventLoopState(knownIndexNames);
999
- let stopped = false;
1000
- function isActive() {
1001
- return electionState.active;
1002
- }
1003
- function stepDown() {
1004
- if (!electionState.active) {
1005
- return;
923
+ const decision = decide(assignment);
924
+ if (decision.outcome === "refuse") {
925
+ return refused;
1006
926
  }
1007
- electionState.active = false;
1008
- clearElectionTimers(electionState);
1009
- clearEventLoopWatchers(eventLoopState);
1010
- if (!stopped) {
1011
- scheduleStandbyRetry(electionState, standbyRetryMs, () => {
1012
- tryElection().catch(() => {
1013
- });
1014
- });
927
+ if (decision.outcome === "accept") {
928
+ return { ...refused, accepted: true };
929
+ }
930
+ const updatedAssignments = new Map(table.assignments);
931
+ updatedAssignments.set(request.partitionId, decision.assignment);
932
+ const written = await coordinator.putAllocation(
933
+ request.indexName,
934
+ { ...table, version: table.version + 1, assignments: updatedAssignments },
935
+ table.version
936
+ );
937
+ if (written) {
938
+ return { ...refused, accepted: true };
1015
939
  }
1016
940
  }
1017
- async function becomeActive() {
1018
- if (electionState.active) {
1019
- return;
941
+ return refused;
942
+ }
943
+ async function requestInsyncRemoval(indexName, partitionId, replicaNodeId, primaryTerm, controllerNodeId, transport, sourceNodeId) {
944
+ const payload = {
945
+ indexName,
946
+ partitionId,
947
+ replicaNodeId,
948
+ primaryTerm
949
+ };
950
+ const message = createInsyncRemoveMessage(payload, sourceNodeId);
951
+ const response = await transport.send(controllerNodeId, message);
952
+ const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
953
+ return { accepted: confirmPayload.accepted };
954
+ }
955
+ function handleInsyncRemoval(payload, coordinator) {
956
+ return updateInsyncSet(payload, coordinator, (assignment) => {
957
+ if (!assignment.inSyncSet.includes(payload.replicaNodeId)) {
958
+ return ACCEPT;
1020
959
  }
1021
- electionState.active = true;
1022
- startRenewalInterval(electionState, coordinator, nodeId, leaseTtlMs, stepDown);
1023
- try {
1024
- for (const indexName of await coordinator.listSchemas()) {
1025
- eventLoopState.knownIndexes.add(indexName);
960
+ return {
961
+ outcome: "write",
962
+ assignment: {
963
+ ...assignment,
964
+ inSyncSet: assignment.inSyncSet.filter((nodeId) => nodeId !== payload.replicaNodeId)
1026
965
  }
1027
- } catch (_) {
966
+ };
967
+ });
968
+ }
969
+ async function requestInsyncAdmission(payload, controllerNodeId, transport, sourceNodeId) {
970
+ const message = createInsyncAddMessage(payload, sourceNodeId);
971
+ const response = await transport.send(controllerNodeId, message);
972
+ const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
973
+ return { accepted: confirmPayload.accepted };
974
+ }
975
+ function handleInsyncAdmission(payload, coordinator) {
976
+ return updateInsyncSet(payload, coordinator, (assignment) => {
977
+ if (assignment.state !== "ACTIVE") {
978
+ return REFUSE;
1028
979
  }
1029
- await startEventLoop(eventLoopState, coordinator, transport, nodeId, isActive, onError);
1030
- }
1031
- async function tryElection() {
1032
- if (stopped) {
1033
- return;
980
+ if (!assignment.replicas.includes(payload.replicaNodeId)) {
981
+ return REFUSE;
1034
982
  }
1035
- const acquired = await tryAcquireLease(coordinator, nodeId, leaseTtlMs);
1036
- if (acquired) {
1037
- await becomeActive();
1038
- } else {
1039
- scheduleStandbyRetry(electionState, standbyRetryMs, () => {
1040
- tryElection().catch(() => {
1041
- });
1042
- });
983
+ if (payload.appliedSeqNo < assignment.commitPoint || payload.appliedSeqNo < payload.commitPoint) {
984
+ return REFUSE;
1043
985
  }
1044
- }
1045
- const controller = {
1046
- get isActive() {
1047
- return electionState.active;
1048
- },
1049
- async start() {
1050
- stopped = false;
1051
- await tryElection();
1052
- },
1053
- async stop() {
1054
- stopped = true;
1055
- clearElectionTimers(electionState);
1056
- clearEventLoopWatchers(eventLoopState);
1057
- if (electionState.active) {
1058
- electionState.active = false;
1059
- try {
1060
- await releaseLease(coordinator);
1061
- } catch (_) {
1062
- }
1063
- }
1064
- },
1065
- async shutdown() {
1066
- await controller.stop();
1067
- eventLoopState = createEventLoopState([]);
1068
- electionState = createElectionState();
986
+ const alreadyInSync = assignment.inSyncSet.includes(payload.replicaNodeId);
987
+ const commitPoint = Math.max(assignment.commitPoint, payload.commitPoint);
988
+ if (alreadyInSync && commitPoint === assignment.commitPoint) {
989
+ return ACCEPT;
1069
990
  }
1070
- };
1071
- return controller;
1072
- }
1073
- function computeBackoffMs(baseMs, maxMs, retryCount) {
1074
- const exponential = baseMs * 2 ** retryCount;
1075
- const capped = Math.min(exponential, maxMs);
1076
- const jitter = capped * (0.5 + Math.random() * 0.5);
1077
- return Math.floor(jitter);
991
+ return {
992
+ outcome: "write",
993
+ assignment: {
994
+ ...assignment,
995
+ inSyncSet: alreadyInSync ? assignment.inSyncSet : [...assignment.inSyncSet, payload.replicaNodeId],
996
+ commitPoint
997
+ }
998
+ };
999
+ });
1078
1000
  }
1079
- async function resolvePrimaryTerm(indexName, partitionId, coordinator) {
1080
- const table = await coordinator.getAllocation(indexName);
1081
- if (table === null) {
1001
+ function validateBootstrapCompletePayload(decoded) {
1002
+ if (!isRecord(decoded)) {
1082
1003
  return null;
1083
1004
  }
1084
- const assignment = table.assignments.get(partitionId);
1085
- if (assignment === void 0) {
1005
+ if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
1086
1006
  return null;
1087
1007
  }
1088
- return assignment.primaryTerm;
1089
- }
1090
- async function resolveControllerTargets(coordinator, controllerNodeId) {
1091
- const targets = [controllerNodeId];
1092
- try {
1093
- const nodes = await coordinator.listNodes();
1094
- const registration = nodes.find((entry) => entry.nodeId === controllerNodeId);
1095
- if (registration !== void 0 && registration.address.length > 0 && registration.address !== controllerNodeId) {
1096
- targets.push(registration.address);
1097
- }
1098
- } catch (_) {
1099
- return targets;
1008
+ if (!isValidInteger(decoded.partitionId) || decoded.partitionId < 0) {
1009
+ return null;
1100
1010
  }
1101
- return targets;
1102
- }
1103
- async function reportBootstrapComplete(indexName, partitionId, nodeId, coordinator, transport) {
1104
- const controllerNodeId = await coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
1105
- if (controllerNodeId === null) {
1106
- return false;
1011
+ if (typeof decoded.nodeId !== "string") {
1012
+ return null;
1107
1013
  }
1108
- const primaryTerm = await resolvePrimaryTerm(indexName, partitionId, coordinator);
1109
- if (primaryTerm === null) {
1110
- return false;
1014
+ if (!isValidInteger(decoded.primaryTerm)) {
1015
+ return null;
1111
1016
  }
1112
- const payload = {
1017
+ return {
1018
+ indexName: decoded.indexName,
1019
+ partitionId: decoded.partitionId,
1020
+ nodeId: decoded.nodeId,
1021
+ primaryTerm: decoded.primaryTerm
1022
+ };
1023
+ }
1024
+ async function sendRejection(respond, controllerNodeId, requestId, indexName, partitionId) {
1025
+ const resultPayload = {
1113
1026
  indexName,
1114
1027
  partitionId,
1115
- nodeId,
1116
- primaryTerm
1028
+ accepted: false
1117
1029
  };
1118
- const message = {
1030
+ const response = {
1119
1031
  type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
1120
- sourceId: nodeId,
1121
- requestId: generateId(),
1122
- payload: encode(payload)
1032
+ sourceId: controllerNodeId,
1033
+ requestId,
1034
+ payload: encode(resultPayload)
1123
1035
  };
1124
- for (const target of await resolveControllerTargets(coordinator, controllerNodeId)) {
1125
- try {
1126
- const response = await transport.send(target, message);
1127
- return isAcceptedResponse(response);
1128
- } catch (_) {
1129
- }
1130
- }
1131
- return false;
1036
+ await respond(response);
1132
1037
  }
1133
- function isAcceptedResponse(response) {
1038
+ async function rejectBootstrapComplete(message, respond, controllerNodeId) {
1039
+ let decoded;
1134
1040
  try {
1135
- const decoded = decode(response.payload);
1136
- if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
1137
- return false;
1138
- }
1139
- return decoded.accepted === true;
1041
+ decoded = decode(message.payload);
1140
1042
  } catch (_) {
1141
- return false;
1043
+ await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
1044
+ return;
1045
+ }
1046
+ const payload = validateBootstrapCompletePayload(decoded);
1047
+ if (payload === null) {
1048
+ await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
1049
+ return;
1142
1050
  }
1051
+ await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
1143
1052
  }
1144
- async function bootstrapPartition(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onBootstrapPartition, onError) {
1145
- for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
1146
- if (state.aborted) {
1147
- return false;
1148
- }
1149
- const synced = await runBootstrapSyncAttempt(state, onBootstrapPartition, onError);
1150
- if (state.aborted) {
1151
- return false;
1152
- }
1153
- if (synced) {
1154
- return retryReportBootstrapComplete(
1155
- state,
1156
- coordinator,
1157
- transport,
1158
- nodeId,
1159
- bootstrapRetryBaseMs,
1160
- bootstrapRetryMaxMs,
1161
- bootstrapMaxRetries,
1162
- onError
1163
- );
1164
- }
1165
- if (attempt < bootstrapMaxRetries) {
1166
- const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
1167
- await waitWithAbort(state, backoffMs);
1168
- }
1053
+ async function handleBootstrapCompleteMessage(message, respond, coordinator, controllerNodeId) {
1054
+ let decoded;
1055
+ try {
1056
+ decoded = decode(message.payload);
1057
+ } catch (_) {
1058
+ await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
1059
+ return;
1169
1060
  }
1170
- return false;
1171
- }
1172
- async function runBootstrapSyncAttempt(state, onBootstrapPartition, onError) {
1061
+ const payload = validateBootstrapCompletePayload(decoded);
1062
+ if (payload === null) {
1063
+ await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
1064
+ return;
1065
+ }
1066
+ if (message.sourceId !== payload.nodeId) {
1067
+ await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
1068
+ return;
1069
+ }
1070
+ let accepted;
1173
1071
  try {
1174
- return await onBootstrapPartition(state.indexName, state.partitionId, state.primaryNodeId);
1175
- } catch (error) {
1176
- if (state.aborted) {
1072
+ accepted = await processBootstrapComplete(payload, coordinator);
1073
+ } catch (_) {
1074
+ await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
1075
+ return;
1076
+ }
1077
+ const resultPayload = {
1078
+ indexName: payload.indexName,
1079
+ partitionId: payload.partitionId,
1080
+ accepted
1081
+ };
1082
+ await respond({
1083
+ type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
1084
+ sourceId: controllerNodeId,
1085
+ requestId: message.requestId,
1086
+ payload: encode(resultPayload)
1087
+ });
1088
+ }
1089
+ var BOOTSTRAP_CAS_ATTEMPTS = 5;
1090
+ async function processBootstrapComplete(payload, coordinator) {
1091
+ for (let attempt = 0; attempt < BOOTSTRAP_CAS_ATTEMPTS; attempt++) {
1092
+ const table = await coordinator.getAllocation(payload.indexName);
1093
+ if (table === null) {
1177
1094
  return false;
1178
1095
  }
1179
- if (onError !== void 0) {
1180
- const cause = error instanceof Error ? error.message : String(error);
1181
- onError(
1182
- new NarsilError(
1183
- ErrorCodes.NODE_BOOTSTRAP_FAILED,
1184
- `Bootstrap sync failed for ${state.indexName}:${state.partitionId}`,
1185
- { indexName: state.indexName, partitionId: state.partitionId, cause }
1186
- )
1187
- );
1096
+ const assignment = table.assignments.get(payload.partitionId);
1097
+ if (assignment === void 0) {
1098
+ return false;
1188
1099
  }
1189
- return false;
1190
- }
1191
- }
1192
- async function retryReportBootstrapComplete(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onError) {
1193
- for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
1194
- if (state.aborted) {
1100
+ if (payload.primaryTerm !== assignment.primaryTerm) {
1195
1101
  return false;
1196
1102
  }
1197
- const accepted = await reportBootstrapComplete(state.indexName, state.partitionId, nodeId, coordinator, transport);
1198
- if (accepted) {
1199
- return true;
1103
+ if (assignment.primary !== payload.nodeId && !assignment.replicas.includes(payload.nodeId)) {
1104
+ return false;
1200
1105
  }
1201
- if (attempt < bootstrapMaxRetries && !state.aborted) {
1202
- const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
1203
- await waitWithAbort(state, backoffMs);
1106
+ if (assignment.state !== "INITIALISING") {
1107
+ return false;
1204
1108
  }
1205
- }
1206
- if (onError !== void 0) {
1207
- onError(
1208
- new NarsilError(
1209
- ErrorCodes.NODE_BOOTSTRAP_FAILED,
1210
- `Failed to report bootstrap completion for ${state.indexName}:${state.partitionId} after ${bootstrapMaxRetries + 1} attempts`,
1211
- { indexName: state.indexName, partitionId: state.partitionId }
1212
- )
1109
+ const updatedAssignments = new Map(table.assignments);
1110
+ updatedAssignments.set(payload.partitionId, { ...assignment, state: "ACTIVE" });
1111
+ const written = await coordinator.putAllocation(
1112
+ payload.indexName,
1113
+ { ...table, version: table.version + 1, assignments: updatedAssignments },
1114
+ table.version
1213
1115
  );
1116
+ if (written) {
1117
+ return true;
1118
+ }
1214
1119
  }
1215
1120
  return false;
1216
1121
  }
1217
- function waitWithAbort(state, ms) {
1218
- return new Promise((resolve) => {
1219
- if (state.aborted) {
1220
- resolve();
1221
- return;
1222
- }
1223
- state.abortResolve = resolve;
1224
- state.retryTimer = setTimeout(() => {
1225
- state.retryTimer = null;
1226
- state.abortResolve = null;
1227
- resolve();
1228
- }, ms);
1229
- });
1230
- }
1231
- function abortBootstrapState(state) {
1232
- state.aborted = true;
1233
- if (state.retryTimer !== null) {
1234
- clearTimeout(state.retryTimer);
1235
- state.retryTimer = null;
1122
+
1123
+ // src/distribution/cluster/controller/event-loop/insync-messages.ts
1124
+ var UNKNOWN_PARTITION_ID = -1;
1125
+ function validateInsyncRemovePayload(decoded) {
1126
+ if (!isRecord(decoded)) {
1127
+ return null;
1236
1128
  }
1237
- if (state.abortResolve !== null) {
1238
- state.abortResolve();
1239
- state.abortResolve = null;
1129
+ if (typeof decoded.indexName !== "string") {
1130
+ return null;
1131
+ }
1132
+ if (!isValidInteger(decoded.partitionId)) {
1133
+ return null;
1134
+ }
1135
+ if (typeof decoded.replicaNodeId !== "string") {
1136
+ return null;
1137
+ }
1138
+ if (!isValidInteger(decoded.primaryTerm)) {
1139
+ return null;
1240
1140
  }
1241
- }
1242
- function createBootstrapState(indexName, partitionId, primaryNodeId) {
1243
1141
  return {
1244
- indexName,
1245
- partitionId,
1246
- primaryNodeId,
1247
- retryCount: 0,
1248
- retryTimer: null,
1249
- aborted: false,
1250
- abortResolve: null
1142
+ indexName: decoded.indexName,
1143
+ partitionId: decoded.partitionId,
1144
+ replicaNodeId: decoded.replicaNodeId,
1145
+ primaryTerm: decoded.primaryTerm
1251
1146
  };
1252
1147
  }
1253
-
1254
- // src/distribution/cluster/node-lifecycle/allocation-watcher.ts
1255
- function partitionKey(indexName, partitionId) {
1256
- return `${indexName}:${partitionId}`;
1148
+ function validateInsyncAddPayload(decoded) {
1149
+ const base = validateInsyncRemovePayload(decoded);
1150
+ if (base === null || !isRecord(decoded)) {
1151
+ return null;
1152
+ }
1153
+ if (!isValidInteger(decoded.appliedSeqNo) || decoded.appliedSeqNo < 0) {
1154
+ return null;
1155
+ }
1156
+ if (!isValidInteger(decoded.commitPoint) || decoded.commitPoint < 0) {
1157
+ return null;
1158
+ }
1159
+ return { ...base, appliedSeqNo: decoded.appliedSeqNo, commitPoint: decoded.commitPoint };
1257
1160
  }
1258
- function createAllocationWatcherState() {
1161
+ function decodeMessage(message) {
1162
+ try {
1163
+ return decode(message.payload);
1164
+ } catch (_) {
1165
+ return null;
1166
+ }
1167
+ }
1168
+ function describeRequest(decoded) {
1169
+ if (!isRecord(decoded)) {
1170
+ return { indexName: "", partitionId: UNKNOWN_PARTITION_ID };
1171
+ }
1259
1172
  return {
1260
- unwatchAllocation: null,
1261
- debounceTimer: null,
1262
- activeBootstraps: /* @__PURE__ */ new Map(),
1263
- trackedPartitions: /* @__PURE__ */ new Map(),
1264
- pendingTables: /* @__PURE__ */ new Map(),
1265
- stopped: false
1173
+ indexName: typeof decoded.indexName === "string" ? decoded.indexName : "",
1174
+ partitionId: isValidInteger(decoded.partitionId) ? decoded.partitionId : UNKNOWN_PARTITION_ID
1266
1175
  };
1267
1176
  }
1268
- async function startAllocationWatcher(state, config) {
1269
- const unwatchAllocation = await config.coordinator.watchAllocation((event) => {
1270
- if (state.stopped) {
1271
- return;
1272
- }
1273
- scheduleProcessAllocation(state, config, event.table);
1177
+ async function assertSenderLeadsPartition(coordinator, indexName, partitionId, sourceId) {
1178
+ const table = await coordinator.getAllocation(indexName);
1179
+ const assignment = table?.assignments.get(partitionId);
1180
+ if (assignment === void 0) {
1181
+ return false;
1182
+ }
1183
+ return assignment.primary === sourceId;
1184
+ }
1185
+ function enqueue(state, task) {
1186
+ state.insyncQueue = state.insyncQueue.then(task).catch(() => {
1274
1187
  });
1275
- state.unwatchAllocation = unwatchAllocation;
1276
1188
  }
1277
- function scheduleProcessAllocation(state, config, table) {
1278
- state.pendingTables.set(table.indexName, table);
1279
- if (state.debounceTimer !== null) {
1280
- clearTimeout(state.debounceTimer);
1189
+ async function respondRefusal(respond, request, nodeId, requestId) {
1190
+ const payload = {
1191
+ indexName: request.indexName,
1192
+ partitionId: request.partitionId,
1193
+ accepted: false
1194
+ };
1195
+ try {
1196
+ await respond(createInsyncConfirmMessage(payload, nodeId, requestId));
1197
+ } catch (_) {
1281
1198
  }
1282
- state.debounceTimer = setTimeout(() => {
1283
- state.debounceTimer = null;
1284
- const tablesToProcess = Array.from(state.pendingTables.values());
1285
- state.pendingTables.clear();
1286
- for (const pending of tablesToProcess) {
1287
- processAllocationChange(state, config, pending);
1288
- }
1289
- }, config.allocationDebounceMs);
1290
1199
  }
1291
- function processAllocationChange(state, config, table) {
1292
- if (state.stopped) {
1200
+ function handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId, isActive) {
1201
+ const decoded = decodeMessage(message);
1202
+ const payload = validateInsyncRemovePayload(decoded);
1203
+ if (payload === null) {
1204
+ void respondRefusal(respond, describeRequest(decoded), nodeId, message.requestId);
1293
1205
  return;
1294
1206
  }
1295
- const nodeId = config.registration.nodeId;
1296
- const currentKeys = /* @__PURE__ */ new Set();
1297
- for (const [partitionId, assignment] of table.assignments) {
1298
- if (!isNodeAssigned(assignment, nodeId)) {
1299
- continue;
1300
- }
1301
- const key = partitionKey(table.indexName, partitionId);
1302
- currentKeys.add(key);
1303
- const existing = state.trackedPartitions.get(key);
1304
- const isPrimary = assignment.primary === nodeId;
1305
- if (existing === void 0) {
1306
- handleNewAssignment(state, config, table.indexName, partitionId, assignment, nodeId);
1307
- } else {
1308
- handleExistingAssignment(state, config, existing, assignment, nodeId);
1207
+ enqueue(state, async () => {
1208
+ const leadsPartition = await assertSenderLeadsPartition(
1209
+ coordinator,
1210
+ payload.indexName,
1211
+ payload.partitionId,
1212
+ message.sourceId
1213
+ );
1214
+ if (!leadsPartition || !isActive()) {
1215
+ await respondRefusal(respond, payload, nodeId, message.requestId);
1216
+ return;
1309
1217
  }
1310
- state.trackedPartitions.set(key, {
1311
- indexName: table.indexName,
1312
- partitionId,
1313
- isPrimary,
1314
- primaryNodeId: assignment.primary,
1315
- primaryTerm: assignment.primaryTerm
1316
- });
1317
- }
1318
- removeUnassignedPartitions(state, config, table.indexName, currentKeys);
1319
- }
1320
- function isNodeAssigned(assignment, nodeId) {
1321
- if (assignment.primary === nodeId) {
1322
- return true;
1323
- }
1324
- return assignment.replicas.includes(nodeId);
1218
+ const confirmPayload = await handleInsyncRemoval(payload, coordinator);
1219
+ await respond(createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId));
1220
+ });
1325
1221
  }
1326
- function handleNewAssignment(state, config, indexName, partitionId, assignment, nodeId) {
1327
- if (assignment.state === "ACTIVE" && assignment.primary === nodeId) {
1328
- return;
1329
- }
1330
- if (assignment.primary === null) {
1222
+ function handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive, onError) {
1223
+ const decoded = decodeMessage(message);
1224
+ const payload = validateInsyncAddPayload(decoded);
1225
+ if (payload === null) {
1226
+ void respondRefusal(respond, describeRequest(decoded), nodeId, message.requestId);
1331
1227
  return;
1332
1228
  }
1333
- if (assignment.primary === nodeId) {
1334
- if (assignment.state === "INITIALISING") {
1335
- startBootstrap(state, config, indexName, partitionId, nodeId);
1229
+ enqueue(state, async () => {
1230
+ const leadsPartition = await assertSenderLeadsPartition(
1231
+ coordinator,
1232
+ payload.indexName,
1233
+ payload.partitionId,
1234
+ message.sourceId
1235
+ );
1236
+ if (!leadsPartition || !isActive()) {
1237
+ await respondRefusal(respond, payload, nodeId, message.requestId);
1238
+ return;
1336
1239
  }
1337
- return;
1338
- }
1339
- startBootstrap(state, config, indexName, partitionId, assignment.primary);
1240
+ const confirmPayload = await handleInsyncAdmission(payload, coordinator);
1241
+ await respond(createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId));
1242
+ if (confirmPayload.accepted && isActive()) {
1243
+ scheduleDebouncedAllocation(state, coordinator, isActive, onError);
1244
+ }
1245
+ });
1340
1246
  }
1341
- function handleExistingAssignment(state, config, existing, assignment, nodeId) {
1342
- if (existing.isPrimary && assignment.primary !== nodeId && assignment.primaryTerm > existing.primaryTerm) {
1343
- if (assignment.primary !== null) {
1344
- if (config.onPrimaryDemotion !== void 0) {
1345
- config.onPrimaryDemotion(existing.indexName, existing.partitionId, assignment.primary);
1346
- }
1347
- startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
1247
+ function handleQueuedBootstrapComplete(state, message, respond, coordinator, nodeId, isActive) {
1248
+ enqueue(state, () => {
1249
+ if (!isActive()) {
1250
+ return rejectBootstrapComplete(message, respond, nodeId);
1348
1251
  }
1349
- return;
1350
- }
1351
- if (assignment.primary !== null && replicaNeedsResync(state, existing, assignment, nodeId)) {
1352
- startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
1353
- }
1252
+ return handleBootstrapCompleteMessage(message, respond, coordinator, nodeId);
1253
+ });
1354
1254
  }
1355
- function replicaNeedsResync(state, existing, assignment, nodeId) {
1356
- if (assignment.state !== "ACTIVE") {
1357
- return false;
1358
- }
1359
- if (assignment.primary === null || assignment.primary === nodeId) {
1360
- return false;
1361
- }
1362
- if (!assignment.replicas.includes(nodeId)) {
1363
- return false;
1255
+
1256
+ // src/distribution/cluster/controller/event-loop/state.ts
1257
+ function createEventLoopState(initialIndexNames) {
1258
+ return {
1259
+ unwatchNodes: null,
1260
+ unwatchSchemas: null,
1261
+ unwatchTransport: null,
1262
+ knownIndexes: new Set(initialIndexNames),
1263
+ debounceTimer: null,
1264
+ teardownTimers: /* @__PURE__ */ new Map(),
1265
+ insyncQueue: Promise.resolve()
1266
+ };
1267
+ }
1268
+ function clearEventLoopWatchers(state) {
1269
+ if (state.unwatchNodes !== null) {
1270
+ state.unwatchNodes();
1271
+ state.unwatchNodes = null;
1272
+ }
1273
+ if (state.unwatchSchemas !== null) {
1274
+ state.unwatchSchemas();
1275
+ state.unwatchSchemas = null;
1276
+ }
1277
+ if (state.unwatchTransport !== null) {
1278
+ state.unwatchTransport();
1279
+ state.unwatchTransport = null;
1280
+ }
1281
+ if (state.debounceTimer !== null) {
1282
+ clearTimeout(state.debounceTimer);
1283
+ state.debounceTimer = null;
1284
+ }
1285
+ for (const timer of state.teardownTimers.values()) {
1286
+ clearTimeout(timer);
1287
+ }
1288
+ state.teardownTimers.clear();
1289
+ }
1290
+
1291
+ // src/distribution/cluster/controller/event-loop/index.ts
1292
+ async function startEventLoop(state, coordinator, transport, nodeId, isActive, onError) {
1293
+ clearEventLoopWatchers(state);
1294
+ try {
1295
+ state.unwatchNodes = await coordinator.watchNodes((_event) => {
1296
+ scheduleDebouncedAllocation(state, coordinator, isActive, onError);
1297
+ });
1298
+ state.unwatchSchemas = await coordinator.watchSchemas((event) => {
1299
+ handleSchemaEvent(event, coordinator, state, isActive, onError).catch((error) => {
1300
+ if (onError !== void 0) {
1301
+ onError(event.indexName, error);
1302
+ }
1303
+ });
1304
+ });
1305
+ state.unwatchTransport = await transport.listen((message, respond) => {
1306
+ if (!isActive()) {
1307
+ return;
1308
+ }
1309
+ if (message.type === ReplicationMessageTypes.INSYNC_REMOVE) {
1310
+ handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId, isActive);
1311
+ } else if (message.type === ReplicationMessageTypes.INSYNC_ADD) {
1312
+ handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive, onError);
1313
+ } else if (message.type === ClusterMessageTypes.BOOTSTRAP_COMPLETE) {
1314
+ handleQueuedBootstrapComplete(state, message, respond, coordinator, nodeId, isActive);
1315
+ }
1316
+ });
1317
+ } catch (error) {
1318
+ clearEventLoopWatchers(state);
1319
+ throw error;
1320
+ }
1321
+ scheduleDebouncedAllocation(state, coordinator, isActive, onError);
1322
+ }
1323
+
1324
+ // src/distribution/cluster/controller/index.ts
1325
+ function createController(config) {
1326
+ const { nodeId, coordinator, transport, leaseTtlMs, standbyRetryMs, knownIndexNames, onError } = config;
1327
+ let electionState = createElectionState();
1328
+ let eventLoopState = createEventLoopState(knownIndexNames);
1329
+ let stopped = false;
1330
+ function isActive() {
1331
+ return electionState.active;
1332
+ }
1333
+ function stepDown() {
1334
+ if (!electionState.active) {
1335
+ return;
1336
+ }
1337
+ electionState.active = false;
1338
+ clearElectionTimers(electionState);
1339
+ clearEventLoopWatchers(eventLoopState);
1340
+ if (!stopped) {
1341
+ scheduleStandbyRetry(electionState, standbyRetryMs, () => {
1342
+ tryElection().catch(() => {
1343
+ });
1344
+ });
1345
+ }
1346
+ }
1347
+ async function becomeActive() {
1348
+ if (electionState.active) {
1349
+ return;
1350
+ }
1351
+ electionState.active = true;
1352
+ startRenewalInterval(electionState, coordinator, nodeId, leaseTtlMs, stepDown);
1353
+ try {
1354
+ for (const indexName of await coordinator.listSchemas()) {
1355
+ eventLoopState.knownIndexes.add(indexName);
1356
+ }
1357
+ } catch (_) {
1358
+ }
1359
+ try {
1360
+ await startEventLoop(eventLoopState, coordinator, transport, nodeId, isActive, onError);
1361
+ } catch (error) {
1362
+ electionState.active = false;
1363
+ clearElectionTimers(electionState);
1364
+ try {
1365
+ await releaseLease(coordinator);
1366
+ } catch (_) {
1367
+ }
1368
+ throw error;
1369
+ }
1370
+ }
1371
+ async function tryElection() {
1372
+ if (stopped) {
1373
+ return;
1374
+ }
1375
+ const acquired = await tryAcquireLease(coordinator, nodeId, leaseTtlMs);
1376
+ if (acquired) {
1377
+ await becomeActive();
1378
+ } else {
1379
+ scheduleStandbyRetry(electionState, standbyRetryMs, () => {
1380
+ tryElection().catch(() => {
1381
+ });
1382
+ });
1383
+ }
1384
+ }
1385
+ const controller = {
1386
+ get isActive() {
1387
+ return electionState.active;
1388
+ },
1389
+ async start() {
1390
+ stopped = false;
1391
+ await tryElection();
1392
+ },
1393
+ async stop() {
1394
+ stopped = true;
1395
+ clearElectionTimers(electionState);
1396
+ clearEventLoopWatchers(eventLoopState);
1397
+ if (electionState.active) {
1398
+ electionState.active = false;
1399
+ try {
1400
+ await releaseLease(coordinator);
1401
+ } catch (_) {
1402
+ }
1403
+ }
1404
+ },
1405
+ async shutdown() {
1406
+ await controller.stop();
1407
+ eventLoopState = createEventLoopState([]);
1408
+ electionState = createElectionState();
1409
+ }
1410
+ };
1411
+ return controller;
1412
+ }
1413
+ function computeBackoffMs(baseMs, maxMs, retryCount) {
1414
+ const exponential = baseMs * 2 ** retryCount;
1415
+ const capped = Math.min(exponential, maxMs);
1416
+ const jitter = capped * (0.5 + Math.random() * 0.5);
1417
+ return Math.floor(jitter);
1418
+ }
1419
+ async function resolvePrimaryTerm(indexName, partitionId, coordinator) {
1420
+ const table = await coordinator.getAllocation(indexName);
1421
+ if (table === null) {
1422
+ return null;
1423
+ }
1424
+ const assignment = table.assignments.get(partitionId);
1425
+ if (assignment === void 0) {
1426
+ return null;
1427
+ }
1428
+ return assignment.primaryTerm;
1429
+ }
1430
+ async function resolveControllerTargets(coordinator, controllerNodeId) {
1431
+ const targets = [controllerNodeId];
1432
+ try {
1433
+ const nodes = await coordinator.listNodes();
1434
+ const registration = nodes.find((entry) => entry.nodeId === controllerNodeId);
1435
+ if (registration !== void 0 && registration.address.length > 0 && registration.address !== controllerNodeId) {
1436
+ targets.push(registration.address);
1437
+ }
1438
+ } catch (_) {
1439
+ return targets;
1440
+ }
1441
+ return targets;
1442
+ }
1443
+ async function reportBootstrapComplete(indexName, partitionId, nodeId, coordinator, transport) {
1444
+ const controllerNodeId = await coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
1445
+ if (controllerNodeId === null) {
1446
+ return false;
1447
+ }
1448
+ const primaryTerm = await resolvePrimaryTerm(indexName, partitionId, coordinator);
1449
+ if (primaryTerm === null) {
1450
+ return false;
1451
+ }
1452
+ const payload = {
1453
+ indexName,
1454
+ partitionId,
1455
+ nodeId,
1456
+ primaryTerm
1457
+ };
1458
+ const message = {
1459
+ type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
1460
+ sourceId: nodeId,
1461
+ requestId: generateId(),
1462
+ payload: encode(payload)
1463
+ };
1464
+ for (const target of await resolveControllerTargets(coordinator, controllerNodeId)) {
1465
+ try {
1466
+ const response = await transport.send(target, message);
1467
+ return isAcceptedResponse(response);
1468
+ } catch (_) {
1469
+ }
1470
+ }
1471
+ return false;
1472
+ }
1473
+ function isAcceptedResponse(response) {
1474
+ try {
1475
+ const decoded = decode(response.payload);
1476
+ if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
1477
+ return false;
1478
+ }
1479
+ return decoded.accepted === true;
1480
+ } catch (_) {
1481
+ return false;
1482
+ }
1483
+ }
1484
+ async function partitionAwaitsBootstrapReport(state, coordinator) {
1485
+ try {
1486
+ const table = await coordinator.getAllocation(state.indexName);
1487
+ const assignment = table?.assignments.get(state.partitionId);
1488
+ if (assignment === void 0) {
1489
+ return true;
1490
+ }
1491
+ return assignment.state === "INITIALISING";
1492
+ } catch (_) {
1493
+ return true;
1494
+ }
1495
+ }
1496
+ async function bootstrapPartition(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onBootstrapPartition, onError) {
1497
+ for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
1498
+ if (state.aborted) {
1499
+ return false;
1500
+ }
1501
+ const synced = await runBootstrapSyncAttempt(state, onBootstrapPartition, onError);
1502
+ if (state.aborted) {
1503
+ return false;
1504
+ }
1505
+ if (synced) {
1506
+ if (!await partitionAwaitsBootstrapReport(state, coordinator)) {
1507
+ return true;
1508
+ }
1509
+ return retryReportBootstrapComplete(
1510
+ state,
1511
+ coordinator,
1512
+ transport,
1513
+ nodeId,
1514
+ bootstrapRetryBaseMs,
1515
+ bootstrapRetryMaxMs,
1516
+ bootstrapMaxRetries,
1517
+ onError
1518
+ );
1519
+ }
1520
+ if (attempt < bootstrapMaxRetries) {
1521
+ const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
1522
+ await waitWithAbort(state, backoffMs);
1523
+ }
1524
+ }
1525
+ return false;
1526
+ }
1527
+ async function runBootstrapSyncAttempt(state, onBootstrapPartition, onError) {
1528
+ try {
1529
+ return await onBootstrapPartition(state.indexName, state.partitionId, state.primaryNodeId);
1530
+ } catch (error) {
1531
+ if (state.aborted) {
1532
+ return false;
1533
+ }
1534
+ if (onError !== void 0) {
1535
+ const cause = error instanceof Error ? error.message : String(error);
1536
+ onError(
1537
+ new NarsilError(
1538
+ ErrorCodes.NODE_BOOTSTRAP_FAILED,
1539
+ `Bootstrap sync failed for ${state.indexName}:${state.partitionId}`,
1540
+ { indexName: state.indexName, partitionId: state.partitionId, cause }
1541
+ )
1542
+ );
1543
+ }
1544
+ return false;
1545
+ }
1546
+ }
1547
+ async function retryReportBootstrapComplete(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onError) {
1548
+ for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
1549
+ if (state.aborted) {
1550
+ return false;
1551
+ }
1552
+ if (attempt > 0 && !await partitionAwaitsBootstrapReport(state, coordinator)) {
1553
+ return true;
1554
+ }
1555
+ const accepted = await reportBootstrapComplete(state.indexName, state.partitionId, nodeId, coordinator, transport);
1556
+ if (accepted) {
1557
+ return true;
1558
+ }
1559
+ if (!await partitionAwaitsBootstrapReport(state, coordinator)) {
1560
+ return true;
1561
+ }
1562
+ if (attempt < bootstrapMaxRetries && !state.aborted) {
1563
+ const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
1564
+ await waitWithAbort(state, backoffMs);
1565
+ }
1566
+ }
1567
+ if (onError !== void 0) {
1568
+ onError(
1569
+ new NarsilError(
1570
+ ErrorCodes.NODE_BOOTSTRAP_FAILED,
1571
+ `Failed to report bootstrap completion for ${state.indexName}:${state.partitionId} after ${bootstrapMaxRetries + 1} attempts`,
1572
+ { indexName: state.indexName, partitionId: state.partitionId }
1573
+ )
1574
+ );
1575
+ }
1576
+ return false;
1577
+ }
1578
+ function waitWithAbort(state, ms) {
1579
+ return new Promise((resolve) => {
1580
+ if (state.aborted) {
1581
+ resolve();
1582
+ return;
1583
+ }
1584
+ state.abortResolve = resolve;
1585
+ state.retryTimer = setTimeout(() => {
1586
+ state.retryTimer = null;
1587
+ state.abortResolve = null;
1588
+ resolve();
1589
+ }, ms);
1590
+ });
1591
+ }
1592
+ function abortBootstrapState(state) {
1593
+ state.aborted = true;
1594
+ if (state.retryTimer !== null) {
1595
+ clearTimeout(state.retryTimer);
1596
+ state.retryTimer = null;
1597
+ }
1598
+ if (state.abortResolve !== null) {
1599
+ state.abortResolve();
1600
+ state.abortResolve = null;
1601
+ }
1602
+ }
1603
+ function createBootstrapState(indexName, partitionId, primaryNodeId) {
1604
+ return {
1605
+ indexName,
1606
+ partitionId,
1607
+ primaryNodeId,
1608
+ retryCount: 0,
1609
+ retryTimer: null,
1610
+ aborted: false,
1611
+ abortResolve: null
1612
+ };
1613
+ }
1614
+
1615
+ // src/distribution/cluster/node-lifecycle/allocation-watcher.ts
1616
+ function partitionKey(indexName, partitionId) {
1617
+ return `${indexName}:${partitionId}`;
1618
+ }
1619
+ function createAllocationWatcherState() {
1620
+ return {
1621
+ unwatchAllocation: null,
1622
+ debounceTimer: null,
1623
+ activeBootstraps: /* @__PURE__ */ new Map(),
1624
+ trackedPartitions: /* @__PURE__ */ new Map(),
1625
+ pendingTables: /* @__PURE__ */ new Map(),
1626
+ restartWaiters: /* @__PURE__ */ new Map(),
1627
+ stopped: false
1628
+ };
1629
+ }
1630
+ async function startAllocationWatcher(state, config) {
1631
+ const unwatchAllocation = await config.coordinator.watchAllocation((event) => {
1632
+ if (state.stopped) {
1633
+ return;
1634
+ }
1635
+ scheduleProcessAllocation(state, config, event.table);
1636
+ });
1637
+ state.unwatchAllocation = unwatchAllocation;
1638
+ }
1639
+ function scheduleProcessAllocation(state, config, table) {
1640
+ state.pendingTables.set(table.indexName, table);
1641
+ if (state.debounceTimer !== null) {
1642
+ clearTimeout(state.debounceTimer);
1643
+ }
1644
+ state.debounceTimer = setTimeout(() => {
1645
+ state.debounceTimer = null;
1646
+ const tablesToProcess = Array.from(state.pendingTables.values());
1647
+ state.pendingTables.clear();
1648
+ for (const pending of tablesToProcess) {
1649
+ processAllocationChange(state, config, pending);
1650
+ }
1651
+ }, config.allocationDebounceMs);
1652
+ }
1653
+ function processAllocationChange(state, config, table) {
1654
+ if (state.stopped) {
1655
+ return;
1656
+ }
1657
+ const nodeId = config.registration.nodeId;
1658
+ const currentKeys = /* @__PURE__ */ new Set();
1659
+ for (const [partitionId, assignment] of table.assignments) {
1660
+ if (!isNodeAssigned(assignment, nodeId)) {
1661
+ continue;
1662
+ }
1663
+ const key = partitionKey(table.indexName, partitionId);
1664
+ currentKeys.add(key);
1665
+ const existing = state.trackedPartitions.get(key);
1666
+ const isPrimary = assignment.primary === nodeId;
1667
+ if (existing === void 0) {
1668
+ handleNewAssignment(state, config, table.indexName, partitionId, assignment, nodeId);
1669
+ } else {
1670
+ handleExistingAssignment(state, config, existing, assignment, nodeId);
1671
+ }
1672
+ state.trackedPartitions.set(key, {
1673
+ indexName: table.indexName,
1674
+ partitionId,
1675
+ isPrimary,
1676
+ primaryNodeId: assignment.primary,
1677
+ primaryTerm: assignment.primaryTerm
1678
+ });
1679
+ }
1680
+ removeUnassignedPartitions(state, config, table.indexName, currentKeys);
1681
+ }
1682
+ function isNodeAssigned(assignment, nodeId) {
1683
+ if (assignment.primary === nodeId) {
1684
+ return true;
1685
+ }
1686
+ return assignment.replicas.includes(nodeId);
1687
+ }
1688
+ function handleNewAssignment(state, config, indexName, partitionId, assignment, nodeId) {
1689
+ if (assignment.state === "ACTIVE" && assignment.primary === nodeId) {
1690
+ return;
1691
+ }
1692
+ if (assignment.primary === null) {
1693
+ return;
1694
+ }
1695
+ if (assignment.primary === nodeId) {
1696
+ if (assignment.state === "INITIALISING") {
1697
+ startBootstrap(state, config, indexName, partitionId, nodeId);
1698
+ }
1699
+ return;
1700
+ }
1701
+ startBootstrap(state, config, indexName, partitionId, assignment.primary);
1702
+ }
1703
+ function handleExistingAssignment(state, config, existing, assignment, nodeId) {
1704
+ if (existing.isPrimary && assignment.primary !== nodeId && assignment.primaryTerm > existing.primaryTerm) {
1705
+ if (assignment.primary !== null) {
1706
+ if (config.onPrimaryDemotion !== void 0) {
1707
+ config.onPrimaryDemotion(existing.indexName, existing.partitionId, assignment.primary);
1708
+ }
1709
+ startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
1710
+ }
1711
+ return;
1712
+ }
1713
+ if (assignment.primary !== null && replicaNeedsResync(state, existing, assignment, nodeId)) {
1714
+ startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
1715
+ }
1716
+ }
1717
+ function replicaNeedsResync(state, existing, assignment, nodeId) {
1718
+ if (assignment.state !== "ACTIVE") {
1719
+ return false;
1720
+ }
1721
+ if (assignment.primary === null || assignment.primary === nodeId) {
1722
+ return false;
1723
+ }
1724
+ if (!assignment.replicas.includes(nodeId)) {
1725
+ return false;
1364
1726
  }
1365
1727
  if (assignment.inSyncSet.includes(nodeId)) {
1366
1728
  return false;
@@ -1391,12 +1753,14 @@ function startBootstrap(state, config, indexName, partitionId, primaryNodeId) {
1391
1753
  }
1392
1754
  if (!succeeded) {
1393
1755
  state.trackedPartitions.delete(key);
1756
+ void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
1394
1757
  }
1395
1758
  }).catch(() => {
1396
1759
  if (state.activeBootstraps.get(key) === bootstrapState) {
1397
1760
  state.activeBootstraps.delete(key);
1398
1761
  }
1399
1762
  state.trackedPartitions.delete(key);
1763
+ void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
1400
1764
  });
1401
1765
  }
1402
1766
  function removeUnassignedPartitions(state, config, indexName, currentKeys) {
@@ -1428,6 +1792,11 @@ function stopAllocationWatcher(state) {
1428
1792
  clearTimeout(state.debounceTimer);
1429
1793
  state.debounceTimer = null;
1430
1794
  }
1795
+ for (const [timer, resolve] of state.restartWaiters) {
1796
+ clearTimeout(timer);
1797
+ resolve();
1798
+ }
1799
+ state.restartWaiters.clear();
1431
1800
  for (const bootstrapState of state.activeBootstraps.values()) {
1432
1801
  abortBootstrapState(bootstrapState);
1433
1802
  }
@@ -1440,10 +1809,90 @@ function processInitialAllocations(state, config, tables) {
1440
1809
  processAllocationChange(state, config, table);
1441
1810
  }
1442
1811
  }
1812
+ function waitBeforeBootstrapRestart(state, config) {
1813
+ const delayMs = Math.min(config.bootstrapRetryMaxMs, config.bootstrapRetryBaseMs);
1814
+ if (delayMs <= 0) {
1815
+ return Promise.resolve();
1816
+ }
1817
+ return new Promise((resolve) => {
1818
+ const timer = setTimeout(() => {
1819
+ state.restartWaiters.delete(timer);
1820
+ resolve();
1821
+ }, delayMs);
1822
+ timer.unref?.();
1823
+ state.restartWaiters.set(timer, resolve);
1824
+ });
1825
+ }
1826
+ async function restartBootstrapWhileOutOfSync(state, config, indexName, partitionId) {
1827
+ if (state.stopped) {
1828
+ return;
1829
+ }
1830
+ await waitBeforeBootstrapRestart(state, config);
1831
+ if (state.stopped) {
1832
+ return;
1833
+ }
1834
+ const nodeId = config.registration.nodeId;
1835
+ let table;
1836
+ try {
1837
+ table = await config.coordinator.getAllocation(indexName);
1838
+ } catch (error) {
1839
+ if (config.onError !== void 0) {
1840
+ config.onError(error);
1841
+ }
1842
+ if (!state.stopped) {
1843
+ void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
1844
+ }
1845
+ return;
1846
+ }
1847
+ if (state.stopped || table === null) {
1848
+ return;
1849
+ }
1850
+ const assignment = table.assignments.get(partitionId);
1851
+ if (assignment === void 0 || assignment.primary === null || assignment.primary === nodeId) {
1852
+ return;
1853
+ }
1854
+ if (!assignment.replicas.includes(nodeId) || assignment.inSyncSet.includes(nodeId)) {
1855
+ return;
1856
+ }
1857
+ if (assignment.state !== "ACTIVE" && assignment.state !== "INITIALISING" && assignment.state !== "MIGRATING") {
1858
+ return;
1859
+ }
1860
+ if (state.activeBootstraps.has(partitionKey(indexName, partitionId))) {
1861
+ return;
1862
+ }
1863
+ startBootstrap(state, config, indexName, partitionId, assignment.primary);
1864
+ }
1865
+
1866
+ // src/distribution/cluster/node-lifecycle/heartbeat.ts
1867
+ function createRegistrationHeartbeatState() {
1868
+ return { timer: null, inFlight: Promise.resolve() };
1869
+ }
1870
+ function clearHeartbeatTimer(state) {
1871
+ if (state.timer !== null) {
1872
+ clearInterval(state.timer);
1873
+ state.timer = null;
1874
+ }
1875
+ }
1876
+ function startRegistrationHeartbeat(state, config) {
1877
+ clearHeartbeatTimer(state);
1878
+ state.timer = setInterval(() => {
1879
+ state.inFlight = state.inFlight.then(() => config.coordinator.registerNode(config.registration)).catch((error) => {
1880
+ if (config.onError !== void 0) {
1881
+ config.onError(error);
1882
+ }
1883
+ });
1884
+ }, config.nodeHeartbeatIntervalMs);
1885
+ state.timer.unref?.();
1886
+ }
1887
+ async function stopRegistrationHeartbeat(state) {
1888
+ clearHeartbeatTimer(state);
1889
+ await state.inFlight;
1890
+ }
1443
1891
 
1444
1892
  // src/distribution/cluster/node-lifecycle/join.ts
1445
- async function joinCluster(config, watcherState) {
1893
+ async function joinCluster(config, watcherState, heartbeatState) {
1446
1894
  await config.coordinator.registerNode(config.registration);
1895
+ startRegistrationHeartbeat(heartbeatState, config);
1447
1896
  const initialTables = await loadInitialAllocations(config);
1448
1897
  processInitialAllocations(watcherState, config, initialTables);
1449
1898
  await startAllocationWatcher(watcherState, config);
@@ -1465,15 +1914,17 @@ async function leaveCluster(config) {
1465
1914
  // src/distribution/cluster/node-lifecycle/types.ts
1466
1915
  var DEFAULT_NODE_LIFECYCLE_CONFIG = {
1467
1916
  bootstrapRetryBaseMs: 1e3,
1468
- bootstrapRetryMaxMs: 3e4,
1917
+ bootstrapRetryMaxMs: 5e3,
1469
1918
  bootstrapMaxRetries: 10,
1470
- allocationDebounceMs: 250
1919
+ allocationDebounceMs: 250,
1920
+ nodeHeartbeatIntervalMs: 1e4
1471
1921
  };
1472
1922
 
1473
1923
  // src/distribution/cluster/node-lifecycle/index.ts
1474
1924
  function createDataNodeLifecycle(config) {
1475
1925
  let status = "stopped";
1476
1926
  let watcherState = createAllocationWatcherState();
1927
+ const heartbeatState = createRegistrationHeartbeatState();
1477
1928
  let operationLock = Promise.resolve();
1478
1929
  function withLock(fn) {
1479
1930
  const prev = operationLock;
@@ -1516,9 +1967,10 @@ function createDataNodeLifecycle(config) {
1516
1967
  }
1517
1968
  status = "joining";
1518
1969
  try {
1519
- await joinCluster(config, watcherState);
1970
+ await joinCluster(config, watcherState, heartbeatState);
1520
1971
  status = "active";
1521
1972
  } catch (error) {
1973
+ await stopRegistrationHeartbeat(heartbeatState);
1522
1974
  status = "stopped";
1523
1975
  throw error;
1524
1976
  }
@@ -1530,6 +1982,7 @@ function createDataNodeLifecycle(config) {
1530
1982
  return;
1531
1983
  }
1532
1984
  status = "leaving";
1985
+ await stopRegistrationHeartbeat(heartbeatState);
1533
1986
  stopAllocationWatcher(watcherState);
1534
1987
  try {
1535
1988
  await leaveCluster(config);
@@ -1544,6 +1997,7 @@ function createDataNodeLifecycle(config) {
1544
1997
  if (status === "shutdown") {
1545
1998
  return;
1546
1999
  }
2000
+ await stopRegistrationHeartbeat(heartbeatState);
1547
2001
  stopAllocationWatcher(watcherState);
1548
2002
  if (status === "active" || status === "joining") {
1549
2003
  try {
@@ -2818,11 +3272,47 @@ function validateLocalPartitionCount(engine, indexName, primaryNodeId, partition
2818
3272
  // src/distribution/cluster-node/bootstrap-sync/targets.ts
2819
3273
  function readLocalLogState(indexName, partitionId, deps) {
2820
3274
  const log = deps.getReplicationLog(indexName, partitionId);
2821
- const lastSeqNo = log.committedSeqNo;
2822
- const lastPrimaryTerm = log.committedPrimaryTerm;
3275
+ const lastSeqNo = log.localLogEnd;
3276
+ const lastPrimaryTerm = log.localLogEndPrimaryTerm;
2823
3277
  const newestEntry = log.getEntry(lastSeqNo);
2824
3278
  return { lastSeqNo, lastPrimaryTerm: newestEntry?.primaryTerm ?? lastPrimaryTerm };
2825
3279
  }
3280
+ var SYNCED_POSITION_REPORT_TIMEOUT_MS = 5e3;
3281
+ async function reportSyncedPosition(indexName, partitionId, target, knownSeqNo, deadline, deps) {
3282
+ const logState = readLocalLogState(indexName, partitionId, deps);
3283
+ if (logState.lastSeqNo <= knownSeqNo) {
3284
+ return;
3285
+ }
3286
+ const budgetMs = Math.min(SYNCED_POSITION_REPORT_TIMEOUT_MS, deadline - Date.now());
3287
+ if (budgetMs <= 0) {
3288
+ return;
3289
+ }
3290
+ const payload = {
3291
+ indexName,
3292
+ partitionId,
3293
+ lastSeqNo: logState.lastSeqNo,
3294
+ lastPrimaryTerm: logState.lastPrimaryTerm
3295
+ };
3296
+ const request = {
3297
+ type: ReplicationMessageTypes.SYNC_REQUEST,
3298
+ sourceId: deps.sourceNodeId,
3299
+ requestId: generateId(),
3300
+ payload: encode(payload)
3301
+ };
3302
+ try {
3303
+ await withDeadline(
3304
+ deps.transport.stream(target, request, () => {
3305
+ }),
3306
+ budgetMs,
3307
+ indexName,
3308
+ "synced-position-report"
3309
+ );
3310
+ } catch (error) {
3311
+ if (deps.onError !== void 0) {
3312
+ deps.onError(error);
3313
+ }
3314
+ }
3315
+ }
2826
3316
  async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNodeId, targets, coordinatorSchema, partitionCount, deadline, deps, abortCheck) {
2827
3317
  let lastError = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "no live sync attempts were made", {
2828
3318
  indexName,
@@ -2856,6 +3346,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
2856
3346
  )
2857
3347
  };
2858
3348
  }
3349
+ const positionBeforeAttempt = readLocalLogState(indexName, partitionId, deps).lastSeqNo;
2859
3350
  const attempt = await syncFromTarget(
2860
3351
  state,
2861
3352
  entry,
@@ -2869,6 +3360,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
2869
3360
  abortCheck
2870
3361
  );
2871
3362
  if (attempt.ok) {
3363
+ await reportSyncedPosition(indexName, partitionId, target, positionBeforeAttempt, deadline, deps);
2872
3364
  return attempt;
2873
3365
  }
2874
3366
  lastError = new NarsilError(attempt.error.code, attempt.error.message, {
@@ -3156,171 +3648,586 @@ async function applyRestore(state, entry, indexName, primaryNodeId, bytes, coord
3156
3648
  if (!anotherBootstrapOwnsKey(state, key, entry)) {
3157
3649
  await dropRestoredIndexQuietly(deps.engine, indexName, deps);
3158
3650
  }
3159
- surfaceAborted(deps, indexName, primaryNodeId);
3160
- return false;
3651
+ surfaceAborted(deps, indexName, primaryNodeId);
3652
+ return false;
3653
+ }
3654
+ const schemaError = validateRestoredSchema(deps.engine, indexName, primaryNodeId, coordinatorSchema);
3655
+ if (schemaError !== null) {
3656
+ await dropRestoredIndexQuietly(deps.engine, indexName, deps);
3657
+ surfaceError(deps, indexName, primaryNodeId, schemaError);
3658
+ return false;
3659
+ }
3660
+ return true;
3661
+ }
3662
+
3663
+ // src/distribution/cluster-node/bootstrap-sync/index.ts
3664
+ async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
3665
+ const validationError = validateArguments(indexName, partitionId, primaryNodeId);
3666
+ if (validationError !== null) {
3667
+ surfaceError(deps, indexName, primaryNodeId, validationError);
3668
+ return false;
3669
+ }
3670
+ const key = entryKey(indexName, partitionId);
3671
+ if (state.completed.has(key)) {
3672
+ if (!await replicaResyncRequired(indexName, partitionId, deps)) {
3673
+ return true;
3674
+ }
3675
+ clearBootstrapSyncIndex(state, indexName, partitionId);
3676
+ }
3677
+ const existing = state.inFlight.get(key);
3678
+ if (existing !== void 0) {
3679
+ const result = await Promise.race([existing.promise, existing.abortPromise]);
3680
+ if (result === ABORT_SENTINEL) {
3681
+ return false;
3682
+ }
3683
+ return result;
3684
+ }
3685
+ const entry = createEntry(state, indexName, partitionId, key);
3686
+ entry.promise = executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps).finally(() => {
3687
+ const current = state.inFlight.get(key);
3688
+ if (current === entry) {
3689
+ state.inFlight.delete(key);
3690
+ }
3691
+ if (!state.completed.has(key) && !state.inFlight.has(key)) {
3692
+ const currentGen = state.generations.get(key);
3693
+ if (currentGen !== void 0 && currentGen !== entry.generation) {
3694
+ state.generations.delete(key);
3695
+ }
3696
+ }
3697
+ });
3698
+ state.inFlight.set(key, entry);
3699
+ return entry.promise;
3700
+ }
3701
+ async function replicaResyncRequired(indexName, partitionId, deps) {
3702
+ const allocation = await deps.coordinator.getAllocation(indexName);
3703
+ const assignment = allocation?.assignments.get(partitionId);
3704
+ if (assignment === void 0) {
3705
+ return false;
3706
+ }
3707
+ return assignment.state === "ACTIVE" && assignment.replicas.includes(deps.sourceNodeId) && !assignment.inSyncSet.includes(deps.sourceNodeId);
3708
+ }
3709
+ async function executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps) {
3710
+ const deadlineMs = deps.deadlineMs ?? DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS;
3711
+ const deadline = Date.now() + deadlineMs;
3712
+ const key = entryKey(indexName, partitionId);
3713
+ const abortCheck = () => {
3714
+ const currentGeneration = state.generations.get(key) ?? 0;
3715
+ if (currentGeneration !== entry.generation) {
3716
+ entry.aborted = true;
3717
+ }
3718
+ return entry.aborted;
3719
+ };
3720
+ if (hasLiveBootstrapSyncDeps(deps)) {
3721
+ return executeLiveBootstrapSync(
3722
+ state,
3723
+ entry,
3724
+ indexName,
3725
+ partitionId,
3726
+ primaryNodeId,
3727
+ deps,
3728
+ deadlineMs,
3729
+ deadline,
3730
+ abortCheck
3731
+ );
3732
+ }
3733
+ const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
3734
+ if (schemaResult === "aborted") {
3735
+ surfaceAborted(deps, indexName, primaryNodeId);
3736
+ return false;
3737
+ }
3738
+ if (schemaResult === null) {
3739
+ return false;
3740
+ }
3741
+ if (abortCheck()) {
3742
+ surfaceAborted(deps, indexName, primaryNodeId);
3743
+ return false;
3744
+ }
3745
+ if (Date.now() >= deadline) {
3746
+ surfaceError(
3747
+ deps,
3748
+ indexName,
3749
+ primaryNodeId,
3750
+ new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
3751
+ indexName,
3752
+ deadlineMs
3753
+ })
3754
+ );
3755
+ return false;
3756
+ }
3757
+ const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
3758
+ if (targetsResult instanceof NarsilError) {
3759
+ surfaceError(deps, indexName, primaryNodeId, targetsResult);
3760
+ return false;
3761
+ }
3762
+ const fetchDeps = { transport: deps.transport, sourceNodeId: deps.sourceNodeId };
3763
+ const fetchResult = await fetchSnapshotFromAnyTarget(
3764
+ indexName,
3765
+ primaryNodeId,
3766
+ targetsResult,
3767
+ deadline,
3768
+ fetchDeps,
3769
+ abortCheck,
3770
+ partitionId
3771
+ );
3772
+ if (!fetchResult.ok) {
3773
+ if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
3774
+ surfaceAborted(deps, indexName, primaryNodeId);
3775
+ return false;
3776
+ }
3777
+ surfaceError(
3778
+ deps,
3779
+ indexName,
3780
+ primaryNodeId,
3781
+ new NarsilError(fetchResult.code, fetchResult.message, {
3782
+ indexName,
3783
+ primaryNodeId,
3784
+ ...fetchResult.details
3785
+ })
3786
+ );
3787
+ return false;
3788
+ }
3789
+ if (abortCheck()) {
3790
+ surfaceAborted(deps, indexName, primaryNodeId);
3791
+ return false;
3792
+ }
3793
+ const restoreSucceeded = await applyRestore(
3794
+ state,
3795
+ entry,
3796
+ indexName,
3797
+ primaryNodeId,
3798
+ fetchResult.bytes,
3799
+ schemaResult,
3800
+ deadline,
3801
+ deps
3802
+ );
3803
+ if (!restoreSucceeded) {
3804
+ return false;
3805
+ }
3806
+ if (abortCheck()) {
3807
+ if (!anotherBootstrapOwnsKey(state, key, entry)) {
3808
+ await dropRestoredIndexQuietly(deps.engine, indexName, deps);
3809
+ }
3810
+ surfaceAborted(deps, indexName, primaryNodeId);
3811
+ return false;
3812
+ }
3813
+ deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
3814
+ state.completed.add(key);
3815
+ return true;
3816
+ }
3817
+
3818
+ // src/distribution/cluster-node/write-routing/assignment.ts
3819
+ function resolvePartitionId(docId, partitionCount) {
3820
+ return fnv1a(docId) % partitionCount;
3821
+ }
3822
+ async function resolveNodeTargets(nodeId, deps) {
3823
+ if (deps.resolveNodeTargets === void 0) {
3824
+ return [nodeId];
3825
+ }
3826
+ const targets = await deps.resolveNodeTargets(nodeId);
3827
+ return targets.length > 0 ? targets : [nodeId];
3828
+ }
3829
+ function requireAssignedPrimary(assignment, indexName, partitionId) {
3830
+ if (assignment === void 0 || assignment.primary === null) {
3831
+ throw new NarsilError(
3832
+ ErrorCodes.PARTITION_UNASSIGNED,
3833
+ `No primary assigned for partition ${partitionId} of index '${indexName}'`,
3834
+ { indexName, partitionId }
3835
+ );
3836
+ }
3837
+ return assignment;
3838
+ }
3839
+ async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrimary) {
3840
+ const table = await deps.coordinator.getAllocation(indexName);
3841
+ if (table === null || table.assignments.size === 0) {
3842
+ if (requireLocalPrimary) {
3843
+ throw new NarsilError(
3844
+ ErrorCodes.QUERY_ROUTING_FAILED,
3845
+ `No allocation table is available for forwarded write to index '${indexName}'`,
3846
+ { indexName }
3847
+ );
3848
+ }
3849
+ return null;
3850
+ }
3851
+ const partitionCount = table.assignments.size;
3852
+ const partitionId = resolvePartitionId(docId, partitionCount);
3853
+ const assignment = requireAssignedPrimary(table.assignments.get(partitionId), indexName, partitionId);
3854
+ if (requireLocalPrimary && assignment.primary !== deps.nodeId) {
3855
+ throw new NarsilError(
3856
+ ErrorCodes.PARTITION_NOT_PRIMARY,
3857
+ `Node '${deps.nodeId}' is not primary for partition ${partitionId} of index '${indexName}'`,
3858
+ { indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
3859
+ );
3860
+ }
3861
+ return { partitionId, assignment };
3862
+ }
3863
+ function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
3864
+ const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
3865
+ if (requiredActiveCopies <= 1) {
3866
+ return;
3867
+ }
3868
+ const activeCopies = 1 + getInSyncReplicaTargets(assignment, deps.nodeId).length;
3869
+ if (activeCopies < requiredActiveCopies) {
3870
+ throw new NarsilError(
3871
+ ErrorCodes.INSUFFICIENT_REPLICAS,
3872
+ `Write to partition ${partitionId} of index '${indexName}' requires ${requiredActiveCopies} active copies but only ${activeCopies} are available`,
3873
+ { indexName, partitionId, requiredActiveCopies, activeCopies }
3874
+ );
3875
+ }
3876
+ }
3877
+ function getInSyncReplicaTargets(assignment, localNodeId, pendingAdmissions = []) {
3878
+ const configuredReplicas = new Set(assignment.replicas);
3879
+ const targets = [];
3880
+ for (const nodeId of [...assignment.inSyncSet, ...pendingAdmissions]) {
3881
+ if (nodeId !== localNodeId && configuredReplicas.has(nodeId) && !targets.includes(nodeId)) {
3882
+ targets.push(nodeId);
3883
+ }
3161
3884
  }
3162
- const schemaError = validateRestoredSchema(deps.engine, indexName, primaryNodeId, coordinatorSchema);
3163
- if (schemaError !== null) {
3164
- await dropRestoredIndexQuietly(deps.engine, indexName, deps);
3165
- surfaceError(deps, indexName, primaryNodeId, schemaError);
3885
+ return targets;
3886
+ }
3887
+
3888
+ // src/distribution/cluster-node/catch-up/state.ts
3889
+ var CATCH_UP_TICK_MS = 1e3;
3890
+ var CATCH_UP_IN_FLIGHT_BYTE_CEILING = 67108864;
3891
+ function partitionKey2(indexName, partitionId) {
3892
+ return `${indexName}:${partitionId}`;
3893
+ }
3894
+ function createCatchUpState() {
3895
+ return {
3896
+ cursors: /* @__PURE__ */ new Map(),
3897
+ pendingAdmissions: /* @__PURE__ */ new Map(),
3898
+ inFlightBytes: 0,
3899
+ timer: null,
3900
+ activeTick: null,
3901
+ stopped: false
3902
+ };
3903
+ }
3904
+ function recordReplicaPosition(state, indexName, partitionId, replicaNodeId, appliedSeqNo) {
3905
+ const key = partitionKey2(indexName, partitionId);
3906
+ const partition = state.cursors.get(key) ?? /* @__PURE__ */ new Map();
3907
+ const existing = partition.get(replicaNodeId);
3908
+ if (existing === void 0) {
3909
+ partition.set(replicaNodeId, { appliedSeqNo, sending: false, syncEpoch: 0 });
3910
+ } else if (appliedSeqNo > existing.appliedSeqNo) {
3911
+ existing.appliedSeqNo = appliedSeqNo;
3912
+ } else if (appliedSeqNo < existing.appliedSeqNo) {
3913
+ existing.appliedSeqNo = appliedSeqNo;
3914
+ existing.syncEpoch += 1;
3915
+ }
3916
+ state.cursors.set(key, partition);
3917
+ }
3918
+ function forgetReplica(state, indexName, partitionId, replicaNodeId) {
3919
+ const key = partitionKey2(indexName, partitionId);
3920
+ const partition = state.cursors.get(key);
3921
+ if (partition !== void 0) {
3922
+ partition.delete(replicaNodeId);
3923
+ if (partition.size === 0) {
3924
+ state.cursors.delete(key);
3925
+ }
3926
+ }
3927
+ clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
3928
+ }
3929
+ function getPendingAdmissions(state, indexName, partitionId) {
3930
+ const pending = state.pendingAdmissions.get(partitionKey2(indexName, partitionId));
3931
+ return pending === void 0 ? [] : [...pending];
3932
+ }
3933
+ function markPendingAdmission(state, indexName, partitionId, replicaNodeId) {
3934
+ const key = partitionKey2(indexName, partitionId);
3935
+ const pending = state.pendingAdmissions.get(key) ?? /* @__PURE__ */ new Set();
3936
+ if (pending.has(replicaNodeId)) {
3166
3937
  return false;
3167
3938
  }
3939
+ pending.add(replicaNodeId);
3940
+ state.pendingAdmissions.set(key, pending);
3168
3941
  return true;
3169
3942
  }
3170
-
3171
- // src/distribution/cluster-node/bootstrap-sync/index.ts
3172
- async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
3173
- const validationError = validateArguments(indexName, partitionId, primaryNodeId);
3174
- if (validationError !== null) {
3175
- surfaceError(deps, indexName, primaryNodeId, validationError);
3176
- return false;
3943
+ function clearPendingAdmission(state, indexName, partitionId, replicaNodeId) {
3944
+ const key = partitionKey2(indexName, partitionId);
3945
+ const pending = state.pendingAdmissions.get(key);
3946
+ if (pending === void 0) {
3947
+ return;
3177
3948
  }
3178
- const key = entryKey(indexName, partitionId);
3179
- if (state.completed.has(key)) {
3180
- if (!await replicaResyncRequired(indexName, partitionId, deps)) {
3181
- return true;
3182
- }
3183
- clearBootstrapSyncIndex(state, indexName, partitionId);
3949
+ pending.delete(replicaNodeId);
3950
+ if (pending.size === 0) {
3951
+ state.pendingAdmissions.delete(key);
3184
3952
  }
3185
- const existing = state.inFlight.get(key);
3186
- if (existing !== void 0) {
3187
- const result = await Promise.race([existing.promise, existing.abortPromise]);
3188
- if (result === ABORT_SENTINEL) {
3189
- return false;
3953
+ }
3954
+
3955
+ // src/distribution/cluster-node/catch-up/admission.ts
3956
+ var ADMISSION_TIMEOUT_MS = 1e4;
3957
+ async function sendToController(payload, controllerNodeId, deps) {
3958
+ const targets = await resolveNodeTargets(controllerNodeId, deps);
3959
+ for (const target of targets) {
3960
+ try {
3961
+ const result = await requestInsyncAdmission(payload, target, deps.transport, deps.nodeId);
3962
+ return result.accepted;
3963
+ } catch (_) {
3190
3964
  }
3191
- return result;
3192
3965
  }
3193
- const entry = createEntry(state, indexName, partitionId, key);
3194
- entry.promise = executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps).finally(() => {
3195
- const current = state.inFlight.get(key);
3196
- if (current === entry) {
3197
- state.inFlight.delete(key);
3198
- }
3199
- if (!state.completed.has(key) && !state.inFlight.has(key)) {
3200
- const currentGen = state.generations.get(key);
3201
- if (currentGen !== void 0 && currentGen !== entry.generation) {
3202
- state.generations.delete(key);
3203
- }
3204
- }
3966
+ return false;
3967
+ }
3968
+ function withTimeout(work, timeoutMs, fallback) {
3969
+ return new Promise((resolve) => {
3970
+ const timer = setTimeout(() => resolve(fallback), timeoutMs);
3971
+ timer.unref?.();
3972
+ work.then((value) => {
3973
+ clearTimeout(timer);
3974
+ resolve(value);
3975
+ }).catch(() => {
3976
+ clearTimeout(timer);
3977
+ resolve(fallback);
3978
+ });
3205
3979
  });
3206
- state.inFlight.set(key, entry);
3207
- return entry.promise;
3208
3980
  }
3209
- async function replicaResyncRequired(indexName, partitionId, deps) {
3210
- const allocation = await deps.coordinator.getAllocation(indexName);
3211
- const assignment = allocation?.assignments.get(partitionId);
3212
- if (assignment === void 0) {
3213
- return false;
3981
+ async function withdrawAdmission(payload, controllerNodeId, deps) {
3982
+ const targets = await resolveNodeTargets(controllerNodeId, deps);
3983
+ for (const target of targets) {
3984
+ try {
3985
+ const result = await requestInsyncRemoval(
3986
+ payload.indexName,
3987
+ payload.partitionId,
3988
+ payload.replicaNodeId,
3989
+ payload.primaryTerm,
3990
+ target,
3991
+ deps.transport,
3992
+ deps.nodeId
3993
+ );
3994
+ if (result.accepted) {
3995
+ return;
3996
+ }
3997
+ } catch (_) {
3998
+ }
3214
3999
  }
3215
- return assignment.state === "ACTIVE" && assignment.replicas.includes(deps.sourceNodeId) && !assignment.inSyncSet.includes(deps.sourceNodeId);
3216
4000
  }
3217
- async function executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps) {
3218
- const deadlineMs = deps.deadlineMs ?? DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS;
3219
- const deadline = Date.now() + deadlineMs;
3220
- const key = entryKey(indexName, partitionId);
3221
- const abortCheck = () => {
3222
- const currentGeneration = state.generations.get(key) ?? 0;
3223
- if (currentGeneration !== entry.generation) {
3224
- entry.aborted = true;
4001
+ async function proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
4002
+ const log = deps.getReplicationLog(indexName, partitionId);
4003
+ if (cursor.appliedSeqNo < log.commitPoint) {
4004
+ return;
4005
+ }
4006
+ if (!markPendingAdmission(state, indexName, partitionId, replicaNodeId)) {
4007
+ return;
4008
+ }
4009
+ const proposedEpoch = cursor.syncEpoch;
4010
+ try {
4011
+ const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
4012
+ if (controllerNodeId === null) {
4013
+ return;
3225
4014
  }
3226
- return entry.aborted;
3227
- };
3228
- if (hasLiveBootstrapSyncDeps(deps)) {
3229
- return executeLiveBootstrapSync(
3230
- state,
3231
- entry,
4015
+ const payload = {
3232
4016
  indexName,
3233
4017
  partitionId,
3234
- primaryNodeId,
3235
- deps,
3236
- deadlineMs,
3237
- deadline,
3238
- abortCheck
3239
- );
4018
+ replicaNodeId,
4019
+ primaryTerm: assignment.primaryTerm,
4020
+ appliedSeqNo: cursor.appliedSeqNo,
4021
+ commitPoint: log.commitPoint
4022
+ };
4023
+ const accepted = await withTimeout(sendToController(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
4024
+ if (accepted && cursor.syncEpoch !== proposedEpoch) {
4025
+ await withTimeout(withdrawAdmission(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, void 0);
4026
+ }
4027
+ } finally {
4028
+ clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
3240
4029
  }
3241
- const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
3242
- if (schemaResult === "aborted") {
3243
- surfaceAborted(deps, indexName, primaryNodeId);
3244
- return false;
4030
+ }
4031
+ async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
4032
+ const message = createEntryMessage(entry, sourceNodeId);
4033
+ return sendAndCollectAcks(message, entry, inSyncReplicas, transport, resolveNodeTargets3);
4034
+ }
4035
+ async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
4036
+ if (entries.length === 0) {
4037
+ return { acknowledged: [], failed: [] };
3245
4038
  }
3246
- if (schemaResult === null) {
3247
- return false;
4039
+ if (entries.length === 1) {
4040
+ return replicateToReplicas(entries[0], inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3);
3248
4041
  }
3249
- if (abortCheck()) {
3250
- surfaceAborted(deps, indexName, primaryNodeId);
4042
+ const message = createEntryBatchMessage(entries, sourceNodeId);
4043
+ const lastEntry = entries[entries.length - 1];
4044
+ return sendAndCollectAcks(message, lastEntry, inSyncReplicas, transport, resolveNodeTargets3);
4045
+ }
4046
+ async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport, resolveNodeTargets3) {
4047
+ const uniqueReplicas = [...new Set(inSyncReplicas)];
4048
+ if (uniqueReplicas.length === 0) {
4049
+ return { acknowledged: [], failed: [] };
4050
+ }
4051
+ const sendResults = await Promise.allSettled(
4052
+ uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
4053
+ );
4054
+ const acknowledged = [];
4055
+ const failed = [];
4056
+ for (let i = 0; i < sendResults.length; i++) {
4057
+ const result = sendResults[i];
4058
+ const replicaNodeId = uniqueReplicas[i];
4059
+ if (result.status === "rejected") {
4060
+ if (result.reason instanceof TransportError) {
4061
+ failed.push(replicaNodeId);
4062
+ continue;
4063
+ }
4064
+ throw result.reason;
4065
+ }
4066
+ const response = result.value;
4067
+ if (isMatchingAck(response, ackEntry)) {
4068
+ acknowledged.push(replicaNodeId);
4069
+ } else {
4070
+ failed.push(replicaNodeId);
4071
+ }
4072
+ }
4073
+ return { acknowledged, failed };
4074
+ }
4075
+ async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
4076
+ const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
4077
+ const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
4078
+ let lastError;
4079
+ for (const target of resolvedTargets) {
4080
+ try {
4081
+ return await transport.send(target, message);
4082
+ } catch (error) {
4083
+ lastError = error;
4084
+ }
4085
+ }
4086
+ throw lastError instanceof Error ? lastError : new Error(String(lastError));
4087
+ }
4088
+ function isMatchingAck(response, entry) {
4089
+ if (response.type !== ReplicationMessageTypes.ACK) {
3251
4090
  return false;
3252
4091
  }
3253
- if (Date.now() >= deadline) {
3254
- surfaceError(
3255
- deps,
3256
- indexName,
3257
- primaryNodeId,
3258
- new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
3259
- indexName,
3260
- deadlineMs
3261
- })
3262
- );
4092
+ try {
4093
+ const payload = validateAckPayload(decode(response.payload));
4094
+ return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
4095
+ } catch {
3263
4096
  return false;
3264
4097
  }
3265
- const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
3266
- if (targetsResult instanceof NarsilError) {
3267
- surfaceError(deps, indexName, primaryNodeId, targetsResult);
4098
+ }
4099
+
4100
+ // src/distribution/cluster-node/catch-up/pump.ts
4101
+ var ENTRY_FIXED_OVERHEAD_BYTES = 40;
4102
+ function entryBytes(entry) {
4103
+ return ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
4104
+ }
4105
+ function batchBytes(entries) {
4106
+ let total = 0;
4107
+ for (const entry of entries) {
4108
+ total += entryBytes(entry);
4109
+ }
4110
+ return total;
4111
+ }
4112
+ function replicaHasFallenOutOfRetention(log, cursor) {
4113
+ const oldest = log.oldestSeqNo;
4114
+ if (oldest === void 0) {
3268
4115
  return false;
3269
4116
  }
3270
- const fetchDeps = { transport: deps.transport, sourceNodeId: deps.sourceNodeId };
3271
- const fetchResult = await fetchSnapshotFromAnyTarget(
3272
- indexName,
3273
- primaryNodeId,
3274
- targetsResult,
3275
- deadline,
3276
- fetchDeps,
3277
- abortCheck,
3278
- partitionId
3279
- );
3280
- if (!fetchResult.ok) {
3281
- if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
3282
- surfaceAborted(deps, indexName, primaryNodeId);
3283
- return false;
3284
- }
3285
- surfaceError(
3286
- deps,
3287
- indexName,
3288
- primaryNodeId,
3289
- new NarsilError(fetchResult.code, fetchResult.message, {
3290
- indexName,
3291
- primaryNodeId,
3292
- ...fetchResult.details
3293
- })
4117
+ return cursor.appliedSeqNo + 1 < oldest;
4118
+ }
4119
+ function nextWireBatch(state, pending) {
4120
+ const chunks = chunkByBudget(pending, {
4121
+ ...WIRE_BATCH_BUDGET,
4122
+ payloadBytesOf: (entry) => entry.document?.byteLength ?? 0,
4123
+ breaksRun: (entry, previous) => entry.seqNo !== previous.seqNo + 1
4124
+ });
4125
+ const batch = chunks[0] ?? [];
4126
+ if (batch.length === 0) {
4127
+ return [];
4128
+ }
4129
+ const available = CATCH_UP_IN_FLIGHT_BYTE_CEILING - state.inFlightBytes;
4130
+ return batchBytes(batch) > available ? [] : batch;
4131
+ }
4132
+ async function pumpReplica(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
4133
+ if (cursor.sending) {
4134
+ return;
4135
+ }
4136
+ const log = deps.getReplicationLog(indexName, partitionId);
4137
+ if (replicaHasFallenOutOfRetention(log, cursor)) {
4138
+ forgetReplica(state, indexName, partitionId, replicaNodeId);
4139
+ return;
4140
+ }
4141
+ if (cursor.appliedSeqNo >= log.localLogEnd) {
4142
+ await proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
4143
+ return;
4144
+ }
4145
+ const batch = nextWireBatch(state, log.getEntriesFrom(cursor.appliedSeqNo + 1));
4146
+ if (batch.length === 0) {
4147
+ return;
4148
+ }
4149
+ const reservedBytes = batchBytes(batch);
4150
+ const sendingEpoch = cursor.syncEpoch;
4151
+ cursor.sending = true;
4152
+ state.inFlightBytes += reservedBytes;
4153
+ try {
4154
+ const result = await replicateBatchToReplicas(
4155
+ batch,
4156
+ [replicaNodeId],
4157
+ deps.transport,
4158
+ deps.nodeId,
4159
+ deps.resolveNodeTargets
3294
4160
  );
3295
- return false;
4161
+ if (result.acknowledged.includes(replicaNodeId) && cursor.syncEpoch === sendingEpoch) {
4162
+ cursor.appliedSeqNo = batch[batch.length - 1].seqNo;
4163
+ }
4164
+ } finally {
4165
+ state.inFlightBytes -= reservedBytes;
4166
+ cursor.sending = false;
4167
+ }
4168
+ }
4169
+ async function pumpPartition(state, deps, key, replicas) {
4170
+ const separator = key.lastIndexOf(":");
4171
+ const indexName = key.slice(0, separator);
4172
+ const partitionId = Number(key.slice(separator + 1));
4173
+ const table = await deps.coordinator.getAllocation(indexName);
4174
+ const assignment = table?.assignments.get(partitionId);
4175
+ if (assignment === void 0 || assignment.primary !== deps.nodeId) {
4176
+ state.cursors.delete(key);
4177
+ state.pendingAdmissions.delete(key);
4178
+ return;
3296
4179
  }
3297
- if (abortCheck()) {
3298
- surfaceAborted(deps, indexName, primaryNodeId);
3299
- return false;
4180
+ for (const [replicaNodeId, cursor] of [...replicas]) {
4181
+ if (state.stopped) {
4182
+ return;
4183
+ }
4184
+ if (!assignment.replicas.includes(replicaNodeId) || assignment.inSyncSet.includes(replicaNodeId)) {
4185
+ forgetReplica(state, indexName, partitionId, replicaNodeId);
4186
+ continue;
4187
+ }
4188
+ try {
4189
+ await pumpReplica(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
4190
+ } catch (_) {
4191
+ }
3300
4192
  }
3301
- const restoreSucceeded = await applyRestore(
3302
- state,
3303
- entry,
3304
- indexName,
3305
- primaryNodeId,
3306
- fetchResult.bytes,
3307
- schemaResult,
3308
- deadline,
3309
- deps
3310
- );
3311
- if (!restoreSucceeded) {
3312
- return false;
4193
+ }
4194
+ function runCatchUpTick(state, deps) {
4195
+ if (state.stopped || state.activeTick !== null || state.cursors.size === 0) {
4196
+ return state.activeTick ?? Promise.resolve();
3313
4197
  }
3314
- if (abortCheck()) {
3315
- if (!anotherBootstrapOwnsKey(state, key, entry)) {
3316
- await dropRestoredIndexQuietly(deps.engine, indexName, deps);
4198
+ const tick = (async () => {
4199
+ for (const [key, replicas] of [...state.cursors]) {
4200
+ if (state.stopped) {
4201
+ return;
4202
+ }
4203
+ try {
4204
+ await pumpPartition(state, deps, key, replicas);
4205
+ } catch (_) {
4206
+ }
3317
4207
  }
3318
- surfaceAborted(deps, indexName, primaryNodeId);
3319
- return false;
4208
+ })().finally(() => {
4209
+ state.activeTick = null;
4210
+ });
4211
+ state.activeTick = tick;
4212
+ return tick;
4213
+ }
4214
+ function startCatchUpPump(state, deps) {
4215
+ void stopCatchUpPump(state);
4216
+ state.stopped = false;
4217
+ state.timer = setInterval(() => {
4218
+ void runCatchUpTick(state, deps);
4219
+ }, CATCH_UP_TICK_MS);
4220
+ state.timer.unref?.();
4221
+ }
4222
+ async function stopCatchUpPump(state) {
4223
+ state.stopped = true;
4224
+ if (state.timer !== null) {
4225
+ clearInterval(state.timer);
4226
+ state.timer = null;
4227
+ }
4228
+ if (state.activeTick !== null) {
4229
+ await state.activeTick;
3320
4230
  }
3321
- deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
3322
- state.completed.add(key);
3323
- return true;
3324
4231
  }
3325
4232
 
3326
4233
  // src/distribution/cluster-node/local-engine.ts
@@ -3663,21 +4570,14 @@ async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
3663
4570
  }
3664
4571
  let sawAssignment = false;
3665
4572
  for (const assignment of allocation.assignments.values()) {
3666
- const isPrimary = assignment.primary === sourceId;
3667
- const isReplica = assignment.replicas.includes(sourceId);
3668
- if (!isPrimary && !isReplica) {
4573
+ const isAssigned = assignment.primary === sourceId || assignment.replicas.includes(sourceId);
4574
+ if (!isAssigned) {
3669
4575
  continue;
3670
4576
  }
3671
4577
  sawAssignment = true;
3672
4578
  if (!BOOTSTRAPPABLE_STATES.has(assignment.state)) {
3673
4579
  continue;
3674
4580
  }
3675
- if (isPrimary) {
3676
- return { outcome: "authorized" };
3677
- }
3678
- if (!assignmentAllowsReplicaBootstrap(assignment, sourceId)) {
3679
- continue;
3680
- }
3681
4581
  return { outcome: "authorized" };
3682
4582
  }
3683
4583
  if (sawAssignment) {
@@ -3693,12 +4593,6 @@ async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
3693
4593
  reason: `node '${sourceId}' is not an assigned replica for index '${indexName}'`
3694
4594
  };
3695
4595
  }
3696
- function assignmentAllowsReplicaBootstrap(assignment, sourceId) {
3697
- if (assignment.state === "INITIALISING" || assignment.state === "MIGRATING") {
3698
- return true;
3699
- }
3700
- return assignment.inSyncSet.includes(sourceId);
3701
- }
3702
4596
 
3703
4597
  // src/distribution/cluster-node/snapshot-cache.ts
3704
4598
  var DEFAULT_MAX_CONCURRENT_SNAPSHOTS = 2;
@@ -4217,7 +5111,7 @@ var NULL_BYTE = String.fromCharCode(0);
4217
5111
  function throwInvalid(code, message, details) {
4218
5112
  throw new NarsilError(code, message, details);
4219
5113
  }
4220
- function isRecord3(value) {
5114
+ function isRecord2(value) {
4221
5115
  return value !== null && typeof value === "object" && !Array.isArray(value);
4222
5116
  }
4223
5117
  function isInteger(value) {
@@ -4335,14 +5229,14 @@ function validateIndexNameField(value, fieldLabel) {
4335
5229
  return value;
4336
5230
  }
4337
5231
  function validateDocumentId(value, fieldLabel) {
4338
- if (!isRecord3(value)) {
5232
+ if (!isRecord2(value)) {
4339
5233
  throwInvalid(CONFIG_INVALID, `Invalid FetchPayload: "${fieldLabel}" must be an object`);
4340
5234
  }
4341
5235
  validateStringField(value.docId, `${fieldLabel}.docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
4342
5236
  validatePartitionId(value.partitionId, `${fieldLabel}.partitionId`, CONFIG_INVALID);
4343
5237
  }
4344
5238
  function validateHighlightConfig(value) {
4345
- if (!isRecord3(value)) {
5239
+ if (!isRecord2(value)) {
4346
5240
  throwInvalid(CONFIG_INVALID, 'Invalid FetchPayload: "highlight" must be an object or null');
4347
5241
  }
4348
5242
  if (value.fields !== null) {
@@ -4368,7 +5262,7 @@ function validateHighlightConfig(value) {
4368
5262
  }
4369
5263
  }
4370
5264
  function validateFetchPayload(decoded) {
4371
- if (!isRecord3(decoded)) {
5265
+ if (!isRecord2(decoded)) {
4372
5266
  throwInvalid(CONFIG_INVALID, "Invalid FetchPayload: expected an object");
4373
5267
  }
4374
5268
  validateIndexNameField(decoded.indexName, "indexName");
@@ -4394,7 +5288,7 @@ function validateFetchPayload(decoded) {
4394
5288
  return decoded;
4395
5289
  }
4396
5290
  function validateFetchResultPayload(decoded) {
4397
- if (!isRecord3(decoded)) {
5291
+ if (!isRecord2(decoded)) {
4398
5292
  throwInvalid(CONFIG_INVALID, "Invalid FetchResultPayload: expected an object");
4399
5293
  }
4400
5294
  if (!Array.isArray(decoded.documents)) {
@@ -4402,14 +5296,14 @@ function validateFetchResultPayload(decoded) {
4402
5296
  }
4403
5297
  for (let i = 0; i < decoded.documents.length; i++) {
4404
5298
  const doc = decoded.documents[i];
4405
- if (!isRecord3(doc)) {
5299
+ if (!isRecord2(doc)) {
4406
5300
  throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}]" must be an object`);
4407
5301
  }
4408
5302
  validateStringField(doc.docId, `documents[${i}].docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
4409
- if (!isRecord3(doc.document)) {
5303
+ if (!isRecord2(doc.document)) {
4410
5304
  throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}].document" must be an object`);
4411
5305
  }
4412
- if (doc.highlights !== null && !isRecord3(doc.highlights)) {
5306
+ if (doc.highlights !== null && !isRecord2(doc.highlights)) {
4413
5307
  throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}].highlights" must be an object or null`);
4414
5308
  }
4415
5309
  }
@@ -4506,7 +5400,7 @@ function isKnownLeafKey(key) {
4506
5400
  return FIELD_FILTER_OPERATORS.includes(key);
4507
5401
  }
4508
5402
  function validateGeoRadius(value, fieldLabel, errorCode) {
4509
- if (!isRecord3(value)) {
5403
+ if (!isRecord2(value)) {
4510
5404
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
4511
5405
  }
4512
5406
  validateNumericOperand(value.lat, `${fieldLabel}.lat`, errorCode);
@@ -4526,7 +5420,7 @@ function validateGeoRadius(value, fieldLabel, errorCode) {
4526
5420
  }
4527
5421
  }
4528
5422
  function validateGeoPolygon(value, fieldLabel, errorCode) {
4529
- if (!isRecord3(value)) {
5423
+ if (!isRecord2(value)) {
4530
5424
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
4531
5425
  }
4532
5426
  if (!Array.isArray(value.points)) {
@@ -4542,7 +5436,7 @@ function validateGeoPolygon(value, fieldLabel, errorCode) {
4542
5436
  }
4543
5437
  for (let i = 0; i < points.length; i++) {
4544
5438
  const point = points[i];
4545
- if (!isRecord3(point)) {
5439
+ if (!isRecord2(point)) {
4546
5440
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}.points[${i}]" must be an object`);
4547
5441
  }
4548
5442
  validateNumericOperand(point.lat, `${fieldLabel}.points[${i}].lat`, errorCode);
@@ -4583,7 +5477,7 @@ function validateLeafOperator(opKey, opValue, leafLabel, errorCode) {
4583
5477
  return;
4584
5478
  }
4585
5479
  if (opKey === "size") {
4586
- if (!isRecord3(opValue)) {
5480
+ if (!isRecord2(opValue)) {
4587
5481
  throwInvalid(errorCode, `Invalid payload: "${opLabel}" must be an object`);
4588
5482
  }
4589
5483
  for (const [innerKey, innerValue] of Object.entries(opValue)) {
@@ -4605,7 +5499,7 @@ function validateLeafOperator(opKey, opValue, leafLabel, errorCode) {
4605
5499
  throwInvalid(errorCode, `Invalid payload: "${leafLabel}" contains unsupported operator "${opKey}"`);
4606
5500
  }
4607
5501
  function validateFieldFilter(value, fieldLabel, errorCode) {
4608
- if (!isRecord3(value)) {
5502
+ if (!isRecord2(value)) {
4609
5503
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
4610
5504
  }
4611
5505
  const entries = Object.entries(value);
@@ -4623,7 +5517,7 @@ function validateFieldFilter(value, fieldLabel, errorCode) {
4623
5517
  }
4624
5518
  }
4625
5519
  function validateFieldsRecord(value, fieldLabel, errorCode) {
4626
- if (!isRecord3(value)) {
5520
+ if (!isRecord2(value)) {
4627
5521
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
4628
5522
  }
4629
5523
  const entries = Object.entries(value);
@@ -4666,7 +5560,7 @@ function validateFilterExpression(value, fieldLabel, errorCode = SEARCH_INVALID_
4666
5560
  limit: MAX_FILTER_DEPTH
4667
5561
  });
4668
5562
  }
4669
- if (!isRecord3(value)) {
5563
+ if (!isRecord2(value)) {
4670
5564
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
4671
5565
  }
4672
5566
  const entries = Object.entries(value);
@@ -4704,7 +5598,7 @@ function validateIndexNameField2(value, fieldLabel) {
4704
5598
  return value;
4705
5599
  }
4706
5600
  function validateVectorParams(value) {
4707
- if (!isRecord3(value)) {
5601
+ if (!isRecord2(value)) {
4708
5602
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.vector" must be an object');
4709
5603
  }
4710
5604
  validateFieldName(value.field, "params.vector.field", SEARCH_INVALID_FIELD);
@@ -4749,7 +5643,7 @@ function validateVectorParams(value) {
4749
5643
  }
4750
5644
  }
4751
5645
  function validateHybridParams(value) {
4752
- if (!isRecord3(value)) {
5646
+ if (!isRecord2(value)) {
4753
5647
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.hybrid" must be an object');
4754
5648
  }
4755
5649
  if (!ALLOWED_HYBRID_STRATEGIES.includes(value.strategy)) {
@@ -4769,7 +5663,7 @@ function validateHybridParams(value) {
4769
5663
  }
4770
5664
  }
4771
5665
  function validateGroupParams(value) {
4772
- if (!isRecord3(value)) {
5666
+ if (!isRecord2(value)) {
4773
5667
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.group" must be an object');
4774
5668
  }
4775
5669
  validateFieldName(value.field, "params.group.field", SEARCH_INVALID_FIELD);
@@ -4796,7 +5690,7 @@ function validateSortParams(value) {
4796
5690
  }
4797
5691
  for (let i = 0; i < value.length; i++) {
4798
5692
  const entry = value[i];
4799
- if (!isRecord3(entry)) {
5693
+ if (!isRecord2(entry)) {
4800
5694
  throwInvalid(CONFIG_INVALID, `Invalid SearchPayload: "params.sort[${i}]" must be an object`);
4801
5695
  }
4802
5696
  validateFieldName(entry.field, `params.sort[${i}].field`, SEARCH_INVALID_FIELD);
@@ -4809,7 +5703,7 @@ function validateSortParams(value) {
4809
5703
  }
4810
5704
  }
4811
5705
  function validateBoostParams(value) {
4812
- if (!isRecord3(value)) {
5706
+ if (!isRecord2(value)) {
4813
5707
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.boost" must be an object or null');
4814
5708
  }
4815
5709
  const entries = Object.entries(value);
@@ -4828,16 +5722,16 @@ function validateBoostParams(value) {
4828
5722
  }
4829
5723
  }
4830
5724
  function validateGlobalStatsShape(value) {
4831
- if (!isRecord3(value)) {
5725
+ if (!isRecord2(value)) {
4832
5726
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats" must be an object or null');
4833
5727
  }
4834
5728
  if (!isFiniteNumber(value.totalDocuments) || value.totalDocuments < 0) {
4835
5729
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.totalDocuments" must be a non-negative number');
4836
5730
  }
4837
- if (!isRecord3(value.docFrequencies)) {
5731
+ if (!isRecord2(value.docFrequencies)) {
4838
5732
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.docFrequencies" must be an object');
4839
5733
  }
4840
- if (!isRecord3(value.totalFieldLengths)) {
5734
+ if (!isRecord2(value.totalFieldLengths)) {
4841
5735
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.totalFieldLengths" must be an object');
4842
5736
  }
4843
5737
  }
@@ -4927,18 +5821,18 @@ function validateParams(params) {
4927
5821
  }
4928
5822
  }
4929
5823
  function validateWireParams(params) {
4930
- if (!isRecord3(params)) {
5824
+ if (!isRecord2(params)) {
4931
5825
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params" must be an object');
4932
5826
  }
4933
5827
  validateParams(params);
4934
5828
  }
4935
5829
  function validateSearchPayload(decoded) {
4936
- if (!isRecord3(decoded)) {
5830
+ if (!isRecord2(decoded)) {
4937
5831
  throwInvalid(CONFIG_INVALID, "Invalid SearchPayload: expected an object");
4938
5832
  }
4939
5833
  validateIndexNameField2(decoded.indexName, "indexName");
4940
5834
  validatePartitionIdsArray(decoded.partitionIds, "partitionIds", CONFIG_INVALID);
4941
- if (!isRecord3(decoded.params)) {
5835
+ if (!isRecord2(decoded.params)) {
4942
5836
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params" must be an object');
4943
5837
  }
4944
5838
  validateParams(decoded.params);
@@ -4974,7 +5868,7 @@ function validateIndexNameField3(decoded, payloadLabel) {
4974
5868
  validateStringField(decoded.indexName, `${payloadLabel}.indexName`, 255, CONFIG_INVALID);
4975
5869
  }
4976
5870
  function validateCountPayload(decoded) {
4977
- if (!isRecord3(decoded)) {
5871
+ if (!isRecord2(decoded)) {
4978
5872
  throwInvalid(CONFIG_INVALID, "Invalid CountPayload: expected an object");
4979
5873
  }
4980
5874
  validateIndexNameField3(decoded, "CountPayload");
@@ -4982,7 +5876,7 @@ function validateCountPayload(decoded) {
4982
5876
  return decoded;
4983
5877
  }
4984
5878
  function validateCountResultPayload(decoded) {
4985
- if (!isRecord3(decoded)) {
5879
+ if (!isRecord2(decoded)) {
4986
5880
  throwInvalid(CONFIG_INVALID, "Invalid CountResultPayload: expected an object");
4987
5881
  }
4988
5882
  if (!Array.isArray(decoded.partitions)) {
@@ -4990,7 +5884,7 @@ function validateCountResultPayload(decoded) {
4990
5884
  }
4991
5885
  for (let i = 0; i < decoded.partitions.length; i++) {
4992
5886
  const entry = decoded.partitions[i];
4993
- if (!isRecord3(entry)) {
5887
+ if (!isRecord2(entry)) {
4994
5888
  throwInvalid(CONFIG_INVALID, `Invalid CountResultPayload: "partitions[${i}]" must be an object`);
4995
5889
  }
4996
5890
  validateNonNegativeInteger(entry.partitionId, `partitions[${i}].partitionId`, MAX_COUNT_VALUE, CONFIG_INVALID);
@@ -5017,7 +5911,7 @@ function validateListSort(value) {
5017
5911
  }
5018
5912
  for (let i = 0; i < value.length; i++) {
5019
5913
  const entry = value[i];
5020
- if (!isRecord3(entry)) {
5914
+ if (!isRecord2(entry)) {
5021
5915
  throwInvalid(CONFIG_INVALID, `Invalid ListPayload: "sort[${i}]" must be an object`);
5022
5916
  }
5023
5917
  validateFieldName(entry.field, `sort[${i}].field`, SEARCH_INVALID_FIELD);
@@ -5027,7 +5921,7 @@ function validateListSort(value) {
5027
5921
  }
5028
5922
  }
5029
5923
  function validateListPayload(decoded) {
5030
- if (!isRecord3(decoded)) {
5924
+ if (!isRecord2(decoded)) {
5031
5925
  throwInvalid(CONFIG_INVALID, "Invalid ListPayload: expected an object");
5032
5926
  }
5033
5927
  validateIndexNameField3(decoded, "ListPayload");
@@ -5072,7 +5966,7 @@ function validateSortValues(value, fieldLabel) {
5072
5966
  }
5073
5967
  }
5074
5968
  function validateListResultPayload(decoded) {
5075
- if (!isRecord3(decoded)) {
5969
+ if (!isRecord2(decoded)) {
5076
5970
  throwInvalid(CONFIG_INVALID, "Invalid ListResultPayload: expected an object");
5077
5971
  }
5078
5972
  if (!Array.isArray(decoded.entries)) {
@@ -5083,11 +5977,11 @@ function validateListResultPayload(decoded) {
5083
5977
  }
5084
5978
  for (let i = 0; i < decoded.entries.length; i++) {
5085
5979
  const entry = decoded.entries[i];
5086
- if (!isRecord3(entry)) {
5980
+ if (!isRecord2(entry)) {
5087
5981
  throwInvalid(CONFIG_INVALID, `Invalid ListResultPayload: "entries[${i}]" must be an object`);
5088
5982
  }
5089
5983
  validateStringField(entry.docId, `entries[${i}].docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
5090
- if (!isRecord3(entry.document)) {
5984
+ if (!isRecord2(entry.document)) {
5091
5985
  throwInvalid(CONFIG_INVALID, `Invalid ListResultPayload: "entries[${i}].document" must be an object`);
5092
5986
  }
5093
5987
  if (entry.sortValues !== null) {
@@ -5101,7 +5995,7 @@ function validateListResultPayload(decoded) {
5101
5995
  return decoded;
5102
5996
  }
5103
5997
  function validateSuggestPayload(decoded) {
5104
- if (!isRecord3(decoded)) {
5998
+ if (!isRecord2(decoded)) {
5105
5999
  throwInvalid(CONFIG_INVALID, "Invalid SuggestPayload: expected an object");
5106
6000
  }
5107
6001
  validateIndexNameField3(decoded, "SuggestPayload");
@@ -5111,7 +6005,7 @@ function validateSuggestPayload(decoded) {
5111
6005
  return decoded;
5112
6006
  }
5113
6007
  function validateSuggestResultPayload(decoded) {
5114
- if (!isRecord3(decoded)) {
6008
+ if (!isRecord2(decoded)) {
5115
6009
  throwInvalid(CONFIG_INVALID, "Invalid SuggestResultPayload: expected an object");
5116
6010
  }
5117
6011
  if (!Array.isArray(decoded.terms)) {
@@ -5125,7 +6019,7 @@ function validateSuggestResultPayload(decoded) {
5125
6019
  }
5126
6020
  for (let i = 0; i < decoded.terms.length; i++) {
5127
6021
  const entry = decoded.terms[i];
5128
- if (!isRecord3(entry)) {
6022
+ if (!isRecord2(entry)) {
5129
6023
  throwInvalid(CONFIG_INVALID, `Invalid SuggestResultPayload: "terms[${i}]" must be an object`);
5130
6024
  }
5131
6025
  validateStringField(entry.term, `terms[${i}].term`, MAX_TERM_LENGTH, CONFIG_INVALID);
@@ -5142,7 +6036,7 @@ function validateSuggestResultPayload(decoded) {
5142
6036
  return decoded;
5143
6037
  }
5144
6038
  function validatePreflightPayload(decoded) {
5145
- if (!isRecord3(decoded)) {
6039
+ if (!isRecord2(decoded)) {
5146
6040
  throwInvalid(CONFIG_INVALID, "Invalid PreflightPayload: expected an object");
5147
6041
  }
5148
6042
  validateIndexNameField3(decoded, "PreflightPayload");
@@ -5151,7 +6045,7 @@ function validatePreflightPayload(decoded) {
5151
6045
  return decoded;
5152
6046
  }
5153
6047
  function validatePreflightResultPayload(decoded) {
5154
- if (!isRecord3(decoded)) {
6048
+ if (!isRecord2(decoded)) {
5155
6049
  throwInvalid(CONFIG_INVALID, "Invalid PreflightResultPayload: expected an object");
5156
6050
  }
5157
6051
  if (!isInteger(decoded.count) || decoded.count < 0) {
@@ -5170,7 +6064,7 @@ var MAX_FACET_FIELDS = 64;
5170
6064
  var MAX_FACET_BUCKETS = 1e4;
5171
6065
  var MAX_FACET_VALUE_LENGTH = 1024;
5172
6066
  function validateScoredEntry(value, fieldLabel) {
5173
- if (!isRecord3(value)) {
6067
+ if (!isRecord2(value)) {
5174
6068
  throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
5175
6069
  }
5176
6070
  if (typeof value.docId !== "string") {
@@ -5216,7 +6110,7 @@ function validateScoredEntry(value, fieldLabel) {
5216
6110
  }
5217
6111
  }
5218
6112
  function validatePartitionSearchResult(value, fieldLabel) {
5219
- if (!isRecord3(value)) {
6113
+ if (!isRecord2(value)) {
5220
6114
  throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
5221
6115
  }
5222
6116
  validatePartitionId(value.partitionId, `${fieldLabel}.partitionId`, CONFIG_INVALID);
@@ -5241,7 +6135,7 @@ function validatePartitionSearchResult(value, fieldLabel) {
5241
6135
  }
5242
6136
  }
5243
6137
  function validateFacetBucket(value, fieldLabel) {
5244
- if (!isRecord3(value)) {
6138
+ if (!isRecord2(value)) {
5245
6139
  throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
5246
6140
  }
5247
6141
  if (typeof value.value !== "string") {
@@ -5260,7 +6154,7 @@ function validateFacetBucket(value, fieldLabel) {
5260
6154
  return { value: value.value, count: value.count };
5261
6155
  }
5262
6156
  function validateFacets(value) {
5263
- if (!isRecord3(value)) {
6157
+ if (!isRecord2(value)) {
5264
6158
  throwInvalid(CONFIG_INVALID, 'Invalid SearchResultPayload: "facets" must be an object or null');
5265
6159
  }
5266
6160
  const entries = Object.entries(value);
@@ -5291,7 +6185,7 @@ function validateFacets(value) {
5291
6185
  }
5292
6186
  }
5293
6187
  function validateFacetErrorBounds(value) {
5294
- if (!isRecord3(value)) {
6188
+ if (!isRecord2(value)) {
5295
6189
  throwInvalid(CONFIG_INVALID, 'Invalid SearchResultPayload: "facetErrorBounds" must be an object or null');
5296
6190
  }
5297
6191
  const entries = Object.entries(value);
@@ -5312,7 +6206,7 @@ function validateFacetErrorBounds(value) {
5312
6206
  }
5313
6207
  }
5314
6208
  function validateSearchResultPayload(decoded) {
5315
- if (!isRecord3(decoded)) {
6209
+ if (!isRecord2(decoded)) {
5316
6210
  throwInvalid(CONFIG_INVALID, "Invalid SearchResultPayload: expected an object");
5317
6211
  }
5318
6212
  if (!Array.isArray(decoded.results)) {
@@ -5339,7 +6233,7 @@ function validateIndexNameField4(value, fieldLabel) {
5339
6233
  return value;
5340
6234
  }
5341
6235
  function validateNumericRecord(value, fieldLabel) {
5342
- if (!isRecord3(value)) {
6236
+ if (!isRecord2(value)) {
5343
6237
  throwInvalid(CONFIG_INVALID, `Invalid payload: "${fieldLabel}" must be an object`);
5344
6238
  }
5345
6239
  for (const [key, entry] of Object.entries(value)) {
@@ -5352,7 +6246,7 @@ function validateNumericRecord(value, fieldLabel) {
5352
6246
  }
5353
6247
  }
5354
6248
  function validateStatsPayload(decoded) {
5355
- if (!isRecord3(decoded)) {
6249
+ if (!isRecord2(decoded)) {
5356
6250
  throwInvalid(CONFIG_INVALID, "Invalid StatsPayload: expected an object");
5357
6251
  }
5358
6252
  validateIndexNameField4(decoded.indexName, "indexName");
@@ -5361,7 +6255,7 @@ function validateStatsPayload(decoded) {
5361
6255
  return decoded;
5362
6256
  }
5363
6257
  function validateStatsResultPayload(decoded) {
5364
- if (!isRecord3(decoded)) {
6258
+ if (!isRecord2(decoded)) {
5365
6259
  throwInvalid(CONFIG_INVALID, "Invalid StatsResultPayload: expected an object");
5366
6260
  }
5367
6261
  if (!isInteger(decoded.totalDocuments) || decoded.totalDocuments < 0) {
@@ -5535,7 +6429,8 @@ function distributedResultToLocal(result, documents = /* @__PURE__ */ new Map())
5535
6429
  count: result.totalHits,
5536
6430
  elapsed: 0,
5537
6431
  cursor: result.cursor ?? void 0,
5538
- facets: result.facets !== null ? convertWireFacetsToLocal(result.facets, result.facetErrorBounds) : void 0
6432
+ facets: result.facets !== null ? convertWireFacetsToLocal(result.facets, result.facetErrorBounds) : void 0,
6433
+ coverage: result.coverage
5539
6434
  };
5540
6435
  }
5541
6436
  function convertLocalSortToWire(sort) {
@@ -5761,10 +6656,10 @@ function toWireSortValue2(value) {
5761
6656
  function decideSyncTier(log, replicaLastSeqNo) {
5762
6657
  const oldest = log.oldestSeqNo;
5763
6658
  if (oldest === void 0) {
5764
- if (log.committedSeqNo === 0) {
6659
+ if (log.localLogEnd === 0) {
5765
6660
  return replicaLastSeqNo === 0 ? "incremental" : "snapshot";
5766
6661
  }
5767
- return replicaLastSeqNo >= log.committedSeqNo ? "incremental" : "snapshot";
6662
+ return replicaLastSeqNo >= log.localLogEnd ? "incremental" : "snapshot";
5768
6663
  }
5769
6664
  if (oldest <= replicaLastSeqNo + 1) {
5770
6665
  return "incremental";
@@ -5843,13 +6738,22 @@ async function handleSyncRequestMessage(message, respond, deps) {
5843
6738
  }
5844
6739
  );
5845
6740
  }
6741
+ if (message.sourceId !== deps.nodeId) {
6742
+ recordReplicaPosition(
6743
+ deps.writeDeps.catchUp,
6744
+ request.indexName,
6745
+ request.partitionId,
6746
+ message.sourceId,
6747
+ request.lastSeqNo
6748
+ );
6749
+ }
5846
6750
  const log = deps.writeDeps.getReplicationLog(request.indexName, request.partitionId);
5847
6751
  const tier = decideSyncTier(log, request.lastSeqNo);
5848
6752
  if (tier === "incremental") {
5849
6753
  await sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1));
5850
6754
  return;
5851
6755
  }
5852
- const snapshotSeqNo = log.committedSeqNo;
6756
+ const snapshotSeqNo = log.localLogEnd;
5853
6757
  const sink = createSingleResponseSink(respond);
5854
6758
  await streamValidatedSnapshotRequest(
5855
6759
  message,
@@ -5902,76 +6806,6 @@ var DEFAULT_CAPACITY = {
5902
6806
  cpuCores: 4,
5903
6807
  diskBytes: null
5904
6808
  };
5905
-
5906
- // src/distribution/cluster-node/write-routing/assignment.ts
5907
- function resolvePartitionId(docId, partitionCount) {
5908
- return fnv1a(docId) % partitionCount;
5909
- }
5910
- async function resolveNodeTargets(nodeId, deps) {
5911
- if (deps.resolveNodeTargets === void 0) {
5912
- return [nodeId];
5913
- }
5914
- const targets = await deps.resolveNodeTargets(nodeId);
5915
- return targets.length > 0 ? targets : [nodeId];
5916
- }
5917
- function requireAssignedPrimary(assignment, indexName, partitionId) {
5918
- if (assignment === void 0 || assignment.primary === null) {
5919
- throw new NarsilError(
5920
- ErrorCodes.PARTITION_UNASSIGNED,
5921
- `No primary assigned for partition ${partitionId} of index '${indexName}'`,
5922
- { indexName, partitionId }
5923
- );
5924
- }
5925
- return assignment;
5926
- }
5927
- async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrimary) {
5928
- const table = await deps.coordinator.getAllocation(indexName);
5929
- if (table === null || table.assignments.size === 0) {
5930
- if (requireLocalPrimary) {
5931
- throw new NarsilError(
5932
- ErrorCodes.QUERY_ROUTING_FAILED,
5933
- `No allocation table is available for forwarded write to index '${indexName}'`,
5934
- { indexName }
5935
- );
5936
- }
5937
- return null;
5938
- }
5939
- const partitionCount = table.assignments.size;
5940
- const partitionId = resolvePartitionId(docId, partitionCount);
5941
- const assignment = requireAssignedPrimary(table.assignments.get(partitionId), indexName, partitionId);
5942
- if (requireLocalPrimary && assignment.primary !== deps.nodeId) {
5943
- throw new NarsilError(
5944
- ErrorCodes.PARTITION_NOT_PRIMARY,
5945
- `Node '${deps.nodeId}' is not primary for partition ${partitionId} of index '${indexName}'`,
5946
- { indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
5947
- );
5948
- }
5949
- return { partitionId, assignment };
5950
- }
5951
- function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
5952
- const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
5953
- if (requiredActiveCopies <= 1) {
5954
- return;
5955
- }
5956
- const activeCopies = 1 + getInSyncReplicaTargets(assignment, deps.nodeId).length;
5957
- if (activeCopies < requiredActiveCopies) {
5958
- throw new NarsilError(
5959
- ErrorCodes.INSUFFICIENT_REPLICAS,
5960
- `Write to partition ${partitionId} of index '${indexName}' requires ${requiredActiveCopies} active copies but only ${activeCopies} are available`,
5961
- { indexName, partitionId, requiredActiveCopies, activeCopies }
5962
- );
5963
- }
5964
- }
5965
- function getInSyncReplicaTargets(assignment, localNodeId) {
5966
- const configuredReplicas = new Set(assignment.replicas);
5967
- const targets = [];
5968
- for (const nodeId of assignment.inSyncSet) {
5969
- if (nodeId !== localNodeId && configuredReplicas.has(nodeId)) {
5970
- targets.push(nodeId);
5971
- }
5972
- }
5973
- return targets;
5974
- }
5975
6809
  function assertForwardResponse(response, indexName, primaryNodeId) {
5976
6810
  const decoded = decode(response.payload);
5977
6811
  if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
@@ -6039,76 +6873,6 @@ async function sendToNode(nodeId, message, deps) {
6039
6873
  }
6040
6874
  throw lastError instanceof Error ? lastError : new Error(String(lastError));
6041
6875
  }
6042
- async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
6043
- const message = createEntryMessage(entry, sourceNodeId);
6044
- return sendAndCollectAcks(message, entry, inSyncReplicas, transport, resolveNodeTargets3);
6045
- }
6046
- async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
6047
- if (entries.length === 0) {
6048
- return { acknowledged: [], failed: [] };
6049
- }
6050
- if (entries.length === 1) {
6051
- return replicateToReplicas(entries[0], inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3);
6052
- }
6053
- const message = createEntryBatchMessage(entries, sourceNodeId);
6054
- const lastEntry = entries[entries.length - 1];
6055
- return sendAndCollectAcks(message, lastEntry, inSyncReplicas, transport, resolveNodeTargets3);
6056
- }
6057
- async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport, resolveNodeTargets3) {
6058
- const uniqueReplicas = [...new Set(inSyncReplicas)];
6059
- if (uniqueReplicas.length === 0) {
6060
- return { acknowledged: [], failed: [] };
6061
- }
6062
- const sendResults = await Promise.allSettled(
6063
- uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
6064
- );
6065
- const acknowledged = [];
6066
- const failed = [];
6067
- for (let i = 0; i < sendResults.length; i++) {
6068
- const result = sendResults[i];
6069
- const replicaNodeId = uniqueReplicas[i];
6070
- if (result.status === "rejected") {
6071
- if (result.reason instanceof TransportError) {
6072
- failed.push(replicaNodeId);
6073
- continue;
6074
- }
6075
- throw result.reason;
6076
- }
6077
- const response = result.value;
6078
- if (isMatchingAck(response, ackEntry)) {
6079
- acknowledged.push(replicaNodeId);
6080
- } else {
6081
- failed.push(replicaNodeId);
6082
- }
6083
- }
6084
- return { acknowledged, failed };
6085
- }
6086
- async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
6087
- const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
6088
- const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
6089
- let lastError;
6090
- for (const target of resolvedTargets) {
6091
- try {
6092
- return await transport.send(target, message);
6093
- } catch (error) {
6094
- lastError = error;
6095
- }
6096
- }
6097
- throw lastError instanceof Error ? lastError : new Error(String(lastError));
6098
- }
6099
- function isMatchingAck(response, entry) {
6100
- if (response.type !== ReplicationMessageTypes.ACK) {
6101
- return false;
6102
- }
6103
- try {
6104
- const payload = validateAckPayload(decode(response.payload));
6105
- return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
6106
- } catch {
6107
- return false;
6108
- }
6109
- }
6110
-
6111
- // src/distribution/cluster-node/write-routing/replication.ts
6112
6876
  async function assertPrimaryWriteAuthority(entry, deps) {
6113
6877
  const table = await deps.coordinator.getAllocation(entry.indexName);
6114
6878
  const currentAssignment = table?.assignments.get(entry.partitionId);
@@ -6204,17 +6968,32 @@ async function requestInsyncRemovalWithTargets(indexName, partitionId, replicaNo
6204
6968
  throw lastError instanceof Error ? lastError : new Error(String(lastError));
6205
6969
  }
6206
6970
  async function replicateEntry(entry, assignment, deps) {
6207
- const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId);
6971
+ const pending = getPendingAdmissions(deps.catchUp, entry.indexName, entry.partitionId);
6972
+ const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId, pending);
6208
6973
  const result = await replicateToReplicas(entry, replicaTargets, deps.transport, deps.nodeId, deps.resolveNodeTargets);
6209
- await removeFailedReplicasFromInsync(entry, result.failed, deps);
6974
+ const inSyncFailures = cancelAdmissionsForFailures(entry, result.failed, assignment, deps);
6975
+ await removeFailedReplicasFromInsync(entry, inSyncFailures, deps);
6210
6976
  await assertPrimaryWriteAuthority(entry, deps);
6977
+ deps.getReplicationLog(entry.indexName, entry.partitionId).advanceCommitPoint(entry.seqNo);
6978
+ }
6979
+ function cancelAdmissionsForFailures(entry, failedReplicas, assignment, deps) {
6980
+ const inSyncFailures = [];
6981
+ for (const replicaNodeId of failedReplicas) {
6982
+ if (assignment.inSyncSet.includes(replicaNodeId)) {
6983
+ inSyncFailures.push(replicaNodeId);
6984
+ continue;
6985
+ }
6986
+ clearPendingAdmission(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId);
6987
+ }
6988
+ return inSyncFailures;
6211
6989
  }
6212
6990
  async function replicateEntryBatch(entries, assignment, deps) {
6213
6991
  if (entries.length === 0) {
6214
6992
  return;
6215
6993
  }
6216
6994
  const lastEntry = entries[entries.length - 1];
6217
- const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId);
6995
+ const pending = getPendingAdmissions(deps.catchUp, lastEntry.indexName, lastEntry.partitionId);
6996
+ const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId, pending);
6218
6997
  const result = await replicateBatchToReplicas(
6219
6998
  entries,
6220
6999
  replicaTargets,
@@ -6222,8 +7001,10 @@ async function replicateEntryBatch(entries, assignment, deps) {
6222
7001
  deps.nodeId,
6223
7002
  deps.resolveNodeTargets
6224
7003
  );
6225
- await removeFailedReplicasFromInsync(lastEntry, result.failed, deps);
7004
+ const inSyncFailures = cancelAdmissionsForFailures(lastEntry, result.failed, assignment, deps);
7005
+ await removeFailedReplicasFromInsync(lastEntry, inSyncFailures, deps);
6226
7006
  await assertPrimaryWriteAuthority(lastEntry, deps);
7007
+ deps.getReplicationLog(lastEntry.indexName, lastEntry.partitionId).advanceCommitPoint(lastEntry.seqNo);
6227
7008
  }
6228
7009
  function chunkReplicationEntries(items) {
6229
7010
  return chunkByBudget(items, {
@@ -7174,17 +7955,17 @@ async function handleForwardBatch(message, respond, deps) {
7174
7955
  async function handleReplicationEntry(message, respond, deps) {
7175
7956
  const payload = validateEntryPayload(decode(message.payload));
7176
7957
  const { entry } = payload;
7177
- const assignment = await resolveValidatedAssignment(entry, message.sourceId, deps);
7178
- await applyEntryToLog(entry, assignment, deps);
7958
+ await resolveValidatedAssignment(entry, message.sourceId, deps);
7959
+ await applyEntryToLog(entry, deps);
7179
7960
  await respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
7180
7961
  }
7181
7962
  async function handleReplicationEntryBatch(message, respond, deps) {
7182
7963
  const payload = validateEntryBatchPayload(decode(message.payload));
7183
7964
  const firstEntry = payload.entries[0];
7184
7965
  const lastEntry = payload.entries[payload.entries.length - 1];
7185
- const assignment = await resolveValidatedAssignment(firstEntry, message.sourceId, deps);
7966
+ await resolveValidatedAssignment(firstEntry, message.sourceId, deps);
7186
7967
  for (const entry of payload.entries) {
7187
- await applyEntryToLog(entry, assignment, deps);
7968
+ await applyEntryToLog(entry, deps);
7188
7969
  }
7189
7970
  await respond(
7190
7971
  createAckMessage(lastEntry.seqNo, lastEntry.partitionId, lastEntry.indexName, deps.nodeId, message.requestId)
@@ -7219,23 +8000,23 @@ async function resolveValidatedAssignment(entry, sourceNodeId, deps) {
7219
8000
  }
7220
8001
  );
7221
8002
  }
7222
- if (assignment.primaryTerm !== entry.primaryTerm) {
8003
+ if (entry.primaryTerm > assignment.primaryTerm) {
7223
8004
  throw new NarsilError(
7224
8005
  ErrorCodes.REPLICATION_TERM_MISMATCH,
7225
- `Replication entry term ${entry.primaryTerm} does not match allocation term ${assignment.primaryTerm}`,
8006
+ `Replication entry term ${entry.primaryTerm} is newer than allocation term ${assignment.primaryTerm}`,
7226
8007
  { indexName: entry.indexName, partitionId: entry.partitionId }
7227
8008
  );
7228
8009
  }
7229
- if (!assignment.replicas.includes(deps.nodeId) || !assignment.inSyncSet.includes(deps.nodeId)) {
8010
+ if (!assignment.replicas.includes(deps.nodeId)) {
7230
8011
  throw new NarsilError(
7231
8012
  ErrorCodes.REPLICATION_ENTRY_INVALID,
7232
- `Node '${deps.nodeId}' is not an in-sync replica for index '${entry.indexName}' partition ${entry.partitionId}`,
8013
+ `Node '${deps.nodeId}' is not an assigned replica for index '${entry.indexName}' partition ${entry.partitionId}`,
7233
8014
  { indexName: entry.indexName, partitionId: entry.partitionId, nodeId: deps.nodeId }
7234
8015
  );
7235
8016
  }
7236
8017
  return assignment;
7237
8018
  }
7238
- async function applyEntryToLog(entry, assignment, deps) {
8019
+ async function applyEntryToLog(entry, deps) {
7239
8020
  let log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
7240
8021
  const existing = log.getEntry(entry.seqNo);
7241
8022
  if (existing !== void 0) {
@@ -7261,7 +8042,7 @@ async function applyEntryToLog(entry, assignment, deps) {
7261
8042
  deps.writeDeps.resetReplicationLog(entry.indexName, entry.partitionId, entry.seqNo, entry.primaryTerm);
7262
8043
  log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
7263
8044
  }
7264
- const validation = validateReplicationEntry(entry, assignment.primaryTerm, log);
8045
+ const validation = validateReplicationEntry(entry, log.localLogEndPrimaryTerm, log);
7265
8046
  if (!validation.valid) {
7266
8047
  throw new NarsilError(
7267
8048
  validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
@@ -7950,9 +8731,11 @@ async function distributedQuery(indexName, params, deps, config, selector) {
7950
8731
  const routing = selectReplicasForQuery(allocationTable, randomSelector);
7951
8732
  const totalPartitions = allocationTable.assignments.size;
7952
8733
  if (routing.unavailablePartitions.length > 0 && !resolvedConfig.allowPartialResults) {
7953
- throw new NarsilError(ErrorCodes.QUERY_NO_ACTIVE_REPLICA, "No active replica for one or more partitions", {
7954
- unavailablePartitions: routing.unavailablePartitions
7955
- });
8734
+ throw new NarsilError(
8735
+ ErrorCodes.QUERY_PARTIAL_FAILURE,
8736
+ "No active replica serves one or more partitions, and this node refuses partial results",
8737
+ { unavailablePartitions: routing.unavailablePartitions }
8738
+ );
7956
8739
  }
7957
8740
  const isHybrid = params.hybrid !== null && params.term !== null && params.vector !== null;
7958
8741
  if (isHybrid && params.searchAfter !== null) {
@@ -8163,17 +8946,20 @@ function worstCaseCoverage(a, b) {
8163
8946
  failedPartitions: failed
8164
8947
  };
8165
8948
  }
8166
- async function activeAllocation(deps, indexName) {
8167
- const allocation = await deps.config.coordinator.getAllocation(indexName);
8949
+ function servesAnyPartition(allocation) {
8168
8950
  if (allocation === null || allocation.assignments.size === 0) {
8169
- return null;
8951
+ return false;
8170
8952
  }
8171
8953
  for (const [, assignment] of allocation.assignments) {
8172
8954
  if (assignment.state === "ACTIVE") {
8173
- return allocation;
8955
+ return true;
8174
8956
  }
8175
8957
  }
8176
- return null;
8958
+ return false;
8959
+ }
8960
+ async function activeAllocation(deps, indexName) {
8961
+ const allocation = await deps.config.coordinator.getAllocation(indexName);
8962
+ return servesAnyPartition(allocation) ? allocation : null;
8177
8963
  }
8178
8964
  function strictScatterGroups(allocation, indexName) {
8179
8965
  const routing = selectReplicasForQuery(allocation, randomSelector);
@@ -8186,8 +8972,30 @@ function strictScatterGroups(allocation, indexName) {
8186
8972
  }
8187
8973
  return Array.from(routing.nodeToPartitions.entries(), ([nodeId, partitionIds]) => ({ nodeId, partitionIds }));
8188
8974
  }
8975
+ var READ_FAILURE_BY_TRANSPORT_CODE = {
8976
+ [TransportErrorCodes.CONNECT_FAILED]: ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
8977
+ [TransportErrorCodes.PEER_UNAVAILABLE]: ErrorCodes.QUERY_NO_ACTIVE_REPLICA,
8978
+ [TransportErrorCodes.TIMEOUT]: ErrorCodes.QUERY_NODE_TIMEOUT,
8979
+ [TransportErrorCodes.MESSAGE_TOO_LARGE]: ErrorCodes.QUERY_ROUTING_FAILED,
8980
+ [TransportErrorCodes.DECODE_FAILED]: ErrorCodes.QUERY_ROUTING_FAILED
8981
+ };
8982
+ function readFailure(error, targetNodeId, indexName) {
8983
+ if (error instanceof NarsilError) {
8984
+ return error;
8985
+ }
8986
+ if (error instanceof TransportError) {
8987
+ return new NarsilError(
8988
+ READ_FAILURE_BY_TRANSPORT_CODE[error.code],
8989
+ `Node '${targetNodeId}' did not answer a read request for index '${indexName}'`,
8990
+ { indexName, targetNodeId, transportCode: error.code }
8991
+ );
8992
+ }
8993
+ return error;
8994
+ }
8189
8995
  async function sendReadRequest(deps, targetNodeId, message, indexName, validate) {
8190
- const response = await sendToNode2(deps.config, targetNodeId, message);
8996
+ const response = await sendToNode2(deps.config, targetNodeId, message).catch((error) => {
8997
+ throw readFailure(error, targetNodeId, indexName);
8998
+ });
8191
8999
  const decoded = decode(response.payload);
8192
9000
  if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
8193
9001
  throw new NarsilError(
@@ -8510,11 +9318,36 @@ async function readClusterDocuments(deps, indexName, docIds) {
8510
9318
  }
8511
9319
  return readDistributedDocuments(deps.config, deps.nodeId, deps.engine, indexName, docIds, allocation);
8512
9320
  }
9321
+ function routingConfig(query) {
9322
+ if (query === void 0) {
9323
+ return void 0;
9324
+ }
9325
+ const config = {};
9326
+ if (query.allowPartialResults !== void 0) {
9327
+ config.allowPartialResults = query.allowPartialResults;
9328
+ }
9329
+ if (query.partitionTimeout !== void 0) {
9330
+ config.partitionTimeout = query.partitionTimeout;
9331
+ }
9332
+ return config;
9333
+ }
9334
+ async function queryLocalCopy(deps, indexName, params, allocation) {
9335
+ const result = await deps.engine.query(indexName, params);
9336
+ if (allocation === null || allocation.assignments.size === 0) {
9337
+ return result;
9338
+ }
9339
+ const totalPartitions = allocation.assignments.size;
9340
+ return {
9341
+ ...result,
9342
+ coverage: { totalPartitions, queriedPartitions: 0, timedOutPartitions: 0, failedPartitions: totalPartitions }
9343
+ };
9344
+ }
8513
9345
  async function queryCluster(deps, indexName, params) {
8514
- const allocation = await activeAllocation(deps, indexName);
8515
- if (allocation === null) {
8516
- return deps.engine.query(indexName, params);
9346
+ const table = await deps.config.coordinator.getAllocation(indexName);
9347
+ if (!servesAnyPartition(table)) {
9348
+ return queryLocalCopy(deps, indexName, params, table);
8517
9349
  }
9350
+ const allocation = table;
8518
9351
  const wireParams = localParamsToWire(params);
8519
9352
  const queryDeps = {
8520
9353
  transport: deps.config.transport,
@@ -8522,7 +9355,7 @@ async function queryCluster(deps, indexName, params) {
8522
9355
  getAllocation: (idx) => deps.config.coordinator.getAllocation(idx),
8523
9356
  resolveNodeTargets: deps.resolveNodeTargets
8524
9357
  };
8525
- const distributed = await distributedQuery(indexName, wireParams, queryDeps);
9358
+ const distributed = await distributedQuery(indexName, wireParams, queryDeps, routingConfig(deps.config.query));
8526
9359
  const documents = await fetchDistributedDocuments(
8527
9360
  deps.config,
8528
9361
  deps.nodeId,
@@ -8652,16 +9485,17 @@ function createClusterNodeOperations(deps) {
8652
9485
  var DEFAULT_LOG_RETENTION_BYTES = 268435456;
8653
9486
 
8654
9487
  // src/distribution/replication/log.ts
8655
- var ENTRY_FIXED_OVERHEAD_BYTES = 40;
9488
+ var ENTRY_FIXED_OVERHEAD_BYTES2 = 40;
8656
9489
  function estimateEntrySize(entry) {
8657
- return ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
9490
+ return ENTRY_FIXED_OVERHEAD_BYTES2 + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
8658
9491
  }
8659
9492
  var computeChecksum = computeEntryChecksum;
8660
9493
  function createReplicationLog(partitionId, config) {
8661
9494
  const retentionBytes = config?.logRetentionBytes ?? DEFAULT_LOG_RETENTION_BYTES;
8662
9495
  let nextSeqNo = config?.startSeqNo ?? 1;
8663
- let committedSeqNo = nextSeqNo - 1;
8664
- let committedPrimaryTerm = config?.lastPrimaryTerm ?? 0;
9496
+ let localLogEnd = nextSeqNo - 1;
9497
+ let localLogEndPrimaryTerm = config?.lastPrimaryTerm ?? 0;
9498
+ let commitPoint = nextSeqNo - 1;
8665
9499
  let entries = [];
8666
9500
  let headIndex = 0;
8667
9501
  let totalSizeBytes = 0;
@@ -8704,7 +9538,7 @@ function createReplicationLog(partitionId, config) {
8704
9538
  }
8705
9539
  }
8706
9540
  function assertEntryFitsRetention(entry) {
8707
- const entrySize = ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
9541
+ const entrySize = ENTRY_FIXED_OVERHEAD_BYTES2 + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
8708
9542
  if (entrySize > retentionBytes) {
8709
9543
  throw new NarsilError(
8710
9544
  ErrorCodes.REPLICATION_LOG_FULL,
@@ -8715,9 +9549,9 @@ function createReplicationLog(partitionId, config) {
8715
9549
  }
8716
9550
  function storeEntry(entry) {
8717
9551
  entries.push(entry);
8718
- if (entry.seqNo >= committedSeqNo) {
8719
- committedSeqNo = entry.seqNo;
8720
- committedPrimaryTerm = entry.primaryTerm;
9552
+ if (entry.seqNo >= localLogEnd) {
9553
+ localLogEnd = entry.seqNo;
9554
+ localLogEndPrimaryTerm = entry.primaryTerm;
8721
9555
  }
8722
9556
  totalSizeBytes += estimateEntrySize(entry);
8723
9557
  evictOldEntries();
@@ -8820,11 +9654,20 @@ function createReplicationLog(partitionId, config) {
8820
9654
  verifyChecksum(entry) {
8821
9655
  return verifyEntryChecksum(entry);
8822
9656
  },
8823
- get committedSeqNo() {
8824
- return committedSeqNo;
9657
+ get localLogEnd() {
9658
+ return localLogEnd;
9659
+ },
9660
+ get localLogEndPrimaryTerm() {
9661
+ return localLogEndPrimaryTerm;
9662
+ },
9663
+ get commitPoint() {
9664
+ return commitPoint;
8825
9665
  },
8826
- get committedPrimaryTerm() {
8827
- return committedPrimaryTerm;
9666
+ advanceCommitPoint(seqNo) {
9667
+ const bounded = Math.min(seqNo, localLogEnd);
9668
+ if (bounded > commitPoint) {
9669
+ commitPoint = bounded;
9670
+ }
8828
9671
  },
8829
9672
  get oldestSeqNo() {
8830
9673
  if (liveCount() === 0) return void 0;
@@ -8844,7 +9687,8 @@ function createReplicationLog(partitionId, config) {
8844
9687
  entries = [];
8845
9688
  headIndex = 0;
8846
9689
  totalSizeBytes = 0;
8847
- committedSeqNo = nextSeqNo - 1;
9690
+ localLogEnd = nextSeqNo - 1;
9691
+ commitPoint = nextSeqNo - 1;
8848
9692
  }
8849
9693
  };
8850
9694
  }
@@ -8875,6 +9719,7 @@ function seedReplicationLog(replicationLogs, indexName, partitionId, startSeqNo,
8875
9719
 
8876
9720
  // src/distribution/cluster-node/transport-listener.ts
8877
9721
  var CONTROLLER_MESSAGE_TYPES = /* @__PURE__ */ new Set([
9722
+ ReplicationMessageTypes.INSYNC_ADD,
8878
9723
  ReplicationMessageTypes.INSYNC_REMOVE,
8879
9724
  ClusterMessageTypes.BOOTSTRAP_COMPLETE
8880
9725
  ]);
@@ -8971,6 +9816,25 @@ function validateClusterNodeConfig(config) {
8971
9816
  );
8972
9817
  }
8973
9818
  }
9819
+ if (config.query !== void 0) {
9820
+ const { allowPartialResults, partitionTimeout } = config.query;
9821
+ if (allowPartialResults !== void 0 && typeof allowPartialResults !== "boolean") {
9822
+ throw new NarsilError(
9823
+ ErrorCodes.CONFIG_INVALID,
9824
+ "ClusterNodeConfig.query.allowPartialResults must be a boolean",
9825
+ {
9826
+ allowPartialResults
9827
+ }
9828
+ );
9829
+ }
9830
+ if (partitionTimeout !== void 0 && (!Number.isInteger(partitionTimeout) || partitionTimeout <= 0)) {
9831
+ throw new NarsilError(
9832
+ ErrorCodes.CONFIG_INVALID,
9833
+ "ClusterNodeConfig.query.partitionTimeout must be a positive integer",
9834
+ { partitionTimeout }
9835
+ );
9836
+ }
9837
+ }
8974
9838
  }
8975
9839
 
8976
9840
  // src/distribution/cluster-node/index.ts
@@ -9022,6 +9886,7 @@ async function createClusterNode(config) {
9022
9886
  getReplicationLog: getReplicationLog2,
9023
9887
  resetReplicationLog: seedReplicationLog2,
9024
9888
  partitionWriteQueues: createPartitionWriteQueues(),
9889
+ catchUp: createCatchUpState(),
9025
9890
  resolveNodeTargets: resolveNodeTargets3,
9026
9891
  waitForActiveReplicas: config.replication?.waitForActiveReplicas
9027
9892
  };
@@ -9092,6 +9957,7 @@ async function createClusterNode(config) {
9092
9957
  bootstrapRetryMaxMs: DEFAULT_NODE_LIFECYCLE_CONFIG.bootstrapRetryMaxMs,
9093
9958
  bootstrapMaxRetries: DEFAULT_NODE_LIFECYCLE_CONFIG.bootstrapMaxRetries,
9094
9959
  allocationDebounceMs: DEFAULT_NODE_LIFECYCLE_CONFIG.allocationDebounceMs,
9960
+ nodeHeartbeatIntervalMs: DEFAULT_NODE_LIFECYCLE_CONFIG.nodeHeartbeatIntervalMs,
9095
9961
  onBootstrapPartition: bootstrapPartition2,
9096
9962
  onRemovePartition: (indexName, partitionId) => {
9097
9963
  clearBootstrapSyncIndex(bootstrapSyncState, indexName, partitionId);
@@ -9203,9 +10069,11 @@ async function createClusterNode(config) {
9203
10069
  });
9204
10070
  const listener = controllerTransport !== null ? controllerTransport.createHandler(dataHandler) : dataHandler;
9205
10071
  unregisterHandler = await config.transport.listen(listener);
10072
+ startCatchUpPump(writeDeps.catchUp, writeDeps);
9206
10073
  }
9207
10074
  },
9208
10075
  async shutdown() {
10076
+ await stopCatchUpPump(writeDeps.catchUp);
9209
10077
  if (isShutdown) {
9210
10078
  return;
9211
10079
  }