@delali/narsil 0.2.3-next.32 → 0.2.3-next.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
- import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, MAX_PARTITION_COUNT, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, MAX_INDEX_NAME_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum, createInsyncConfirmMessage, createEntryBatchMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload } from './chunk-TEAJO7CU.mjs';
2
- import { ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-C6VVLMRX.mjs';
1
+ import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, MAX_PARTITION_COUNT, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, MAX_INDEX_NAME_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-GN7KF675.mjs';
2
+ import { ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-36ER4WCG.mjs';
3
3
  import { deserializePayloadV2, createPartitionIndex, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, getLanguage, DEFAULT_PAGE_SIZE, requireWithinResultWindow, fnv1a, FILTER_EXPRESSION_KEYS, MAX_CURSOR_LENGTH, mergePartitionStats, FIELD_FILTER_OPERATORS, MAX_NESTING_DEPTH } from './chunk-RO4EFWE6.mjs';
4
4
  import './chunk-TGP5M5R7.mjs';
5
5
  import './chunk-OBXS7OZM.mjs';
@@ -60,61 +60,6 @@ function startRenewalInterval(state, coordinator, nodeId, ttlMs, onLeaseLost) {
60
60
  function scheduleStandbyRetry(state, retryMs, onRetry) {
61
61
  state.standbyTimer = setTimeout(onRetry, retryMs);
62
62
  }
63
- async function requestInsyncRemoval(indexName, partitionId, replicaNodeId, primaryTerm, controllerNodeId, transport, sourceNodeId) {
64
- const payload = {
65
- indexName,
66
- partitionId,
67
- replicaNodeId,
68
- primaryTerm
69
- };
70
- const message = createInsyncRemoveMessage(payload, sourceNodeId);
71
- const response = await transport.send(controllerNodeId, message);
72
- const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
73
- return { accepted: confirmPayload.accepted };
74
- }
75
- async function handleInsyncRemoval(payload, coordinator) {
76
- const table = await coordinator.getAllocation(payload.indexName);
77
- if (table === null) {
78
- return {
79
- indexName: payload.indexName,
80
- partitionId: payload.partitionId,
81
- accepted: false
82
- };
83
- }
84
- const assignment = table.assignments.get(payload.partitionId);
85
- if (assignment === void 0) {
86
- return {
87
- indexName: payload.indexName,
88
- partitionId: payload.partitionId,
89
- accepted: false
90
- };
91
- }
92
- if (payload.primaryTerm !== assignment.primaryTerm) {
93
- return {
94
- indexName: payload.indexName,
95
- partitionId: payload.partitionId,
96
- accepted: false
97
- };
98
- }
99
- const updatedInSyncSet = assignment.inSyncSet.filter((nodeId) => nodeId !== payload.replicaNodeId);
100
- const updatedAssignment = {
101
- ...assignment,
102
- inSyncSet: updatedInSyncSet
103
- };
104
- const updatedAssignments = new Map(table.assignments);
105
- updatedAssignments.set(payload.partitionId, updatedAssignment);
106
- const updatedTable = {
107
- ...table,
108
- version: table.version + 1,
109
- assignments: updatedAssignments
110
- };
111
- const written = await coordinator.putAllocation(payload.indexName, updatedTable, table.version);
112
- return {
113
- indexName: payload.indexName,
114
- partitionId: payload.partitionId,
115
- accepted: written
116
- };
117
- }
118
63
 
119
64
  // src/distribution/cluster/allocator/deciders/balance.ts
120
65
  var balanceDecider = {
@@ -392,6 +337,7 @@ function initialAllocate(nodes, indexName, partitionCount, replicationFactor, co
392
337
  primary: primaryNodeId,
393
338
  replicas,
394
339
  inSyncSet: [],
340
+ commitPoint: 0,
395
341
  state: "INITIALISING",
396
342
  primaryTerm: 1
397
343
  };
@@ -442,6 +388,7 @@ function cloneAssignments(assignments) {
442
388
  primary: assignment.primary,
443
389
  replicas: [...assignment.replicas],
444
390
  inSyncSet: [...assignment.inSyncSet],
391
+ commitPoint: assignment.commitPoint,
445
392
  state: assignment.state,
446
393
  primaryTerm: assignment.primaryTerm
447
394
  });
@@ -613,203 +560,14 @@ function allocate(nodes, currentTable, indexName, partitionCount, replicationFac
613
560
  }
614
561
  return rebalanceAllocate(dataNodes, currentTable, constraints, deciders);
615
562
  }
616
- function isRecord(value) {
617
- return value !== null && typeof value === "object" && !Array.isArray(value);
618
- }
619
- function isValidInteger(value) {
620
- return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
621
- }
622
- function validateBootstrapCompletePayload(decoded) {
623
- if (!isRecord(decoded)) {
624
- return null;
625
- }
626
- if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
627
- return null;
628
- }
629
- if (!isValidInteger(decoded.partitionId) || decoded.partitionId < 0) {
630
- return null;
631
- }
632
- if (typeof decoded.nodeId !== "string") {
633
- return null;
634
- }
635
- if (!isValidInteger(decoded.primaryTerm)) {
636
- return null;
637
- }
638
- return {
639
- indexName: decoded.indexName,
640
- partitionId: decoded.partitionId,
641
- nodeId: decoded.nodeId,
642
- primaryTerm: decoded.primaryTerm
643
- };
644
- }
645
- async function sendRejection(respond, controllerNodeId, requestId, indexName, partitionId) {
646
- const resultPayload = {
647
- indexName,
648
- partitionId,
649
- accepted: false
650
- };
651
- const response = {
652
- type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
653
- sourceId: controllerNodeId,
654
- requestId,
655
- payload: encode(resultPayload)
656
- };
657
- await respond(response);
658
- }
659
- function handleBootstrapCompleteMessage(message, respond, coordinator, controllerNodeId) {
660
- let decoded;
661
- try {
662
- decoded = decode(message.payload);
663
- } catch (_) {
664
- void sendRejection(respond, controllerNodeId, message.requestId, "", -1);
665
- return;
666
- }
667
- const payload = validateBootstrapCompletePayload(decoded);
668
- if (payload === null) {
669
- void sendRejection(respond, controllerNodeId, message.requestId, "", -1);
670
- return;
671
- }
672
- if (message.sourceId !== payload.nodeId) {
673
- void sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
674
- return;
675
- }
676
- processBootstrapComplete(payload, coordinator).then(async (accepted) => {
677
- const resultPayload = {
678
- indexName: payload.indexName,
679
- partitionId: payload.partitionId,
680
- accepted
681
- };
682
- const response = {
683
- type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
684
- sourceId: controllerNodeId,
685
- requestId: message.requestId,
686
- payload: encode(resultPayload)
687
- };
688
- await respond(response);
689
- }).catch(() => sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId));
690
- }
691
- function ensurePrimaryInSyncSet(assignment, inSyncSet) {
692
- if (assignment.primary === null) {
693
- return inSyncSet;
694
- }
695
- if (inSyncSet.includes(assignment.primary)) {
696
- return inSyncSet;
697
- }
698
- return [assignment.primary, ...inSyncSet];
699
- }
700
- async function processBootstrapComplete(payload, coordinator) {
701
- const table = await coordinator.getAllocation(payload.indexName);
702
- if (table === null) {
703
- return false;
704
- }
705
- const assignment = table.assignments.get(payload.partitionId);
706
- if (assignment === void 0) {
707
- return false;
708
- }
709
- if (payload.primaryTerm !== assignment.primaryTerm) {
710
- return false;
711
- }
712
- const isAssigned = assignment.primary === payload.nodeId || assignment.replicas.includes(payload.nodeId);
713
- if (!isAssigned) {
714
- return false;
715
- }
716
- if (assignment.state === "ACTIVE") {
717
- if (assignment.inSyncSet.includes(payload.nodeId)) {
718
- return true;
719
- }
720
- const updatedInSyncSet2 = [...assignment.inSyncSet, payload.nodeId];
721
- const updatedAssignment2 = {
722
- ...assignment,
723
- inSyncSet: updatedInSyncSet2
724
- };
725
- const updatedAssignments2 = new Map(table.assignments);
726
- updatedAssignments2.set(payload.partitionId, updatedAssignment2);
727
- const updatedTable2 = {
728
- ...table,
729
- version: table.version + 1,
730
- assignments: updatedAssignments2
731
- };
732
- return coordinator.putAllocation(payload.indexName, updatedTable2, table.version);
733
- }
734
- if (assignment.state !== "INITIALISING") {
735
- return false;
736
- }
737
- const baseInSyncSet = assignment.inSyncSet.includes(payload.nodeId) ? assignment.inSyncSet : [...assignment.inSyncSet, payload.nodeId];
738
- const updatedInSyncSet = ensurePrimaryInSyncSet(assignment, baseInSyncSet);
739
- const updatedAssignment = {
740
- ...assignment,
741
- state: "ACTIVE",
742
- inSyncSet: updatedInSyncSet
743
- };
744
- const updatedAssignments = new Map(table.assignments);
745
- updatedAssignments.set(payload.partitionId, updatedAssignment);
746
- const updatedTable = {
747
- ...table,
748
- version: table.version + 1,
749
- assignments: updatedAssignments
750
- };
751
- return coordinator.putAllocation(payload.indexName, updatedTable, table.version);
752
- }
753
563
 
754
- // src/distribution/cluster/controller/event-loop.ts
755
- function createEventLoopState(initialIndexNames) {
756
- return {
757
- unwatchNodes: null,
758
- unwatchSchemas: null,
759
- unwatchTransport: null,
760
- knownIndexes: new Set(initialIndexNames),
761
- debounceTimer: null,
762
- insyncQueue: Promise.resolve()
763
- };
764
- }
765
- function clearEventLoopWatchers(state) {
766
- if (state.unwatchNodes !== null) {
767
- state.unwatchNodes();
768
- state.unwatchNodes = null;
769
- }
770
- if (state.unwatchSchemas !== null) {
771
- state.unwatchSchemas();
772
- state.unwatchSchemas = null;
773
- }
774
- if (state.unwatchTransport !== null) {
775
- state.unwatchTransport();
776
- state.unwatchTransport = null;
777
- }
778
- if (state.debounceTimer !== null) {
779
- clearTimeout(state.debounceTimer);
780
- state.debounceTimer = null;
781
- }
782
- }
783
- function isRecord2(value) {
784
- return value !== null && typeof value === "object" && !Array.isArray(value);
785
- }
786
- function isValidInteger2(value) {
787
- return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
788
- }
789
- function validateInsyncRemovePayload(decoded) {
790
- if (!isRecord2(decoded)) {
791
- return null;
792
- }
793
- if (typeof decoded.indexName !== "string") {
794
- return null;
795
- }
796
- if (!isValidInteger2(decoded.partitionId)) {
797
- return null;
798
- }
799
- if (typeof decoded.replicaNodeId !== "string") {
800
- return null;
801
- }
802
- if (!isValidInteger2(decoded.primaryTerm)) {
803
- return null;
804
- }
805
- return {
806
- indexName: decoded.indexName,
807
- partitionId: decoded.partitionId,
808
- replicaNodeId: decoded.replicaNodeId,
809
- primaryTerm: decoded.primaryTerm
810
- };
811
- }
812
- async function runAllocatorForIndex(coordinator, indexName, nodes, isActive) {
564
+ // src/distribution/cluster/controller/event-loop/allocation.ts
565
+ var ALLOCATION_CAS_ATTEMPTS = 5;
566
+ var ALLOCATION_RETRY_DELAY_MS = 1e3;
567
+ var TEARDOWN_CAS_ATTEMPTS = 5;
568
+ var TEARDOWN_RETRY_ROUNDS = 3;
569
+ var TEARDOWN_RETRY_DELAY_MS = 500;
570
+ async function runAllocatorForIndex(coordinator, indexName, nodes, isActive, attempt = 0) {
813
571
  if (!isActive()) {
814
572
  return;
815
573
  }
@@ -832,9 +590,20 @@ async function runAllocatorForIndex(coordinator, indexName, nodes, isActive) {
832
590
  if (!isActive()) {
833
591
  return;
834
592
  }
835
- await coordinator.putAllocation(indexName, result.table, currentTable.version);
593
+ if (await coordinator.putAllocation(indexName, result.table, currentTable.version)) {
594
+ return;
595
+ }
596
+ if (attempt + 1 < ALLOCATION_CAS_ATTEMPTS) {
597
+ await runAllocatorForIndex(coordinator, indexName, nodes, isActive, attempt + 1);
598
+ return;
599
+ }
600
+ throw new NarsilError(
601
+ ErrorCodes.ALLOCATION_FAILED,
602
+ `Allocation for index '${indexName}' lost ${ALLOCATION_CAS_ATTEMPTS} compare-and-set attempts`,
603
+ { indexName, attempts: ALLOCATION_CAS_ATTEMPTS }
604
+ );
836
605
  }
837
- async function runAllocatorForAllIndexes(coordinator, knownIndexes, isActive, onError) {
606
+ async function runAllocatorForAllIndexes(state, coordinator, isActive, onError) {
838
607
  if (!isActive()) {
839
608
  return;
840
609
  }
@@ -842,18 +611,23 @@ async function runAllocatorForAllIndexes(coordinator, knownIndexes, isActive, on
842
611
  if (nodes.length === 0 || !isActive()) {
843
612
  return;
844
613
  }
845
- for (const indexName of knownIndexes) {
614
+ let anyIndexFailed = false;
615
+ for (const indexName of state.knownIndexes) {
846
616
  if (!isActive()) {
847
617
  return;
848
618
  }
849
619
  try {
850
620
  await runAllocatorForIndex(coordinator, indexName, nodes, isActive);
851
621
  } catch (error) {
622
+ anyIndexFailed = true;
852
623
  if (onError !== void 0) {
853
624
  onError(indexName, error);
854
625
  }
855
626
  }
856
627
  }
628
+ if (anyIndexFailed && isActive()) {
629
+ scheduleDebouncedAllocation(state, coordinator, isActive, onError, ALLOCATION_RETRY_DELAY_MS);
630
+ }
857
631
  }
858
632
  function scheduleDebouncedAllocation(state, coordinator, isActive, onError, debounceMs = 500) {
859
633
  if (state.debounceTimer !== null) {
@@ -861,9 +635,13 @@ function scheduleDebouncedAllocation(state, coordinator, isActive, onError, debo
861
635
  }
862
636
  state.debounceTimer = setTimeout(() => {
863
637
  state.debounceTimer = null;
864
- runAllocatorForAllIndexes(coordinator, state.knownIndexes, isActive, onError).catch(() => {
638
+ runAllocatorForAllIndexes(state, coordinator, isActive, onError).catch(() => {
639
+ if (isActive()) {
640
+ scheduleDebouncedAllocation(state, coordinator, isActive, onError, ALLOCATION_RETRY_DELAY_MS);
641
+ }
865
642
  });
866
643
  }, debounceMs);
644
+ state.debounceTimer.unref?.();
867
645
  }
868
646
  async function handleSchemaCreated(indexName, coordinator, knownIndexes, isActive) {
869
647
  knownIndexes.add(indexName);
@@ -895,11 +673,28 @@ async function handleSchemaCreated(indexName, coordinator, knownIndexes, isActiv
895
673
  return;
896
674
  }
897
675
  const expectedVersion = storedTable !== null ? storedTable.version : null;
898
- await coordinator.putAllocation(indexName, result.table, expectedVersion);
676
+ if (await coordinator.putAllocation(indexName, result.table, expectedVersion)) {
677
+ return;
678
+ }
679
+ await runAllocatorForIndex(coordinator, indexName, nodes, isActive);
899
680
  }
900
- var TEARDOWN_CAS_ATTEMPTS = 5;
901
- async function handleSchemaDropped(indexName, coordinator, knownIndexes, isActive) {
902
- knownIndexes.delete(indexName);
681
+ function scheduleTeardownRetry(indexName, coordinator, state, isActive, round, onError) {
682
+ const existing = state.teardownTimers.get(indexName);
683
+ if (existing !== void 0) {
684
+ clearTimeout(existing);
685
+ }
686
+ const timer = setTimeout(() => {
687
+ state.teardownTimers.delete(indexName);
688
+ tearDownAllocation(indexName, coordinator, state, isActive, round, onError).catch((error) => {
689
+ if (onError !== void 0) {
690
+ onError(indexName, error);
691
+ }
692
+ });
693
+ }, TEARDOWN_RETRY_DELAY_MS);
694
+ timer.unref?.();
695
+ state.teardownTimers.set(indexName, timer);
696
+ }
697
+ async function tearDownAllocation(indexName, coordinator, state, isActive, round, onError) {
903
698
  for (let attempt = 0; attempt < TEARDOWN_CAS_ATTEMPTS; attempt++) {
904
699
  if (!isActive()) {
905
700
  return;
@@ -929,461 +724,892 @@ async function handleSchemaDropped(indexName, coordinator, knownIndexes, isActiv
929
724
  await coordinator.deleteAllocation(indexName);
930
725
  return;
931
726
  }
932
- }
933
- async function handleSchemaEvent(event, coordinator, knownIndexes, isActive) {
934
- if (event.type === "schema_created") {
935
- await handleSchemaCreated(event.indexName, coordinator, knownIndexes, isActive);
936
- } else if (event.type === "schema_dropped") {
937
- await handleSchemaDropped(event.indexName, coordinator, knownIndexes, isActive);
727
+ if (round + 1 < TEARDOWN_RETRY_ROUNDS && isActive()) {
728
+ scheduleTeardownRetry(indexName, coordinator, state, isActive, round + 1, onError);
729
+ return;
938
730
  }
731
+ throw new NarsilError(
732
+ ErrorCodes.ALLOCATION_FAILED,
733
+ `Teardown of the allocation for dropped index '${indexName}' lost every compare-and-set attempt`,
734
+ { indexName, attempts: TEARDOWN_CAS_ATTEMPTS * TEARDOWN_RETRY_ROUNDS }
735
+ );
939
736
  }
940
- async function startEventLoop(state, coordinator, transport, nodeId, isActive, onError) {
941
- clearEventLoopWatchers(state);
942
- const unwatchNodes = await coordinator.watchNodes((_event) => {
943
- scheduleDebouncedAllocation(state, coordinator, isActive, onError);
944
- });
945
- state.unwatchNodes = unwatchNodes;
946
- const unwatchSchemas = await coordinator.watchSchemas((event) => {
947
- handleSchemaEvent(event, coordinator, state.knownIndexes, isActive).catch(() => {
948
- });
949
- });
950
- state.unwatchSchemas = unwatchSchemas;
951
- const unwatchTransport = await transport.listen((message, respond) => {
952
- if (!isActive()) {
953
- return;
954
- }
955
- if (message.type === ReplicationMessageTypes.INSYNC_REMOVE) {
956
- handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId);
957
- } else if (message.type === ClusterMessageTypes.BOOTSTRAP_COMPLETE) {
958
- handleBootstrapCompleteMessage(message, respond, coordinator, nodeId);
959
- }
960
- });
961
- state.unwatchTransport = unwatchTransport;
962
- scheduleDebouncedAllocation(state, coordinator, isActive, onError);
963
- }
964
- function handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId) {
965
- let decoded;
966
- try {
967
- decoded = decode(message.payload);
968
- } catch (_) {
737
+ async function handleSchemaEvent(event, coordinator, state, isActive, onError) {
738
+ if (event.type === "schema_created") {
739
+ await handleSchemaCreated(event.indexName, coordinator, state.knownIndexes, isActive);
969
740
  return;
970
741
  }
971
- const payload = validateInsyncRemovePayload(decoded);
972
- if (payload === null) {
973
- return;
742
+ if (event.type === "schema_dropped") {
743
+ state.knownIndexes.delete(event.indexName);
744
+ await tearDownAllocation(event.indexName, coordinator, state, isActive, 0, onError);
974
745
  }
975
- state.insyncQueue = state.insyncQueue.then(async () => {
976
- const table = await coordinator.getAllocation(payload.indexName);
746
+ }
747
+
748
+ // src/distribution/payload-guards.ts
749
+ function isRecord(value) {
750
+ return value !== null && typeof value === "object" && !Array.isArray(value);
751
+ }
752
+ function isValidInteger(value) {
753
+ return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
754
+ }
755
+ var INSYNC_CAS_ATTEMPTS = 5;
756
+ var REFUSE = { outcome: "refuse" };
757
+ var ACCEPT = { outcome: "accept" };
758
+ async function updateInsyncSet(request, coordinator, decide) {
759
+ const refused = {
760
+ indexName: request.indexName,
761
+ partitionId: request.partitionId,
762
+ accepted: false
763
+ };
764
+ for (let attempt = 0; attempt < INSYNC_CAS_ATTEMPTS; attempt++) {
765
+ const table = await coordinator.getAllocation(request.indexName);
977
766
  if (table === null) {
978
- return;
767
+ return refused;
979
768
  }
980
- const assignment = table.assignments.get(payload.partitionId);
769
+ const assignment = table.assignments.get(request.partitionId);
981
770
  if (assignment === void 0) {
982
- return;
771
+ return refused;
983
772
  }
984
- if (assignment.primary !== message.sourceId) {
985
- return;
773
+ if (request.primaryTerm !== assignment.primaryTerm) {
774
+ return refused;
986
775
  }
987
- const confirmPayload = await handleInsyncRemoval(payload, coordinator);
988
- const response = createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId);
989
- await respond(response);
990
- }).catch(() => {
991
- });
992
- }
993
-
994
- // src/distribution/cluster/controller/index.ts
995
- function createController(config) {
996
- const { nodeId, coordinator, transport, leaseTtlMs, standbyRetryMs, knownIndexNames, onError } = config;
997
- let electionState = createElectionState();
998
- let eventLoopState = createEventLoopState(knownIndexNames);
999
- let stopped = false;
1000
- function isActive() {
1001
- return electionState.active;
1002
- }
1003
- function stepDown() {
1004
- if (!electionState.active) {
1005
- return;
776
+ const decision = decide(assignment);
777
+ if (decision.outcome === "refuse") {
778
+ return refused;
1006
779
  }
1007
- electionState.active = false;
1008
- clearElectionTimers(electionState);
1009
- clearEventLoopWatchers(eventLoopState);
1010
- if (!stopped) {
1011
- scheduleStandbyRetry(electionState, standbyRetryMs, () => {
1012
- tryElection().catch(() => {
1013
- });
1014
- });
780
+ if (decision.outcome === "accept") {
781
+ return { ...refused, accepted: true };
782
+ }
783
+ const updatedAssignments = new Map(table.assignments);
784
+ updatedAssignments.set(request.partitionId, decision.assignment);
785
+ const written = await coordinator.putAllocation(
786
+ request.indexName,
787
+ { ...table, version: table.version + 1, assignments: updatedAssignments },
788
+ table.version
789
+ );
790
+ if (written) {
791
+ return { ...refused, accepted: true };
1015
792
  }
1016
793
  }
1017
- async function becomeActive() {
1018
- if (electionState.active) {
1019
- return;
794
+ return refused;
795
+ }
796
+ async function requestInsyncRemoval(indexName, partitionId, replicaNodeId, primaryTerm, controllerNodeId, transport, sourceNodeId) {
797
+ const payload = {
798
+ indexName,
799
+ partitionId,
800
+ replicaNodeId,
801
+ primaryTerm
802
+ };
803
+ const message = createInsyncRemoveMessage(payload, sourceNodeId);
804
+ const response = await transport.send(controllerNodeId, message);
805
+ const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
806
+ return { accepted: confirmPayload.accepted };
807
+ }
808
+ function handleInsyncRemoval(payload, coordinator) {
809
+ return updateInsyncSet(payload, coordinator, (assignment) => {
810
+ if (!assignment.inSyncSet.includes(payload.replicaNodeId)) {
811
+ return ACCEPT;
1020
812
  }
1021
- electionState.active = true;
1022
- startRenewalInterval(electionState, coordinator, nodeId, leaseTtlMs, stepDown);
1023
- try {
1024
- for (const indexName of await coordinator.listSchemas()) {
1025
- eventLoopState.knownIndexes.add(indexName);
813
+ return {
814
+ outcome: "write",
815
+ assignment: {
816
+ ...assignment,
817
+ inSyncSet: assignment.inSyncSet.filter((nodeId) => nodeId !== payload.replicaNodeId)
1026
818
  }
1027
- } catch (_) {
819
+ };
820
+ });
821
+ }
822
+ async function requestInsyncAdmission(payload, controllerNodeId, transport, sourceNodeId) {
823
+ const message = createInsyncAddMessage(payload, sourceNodeId);
824
+ const response = await transport.send(controllerNodeId, message);
825
+ const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
826
+ return { accepted: confirmPayload.accepted };
827
+ }
828
+ function handleInsyncAdmission(payload, coordinator) {
829
+ return updateInsyncSet(payload, coordinator, (assignment) => {
830
+ if (assignment.state !== "ACTIVE") {
831
+ return REFUSE;
1028
832
  }
1029
- await startEventLoop(eventLoopState, coordinator, transport, nodeId, isActive, onError);
1030
- }
1031
- async function tryElection() {
1032
- if (stopped) {
1033
- return;
833
+ if (!assignment.replicas.includes(payload.replicaNodeId)) {
834
+ return REFUSE;
1034
835
  }
1035
- const acquired = await tryAcquireLease(coordinator, nodeId, leaseTtlMs);
1036
- if (acquired) {
1037
- await becomeActive();
1038
- } else {
1039
- scheduleStandbyRetry(electionState, standbyRetryMs, () => {
1040
- tryElection().catch(() => {
1041
- });
1042
- });
836
+ if (payload.appliedSeqNo < assignment.commitPoint || payload.appliedSeqNo < payload.commitPoint) {
837
+ return REFUSE;
1043
838
  }
1044
- }
1045
- const controller = {
1046
- get isActive() {
1047
- return electionState.active;
1048
- },
1049
- async start() {
1050
- stopped = false;
1051
- await tryElection();
1052
- },
1053
- async stop() {
1054
- stopped = true;
1055
- clearElectionTimers(electionState);
1056
- clearEventLoopWatchers(eventLoopState);
1057
- if (electionState.active) {
1058
- electionState.active = false;
1059
- try {
1060
- await releaseLease(coordinator);
1061
- } catch (_) {
1062
- }
1063
- }
1064
- },
1065
- async shutdown() {
1066
- await controller.stop();
1067
- eventLoopState = createEventLoopState([]);
1068
- electionState = createElectionState();
839
+ const alreadyInSync = assignment.inSyncSet.includes(payload.replicaNodeId);
840
+ const commitPoint = Math.max(assignment.commitPoint, payload.commitPoint);
841
+ if (alreadyInSync && commitPoint === assignment.commitPoint) {
842
+ return ACCEPT;
1069
843
  }
1070
- };
1071
- return controller;
1072
- }
1073
- function computeBackoffMs(baseMs, maxMs, retryCount) {
1074
- const exponential = baseMs * 2 ** retryCount;
1075
- const capped = Math.min(exponential, maxMs);
1076
- const jitter = capped * (0.5 + Math.random() * 0.5);
1077
- return Math.floor(jitter);
844
+ return {
845
+ outcome: "write",
846
+ assignment: {
847
+ ...assignment,
848
+ inSyncSet: alreadyInSync ? assignment.inSyncSet : [...assignment.inSyncSet, payload.replicaNodeId],
849
+ commitPoint
850
+ }
851
+ };
852
+ });
1078
853
  }
1079
- async function resolvePrimaryTerm(indexName, partitionId, coordinator) {
1080
- const table = await coordinator.getAllocation(indexName);
1081
- if (table === null) {
854
+ function validateBootstrapCompletePayload(decoded) {
855
+ if (!isRecord(decoded)) {
1082
856
  return null;
1083
857
  }
1084
- const assignment = table.assignments.get(partitionId);
1085
- if (assignment === void 0) {
858
+ if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
1086
859
  return null;
1087
860
  }
1088
- return assignment.primaryTerm;
1089
- }
1090
- async function resolveControllerTargets(coordinator, controllerNodeId) {
1091
- const targets = [controllerNodeId];
1092
- try {
1093
- const nodes = await coordinator.listNodes();
1094
- const registration = nodes.find((entry) => entry.nodeId === controllerNodeId);
1095
- if (registration !== void 0 && registration.address.length > 0 && registration.address !== controllerNodeId) {
1096
- targets.push(registration.address);
1097
- }
1098
- } catch (_) {
1099
- return targets;
861
+ if (!isValidInteger(decoded.partitionId) || decoded.partitionId < 0) {
862
+ return null;
1100
863
  }
1101
- return targets;
1102
- }
1103
- async function reportBootstrapComplete(indexName, partitionId, nodeId, coordinator, transport) {
1104
- const controllerNodeId = await coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
1105
- if (controllerNodeId === null) {
1106
- return false;
864
+ if (typeof decoded.nodeId !== "string") {
865
+ return null;
1107
866
  }
1108
- const primaryTerm = await resolvePrimaryTerm(indexName, partitionId, coordinator);
1109
- if (primaryTerm === null) {
1110
- return false;
867
+ if (!isValidInteger(decoded.primaryTerm)) {
868
+ return null;
1111
869
  }
1112
- const payload = {
870
+ return {
871
+ indexName: decoded.indexName,
872
+ partitionId: decoded.partitionId,
873
+ nodeId: decoded.nodeId,
874
+ primaryTerm: decoded.primaryTerm
875
+ };
876
+ }
877
+ async function sendRejection(respond, controllerNodeId, requestId, indexName, partitionId) {
878
+ const resultPayload = {
1113
879
  indexName,
1114
880
  partitionId,
1115
- nodeId,
1116
- primaryTerm
881
+ accepted: false
1117
882
  };
1118
- const message = {
883
+ const response = {
1119
884
  type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
1120
- sourceId: nodeId,
1121
- requestId: generateId(),
1122
- payload: encode(payload)
885
+ sourceId: controllerNodeId,
886
+ requestId,
887
+ payload: encode(resultPayload)
1123
888
  };
1124
- for (const target of await resolveControllerTargets(coordinator, controllerNodeId)) {
1125
- try {
1126
- const response = await transport.send(target, message);
1127
- return isAcceptedResponse(response);
1128
- } catch (_) {
1129
- }
889
+ await respond(response);
890
+ }
891
+ async function rejectBootstrapComplete(message, respond, controllerNodeId) {
892
+ let decoded;
893
+ try {
894
+ decoded = decode(message.payload);
895
+ } catch (_) {
896
+ await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
897
+ return;
1130
898
  }
1131
- return false;
899
+ const payload = validateBootstrapCompletePayload(decoded);
900
+ if (payload === null) {
901
+ await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
902
+ return;
903
+ }
904
+ await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
1132
905
  }
1133
- function isAcceptedResponse(response) {
906
+ async function handleBootstrapCompleteMessage(message, respond, coordinator, controllerNodeId) {
907
+ let decoded;
1134
908
  try {
1135
- const decoded = decode(response.payload);
1136
- if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
1137
- return false;
1138
- }
1139
- return decoded.accepted === true;
909
+ decoded = decode(message.payload);
1140
910
  } catch (_) {
1141
- return false;
911
+ await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
912
+ return;
913
+ }
914
+ const payload = validateBootstrapCompletePayload(decoded);
915
+ if (payload === null) {
916
+ await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
917
+ return;
918
+ }
919
+ if (message.sourceId !== payload.nodeId) {
920
+ await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
921
+ return;
922
+ }
923
+ let accepted;
924
+ try {
925
+ accepted = await processBootstrapComplete(payload, coordinator);
926
+ } catch (_) {
927
+ await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
928
+ return;
1142
929
  }
930
+ const resultPayload = {
931
+ indexName: payload.indexName,
932
+ partitionId: payload.partitionId,
933
+ accepted
934
+ };
935
+ await respond({
936
+ type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
937
+ sourceId: controllerNodeId,
938
+ requestId: message.requestId,
939
+ payload: encode(resultPayload)
940
+ });
1143
941
  }
1144
- async function bootstrapPartition(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onBootstrapPartition, onError) {
1145
- for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
1146
- if (state.aborted) {
942
+ var BOOTSTRAP_CAS_ATTEMPTS = 5;
943
+ function ensurePrimaryInSyncSet(assignment, inSyncSet) {
944
+ if (assignment.primary === null) {
945
+ return inSyncSet;
946
+ }
947
+ if (inSyncSet.includes(assignment.primary)) {
948
+ return inSyncSet;
949
+ }
950
+ return [assignment.primary, ...inSyncSet];
951
+ }
952
+ async function processBootstrapComplete(payload, coordinator) {
953
+ for (let attempt = 0; attempt < BOOTSTRAP_CAS_ATTEMPTS; attempt++) {
954
+ const table = await coordinator.getAllocation(payload.indexName);
955
+ if (table === null) {
1147
956
  return false;
1148
957
  }
1149
- const synced = await runBootstrapSyncAttempt(state, onBootstrapPartition, onError);
1150
- if (state.aborted) {
958
+ const assignment = table.assignments.get(payload.partitionId);
959
+ if (assignment === void 0) {
1151
960
  return false;
1152
961
  }
1153
- if (synced) {
1154
- return retryReportBootstrapComplete(
1155
- state,
1156
- coordinator,
1157
- transport,
1158
- nodeId,
1159
- bootstrapRetryBaseMs,
1160
- bootstrapRetryMaxMs,
1161
- bootstrapMaxRetries,
1162
- onError
1163
- );
1164
- }
1165
- if (attempt < bootstrapMaxRetries) {
1166
- const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
1167
- await waitWithAbort(state, backoffMs);
1168
- }
1169
- }
1170
- return false;
1171
- }
1172
- async function runBootstrapSyncAttempt(state, onBootstrapPartition, onError) {
1173
- try {
1174
- return await onBootstrapPartition(state.indexName, state.partitionId, state.primaryNodeId);
1175
- } catch (error) {
1176
- if (state.aborted) {
962
+ if (payload.primaryTerm !== assignment.primaryTerm) {
1177
963
  return false;
1178
964
  }
1179
- if (onError !== void 0) {
1180
- const cause = error instanceof Error ? error.message : String(error);
1181
- onError(
1182
- new NarsilError(
1183
- ErrorCodes.NODE_BOOTSTRAP_FAILED,
1184
- `Bootstrap sync failed for ${state.indexName}:${state.partitionId}`,
1185
- { indexName: state.indexName, partitionId: state.partitionId, cause }
1186
- )
1187
- );
965
+ if (assignment.primary !== payload.nodeId && !assignment.replicas.includes(payload.nodeId)) {
966
+ return false;
1188
967
  }
1189
- return false;
1190
- }
1191
- }
1192
- async function retryReportBootstrapComplete(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onError) {
1193
- for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
1194
- if (state.aborted) {
968
+ if (assignment.state !== "INITIALISING") {
1195
969
  return false;
1196
970
  }
1197
- const accepted = await reportBootstrapComplete(state.indexName, state.partitionId, nodeId, coordinator, transport);
1198
- if (accepted) {
971
+ const baseInSyncSet = assignment.inSyncSet.includes(payload.nodeId) ? assignment.inSyncSet : [...assignment.inSyncSet, payload.nodeId];
972
+ const updatedAssignments = new Map(table.assignments);
973
+ updatedAssignments.set(payload.partitionId, {
974
+ ...assignment,
975
+ state: "ACTIVE",
976
+ inSyncSet: ensurePrimaryInSyncSet(assignment, baseInSyncSet)
977
+ });
978
+ const written = await coordinator.putAllocation(
979
+ payload.indexName,
980
+ { ...table, version: table.version + 1, assignments: updatedAssignments },
981
+ table.version
982
+ );
983
+ if (written) {
1199
984
  return true;
1200
985
  }
1201
- if (attempt < bootstrapMaxRetries && !state.aborted) {
1202
- const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
1203
- await waitWithAbort(state, backoffMs);
1204
- }
1205
- }
1206
- if (onError !== void 0) {
1207
- onError(
1208
- new NarsilError(
1209
- ErrorCodes.NODE_BOOTSTRAP_FAILED,
1210
- `Failed to report bootstrap completion for ${state.indexName}:${state.partitionId} after ${bootstrapMaxRetries + 1} attempts`,
1211
- { indexName: state.indexName, partitionId: state.partitionId }
1212
- )
1213
- );
1214
986
  }
1215
987
  return false;
1216
988
  }
1217
- function waitWithAbort(state, ms) {
1218
- return new Promise((resolve) => {
1219
- if (state.aborted) {
1220
- resolve();
1221
- return;
1222
- }
1223
- state.abortResolve = resolve;
1224
- state.retryTimer = setTimeout(() => {
1225
- state.retryTimer = null;
1226
- state.abortResolve = null;
1227
- resolve();
1228
- }, ms);
1229
- });
1230
- }
1231
- function abortBootstrapState(state) {
1232
- state.aborted = true;
1233
- if (state.retryTimer !== null) {
1234
- clearTimeout(state.retryTimer);
1235
- state.retryTimer = null;
989
+
990
+ // src/distribution/cluster/controller/event-loop/insync-messages.ts
991
+ var UNKNOWN_PARTITION_ID = -1;
992
+ function validateInsyncRemovePayload(decoded) {
993
+ if (!isRecord(decoded)) {
994
+ return null;
1236
995
  }
1237
- if (state.abortResolve !== null) {
1238
- state.abortResolve();
1239
- state.abortResolve = null;
996
+ if (typeof decoded.indexName !== "string") {
997
+ return null;
998
+ }
999
+ if (!isValidInteger(decoded.partitionId)) {
1000
+ return null;
1001
+ }
1002
+ if (typeof decoded.replicaNodeId !== "string") {
1003
+ return null;
1004
+ }
1005
+ if (!isValidInteger(decoded.primaryTerm)) {
1006
+ return null;
1240
1007
  }
1241
- }
1242
- function createBootstrapState(indexName, partitionId, primaryNodeId) {
1243
1008
  return {
1244
- indexName,
1245
- partitionId,
1246
- primaryNodeId,
1247
- retryCount: 0,
1248
- retryTimer: null,
1249
- aborted: false,
1250
- abortResolve: null
1009
+ indexName: decoded.indexName,
1010
+ partitionId: decoded.partitionId,
1011
+ replicaNodeId: decoded.replicaNodeId,
1012
+ primaryTerm: decoded.primaryTerm
1251
1013
  };
1252
1014
  }
1253
-
1254
- // src/distribution/cluster/node-lifecycle/allocation-watcher.ts
1255
- function partitionKey(indexName, partitionId) {
1256
- return `${indexName}:${partitionId}`;
1015
+ function validateInsyncAddPayload(decoded) {
1016
+ const base = validateInsyncRemovePayload(decoded);
1017
+ if (base === null || !isRecord(decoded)) {
1018
+ return null;
1019
+ }
1020
+ if (!isValidInteger(decoded.appliedSeqNo) || decoded.appliedSeqNo < 0) {
1021
+ return null;
1022
+ }
1023
+ if (!isValidInteger(decoded.commitPoint) || decoded.commitPoint < 0) {
1024
+ return null;
1025
+ }
1026
+ return { ...base, appliedSeqNo: decoded.appliedSeqNo, commitPoint: decoded.commitPoint };
1257
1027
  }
1258
- function createAllocationWatcherState() {
1028
+ function decodeMessage(message) {
1029
+ try {
1030
+ return decode(message.payload);
1031
+ } catch (_) {
1032
+ return null;
1033
+ }
1034
+ }
1035
+ function describeRequest(decoded) {
1036
+ if (!isRecord(decoded)) {
1037
+ return { indexName: "", partitionId: UNKNOWN_PARTITION_ID };
1038
+ }
1259
1039
  return {
1260
- unwatchAllocation: null,
1261
- debounceTimer: null,
1262
- activeBootstraps: /* @__PURE__ */ new Map(),
1263
- trackedPartitions: /* @__PURE__ */ new Map(),
1264
- pendingTables: /* @__PURE__ */ new Map(),
1265
- stopped: false
1040
+ indexName: typeof decoded.indexName === "string" ? decoded.indexName : "",
1041
+ partitionId: isValidInteger(decoded.partitionId) ? decoded.partitionId : UNKNOWN_PARTITION_ID
1266
1042
  };
1267
1043
  }
1268
- async function startAllocationWatcher(state, config) {
1269
- const unwatchAllocation = await config.coordinator.watchAllocation((event) => {
1270
- if (state.stopped) {
1271
- return;
1272
- }
1273
- scheduleProcessAllocation(state, config, event.table);
1044
+ async function assertSenderLeadsPartition(coordinator, indexName, partitionId, sourceId) {
1045
+ const table = await coordinator.getAllocation(indexName);
1046
+ const assignment = table?.assignments.get(partitionId);
1047
+ if (assignment === void 0) {
1048
+ return false;
1049
+ }
1050
+ return assignment.primary === sourceId;
1051
+ }
1052
+ function enqueue(state, task) {
1053
+ state.insyncQueue = state.insyncQueue.then(task).catch(() => {
1274
1054
  });
1275
- state.unwatchAllocation = unwatchAllocation;
1276
1055
  }
1277
- function scheduleProcessAllocation(state, config, table) {
1278
- state.pendingTables.set(table.indexName, table);
1279
- if (state.debounceTimer !== null) {
1280
- clearTimeout(state.debounceTimer);
1056
+ async function respondRefusal(respond, request, nodeId, requestId) {
1057
+ const payload = {
1058
+ indexName: request.indexName,
1059
+ partitionId: request.partitionId,
1060
+ accepted: false
1061
+ };
1062
+ try {
1063
+ await respond(createInsyncConfirmMessage(payload, nodeId, requestId));
1064
+ } catch (_) {
1281
1065
  }
1282
- state.debounceTimer = setTimeout(() => {
1283
- state.debounceTimer = null;
1284
- const tablesToProcess = Array.from(state.pendingTables.values());
1285
- state.pendingTables.clear();
1286
- for (const pending of tablesToProcess) {
1287
- processAllocationChange(state, config, pending);
1288
- }
1289
- }, config.allocationDebounceMs);
1290
1066
  }
1291
- function processAllocationChange(state, config, table) {
1292
- if (state.stopped) {
1067
+ function handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId, isActive) {
1068
+ const decoded = decodeMessage(message);
1069
+ const payload = validateInsyncRemovePayload(decoded);
1070
+ if (payload === null) {
1071
+ void respondRefusal(respond, describeRequest(decoded), nodeId, message.requestId);
1293
1072
  return;
1294
1073
  }
1295
- const nodeId = config.registration.nodeId;
1296
- const currentKeys = /* @__PURE__ */ new Set();
1297
- for (const [partitionId, assignment] of table.assignments) {
1298
- if (!isNodeAssigned(assignment, nodeId)) {
1299
- continue;
1300
- }
1301
- const key = partitionKey(table.indexName, partitionId);
1302
- currentKeys.add(key);
1303
- const existing = state.trackedPartitions.get(key);
1304
- const isPrimary = assignment.primary === nodeId;
1305
- if (existing === void 0) {
1306
- handleNewAssignment(state, config, table.indexName, partitionId, assignment, nodeId);
1307
- } else {
1308
- handleExistingAssignment(state, config, existing, assignment, nodeId);
1074
+ enqueue(state, async () => {
1075
+ const leadsPartition = await assertSenderLeadsPartition(
1076
+ coordinator,
1077
+ payload.indexName,
1078
+ payload.partitionId,
1079
+ message.sourceId
1080
+ );
1081
+ if (!leadsPartition || !isActive()) {
1082
+ await respondRefusal(respond, payload, nodeId, message.requestId);
1083
+ return;
1309
1084
  }
1310
- state.trackedPartitions.set(key, {
1311
- indexName: table.indexName,
1312
- partitionId,
1313
- isPrimary,
1314
- primaryNodeId: assignment.primary,
1315
- primaryTerm: assignment.primaryTerm
1316
- });
1317
- }
1318
- removeUnassignedPartitions(state, config, table.indexName, currentKeys);
1319
- }
1320
- function isNodeAssigned(assignment, nodeId) {
1321
- if (assignment.primary === nodeId) {
1322
- return true;
1323
- }
1324
- return assignment.replicas.includes(nodeId);
1085
+ const confirmPayload = await handleInsyncRemoval(payload, coordinator);
1086
+ await respond(createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId));
1087
+ });
1325
1088
  }
1326
- function handleNewAssignment(state, config, indexName, partitionId, assignment, nodeId) {
1327
- if (assignment.state === "ACTIVE" && assignment.primary === nodeId) {
1328
- return;
1329
- }
1330
- if (assignment.primary === null) {
1089
+ function handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive) {
1090
+ const decoded = decodeMessage(message);
1091
+ const payload = validateInsyncAddPayload(decoded);
1092
+ if (payload === null) {
1093
+ void respondRefusal(respond, describeRequest(decoded), nodeId, message.requestId);
1331
1094
  return;
1332
1095
  }
1333
- if (assignment.primary === nodeId) {
1334
- if (assignment.state === "INITIALISING") {
1335
- startBootstrap(state, config, indexName, partitionId, nodeId);
1096
+ enqueue(state, async () => {
1097
+ const leadsPartition = await assertSenderLeadsPartition(
1098
+ coordinator,
1099
+ payload.indexName,
1100
+ payload.partitionId,
1101
+ message.sourceId
1102
+ );
1103
+ if (!leadsPartition || !isActive()) {
1104
+ await respondRefusal(respond, payload, nodeId, message.requestId);
1105
+ return;
1336
1106
  }
1337
- return;
1338
- }
1339
- startBootstrap(state, config, indexName, partitionId, assignment.primary);
1107
+ const confirmPayload = await handleInsyncAdmission(payload, coordinator);
1108
+ await respond(createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId));
1109
+ });
1340
1110
  }
1341
- function handleExistingAssignment(state, config, existing, assignment, nodeId) {
1342
- if (existing.isPrimary && assignment.primary !== nodeId && assignment.primaryTerm > existing.primaryTerm) {
1343
- if (assignment.primary !== null) {
1344
- if (config.onPrimaryDemotion !== void 0) {
1345
- config.onPrimaryDemotion(existing.indexName, existing.partitionId, assignment.primary);
1346
- }
1347
- startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
1111
+ function handleQueuedBootstrapComplete(state, message, respond, coordinator, nodeId, isActive) {
1112
+ enqueue(state, () => {
1113
+ if (!isActive()) {
1114
+ return rejectBootstrapComplete(message, respond, nodeId);
1348
1115
  }
1349
- return;
1116
+ return handleBootstrapCompleteMessage(message, respond, coordinator, nodeId);
1117
+ });
1118
+ }
1119
+
1120
+ // src/distribution/cluster/controller/event-loop/state.ts
1121
+ function createEventLoopState(initialIndexNames) {
1122
+ return {
1123
+ unwatchNodes: null,
1124
+ unwatchSchemas: null,
1125
+ unwatchTransport: null,
1126
+ knownIndexes: new Set(initialIndexNames),
1127
+ debounceTimer: null,
1128
+ teardownTimers: /* @__PURE__ */ new Map(),
1129
+ insyncQueue: Promise.resolve()
1130
+ };
1131
+ }
1132
+ function clearEventLoopWatchers(state) {
1133
+ if (state.unwatchNodes !== null) {
1134
+ state.unwatchNodes();
1135
+ state.unwatchNodes = null;
1350
1136
  }
1351
- if (assignment.primary !== null && replicaNeedsResync(state, existing, assignment, nodeId)) {
1352
- startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
1137
+ if (state.unwatchSchemas !== null) {
1138
+ state.unwatchSchemas();
1139
+ state.unwatchSchemas = null;
1353
1140
  }
1354
- }
1355
- function replicaNeedsResync(state, existing, assignment, nodeId) {
1356
- if (assignment.state !== "ACTIVE") {
1357
- return false;
1141
+ if (state.unwatchTransport !== null) {
1142
+ state.unwatchTransport();
1143
+ state.unwatchTransport = null;
1358
1144
  }
1359
- if (assignment.primary === null || assignment.primary === nodeId) {
1360
- return false;
1145
+ if (state.debounceTimer !== null) {
1146
+ clearTimeout(state.debounceTimer);
1147
+ state.debounceTimer = null;
1361
1148
  }
1362
- if (!assignment.replicas.includes(nodeId)) {
1363
- return false;
1149
+ for (const timer of state.teardownTimers.values()) {
1150
+ clearTimeout(timer);
1364
1151
  }
1365
- if (assignment.inSyncSet.includes(nodeId)) {
1366
- return false;
1152
+ state.teardownTimers.clear();
1153
+ }
1154
+
1155
+ // src/distribution/cluster/controller/event-loop/index.ts
1156
+ async function startEventLoop(state, coordinator, transport, nodeId, isActive, onError) {
1157
+ clearEventLoopWatchers(state);
1158
+ try {
1159
+ state.unwatchNodes = await coordinator.watchNodes((_event) => {
1160
+ scheduleDebouncedAllocation(state, coordinator, isActive, onError);
1161
+ });
1162
+ state.unwatchSchemas = await coordinator.watchSchemas((event) => {
1163
+ handleSchemaEvent(event, coordinator, state, isActive, onError).catch((error) => {
1164
+ if (onError !== void 0) {
1165
+ onError(event.indexName, error);
1166
+ }
1167
+ });
1168
+ });
1169
+ state.unwatchTransport = await transport.listen((message, respond) => {
1170
+ if (!isActive()) {
1171
+ return;
1172
+ }
1173
+ if (message.type === ReplicationMessageTypes.INSYNC_REMOVE) {
1174
+ handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId, isActive);
1175
+ } else if (message.type === ReplicationMessageTypes.INSYNC_ADD) {
1176
+ handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive);
1177
+ } else if (message.type === ClusterMessageTypes.BOOTSTRAP_COMPLETE) {
1178
+ handleQueuedBootstrapComplete(state, message, respond, coordinator, nodeId, isActive);
1179
+ }
1180
+ });
1181
+ } catch (error) {
1182
+ clearEventLoopWatchers(state);
1183
+ throw error;
1367
1184
  }
1368
- return !state.activeBootstraps.has(partitionKey(existing.indexName, existing.partitionId));
1185
+ scheduleDebouncedAllocation(state, coordinator, isActive, onError);
1369
1186
  }
1370
- function startBootstrap(state, config, indexName, partitionId, primaryNodeId) {
1371
- const key = partitionKey(indexName, partitionId);
1372
- const existingBootstrap = state.activeBootstraps.get(key);
1373
- if (existingBootstrap !== void 0) {
1374
- abortBootstrapState(existingBootstrap);
1187
+
1188
+ // src/distribution/cluster/controller/index.ts
1189
+ function createController(config) {
1190
+ const { nodeId, coordinator, transport, leaseTtlMs, standbyRetryMs, knownIndexNames, onError } = config;
1191
+ let electionState = createElectionState();
1192
+ let eventLoopState = createEventLoopState(knownIndexNames);
1193
+ let stopped = false;
1194
+ function isActive() {
1195
+ return electionState.active;
1375
1196
  }
1376
- const bootstrapState = createBootstrapState(indexName, partitionId, primaryNodeId);
1377
- state.activeBootstraps.set(key, bootstrapState);
1378
- bootstrapPartition(
1379
- bootstrapState,
1380
- config.coordinator,
1381
- config.transport,
1382
- config.registration.nodeId,
1383
- config.bootstrapRetryBaseMs,
1384
- config.bootstrapRetryMaxMs,
1385
- config.bootstrapMaxRetries,
1386
- config.onBootstrapPartition,
1197
+ function stepDown() {
1198
+ if (!electionState.active) {
1199
+ return;
1200
+ }
1201
+ electionState.active = false;
1202
+ clearElectionTimers(electionState);
1203
+ clearEventLoopWatchers(eventLoopState);
1204
+ if (!stopped) {
1205
+ scheduleStandbyRetry(electionState, standbyRetryMs, () => {
1206
+ tryElection().catch(() => {
1207
+ });
1208
+ });
1209
+ }
1210
+ }
1211
+ async function becomeActive() {
1212
+ if (electionState.active) {
1213
+ return;
1214
+ }
1215
+ electionState.active = true;
1216
+ startRenewalInterval(electionState, coordinator, nodeId, leaseTtlMs, stepDown);
1217
+ try {
1218
+ for (const indexName of await coordinator.listSchemas()) {
1219
+ eventLoopState.knownIndexes.add(indexName);
1220
+ }
1221
+ } catch (_) {
1222
+ }
1223
+ try {
1224
+ await startEventLoop(eventLoopState, coordinator, transport, nodeId, isActive, onError);
1225
+ } catch (error) {
1226
+ electionState.active = false;
1227
+ clearElectionTimers(electionState);
1228
+ try {
1229
+ await releaseLease(coordinator);
1230
+ } catch (_) {
1231
+ }
1232
+ throw error;
1233
+ }
1234
+ }
1235
+ async function tryElection() {
1236
+ if (stopped) {
1237
+ return;
1238
+ }
1239
+ const acquired = await tryAcquireLease(coordinator, nodeId, leaseTtlMs);
1240
+ if (acquired) {
1241
+ await becomeActive();
1242
+ } else {
1243
+ scheduleStandbyRetry(electionState, standbyRetryMs, () => {
1244
+ tryElection().catch(() => {
1245
+ });
1246
+ });
1247
+ }
1248
+ }
1249
+ const controller = {
1250
+ get isActive() {
1251
+ return electionState.active;
1252
+ },
1253
+ async start() {
1254
+ stopped = false;
1255
+ await tryElection();
1256
+ },
1257
+ async stop() {
1258
+ stopped = true;
1259
+ clearElectionTimers(electionState);
1260
+ clearEventLoopWatchers(eventLoopState);
1261
+ if (electionState.active) {
1262
+ electionState.active = false;
1263
+ try {
1264
+ await releaseLease(coordinator);
1265
+ } catch (_) {
1266
+ }
1267
+ }
1268
+ },
1269
+ async shutdown() {
1270
+ await controller.stop();
1271
+ eventLoopState = createEventLoopState([]);
1272
+ electionState = createElectionState();
1273
+ }
1274
+ };
1275
+ return controller;
1276
+ }
1277
+ function computeBackoffMs(baseMs, maxMs, retryCount) {
1278
+ const exponential = baseMs * 2 ** retryCount;
1279
+ const capped = Math.min(exponential, maxMs);
1280
+ const jitter = capped * (0.5 + Math.random() * 0.5);
1281
+ return Math.floor(jitter);
1282
+ }
1283
+ async function resolvePrimaryTerm(indexName, partitionId, coordinator) {
1284
+ const table = await coordinator.getAllocation(indexName);
1285
+ if (table === null) {
1286
+ return null;
1287
+ }
1288
+ const assignment = table.assignments.get(partitionId);
1289
+ if (assignment === void 0) {
1290
+ return null;
1291
+ }
1292
+ return assignment.primaryTerm;
1293
+ }
1294
+ async function resolveControllerTargets(coordinator, controllerNodeId) {
1295
+ const targets = [controllerNodeId];
1296
+ try {
1297
+ const nodes = await coordinator.listNodes();
1298
+ const registration = nodes.find((entry) => entry.nodeId === controllerNodeId);
1299
+ if (registration !== void 0 && registration.address.length > 0 && registration.address !== controllerNodeId) {
1300
+ targets.push(registration.address);
1301
+ }
1302
+ } catch (_) {
1303
+ return targets;
1304
+ }
1305
+ return targets;
1306
+ }
1307
+ async function reportBootstrapComplete(indexName, partitionId, nodeId, coordinator, transport) {
1308
+ const controllerNodeId = await coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
1309
+ if (controllerNodeId === null) {
1310
+ return false;
1311
+ }
1312
+ const primaryTerm = await resolvePrimaryTerm(indexName, partitionId, coordinator);
1313
+ if (primaryTerm === null) {
1314
+ return false;
1315
+ }
1316
+ const payload = {
1317
+ indexName,
1318
+ partitionId,
1319
+ nodeId,
1320
+ primaryTerm
1321
+ };
1322
+ const message = {
1323
+ type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
1324
+ sourceId: nodeId,
1325
+ requestId: generateId(),
1326
+ payload: encode(payload)
1327
+ };
1328
+ for (const target of await resolveControllerTargets(coordinator, controllerNodeId)) {
1329
+ try {
1330
+ const response = await transport.send(target, message);
1331
+ return isAcceptedResponse(response);
1332
+ } catch (_) {
1333
+ }
1334
+ }
1335
+ return false;
1336
+ }
1337
+ function isAcceptedResponse(response) {
1338
+ try {
1339
+ const decoded = decode(response.payload);
1340
+ if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
1341
+ return false;
1342
+ }
1343
+ return decoded.accepted === true;
1344
+ } catch (_) {
1345
+ return false;
1346
+ }
1347
+ }
1348
+ async function partitionAwaitsBootstrapReport(state, coordinator) {
1349
+ try {
1350
+ const table = await coordinator.getAllocation(state.indexName);
1351
+ const assignment = table?.assignments.get(state.partitionId);
1352
+ if (assignment === void 0) {
1353
+ return true;
1354
+ }
1355
+ return assignment.state === "INITIALISING";
1356
+ } catch (_) {
1357
+ return true;
1358
+ }
1359
+ }
1360
+ async function bootstrapPartition(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onBootstrapPartition, onError) {
1361
+ for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
1362
+ if (state.aborted) {
1363
+ return false;
1364
+ }
1365
+ const synced = await runBootstrapSyncAttempt(state, onBootstrapPartition, onError);
1366
+ if (state.aborted) {
1367
+ return false;
1368
+ }
1369
+ if (synced) {
1370
+ if (!await partitionAwaitsBootstrapReport(state, coordinator)) {
1371
+ return true;
1372
+ }
1373
+ return retryReportBootstrapComplete(
1374
+ state,
1375
+ coordinator,
1376
+ transport,
1377
+ nodeId,
1378
+ bootstrapRetryBaseMs,
1379
+ bootstrapRetryMaxMs,
1380
+ bootstrapMaxRetries,
1381
+ onError
1382
+ );
1383
+ }
1384
+ if (attempt < bootstrapMaxRetries) {
1385
+ const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
1386
+ await waitWithAbort(state, backoffMs);
1387
+ }
1388
+ }
1389
+ return false;
1390
+ }
1391
+ async function runBootstrapSyncAttempt(state, onBootstrapPartition, onError) {
1392
+ try {
1393
+ return await onBootstrapPartition(state.indexName, state.partitionId, state.primaryNodeId);
1394
+ } catch (error) {
1395
+ if (state.aborted) {
1396
+ return false;
1397
+ }
1398
+ if (onError !== void 0) {
1399
+ const cause = error instanceof Error ? error.message : String(error);
1400
+ onError(
1401
+ new NarsilError(
1402
+ ErrorCodes.NODE_BOOTSTRAP_FAILED,
1403
+ `Bootstrap sync failed for ${state.indexName}:${state.partitionId}`,
1404
+ { indexName: state.indexName, partitionId: state.partitionId, cause }
1405
+ )
1406
+ );
1407
+ }
1408
+ return false;
1409
+ }
1410
+ }
1411
+ async function retryReportBootstrapComplete(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onError) {
1412
+ for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
1413
+ if (state.aborted) {
1414
+ return false;
1415
+ }
1416
+ if (attempt > 0 && !await partitionAwaitsBootstrapReport(state, coordinator)) {
1417
+ return true;
1418
+ }
1419
+ const accepted = await reportBootstrapComplete(state.indexName, state.partitionId, nodeId, coordinator, transport);
1420
+ if (accepted) {
1421
+ return true;
1422
+ }
1423
+ if (!await partitionAwaitsBootstrapReport(state, coordinator)) {
1424
+ return true;
1425
+ }
1426
+ if (attempt < bootstrapMaxRetries && !state.aborted) {
1427
+ const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
1428
+ await waitWithAbort(state, backoffMs);
1429
+ }
1430
+ }
1431
+ if (onError !== void 0) {
1432
+ onError(
1433
+ new NarsilError(
1434
+ ErrorCodes.NODE_BOOTSTRAP_FAILED,
1435
+ `Failed to report bootstrap completion for ${state.indexName}:${state.partitionId} after ${bootstrapMaxRetries + 1} attempts`,
1436
+ { indexName: state.indexName, partitionId: state.partitionId }
1437
+ )
1438
+ );
1439
+ }
1440
+ return false;
1441
+ }
1442
+ function waitWithAbort(state, ms) {
1443
+ return new Promise((resolve) => {
1444
+ if (state.aborted) {
1445
+ resolve();
1446
+ return;
1447
+ }
1448
+ state.abortResolve = resolve;
1449
+ state.retryTimer = setTimeout(() => {
1450
+ state.retryTimer = null;
1451
+ state.abortResolve = null;
1452
+ resolve();
1453
+ }, ms);
1454
+ });
1455
+ }
1456
+ function abortBootstrapState(state) {
1457
+ state.aborted = true;
1458
+ if (state.retryTimer !== null) {
1459
+ clearTimeout(state.retryTimer);
1460
+ state.retryTimer = null;
1461
+ }
1462
+ if (state.abortResolve !== null) {
1463
+ state.abortResolve();
1464
+ state.abortResolve = null;
1465
+ }
1466
+ }
1467
+ function createBootstrapState(indexName, partitionId, primaryNodeId) {
1468
+ return {
1469
+ indexName,
1470
+ partitionId,
1471
+ primaryNodeId,
1472
+ retryCount: 0,
1473
+ retryTimer: null,
1474
+ aborted: false,
1475
+ abortResolve: null
1476
+ };
1477
+ }
1478
+
1479
+ // src/distribution/cluster/node-lifecycle/allocation-watcher.ts
1480
+ function partitionKey(indexName, partitionId) {
1481
+ return `${indexName}:${partitionId}`;
1482
+ }
1483
+ function createAllocationWatcherState() {
1484
+ return {
1485
+ unwatchAllocation: null,
1486
+ debounceTimer: null,
1487
+ activeBootstraps: /* @__PURE__ */ new Map(),
1488
+ trackedPartitions: /* @__PURE__ */ new Map(),
1489
+ pendingTables: /* @__PURE__ */ new Map(),
1490
+ restartWaiters: /* @__PURE__ */ new Map(),
1491
+ stopped: false
1492
+ };
1493
+ }
1494
+ async function startAllocationWatcher(state, config) {
1495
+ const unwatchAllocation = await config.coordinator.watchAllocation((event) => {
1496
+ if (state.stopped) {
1497
+ return;
1498
+ }
1499
+ scheduleProcessAllocation(state, config, event.table);
1500
+ });
1501
+ state.unwatchAllocation = unwatchAllocation;
1502
+ }
1503
+ function scheduleProcessAllocation(state, config, table) {
1504
+ state.pendingTables.set(table.indexName, table);
1505
+ if (state.debounceTimer !== null) {
1506
+ clearTimeout(state.debounceTimer);
1507
+ }
1508
+ state.debounceTimer = setTimeout(() => {
1509
+ state.debounceTimer = null;
1510
+ const tablesToProcess = Array.from(state.pendingTables.values());
1511
+ state.pendingTables.clear();
1512
+ for (const pending of tablesToProcess) {
1513
+ processAllocationChange(state, config, pending);
1514
+ }
1515
+ }, config.allocationDebounceMs);
1516
+ }
1517
+ function processAllocationChange(state, config, table) {
1518
+ if (state.stopped) {
1519
+ return;
1520
+ }
1521
+ const nodeId = config.registration.nodeId;
1522
+ const currentKeys = /* @__PURE__ */ new Set();
1523
+ for (const [partitionId, assignment] of table.assignments) {
1524
+ if (!isNodeAssigned(assignment, nodeId)) {
1525
+ continue;
1526
+ }
1527
+ const key = partitionKey(table.indexName, partitionId);
1528
+ currentKeys.add(key);
1529
+ const existing = state.trackedPartitions.get(key);
1530
+ const isPrimary = assignment.primary === nodeId;
1531
+ if (existing === void 0) {
1532
+ handleNewAssignment(state, config, table.indexName, partitionId, assignment, nodeId);
1533
+ } else {
1534
+ handleExistingAssignment(state, config, existing, assignment, nodeId);
1535
+ }
1536
+ state.trackedPartitions.set(key, {
1537
+ indexName: table.indexName,
1538
+ partitionId,
1539
+ isPrimary,
1540
+ primaryNodeId: assignment.primary,
1541
+ primaryTerm: assignment.primaryTerm
1542
+ });
1543
+ }
1544
+ removeUnassignedPartitions(state, config, table.indexName, currentKeys);
1545
+ }
1546
+ function isNodeAssigned(assignment, nodeId) {
1547
+ if (assignment.primary === nodeId) {
1548
+ return true;
1549
+ }
1550
+ return assignment.replicas.includes(nodeId);
1551
+ }
1552
+ function handleNewAssignment(state, config, indexName, partitionId, assignment, nodeId) {
1553
+ if (assignment.state === "ACTIVE" && assignment.primary === nodeId) {
1554
+ return;
1555
+ }
1556
+ if (assignment.primary === null) {
1557
+ return;
1558
+ }
1559
+ if (assignment.primary === nodeId) {
1560
+ if (assignment.state === "INITIALISING") {
1561
+ startBootstrap(state, config, indexName, partitionId, nodeId);
1562
+ }
1563
+ return;
1564
+ }
1565
+ startBootstrap(state, config, indexName, partitionId, assignment.primary);
1566
+ }
1567
+ function handleExistingAssignment(state, config, existing, assignment, nodeId) {
1568
+ if (existing.isPrimary && assignment.primary !== nodeId && assignment.primaryTerm > existing.primaryTerm) {
1569
+ if (assignment.primary !== null) {
1570
+ if (config.onPrimaryDemotion !== void 0) {
1571
+ config.onPrimaryDemotion(existing.indexName, existing.partitionId, assignment.primary);
1572
+ }
1573
+ startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
1574
+ }
1575
+ return;
1576
+ }
1577
+ if (assignment.primary !== null && replicaNeedsResync(state, existing, assignment, nodeId)) {
1578
+ startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
1579
+ }
1580
+ }
1581
+ function replicaNeedsResync(state, existing, assignment, nodeId) {
1582
+ if (assignment.state !== "ACTIVE") {
1583
+ return false;
1584
+ }
1585
+ if (assignment.primary === null || assignment.primary === nodeId) {
1586
+ return false;
1587
+ }
1588
+ if (!assignment.replicas.includes(nodeId)) {
1589
+ return false;
1590
+ }
1591
+ if (assignment.inSyncSet.includes(nodeId)) {
1592
+ return false;
1593
+ }
1594
+ return !state.activeBootstraps.has(partitionKey(existing.indexName, existing.partitionId));
1595
+ }
1596
+ function startBootstrap(state, config, indexName, partitionId, primaryNodeId) {
1597
+ const key = partitionKey(indexName, partitionId);
1598
+ const existingBootstrap = state.activeBootstraps.get(key);
1599
+ if (existingBootstrap !== void 0) {
1600
+ abortBootstrapState(existingBootstrap);
1601
+ }
1602
+ const bootstrapState = createBootstrapState(indexName, partitionId, primaryNodeId);
1603
+ state.activeBootstraps.set(key, bootstrapState);
1604
+ bootstrapPartition(
1605
+ bootstrapState,
1606
+ config.coordinator,
1607
+ config.transport,
1608
+ config.registration.nodeId,
1609
+ config.bootstrapRetryBaseMs,
1610
+ config.bootstrapRetryMaxMs,
1611
+ config.bootstrapMaxRetries,
1612
+ config.onBootstrapPartition,
1387
1613
  config.onError
1388
1614
  ).then((succeeded) => {
1389
1615
  if (state.activeBootstraps.get(key) === bootstrapState) {
@@ -1391,12 +1617,14 @@ function startBootstrap(state, config, indexName, partitionId, primaryNodeId) {
1391
1617
  }
1392
1618
  if (!succeeded) {
1393
1619
  state.trackedPartitions.delete(key);
1620
+ void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
1394
1621
  }
1395
1622
  }).catch(() => {
1396
1623
  if (state.activeBootstraps.get(key) === bootstrapState) {
1397
1624
  state.activeBootstraps.delete(key);
1398
1625
  }
1399
1626
  state.trackedPartitions.delete(key);
1627
+ void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
1400
1628
  });
1401
1629
  }
1402
1630
  function removeUnassignedPartitions(state, config, indexName, currentKeys) {
@@ -1413,37 +1641,122 @@ function removeUnassignedPartitions(state, config, indexName, currentKeys) {
1413
1641
  abortBootstrapState(activeBootstrap);
1414
1642
  state.activeBootstraps.delete(key);
1415
1643
  }
1416
- if (config.onRemovePartition !== void 0) {
1417
- config.onRemovePartition(tracked.indexName, tracked.partitionId);
1644
+ if (config.onRemovePartition !== void 0) {
1645
+ config.onRemovePartition(tracked.indexName, tracked.partitionId);
1646
+ }
1647
+ }
1648
+ }
1649
+ function stopAllocationWatcher(state) {
1650
+ state.stopped = true;
1651
+ if (state.unwatchAllocation !== null) {
1652
+ state.unwatchAllocation();
1653
+ state.unwatchAllocation = null;
1654
+ }
1655
+ if (state.debounceTimer !== null) {
1656
+ clearTimeout(state.debounceTimer);
1657
+ state.debounceTimer = null;
1658
+ }
1659
+ for (const [timer, resolve] of state.restartWaiters) {
1660
+ clearTimeout(timer);
1661
+ resolve();
1662
+ }
1663
+ state.restartWaiters.clear();
1664
+ for (const bootstrapState of state.activeBootstraps.values()) {
1665
+ abortBootstrapState(bootstrapState);
1666
+ }
1667
+ state.activeBootstraps.clear();
1668
+ state.trackedPartitions.clear();
1669
+ state.pendingTables.clear();
1670
+ }
1671
+ function processInitialAllocations(state, config, tables) {
1672
+ for (const table of tables) {
1673
+ processAllocationChange(state, config, table);
1674
+ }
1675
+ }
1676
+ function waitBeforeBootstrapRestart(state, config) {
1677
+ const delayMs = Math.min(config.bootstrapRetryMaxMs, config.bootstrapRetryBaseMs);
1678
+ if (delayMs <= 0) {
1679
+ return Promise.resolve();
1680
+ }
1681
+ return new Promise((resolve) => {
1682
+ const timer = setTimeout(() => {
1683
+ state.restartWaiters.delete(timer);
1684
+ resolve();
1685
+ }, delayMs);
1686
+ timer.unref?.();
1687
+ state.restartWaiters.set(timer, resolve);
1688
+ });
1689
+ }
1690
+ async function restartBootstrapWhileOutOfSync(state, config, indexName, partitionId) {
1691
+ if (state.stopped) {
1692
+ return;
1693
+ }
1694
+ await waitBeforeBootstrapRestart(state, config);
1695
+ if (state.stopped) {
1696
+ return;
1697
+ }
1698
+ const nodeId = config.registration.nodeId;
1699
+ let table;
1700
+ try {
1701
+ table = await config.coordinator.getAllocation(indexName);
1702
+ } catch (error) {
1703
+ if (config.onError !== void 0) {
1704
+ config.onError(error);
1705
+ }
1706
+ if (!state.stopped) {
1707
+ void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
1418
1708
  }
1709
+ return;
1419
1710
  }
1420
- }
1421
- function stopAllocationWatcher(state) {
1422
- state.stopped = true;
1423
- if (state.unwatchAllocation !== null) {
1424
- state.unwatchAllocation();
1425
- state.unwatchAllocation = null;
1711
+ if (state.stopped || table === null) {
1712
+ return;
1426
1713
  }
1427
- if (state.debounceTimer !== null) {
1428
- clearTimeout(state.debounceTimer);
1429
- state.debounceTimer = null;
1714
+ const assignment = table.assignments.get(partitionId);
1715
+ if (assignment === void 0 || assignment.primary === null || assignment.primary === nodeId) {
1716
+ return;
1430
1717
  }
1431
- for (const bootstrapState of state.activeBootstraps.values()) {
1432
- abortBootstrapState(bootstrapState);
1718
+ if (!assignment.replicas.includes(nodeId) || assignment.inSyncSet.includes(nodeId)) {
1719
+ return;
1433
1720
  }
1434
- state.activeBootstraps.clear();
1435
- state.trackedPartitions.clear();
1436
- state.pendingTables.clear();
1437
- }
1438
- function processInitialAllocations(state, config, tables) {
1439
- for (const table of tables) {
1440
- processAllocationChange(state, config, table);
1721
+ if (assignment.state !== "ACTIVE" && assignment.state !== "INITIALISING" && assignment.state !== "MIGRATING") {
1722
+ return;
1441
1723
  }
1724
+ if (state.activeBootstraps.has(partitionKey(indexName, partitionId))) {
1725
+ return;
1726
+ }
1727
+ startBootstrap(state, config, indexName, partitionId, assignment.primary);
1728
+ }
1729
+
1730
+ // src/distribution/cluster/node-lifecycle/heartbeat.ts
1731
+ function createRegistrationHeartbeatState() {
1732
+ return { timer: null, inFlight: Promise.resolve() };
1733
+ }
1734
+ function clearHeartbeatTimer(state) {
1735
+ if (state.timer !== null) {
1736
+ clearInterval(state.timer);
1737
+ state.timer = null;
1738
+ }
1739
+ }
1740
+ function startRegistrationHeartbeat(state, config) {
1741
+ clearHeartbeatTimer(state);
1742
+ state.timer = setInterval(() => {
1743
+ state.inFlight = state.inFlight.then(() => config.coordinator.registerNode(config.registration)).catch((error) => {
1744
+ if (config.onError !== void 0) {
1745
+ config.onError(error);
1746
+ }
1747
+ });
1748
+ }, config.nodeHeartbeatIntervalMs);
1749
+ state.timer.unref?.();
1750
+ }
1751
+ async function stopRegistrationHeartbeat(state) {
1752
+ clearHeartbeatTimer(state);
1753
+ await state.inFlight;
1442
1754
  }
1443
1755
 
1444
1756
  // src/distribution/cluster/node-lifecycle/join.ts
1445
- async function joinCluster(config, watcherState) {
1757
+ async function joinCluster(config, watcherState, heartbeatState) {
1446
1758
  await config.coordinator.registerNode(config.registration);
1759
+ startRegistrationHeartbeat(heartbeatState, config);
1447
1760
  const initialTables = await loadInitialAllocations(config);
1448
1761
  processInitialAllocations(watcherState, config, initialTables);
1449
1762
  await startAllocationWatcher(watcherState, config);
@@ -1465,15 +1778,17 @@ async function leaveCluster(config) {
1465
1778
  // src/distribution/cluster/node-lifecycle/types.ts
1466
1779
  var DEFAULT_NODE_LIFECYCLE_CONFIG = {
1467
1780
  bootstrapRetryBaseMs: 1e3,
1468
- bootstrapRetryMaxMs: 3e4,
1781
+ bootstrapRetryMaxMs: 5e3,
1469
1782
  bootstrapMaxRetries: 10,
1470
- allocationDebounceMs: 250
1783
+ allocationDebounceMs: 250,
1784
+ nodeHeartbeatIntervalMs: 1e4
1471
1785
  };
1472
1786
 
1473
1787
  // src/distribution/cluster/node-lifecycle/index.ts
1474
1788
  function createDataNodeLifecycle(config) {
1475
1789
  let status = "stopped";
1476
1790
  let watcherState = createAllocationWatcherState();
1791
+ const heartbeatState = createRegistrationHeartbeatState();
1477
1792
  let operationLock = Promise.resolve();
1478
1793
  function withLock(fn) {
1479
1794
  const prev = operationLock;
@@ -1516,9 +1831,10 @@ function createDataNodeLifecycle(config) {
1516
1831
  }
1517
1832
  status = "joining";
1518
1833
  try {
1519
- await joinCluster(config, watcherState);
1834
+ await joinCluster(config, watcherState, heartbeatState);
1520
1835
  status = "active";
1521
1836
  } catch (error) {
1837
+ await stopRegistrationHeartbeat(heartbeatState);
1522
1838
  status = "stopped";
1523
1839
  throw error;
1524
1840
  }
@@ -1530,6 +1846,7 @@ function createDataNodeLifecycle(config) {
1530
1846
  return;
1531
1847
  }
1532
1848
  status = "leaving";
1849
+ await stopRegistrationHeartbeat(heartbeatState);
1533
1850
  stopAllocationWatcher(watcherState);
1534
1851
  try {
1535
1852
  await leaveCluster(config);
@@ -1544,6 +1861,7 @@ function createDataNodeLifecycle(config) {
1544
1861
  if (status === "shutdown") {
1545
1862
  return;
1546
1863
  }
1864
+ await stopRegistrationHeartbeat(heartbeatState);
1547
1865
  stopAllocationWatcher(watcherState);
1548
1866
  if (status === "active" || status === "joining") {
1549
1867
  try {
@@ -2818,11 +3136,47 @@ function validateLocalPartitionCount(engine, indexName, primaryNodeId, partition
2818
3136
  // src/distribution/cluster-node/bootstrap-sync/targets.ts
2819
3137
  function readLocalLogState(indexName, partitionId, deps) {
2820
3138
  const log = deps.getReplicationLog(indexName, partitionId);
2821
- const lastSeqNo = log.committedSeqNo;
2822
- const lastPrimaryTerm = log.committedPrimaryTerm;
3139
+ const lastSeqNo = log.localLogEnd;
3140
+ const lastPrimaryTerm = log.localLogEndPrimaryTerm;
2823
3141
  const newestEntry = log.getEntry(lastSeqNo);
2824
3142
  return { lastSeqNo, lastPrimaryTerm: newestEntry?.primaryTerm ?? lastPrimaryTerm };
2825
3143
  }
3144
+ var SYNCED_POSITION_REPORT_TIMEOUT_MS = 5e3;
3145
+ async function reportSyncedPosition(indexName, partitionId, target, knownSeqNo, deadline, deps) {
3146
+ const logState = readLocalLogState(indexName, partitionId, deps);
3147
+ if (logState.lastSeqNo <= knownSeqNo) {
3148
+ return;
3149
+ }
3150
+ const budgetMs = Math.min(SYNCED_POSITION_REPORT_TIMEOUT_MS, deadline - Date.now());
3151
+ if (budgetMs <= 0) {
3152
+ return;
3153
+ }
3154
+ const payload = {
3155
+ indexName,
3156
+ partitionId,
3157
+ lastSeqNo: logState.lastSeqNo,
3158
+ lastPrimaryTerm: logState.lastPrimaryTerm
3159
+ };
3160
+ const request = {
3161
+ type: ReplicationMessageTypes.SYNC_REQUEST,
3162
+ sourceId: deps.sourceNodeId,
3163
+ requestId: generateId(),
3164
+ payload: encode(payload)
3165
+ };
3166
+ try {
3167
+ await withDeadline(
3168
+ deps.transport.stream(target, request, () => {
3169
+ }),
3170
+ budgetMs,
3171
+ indexName,
3172
+ "synced-position-report"
3173
+ );
3174
+ } catch (error) {
3175
+ if (deps.onError !== void 0) {
3176
+ deps.onError(error);
3177
+ }
3178
+ }
3179
+ }
2826
3180
  async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNodeId, targets, coordinatorSchema, partitionCount, deadline, deps, abortCheck) {
2827
3181
  let lastError = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "no live sync attempts were made", {
2828
3182
  indexName,
@@ -2856,6 +3210,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
2856
3210
  )
2857
3211
  };
2858
3212
  }
3213
+ const positionBeforeAttempt = readLocalLogState(indexName, partitionId, deps).lastSeqNo;
2859
3214
  const attempt = await syncFromTarget(
2860
3215
  state,
2861
3216
  entry,
@@ -2869,6 +3224,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
2869
3224
  abortCheck
2870
3225
  );
2871
3226
  if (attempt.ok) {
3227
+ await reportSyncedPosition(indexName, partitionId, target, positionBeforeAttempt, deadline, deps);
2872
3228
  return attempt;
2873
3229
  }
2874
3230
  lastError = new NarsilError(attempt.error.code, attempt.error.message, {
@@ -2974,94 +3330,288 @@ async function syncFromTarget(state, entry, indexName, partitionId, target, coor
2974
3330
  };
2975
3331
  }
2976
3332
 
2977
- // src/distribution/cluster-node/bootstrap-sync/live.ts
2978
- async function resolveLivePartitionCount(coordinator, indexName, partitionId, primaryNodeId, abortPromise) {
2979
- try {
2980
- const winner = await Promise.race([coordinator.getAllocation(indexName), abortPromise]);
2981
- if (winner === ABORT_SENTINEL) {
2982
- return "aborted";
3333
+ // src/distribution/cluster-node/bootstrap-sync/live.ts
3334
+ async function resolveLivePartitionCount(coordinator, indexName, partitionId, primaryNodeId, abortPromise) {
3335
+ try {
3336
+ const winner = await Promise.race([coordinator.getAllocation(indexName), abortPromise]);
3337
+ if (winner === ABORT_SENTINEL) {
3338
+ return "aborted";
3339
+ }
3340
+ const allocation = winner;
3341
+ if (allocation === null) {
3342
+ return {
3343
+ error: new NarsilError(
3344
+ ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
3345
+ `coordinator has no allocation for index '${indexName}'`,
3346
+ { indexName, primaryNodeId }
3347
+ )
3348
+ };
3349
+ }
3350
+ const assignment = allocation.assignments.get(partitionId);
3351
+ if (assignment === void 0) {
3352
+ return {
3353
+ error: new NarsilError(
3354
+ ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
3355
+ `No assignment exists for partition ${partitionId} of index '${indexName}'`,
3356
+ { indexName, partitionId, primaryNodeId }
3357
+ )
3358
+ };
3359
+ }
3360
+ if (assignment.primary !== primaryNodeId) {
3361
+ return {
3362
+ error: new NarsilError(
3363
+ ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
3364
+ `Partition ${partitionId} of index '${indexName}' is no longer primary on '${primaryNodeId}'`,
3365
+ { indexName, partitionId, primaryNodeId, currentPrimary: assignment.primary }
3366
+ )
3367
+ };
3368
+ }
3369
+ return { partitionCount: allocation.assignments.size };
3370
+ } catch (err) {
3371
+ const cause = err instanceof Error ? err.message : String(err);
3372
+ return {
3373
+ error: new NarsilError(
3374
+ ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
3375
+ `coordinator.getAllocation failed: ${cause}`,
3376
+ {
3377
+ indexName,
3378
+ partitionId,
3379
+ primaryNodeId,
3380
+ cause
3381
+ }
3382
+ )
3383
+ };
3384
+ }
3385
+ }
3386
+ async function executeLiveBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps, deadlineMs, deadline, abortCheck) {
3387
+ const schemaResult = await loadCoordinatorSchema(deps.coordinator, indexName, primaryNodeId, entry.abortPromise);
3388
+ if (schemaResult === "aborted") {
3389
+ surfaceAborted(deps, indexName, primaryNodeId);
3390
+ return false;
3391
+ }
3392
+ if ("error" in schemaResult) {
3393
+ surfaceError(deps, indexName, primaryNodeId, schemaResult.error);
3394
+ return false;
3395
+ }
3396
+ if (abortCheck()) {
3397
+ surfaceAborted(deps, indexName, primaryNodeId);
3398
+ return false;
3399
+ }
3400
+ const allocationResult = await resolveLivePartitionCount(
3401
+ deps.coordinator,
3402
+ indexName,
3403
+ partitionId,
3404
+ primaryNodeId,
3405
+ entry.abortPromise
3406
+ );
3407
+ if (allocationResult === "aborted") {
3408
+ surfaceAborted(deps, indexName, primaryNodeId);
3409
+ return false;
3410
+ }
3411
+ if ("error" in allocationResult) {
3412
+ surfaceError(deps, indexName, primaryNodeId, allocationResult.error);
3413
+ return false;
3414
+ }
3415
+ if (Date.now() >= deadline) {
3416
+ surfaceError(
3417
+ deps,
3418
+ indexName,
3419
+ primaryNodeId,
3420
+ new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before starting live sync", {
3421
+ indexName,
3422
+ deadlineMs
3423
+ })
3424
+ );
3425
+ return false;
3426
+ }
3427
+ const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
3428
+ if (targetsResult instanceof NarsilError) {
3429
+ surfaceError(deps, indexName, primaryNodeId, targetsResult);
3430
+ return false;
3431
+ }
3432
+ const syncResult = await syncFromAnyTarget(
3433
+ state,
3434
+ entry,
3435
+ indexName,
3436
+ partitionId,
3437
+ primaryNodeId,
3438
+ targetsResult,
3439
+ schemaResult.schema,
3440
+ allocationResult.partitionCount,
3441
+ deadline,
3442
+ deps,
3443
+ abortCheck
3444
+ );
3445
+ if (!syncResult.ok) {
3446
+ if (syncResult.error.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
3447
+ surfaceAborted(deps, indexName, primaryNodeId);
3448
+ return false;
3449
+ }
3450
+ if (syncResult.error.details.alreadySurfaced !== true) {
3451
+ surfaceError(deps, indexName, primaryNodeId, syncResult.error);
3452
+ }
3453
+ return false;
3454
+ }
3455
+ if (abortCheck()) {
3456
+ surfaceAborted(deps, indexName, primaryNodeId);
3457
+ return false;
3458
+ }
3459
+ if (syncResult.snapshotHeader !== null) {
3460
+ deps.onSnapshotApplied?.(indexName, partitionId, syncResult.snapshotHeader);
3461
+ }
3462
+ state.completed.add(entryKey(indexName, partitionId));
3463
+ return true;
3464
+ }
3465
+
3466
+ // src/distribution/cluster-node/bootstrap-sync/snapshot-restore.ts
3467
+ async function fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps) {
3468
+ const dropError = await dropExistingIndex(deps.engine, indexName, primaryNodeId);
3469
+ if (dropError !== null) {
3470
+ surfaceError(deps, indexName, primaryNodeId, dropError);
3471
+ return null;
3472
+ }
3473
+ if (entry.aborted) {
3474
+ return "aborted";
3475
+ }
3476
+ const schemaResult = await loadCoordinatorSchema(deps.coordinator, indexName, primaryNodeId, entry.abortPromise);
3477
+ if (schemaResult === "aborted") {
3478
+ return "aborted";
3479
+ }
3480
+ if ("error" in schemaResult) {
3481
+ surfaceError(deps, indexName, primaryNodeId, schemaResult.error);
3482
+ return null;
3483
+ }
3484
+ return schemaResult.schema;
3485
+ }
3486
+ async function applyRestore(state, entry, indexName, primaryNodeId, bytes, coordinatorSchema, deadline, deps) {
3487
+ const remainingMs = deadline - Date.now();
3488
+ if (remainingMs <= 0) {
3489
+ surfaceError(
3490
+ deps,
3491
+ indexName,
3492
+ primaryNodeId,
3493
+ new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before restore", {
3494
+ indexName
3495
+ })
3496
+ );
3497
+ return false;
3498
+ }
3499
+ const key = entryKey(indexName, entry.partitionId);
3500
+ const generationBeforeRestore = state.generations.get(key) ?? 0;
3501
+ if (generationBeforeRestore !== entry.generation || entry.aborted) {
3502
+ surfaceAborted(deps, indexName, primaryNodeId);
3503
+ return false;
3504
+ }
3505
+ const restoreError = await executeEngineRestore(deps.engine, indexName, primaryNodeId, bytes, remainingMs);
3506
+ if (restoreError !== null) {
3507
+ surfaceError(deps, indexName, primaryNodeId, restoreError);
3508
+ return false;
3509
+ }
3510
+ const generationAfterRestore = state.generations.get(key) ?? 0;
3511
+ if (generationAfterRestore !== entry.generation || entry.aborted) {
3512
+ if (!anotherBootstrapOwnsKey(state, key, entry)) {
3513
+ await dropRestoredIndexQuietly(deps.engine, indexName, deps);
3514
+ }
3515
+ surfaceAborted(deps, indexName, primaryNodeId);
3516
+ return false;
3517
+ }
3518
+ const schemaError = validateRestoredSchema(deps.engine, indexName, primaryNodeId, coordinatorSchema);
3519
+ if (schemaError !== null) {
3520
+ await dropRestoredIndexQuietly(deps.engine, indexName, deps);
3521
+ surfaceError(deps, indexName, primaryNodeId, schemaError);
3522
+ return false;
3523
+ }
3524
+ return true;
3525
+ }
3526
+
3527
+ // src/distribution/cluster-node/bootstrap-sync/index.ts
3528
+ async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
3529
+ const validationError = validateArguments(indexName, partitionId, primaryNodeId);
3530
+ if (validationError !== null) {
3531
+ surfaceError(deps, indexName, primaryNodeId, validationError);
3532
+ return false;
3533
+ }
3534
+ const key = entryKey(indexName, partitionId);
3535
+ if (state.completed.has(key)) {
3536
+ if (!await replicaResyncRequired(indexName, partitionId, deps)) {
3537
+ return true;
2983
3538
  }
2984
- const allocation = winner;
2985
- if (allocation === null) {
2986
- return {
2987
- error: new NarsilError(
2988
- ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
2989
- `coordinator has no allocation for index '${indexName}'`,
2990
- { indexName, primaryNodeId }
2991
- )
2992
- };
3539
+ clearBootstrapSyncIndex(state, indexName, partitionId);
3540
+ }
3541
+ const existing = state.inFlight.get(key);
3542
+ if (existing !== void 0) {
3543
+ const result = await Promise.race([existing.promise, existing.abortPromise]);
3544
+ if (result === ABORT_SENTINEL) {
3545
+ return false;
2993
3546
  }
2994
- const assignment = allocation.assignments.get(partitionId);
2995
- if (assignment === void 0) {
2996
- return {
2997
- error: new NarsilError(
2998
- ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
2999
- `No assignment exists for partition ${partitionId} of index '${indexName}'`,
3000
- { indexName, partitionId, primaryNodeId }
3001
- )
3002
- };
3547
+ return result;
3548
+ }
3549
+ const entry = createEntry(state, indexName, partitionId, key);
3550
+ entry.promise = executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps).finally(() => {
3551
+ const current = state.inFlight.get(key);
3552
+ if (current === entry) {
3553
+ state.inFlight.delete(key);
3003
3554
  }
3004
- if (assignment.primary !== primaryNodeId) {
3005
- return {
3006
- error: new NarsilError(
3007
- ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
3008
- `Partition ${partitionId} of index '${indexName}' is no longer primary on '${primaryNodeId}'`,
3009
- { indexName, partitionId, primaryNodeId, currentPrimary: assignment.primary }
3010
- )
3011
- };
3555
+ if (!state.completed.has(key) && !state.inFlight.has(key)) {
3556
+ const currentGen = state.generations.get(key);
3557
+ if (currentGen !== void 0 && currentGen !== entry.generation) {
3558
+ state.generations.delete(key);
3559
+ }
3012
3560
  }
3013
- return { partitionCount: allocation.assignments.size };
3014
- } catch (err) {
3015
- const cause = err instanceof Error ? err.message : String(err);
3016
- return {
3017
- error: new NarsilError(
3018
- ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
3019
- `coordinator.getAllocation failed: ${cause}`,
3020
- {
3021
- indexName,
3022
- partitionId,
3023
- primaryNodeId,
3024
- cause
3025
- }
3026
- )
3027
- };
3561
+ });
3562
+ state.inFlight.set(key, entry);
3563
+ return entry.promise;
3564
+ }
3565
+ async function replicaResyncRequired(indexName, partitionId, deps) {
3566
+ const allocation = await deps.coordinator.getAllocation(indexName);
3567
+ const assignment = allocation?.assignments.get(partitionId);
3568
+ if (assignment === void 0) {
3569
+ return false;
3028
3570
  }
3571
+ return assignment.state === "ACTIVE" && assignment.replicas.includes(deps.sourceNodeId) && !assignment.inSyncSet.includes(deps.sourceNodeId);
3029
3572
  }
3030
- async function executeLiveBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps, deadlineMs, deadline, abortCheck) {
3031
- const schemaResult = await loadCoordinatorSchema(deps.coordinator, indexName, primaryNodeId, entry.abortPromise);
3573
+ async function executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps) {
3574
+ const deadlineMs = deps.deadlineMs ?? DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS;
3575
+ const deadline = Date.now() + deadlineMs;
3576
+ const key = entryKey(indexName, partitionId);
3577
+ const abortCheck = () => {
3578
+ const currentGeneration = state.generations.get(key) ?? 0;
3579
+ if (currentGeneration !== entry.generation) {
3580
+ entry.aborted = true;
3581
+ }
3582
+ return entry.aborted;
3583
+ };
3584
+ if (hasLiveBootstrapSyncDeps(deps)) {
3585
+ return executeLiveBootstrapSync(
3586
+ state,
3587
+ entry,
3588
+ indexName,
3589
+ partitionId,
3590
+ primaryNodeId,
3591
+ deps,
3592
+ deadlineMs,
3593
+ deadline,
3594
+ abortCheck
3595
+ );
3596
+ }
3597
+ const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
3032
3598
  if (schemaResult === "aborted") {
3033
3599
  surfaceAborted(deps, indexName, primaryNodeId);
3034
3600
  return false;
3035
3601
  }
3036
- if ("error" in schemaResult) {
3037
- surfaceError(deps, indexName, primaryNodeId, schemaResult.error);
3602
+ if (schemaResult === null) {
3038
3603
  return false;
3039
3604
  }
3040
3605
  if (abortCheck()) {
3041
3606
  surfaceAborted(deps, indexName, primaryNodeId);
3042
3607
  return false;
3043
3608
  }
3044
- const allocationResult = await resolveLivePartitionCount(
3045
- deps.coordinator,
3046
- indexName,
3047
- partitionId,
3048
- primaryNodeId,
3049
- entry.abortPromise
3050
- );
3051
- if (allocationResult === "aborted") {
3052
- surfaceAborted(deps, indexName, primaryNodeId);
3053
- return false;
3054
- }
3055
- if ("error" in allocationResult) {
3056
- surfaceError(deps, indexName, primaryNodeId, allocationResult.error);
3057
- return false;
3058
- }
3059
3609
  if (Date.now() >= deadline) {
3060
3610
  surfaceError(
3061
3611
  deps,
3062
3612
  indexName,
3063
3613
  primaryNodeId,
3064
- new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before starting live sync", {
3614
+ new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
3065
3615
  indexName,
3066
3616
  deadlineMs
3067
3617
  })
@@ -3073,254 +3623,475 @@ async function executeLiveBootstrapSync(state, entry, indexName, partitionId, pr
3073
3623
  surfaceError(deps, indexName, primaryNodeId, targetsResult);
3074
3624
  return false;
3075
3625
  }
3076
- const syncResult = await syncFromAnyTarget(
3077
- state,
3078
- entry,
3626
+ const fetchDeps = { transport: deps.transport, sourceNodeId: deps.sourceNodeId };
3627
+ const fetchResult = await fetchSnapshotFromAnyTarget(
3079
3628
  indexName,
3080
- partitionId,
3081
3629
  primaryNodeId,
3082
3630
  targetsResult,
3083
- schemaResult.schema,
3084
- allocationResult.partitionCount,
3085
3631
  deadline,
3086
- deps,
3087
- abortCheck
3632
+ fetchDeps,
3633
+ abortCheck,
3634
+ partitionId
3088
3635
  );
3089
- if (!syncResult.ok) {
3090
- if (syncResult.error.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
3636
+ if (!fetchResult.ok) {
3637
+ if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
3091
3638
  surfaceAborted(deps, indexName, primaryNodeId);
3092
3639
  return false;
3093
3640
  }
3094
- if (syncResult.error.details.alreadySurfaced !== true) {
3095
- surfaceError(deps, indexName, primaryNodeId, syncResult.error);
3641
+ surfaceError(
3642
+ deps,
3643
+ indexName,
3644
+ primaryNodeId,
3645
+ new NarsilError(fetchResult.code, fetchResult.message, {
3646
+ indexName,
3647
+ primaryNodeId,
3648
+ ...fetchResult.details
3649
+ })
3650
+ );
3651
+ return false;
3652
+ }
3653
+ if (abortCheck()) {
3654
+ surfaceAborted(deps, indexName, primaryNodeId);
3655
+ return false;
3656
+ }
3657
+ const restoreSucceeded = await applyRestore(
3658
+ state,
3659
+ entry,
3660
+ indexName,
3661
+ primaryNodeId,
3662
+ fetchResult.bytes,
3663
+ schemaResult,
3664
+ deadline,
3665
+ deps
3666
+ );
3667
+ if (!restoreSucceeded) {
3668
+ return false;
3669
+ }
3670
+ if (abortCheck()) {
3671
+ if (!anotherBootstrapOwnsKey(state, key, entry)) {
3672
+ await dropRestoredIndexQuietly(deps.engine, indexName, deps);
3673
+ }
3674
+ surfaceAborted(deps, indexName, primaryNodeId);
3675
+ return false;
3676
+ }
3677
+ deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
3678
+ state.completed.add(key);
3679
+ return true;
3680
+ }
3681
+
3682
+ // src/distribution/cluster-node/write-routing/assignment.ts
3683
+ function resolvePartitionId(docId, partitionCount) {
3684
+ return fnv1a(docId) % partitionCount;
3685
+ }
3686
+ async function resolveNodeTargets(nodeId, deps) {
3687
+ if (deps.resolveNodeTargets === void 0) {
3688
+ return [nodeId];
3689
+ }
3690
+ const targets = await deps.resolveNodeTargets(nodeId);
3691
+ return targets.length > 0 ? targets : [nodeId];
3692
+ }
3693
+ function requireAssignedPrimary(assignment, indexName, partitionId) {
3694
+ if (assignment === void 0 || assignment.primary === null) {
3695
+ throw new NarsilError(
3696
+ ErrorCodes.PARTITION_UNASSIGNED,
3697
+ `No primary assigned for partition ${partitionId} of index '${indexName}'`,
3698
+ { indexName, partitionId }
3699
+ );
3700
+ }
3701
+ return assignment;
3702
+ }
3703
+ async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrimary) {
3704
+ const table = await deps.coordinator.getAllocation(indexName);
3705
+ if (table === null || table.assignments.size === 0) {
3706
+ if (requireLocalPrimary) {
3707
+ throw new NarsilError(
3708
+ ErrorCodes.QUERY_ROUTING_FAILED,
3709
+ `No allocation table is available for forwarded write to index '${indexName}'`,
3710
+ { indexName }
3711
+ );
3712
+ }
3713
+ return null;
3714
+ }
3715
+ const partitionCount = table.assignments.size;
3716
+ const partitionId = resolvePartitionId(docId, partitionCount);
3717
+ const assignment = requireAssignedPrimary(table.assignments.get(partitionId), indexName, partitionId);
3718
+ if (requireLocalPrimary && assignment.primary !== deps.nodeId) {
3719
+ throw new NarsilError(
3720
+ ErrorCodes.PARTITION_NOT_PRIMARY,
3721
+ `Node '${deps.nodeId}' is not primary for partition ${partitionId} of index '${indexName}'`,
3722
+ { indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
3723
+ );
3724
+ }
3725
+ return { partitionId, assignment };
3726
+ }
3727
+ function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
3728
+ const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
3729
+ if (requiredActiveCopies <= 1) {
3730
+ return;
3731
+ }
3732
+ const activeCopies = 1 + getInSyncReplicaTargets(assignment, deps.nodeId).length;
3733
+ if (activeCopies < requiredActiveCopies) {
3734
+ throw new NarsilError(
3735
+ ErrorCodes.INSUFFICIENT_REPLICAS,
3736
+ `Write to partition ${partitionId} of index '${indexName}' requires ${requiredActiveCopies} active copies but only ${activeCopies} are available`,
3737
+ { indexName, partitionId, requiredActiveCopies, activeCopies }
3738
+ );
3739
+ }
3740
+ }
3741
+ function getInSyncReplicaTargets(assignment, localNodeId, pendingAdmissions = []) {
3742
+ const configuredReplicas = new Set(assignment.replicas);
3743
+ const targets = [];
3744
+ for (const nodeId of [...assignment.inSyncSet, ...pendingAdmissions]) {
3745
+ if (nodeId !== localNodeId && configuredReplicas.has(nodeId) && !targets.includes(nodeId)) {
3746
+ targets.push(nodeId);
3096
3747
  }
3097
- return false;
3098
3748
  }
3099
- if (abortCheck()) {
3100
- surfaceAborted(deps, indexName, primaryNodeId);
3749
+ return targets;
3750
+ }
3751
+
3752
+ // src/distribution/cluster-node/catch-up/state.ts
3753
+ var CATCH_UP_TICK_MS = 1e3;
3754
+ var CATCH_UP_IN_FLIGHT_BYTE_CEILING = 67108864;
3755
+ function partitionKey2(indexName, partitionId) {
3756
+ return `${indexName}:${partitionId}`;
3757
+ }
3758
+ function createCatchUpState() {
3759
+ return {
3760
+ cursors: /* @__PURE__ */ new Map(),
3761
+ pendingAdmissions: /* @__PURE__ */ new Map(),
3762
+ inFlightBytes: 0,
3763
+ timer: null,
3764
+ activeTick: null,
3765
+ stopped: false
3766
+ };
3767
+ }
3768
+ function recordReplicaPosition(state, indexName, partitionId, replicaNodeId, appliedSeqNo) {
3769
+ const key = partitionKey2(indexName, partitionId);
3770
+ const partition = state.cursors.get(key) ?? /* @__PURE__ */ new Map();
3771
+ const existing = partition.get(replicaNodeId);
3772
+ if (existing === void 0) {
3773
+ partition.set(replicaNodeId, { appliedSeqNo, sending: false, syncEpoch: 0 });
3774
+ } else if (appliedSeqNo > existing.appliedSeqNo) {
3775
+ existing.appliedSeqNo = appliedSeqNo;
3776
+ } else if (appliedSeqNo < existing.appliedSeqNo) {
3777
+ existing.appliedSeqNo = appliedSeqNo;
3778
+ existing.syncEpoch += 1;
3779
+ }
3780
+ state.cursors.set(key, partition);
3781
+ }
3782
+ function forgetReplica(state, indexName, partitionId, replicaNodeId) {
3783
+ const key = partitionKey2(indexName, partitionId);
3784
+ const partition = state.cursors.get(key);
3785
+ if (partition !== void 0) {
3786
+ partition.delete(replicaNodeId);
3787
+ if (partition.size === 0) {
3788
+ state.cursors.delete(key);
3789
+ }
3790
+ }
3791
+ clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
3792
+ }
3793
+ function getPendingAdmissions(state, indexName, partitionId) {
3794
+ const pending = state.pendingAdmissions.get(partitionKey2(indexName, partitionId));
3795
+ return pending === void 0 ? [] : [...pending];
3796
+ }
3797
+ function markPendingAdmission(state, indexName, partitionId, replicaNodeId) {
3798
+ const key = partitionKey2(indexName, partitionId);
3799
+ const pending = state.pendingAdmissions.get(key) ?? /* @__PURE__ */ new Set();
3800
+ if (pending.has(replicaNodeId)) {
3101
3801
  return false;
3102
3802
  }
3103
- if (syncResult.snapshotHeader !== null) {
3104
- deps.onSnapshotApplied?.(indexName, partitionId, syncResult.snapshotHeader);
3105
- }
3106
- state.completed.add(entryKey(indexName, partitionId));
3803
+ pending.add(replicaNodeId);
3804
+ state.pendingAdmissions.set(key, pending);
3107
3805
  return true;
3108
3806
  }
3109
-
3110
- // src/distribution/cluster-node/bootstrap-sync/snapshot-restore.ts
3111
- async function fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps) {
3112
- const dropError = await dropExistingIndex(deps.engine, indexName, primaryNodeId);
3113
- if (dropError !== null) {
3114
- surfaceError(deps, indexName, primaryNodeId, dropError);
3115
- return null;
3116
- }
3117
- if (entry.aborted) {
3118
- return "aborted";
3807
+ function clearPendingAdmission(state, indexName, partitionId, replicaNodeId) {
3808
+ const key = partitionKey2(indexName, partitionId);
3809
+ const pending = state.pendingAdmissions.get(key);
3810
+ if (pending === void 0) {
3811
+ return;
3119
3812
  }
3120
- const schemaResult = await loadCoordinatorSchema(deps.coordinator, indexName, primaryNodeId, entry.abortPromise);
3121
- if (schemaResult === "aborted") {
3122
- return "aborted";
3813
+ pending.delete(replicaNodeId);
3814
+ if (pending.size === 0) {
3815
+ state.pendingAdmissions.delete(key);
3123
3816
  }
3124
- if ("error" in schemaResult) {
3125
- surfaceError(deps, indexName, primaryNodeId, schemaResult.error);
3126
- return null;
3817
+ }
3818
+
3819
+ // src/distribution/cluster-node/catch-up/admission.ts
3820
+ var ADMISSION_TIMEOUT_MS = 1e4;
3821
+ async function sendToController(payload, controllerNodeId, deps) {
3822
+ const targets = await resolveNodeTargets(controllerNodeId, deps);
3823
+ for (const target of targets) {
3824
+ try {
3825
+ const result = await requestInsyncAdmission(payload, target, deps.transport, deps.nodeId);
3826
+ return result.accepted;
3827
+ } catch (_) {
3828
+ }
3127
3829
  }
3128
- return schemaResult.schema;
3830
+ return false;
3129
3831
  }
3130
- async function applyRestore(state, entry, indexName, primaryNodeId, bytes, coordinatorSchema, deadline, deps) {
3131
- const remainingMs = deadline - Date.now();
3132
- if (remainingMs <= 0) {
3133
- surfaceError(
3134
- deps,
3135
- indexName,
3136
- primaryNodeId,
3137
- new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before restore", {
3138
- indexName
3139
- })
3140
- );
3141
- return false;
3832
+ function withTimeout(work, timeoutMs, fallback) {
3833
+ return new Promise((resolve) => {
3834
+ const timer = setTimeout(() => resolve(fallback), timeoutMs);
3835
+ timer.unref?.();
3836
+ work.then((value) => {
3837
+ clearTimeout(timer);
3838
+ resolve(value);
3839
+ }).catch(() => {
3840
+ clearTimeout(timer);
3841
+ resolve(fallback);
3842
+ });
3843
+ });
3844
+ }
3845
+ async function withdrawAdmission(payload, controllerNodeId, deps) {
3846
+ const targets = await resolveNodeTargets(controllerNodeId, deps);
3847
+ for (const target of targets) {
3848
+ try {
3849
+ const result = await requestInsyncRemoval(
3850
+ payload.indexName,
3851
+ payload.partitionId,
3852
+ payload.replicaNodeId,
3853
+ payload.primaryTerm,
3854
+ target,
3855
+ deps.transport,
3856
+ deps.nodeId
3857
+ );
3858
+ if (result.accepted) {
3859
+ return;
3860
+ }
3861
+ } catch (_) {
3862
+ }
3142
3863
  }
3143
- const key = entryKey(indexName, entry.partitionId);
3144
- const generationBeforeRestore = state.generations.get(key) ?? 0;
3145
- if (generationBeforeRestore !== entry.generation || entry.aborted) {
3146
- surfaceAborted(deps, indexName, primaryNodeId);
3147
- return false;
3864
+ }
3865
+ async function proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
3866
+ const log = deps.getReplicationLog(indexName, partitionId);
3867
+ if (cursor.appliedSeqNo < log.commitPoint) {
3868
+ return;
3148
3869
  }
3149
- const restoreError = await executeEngineRestore(deps.engine, indexName, primaryNodeId, bytes, remainingMs);
3150
- if (restoreError !== null) {
3151
- surfaceError(deps, indexName, primaryNodeId, restoreError);
3152
- return false;
3870
+ if (!markPendingAdmission(state, indexName, partitionId, replicaNodeId)) {
3871
+ return;
3153
3872
  }
3154
- const generationAfterRestore = state.generations.get(key) ?? 0;
3155
- if (generationAfterRestore !== entry.generation || entry.aborted) {
3156
- if (!anotherBootstrapOwnsKey(state, key, entry)) {
3157
- await dropRestoredIndexQuietly(deps.engine, indexName, deps);
3873
+ const proposedEpoch = cursor.syncEpoch;
3874
+ try {
3875
+ const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
3876
+ if (controllerNodeId === null) {
3877
+ return;
3158
3878
  }
3159
- surfaceAborted(deps, indexName, primaryNodeId);
3160
- return false;
3161
- }
3162
- const schemaError = validateRestoredSchema(deps.engine, indexName, primaryNodeId, coordinatorSchema);
3163
- if (schemaError !== null) {
3164
- await dropRestoredIndexQuietly(deps.engine, indexName, deps);
3165
- surfaceError(deps, indexName, primaryNodeId, schemaError);
3166
- return false;
3879
+ const payload = {
3880
+ indexName,
3881
+ partitionId,
3882
+ replicaNodeId,
3883
+ primaryTerm: assignment.primaryTerm,
3884
+ appliedSeqNo: cursor.appliedSeqNo,
3885
+ commitPoint: log.commitPoint
3886
+ };
3887
+ const accepted = await withTimeout(sendToController(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
3888
+ if (accepted && cursor.syncEpoch !== proposedEpoch) {
3889
+ await withTimeout(withdrawAdmission(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, void 0);
3890
+ }
3891
+ } finally {
3892
+ clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
3167
3893
  }
3168
- return true;
3169
3894
  }
3170
-
3171
- // src/distribution/cluster-node/bootstrap-sync/index.ts
3172
- async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
3173
- const validationError = validateArguments(indexName, partitionId, primaryNodeId);
3174
- if (validationError !== null) {
3175
- surfaceError(deps, indexName, primaryNodeId, validationError);
3176
- return false;
3895
+ async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
3896
+ const message = createEntryMessage(entry, sourceNodeId);
3897
+ return sendAndCollectAcks(message, entry, inSyncReplicas, transport, resolveNodeTargets3);
3898
+ }
3899
+ async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
3900
+ if (entries.length === 0) {
3901
+ return { acknowledged: [], failed: [] };
3177
3902
  }
3178
- const key = entryKey(indexName, partitionId);
3179
- if (state.completed.has(key)) {
3180
- if (!await replicaResyncRequired(indexName, partitionId, deps)) {
3181
- return true;
3182
- }
3183
- clearBootstrapSyncIndex(state, indexName, partitionId);
3903
+ if (entries.length === 1) {
3904
+ return replicateToReplicas(entries[0], inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3);
3184
3905
  }
3185
- const existing = state.inFlight.get(key);
3186
- if (existing !== void 0) {
3187
- const result = await Promise.race([existing.promise, existing.abortPromise]);
3188
- if (result === ABORT_SENTINEL) {
3189
- return false;
3190
- }
3191
- return result;
3906
+ const message = createEntryBatchMessage(entries, sourceNodeId);
3907
+ const lastEntry = entries[entries.length - 1];
3908
+ return sendAndCollectAcks(message, lastEntry, inSyncReplicas, transport, resolveNodeTargets3);
3909
+ }
3910
+ async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport, resolveNodeTargets3) {
3911
+ const uniqueReplicas = [...new Set(inSyncReplicas)];
3912
+ if (uniqueReplicas.length === 0) {
3913
+ return { acknowledged: [], failed: [] };
3192
3914
  }
3193
- const entry = createEntry(state, indexName, partitionId, key);
3194
- entry.promise = executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps).finally(() => {
3195
- const current = state.inFlight.get(key);
3196
- if (current === entry) {
3197
- state.inFlight.delete(key);
3198
- }
3199
- if (!state.completed.has(key) && !state.inFlight.has(key)) {
3200
- const currentGen = state.generations.get(key);
3201
- if (currentGen !== void 0 && currentGen !== entry.generation) {
3202
- state.generations.delete(key);
3915
+ const sendResults = await Promise.allSettled(
3916
+ uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
3917
+ );
3918
+ const acknowledged = [];
3919
+ const failed = [];
3920
+ for (let i = 0; i < sendResults.length; i++) {
3921
+ const result = sendResults[i];
3922
+ const replicaNodeId = uniqueReplicas[i];
3923
+ if (result.status === "rejected") {
3924
+ if (result.reason instanceof TransportError) {
3925
+ failed.push(replicaNodeId);
3926
+ continue;
3203
3927
  }
3928
+ throw result.reason;
3929
+ }
3930
+ const response = result.value;
3931
+ if (isMatchingAck(response, ackEntry)) {
3932
+ acknowledged.push(replicaNodeId);
3933
+ } else {
3934
+ failed.push(replicaNodeId);
3204
3935
  }
3205
- });
3206
- state.inFlight.set(key, entry);
3207
- return entry.promise;
3208
- }
3209
- async function replicaResyncRequired(indexName, partitionId, deps) {
3210
- const allocation = await deps.coordinator.getAllocation(indexName);
3211
- const assignment = allocation?.assignments.get(partitionId);
3212
- if (assignment === void 0) {
3213
- return false;
3214
3936
  }
3215
- return assignment.state === "ACTIVE" && assignment.replicas.includes(deps.sourceNodeId) && !assignment.inSyncSet.includes(deps.sourceNodeId);
3937
+ return { acknowledged, failed };
3216
3938
  }
3217
- async function executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps) {
3218
- const deadlineMs = deps.deadlineMs ?? DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS;
3219
- const deadline = Date.now() + deadlineMs;
3220
- const key = entryKey(indexName, partitionId);
3221
- const abortCheck = () => {
3222
- const currentGeneration = state.generations.get(key) ?? 0;
3223
- if (currentGeneration !== entry.generation) {
3224
- entry.aborted = true;
3939
+ async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
3940
+ const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
3941
+ const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
3942
+ let lastError;
3943
+ for (const target of resolvedTargets) {
3944
+ try {
3945
+ return await transport.send(target, message);
3946
+ } catch (error) {
3947
+ lastError = error;
3225
3948
  }
3226
- return entry.aborted;
3227
- };
3228
- if (hasLiveBootstrapSyncDeps(deps)) {
3229
- return executeLiveBootstrapSync(
3230
- state,
3231
- entry,
3232
- indexName,
3233
- partitionId,
3234
- primaryNodeId,
3235
- deps,
3236
- deadlineMs,
3237
- deadline,
3238
- abortCheck
3239
- );
3240
3949
  }
3241
- const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
3242
- if (schemaResult === "aborted") {
3243
- surfaceAborted(deps, indexName, primaryNodeId);
3950
+ throw lastError instanceof Error ? lastError : new Error(String(lastError));
3951
+ }
3952
+ function isMatchingAck(response, entry) {
3953
+ if (response.type !== ReplicationMessageTypes.ACK) {
3244
3954
  return false;
3245
3955
  }
3246
- if (schemaResult === null) {
3956
+ try {
3957
+ const payload = validateAckPayload(decode(response.payload));
3958
+ return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
3959
+ } catch {
3247
3960
  return false;
3248
3961
  }
3249
- if (abortCheck()) {
3250
- surfaceAborted(deps, indexName, primaryNodeId);
3962
+ }
3963
+
3964
+ // src/distribution/cluster-node/catch-up/pump.ts
3965
+ var ENTRY_FIXED_OVERHEAD_BYTES = 40;
3966
+ function entryBytes(entry) {
3967
+ return ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
3968
+ }
3969
+ function batchBytes(entries) {
3970
+ let total = 0;
3971
+ for (const entry of entries) {
3972
+ total += entryBytes(entry);
3973
+ }
3974
+ return total;
3975
+ }
3976
+ function replicaHasFallenOutOfRetention(log, cursor) {
3977
+ const oldest = log.oldestSeqNo;
3978
+ if (oldest === void 0) {
3251
3979
  return false;
3252
3980
  }
3253
- if (Date.now() >= deadline) {
3254
- surfaceError(
3255
- deps,
3256
- indexName,
3257
- primaryNodeId,
3258
- new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
3259
- indexName,
3260
- deadlineMs
3261
- })
3981
+ return cursor.appliedSeqNo + 1 < oldest;
3982
+ }
3983
+ function nextWireBatch(state, pending) {
3984
+ const chunks = chunkByBudget(pending, {
3985
+ ...WIRE_BATCH_BUDGET,
3986
+ payloadBytesOf: (entry) => entry.document?.byteLength ?? 0,
3987
+ breaksRun: (entry, previous) => entry.seqNo !== previous.seqNo + 1
3988
+ });
3989
+ const batch = chunks[0] ?? [];
3990
+ if (batch.length === 0) {
3991
+ return [];
3992
+ }
3993
+ const available = CATCH_UP_IN_FLIGHT_BYTE_CEILING - state.inFlightBytes;
3994
+ return batchBytes(batch) > available ? [] : batch;
3995
+ }
3996
+ async function pumpReplica(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
3997
+ if (cursor.sending) {
3998
+ return;
3999
+ }
4000
+ const log = deps.getReplicationLog(indexName, partitionId);
4001
+ if (replicaHasFallenOutOfRetention(log, cursor)) {
4002
+ forgetReplica(state, indexName, partitionId, replicaNodeId);
4003
+ return;
4004
+ }
4005
+ if (cursor.appliedSeqNo >= log.localLogEnd) {
4006
+ await proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
4007
+ return;
4008
+ }
4009
+ const batch = nextWireBatch(state, log.getEntriesFrom(cursor.appliedSeqNo + 1));
4010
+ if (batch.length === 0) {
4011
+ return;
4012
+ }
4013
+ const reservedBytes = batchBytes(batch);
4014
+ const sendingEpoch = cursor.syncEpoch;
4015
+ cursor.sending = true;
4016
+ state.inFlightBytes += reservedBytes;
4017
+ try {
4018
+ const result = await replicateBatchToReplicas(
4019
+ batch,
4020
+ [replicaNodeId],
4021
+ deps.transport,
4022
+ deps.nodeId,
4023
+ deps.resolveNodeTargets
3262
4024
  );
3263
- return false;
4025
+ if (result.acknowledged.includes(replicaNodeId) && cursor.syncEpoch === sendingEpoch) {
4026
+ cursor.appliedSeqNo = batch[batch.length - 1].seqNo;
4027
+ }
4028
+ } finally {
4029
+ state.inFlightBytes -= reservedBytes;
4030
+ cursor.sending = false;
3264
4031
  }
3265
- const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
3266
- if (targetsResult instanceof NarsilError) {
3267
- surfaceError(deps, indexName, primaryNodeId, targetsResult);
3268
- return false;
4032
+ }
4033
+ async function pumpPartition(state, deps, key, replicas) {
4034
+ const separator = key.lastIndexOf(":");
4035
+ const indexName = key.slice(0, separator);
4036
+ const partitionId = Number(key.slice(separator + 1));
4037
+ const table = await deps.coordinator.getAllocation(indexName);
4038
+ const assignment = table?.assignments.get(partitionId);
4039
+ if (assignment === void 0 || assignment.primary !== deps.nodeId) {
4040
+ state.cursors.delete(key);
4041
+ state.pendingAdmissions.delete(key);
4042
+ return;
3269
4043
  }
3270
- const fetchDeps = { transport: deps.transport, sourceNodeId: deps.sourceNodeId };
3271
- const fetchResult = await fetchSnapshotFromAnyTarget(
3272
- indexName,
3273
- primaryNodeId,
3274
- targetsResult,
3275
- deadline,
3276
- fetchDeps,
3277
- abortCheck,
3278
- partitionId
3279
- );
3280
- if (!fetchResult.ok) {
3281
- if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
3282
- surfaceAborted(deps, indexName, primaryNodeId);
3283
- return false;
4044
+ for (const [replicaNodeId, cursor] of [...replicas]) {
4045
+ if (state.stopped) {
4046
+ return;
4047
+ }
4048
+ if (!assignment.replicas.includes(replicaNodeId) || assignment.inSyncSet.includes(replicaNodeId)) {
4049
+ forgetReplica(state, indexName, partitionId, replicaNodeId);
4050
+ continue;
4051
+ }
4052
+ try {
4053
+ await pumpReplica(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
4054
+ } catch (_) {
3284
4055
  }
3285
- surfaceError(
3286
- deps,
3287
- indexName,
3288
- primaryNodeId,
3289
- new NarsilError(fetchResult.code, fetchResult.message, {
3290
- indexName,
3291
- primaryNodeId,
3292
- ...fetchResult.details
3293
- })
3294
- );
3295
- return false;
3296
- }
3297
- if (abortCheck()) {
3298
- surfaceAborted(deps, indexName, primaryNodeId);
3299
- return false;
3300
4056
  }
3301
- const restoreSucceeded = await applyRestore(
3302
- state,
3303
- entry,
3304
- indexName,
3305
- primaryNodeId,
3306
- fetchResult.bytes,
3307
- schemaResult,
3308
- deadline,
3309
- deps
3310
- );
3311
- if (!restoreSucceeded) {
3312
- return false;
4057
+ }
4058
+ function runCatchUpTick(state, deps) {
4059
+ if (state.stopped || state.activeTick !== null || state.cursors.size === 0) {
4060
+ return state.activeTick ?? Promise.resolve();
3313
4061
  }
3314
- if (abortCheck()) {
3315
- if (!anotherBootstrapOwnsKey(state, key, entry)) {
3316
- await dropRestoredIndexQuietly(deps.engine, indexName, deps);
4062
+ const tick = (async () => {
4063
+ for (const [key, replicas] of [...state.cursors]) {
4064
+ if (state.stopped) {
4065
+ return;
4066
+ }
4067
+ try {
4068
+ await pumpPartition(state, deps, key, replicas);
4069
+ } catch (_) {
4070
+ }
3317
4071
  }
3318
- surfaceAborted(deps, indexName, primaryNodeId);
3319
- return false;
4072
+ })().finally(() => {
4073
+ state.activeTick = null;
4074
+ });
4075
+ state.activeTick = tick;
4076
+ return tick;
4077
+ }
4078
+ function startCatchUpPump(state, deps) {
4079
+ void stopCatchUpPump(state);
4080
+ state.stopped = false;
4081
+ state.timer = setInterval(() => {
4082
+ void runCatchUpTick(state, deps);
4083
+ }, CATCH_UP_TICK_MS);
4084
+ state.timer.unref?.();
4085
+ }
4086
+ async function stopCatchUpPump(state) {
4087
+ state.stopped = true;
4088
+ if (state.timer !== null) {
4089
+ clearInterval(state.timer);
4090
+ state.timer = null;
4091
+ }
4092
+ if (state.activeTick !== null) {
4093
+ await state.activeTick;
3320
4094
  }
3321
- deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
3322
- state.completed.add(key);
3323
- return true;
3324
4095
  }
3325
4096
 
3326
4097
  // src/distribution/cluster-node/local-engine.ts
@@ -3663,21 +4434,14 @@ async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
3663
4434
  }
3664
4435
  let sawAssignment = false;
3665
4436
  for (const assignment of allocation.assignments.values()) {
3666
- const isPrimary = assignment.primary === sourceId;
3667
- const isReplica = assignment.replicas.includes(sourceId);
3668
- if (!isPrimary && !isReplica) {
4437
+ const isAssigned = assignment.primary === sourceId || assignment.replicas.includes(sourceId);
4438
+ if (!isAssigned) {
3669
4439
  continue;
3670
4440
  }
3671
4441
  sawAssignment = true;
3672
4442
  if (!BOOTSTRAPPABLE_STATES.has(assignment.state)) {
3673
4443
  continue;
3674
4444
  }
3675
- if (isPrimary) {
3676
- return { outcome: "authorized" };
3677
- }
3678
- if (!assignmentAllowsReplicaBootstrap(assignment, sourceId)) {
3679
- continue;
3680
- }
3681
4445
  return { outcome: "authorized" };
3682
4446
  }
3683
4447
  if (sawAssignment) {
@@ -3693,12 +4457,6 @@ async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
3693
4457
  reason: `node '${sourceId}' is not an assigned replica for index '${indexName}'`
3694
4458
  };
3695
4459
  }
3696
- function assignmentAllowsReplicaBootstrap(assignment, sourceId) {
3697
- if (assignment.state === "INITIALISING" || assignment.state === "MIGRATING") {
3698
- return true;
3699
- }
3700
- return assignment.inSyncSet.includes(sourceId);
3701
- }
3702
4460
 
3703
4461
  // src/distribution/cluster-node/snapshot-cache.ts
3704
4462
  var DEFAULT_MAX_CONCURRENT_SNAPSHOTS = 2;
@@ -4217,7 +4975,7 @@ var NULL_BYTE = String.fromCharCode(0);
4217
4975
  function throwInvalid(code, message, details) {
4218
4976
  throw new NarsilError(code, message, details);
4219
4977
  }
4220
- function isRecord3(value) {
4978
+ function isRecord2(value) {
4221
4979
  return value !== null && typeof value === "object" && !Array.isArray(value);
4222
4980
  }
4223
4981
  function isInteger(value) {
@@ -4335,14 +5093,14 @@ function validateIndexNameField(value, fieldLabel) {
4335
5093
  return value;
4336
5094
  }
4337
5095
  function validateDocumentId(value, fieldLabel) {
4338
- if (!isRecord3(value)) {
5096
+ if (!isRecord2(value)) {
4339
5097
  throwInvalid(CONFIG_INVALID, `Invalid FetchPayload: "${fieldLabel}" must be an object`);
4340
5098
  }
4341
5099
  validateStringField(value.docId, `${fieldLabel}.docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
4342
5100
  validatePartitionId(value.partitionId, `${fieldLabel}.partitionId`, CONFIG_INVALID);
4343
5101
  }
4344
5102
  function validateHighlightConfig(value) {
4345
- if (!isRecord3(value)) {
5103
+ if (!isRecord2(value)) {
4346
5104
  throwInvalid(CONFIG_INVALID, 'Invalid FetchPayload: "highlight" must be an object or null');
4347
5105
  }
4348
5106
  if (value.fields !== null) {
@@ -4368,7 +5126,7 @@ function validateHighlightConfig(value) {
4368
5126
  }
4369
5127
  }
4370
5128
  function validateFetchPayload(decoded) {
4371
- if (!isRecord3(decoded)) {
5129
+ if (!isRecord2(decoded)) {
4372
5130
  throwInvalid(CONFIG_INVALID, "Invalid FetchPayload: expected an object");
4373
5131
  }
4374
5132
  validateIndexNameField(decoded.indexName, "indexName");
@@ -4394,7 +5152,7 @@ function validateFetchPayload(decoded) {
4394
5152
  return decoded;
4395
5153
  }
4396
5154
  function validateFetchResultPayload(decoded) {
4397
- if (!isRecord3(decoded)) {
5155
+ if (!isRecord2(decoded)) {
4398
5156
  throwInvalid(CONFIG_INVALID, "Invalid FetchResultPayload: expected an object");
4399
5157
  }
4400
5158
  if (!Array.isArray(decoded.documents)) {
@@ -4402,14 +5160,14 @@ function validateFetchResultPayload(decoded) {
4402
5160
  }
4403
5161
  for (let i = 0; i < decoded.documents.length; i++) {
4404
5162
  const doc = decoded.documents[i];
4405
- if (!isRecord3(doc)) {
5163
+ if (!isRecord2(doc)) {
4406
5164
  throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}]" must be an object`);
4407
5165
  }
4408
5166
  validateStringField(doc.docId, `documents[${i}].docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
4409
- if (!isRecord3(doc.document)) {
5167
+ if (!isRecord2(doc.document)) {
4410
5168
  throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}].document" must be an object`);
4411
5169
  }
4412
- if (doc.highlights !== null && !isRecord3(doc.highlights)) {
5170
+ if (doc.highlights !== null && !isRecord2(doc.highlights)) {
4413
5171
  throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}].highlights" must be an object or null`);
4414
5172
  }
4415
5173
  }
@@ -4506,7 +5264,7 @@ function isKnownLeafKey(key) {
4506
5264
  return FIELD_FILTER_OPERATORS.includes(key);
4507
5265
  }
4508
5266
  function validateGeoRadius(value, fieldLabel, errorCode) {
4509
- if (!isRecord3(value)) {
5267
+ if (!isRecord2(value)) {
4510
5268
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
4511
5269
  }
4512
5270
  validateNumericOperand(value.lat, `${fieldLabel}.lat`, errorCode);
@@ -4526,7 +5284,7 @@ function validateGeoRadius(value, fieldLabel, errorCode) {
4526
5284
  }
4527
5285
  }
4528
5286
  function validateGeoPolygon(value, fieldLabel, errorCode) {
4529
- if (!isRecord3(value)) {
5287
+ if (!isRecord2(value)) {
4530
5288
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
4531
5289
  }
4532
5290
  if (!Array.isArray(value.points)) {
@@ -4542,7 +5300,7 @@ function validateGeoPolygon(value, fieldLabel, errorCode) {
4542
5300
  }
4543
5301
  for (let i = 0; i < points.length; i++) {
4544
5302
  const point = points[i];
4545
- if (!isRecord3(point)) {
5303
+ if (!isRecord2(point)) {
4546
5304
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}.points[${i}]" must be an object`);
4547
5305
  }
4548
5306
  validateNumericOperand(point.lat, `${fieldLabel}.points[${i}].lat`, errorCode);
@@ -4583,7 +5341,7 @@ function validateLeafOperator(opKey, opValue, leafLabel, errorCode) {
4583
5341
  return;
4584
5342
  }
4585
5343
  if (opKey === "size") {
4586
- if (!isRecord3(opValue)) {
5344
+ if (!isRecord2(opValue)) {
4587
5345
  throwInvalid(errorCode, `Invalid payload: "${opLabel}" must be an object`);
4588
5346
  }
4589
5347
  for (const [innerKey, innerValue] of Object.entries(opValue)) {
@@ -4605,7 +5363,7 @@ function validateLeafOperator(opKey, opValue, leafLabel, errorCode) {
4605
5363
  throwInvalid(errorCode, `Invalid payload: "${leafLabel}" contains unsupported operator "${opKey}"`);
4606
5364
  }
4607
5365
  function validateFieldFilter(value, fieldLabel, errorCode) {
4608
- if (!isRecord3(value)) {
5366
+ if (!isRecord2(value)) {
4609
5367
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
4610
5368
  }
4611
5369
  const entries = Object.entries(value);
@@ -4623,7 +5381,7 @@ function validateFieldFilter(value, fieldLabel, errorCode) {
4623
5381
  }
4624
5382
  }
4625
5383
  function validateFieldsRecord(value, fieldLabel, errorCode) {
4626
- if (!isRecord3(value)) {
5384
+ if (!isRecord2(value)) {
4627
5385
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
4628
5386
  }
4629
5387
  const entries = Object.entries(value);
@@ -4666,7 +5424,7 @@ function validateFilterExpression(value, fieldLabel, errorCode = SEARCH_INVALID_
4666
5424
  limit: MAX_FILTER_DEPTH
4667
5425
  });
4668
5426
  }
4669
- if (!isRecord3(value)) {
5427
+ if (!isRecord2(value)) {
4670
5428
  throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
4671
5429
  }
4672
5430
  const entries = Object.entries(value);
@@ -4704,7 +5462,7 @@ function validateIndexNameField2(value, fieldLabel) {
4704
5462
  return value;
4705
5463
  }
4706
5464
  function validateVectorParams(value) {
4707
- if (!isRecord3(value)) {
5465
+ if (!isRecord2(value)) {
4708
5466
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.vector" must be an object');
4709
5467
  }
4710
5468
  validateFieldName(value.field, "params.vector.field", SEARCH_INVALID_FIELD);
@@ -4749,7 +5507,7 @@ function validateVectorParams(value) {
4749
5507
  }
4750
5508
  }
4751
5509
  function validateHybridParams(value) {
4752
- if (!isRecord3(value)) {
5510
+ if (!isRecord2(value)) {
4753
5511
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.hybrid" must be an object');
4754
5512
  }
4755
5513
  if (!ALLOWED_HYBRID_STRATEGIES.includes(value.strategy)) {
@@ -4769,7 +5527,7 @@ function validateHybridParams(value) {
4769
5527
  }
4770
5528
  }
4771
5529
  function validateGroupParams(value) {
4772
- if (!isRecord3(value)) {
5530
+ if (!isRecord2(value)) {
4773
5531
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.group" must be an object');
4774
5532
  }
4775
5533
  validateFieldName(value.field, "params.group.field", SEARCH_INVALID_FIELD);
@@ -4796,7 +5554,7 @@ function validateSortParams(value) {
4796
5554
  }
4797
5555
  for (let i = 0; i < value.length; i++) {
4798
5556
  const entry = value[i];
4799
- if (!isRecord3(entry)) {
5557
+ if (!isRecord2(entry)) {
4800
5558
  throwInvalid(CONFIG_INVALID, `Invalid SearchPayload: "params.sort[${i}]" must be an object`);
4801
5559
  }
4802
5560
  validateFieldName(entry.field, `params.sort[${i}].field`, SEARCH_INVALID_FIELD);
@@ -4809,7 +5567,7 @@ function validateSortParams(value) {
4809
5567
  }
4810
5568
  }
4811
5569
  function validateBoostParams(value) {
4812
- if (!isRecord3(value)) {
5570
+ if (!isRecord2(value)) {
4813
5571
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.boost" must be an object or null');
4814
5572
  }
4815
5573
  const entries = Object.entries(value);
@@ -4828,16 +5586,16 @@ function validateBoostParams(value) {
4828
5586
  }
4829
5587
  }
4830
5588
  function validateGlobalStatsShape(value) {
4831
- if (!isRecord3(value)) {
5589
+ if (!isRecord2(value)) {
4832
5590
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats" must be an object or null');
4833
5591
  }
4834
5592
  if (!isFiniteNumber(value.totalDocuments) || value.totalDocuments < 0) {
4835
5593
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.totalDocuments" must be a non-negative number');
4836
5594
  }
4837
- if (!isRecord3(value.docFrequencies)) {
5595
+ if (!isRecord2(value.docFrequencies)) {
4838
5596
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.docFrequencies" must be an object');
4839
5597
  }
4840
- if (!isRecord3(value.totalFieldLengths)) {
5598
+ if (!isRecord2(value.totalFieldLengths)) {
4841
5599
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.totalFieldLengths" must be an object');
4842
5600
  }
4843
5601
  }
@@ -4927,18 +5685,18 @@ function validateParams(params) {
4927
5685
  }
4928
5686
  }
4929
5687
  function validateWireParams(params) {
4930
- if (!isRecord3(params)) {
5688
+ if (!isRecord2(params)) {
4931
5689
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params" must be an object');
4932
5690
  }
4933
5691
  validateParams(params);
4934
5692
  }
4935
5693
  function validateSearchPayload(decoded) {
4936
- if (!isRecord3(decoded)) {
5694
+ if (!isRecord2(decoded)) {
4937
5695
  throwInvalid(CONFIG_INVALID, "Invalid SearchPayload: expected an object");
4938
5696
  }
4939
5697
  validateIndexNameField2(decoded.indexName, "indexName");
4940
5698
  validatePartitionIdsArray(decoded.partitionIds, "partitionIds", CONFIG_INVALID);
4941
- if (!isRecord3(decoded.params)) {
5699
+ if (!isRecord2(decoded.params)) {
4942
5700
  throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params" must be an object');
4943
5701
  }
4944
5702
  validateParams(decoded.params);
@@ -4974,7 +5732,7 @@ function validateIndexNameField3(decoded, payloadLabel) {
4974
5732
  validateStringField(decoded.indexName, `${payloadLabel}.indexName`, 255, CONFIG_INVALID);
4975
5733
  }
4976
5734
  function validateCountPayload(decoded) {
4977
- if (!isRecord3(decoded)) {
5735
+ if (!isRecord2(decoded)) {
4978
5736
  throwInvalid(CONFIG_INVALID, "Invalid CountPayload: expected an object");
4979
5737
  }
4980
5738
  validateIndexNameField3(decoded, "CountPayload");
@@ -4982,7 +5740,7 @@ function validateCountPayload(decoded) {
4982
5740
  return decoded;
4983
5741
  }
4984
5742
  function validateCountResultPayload(decoded) {
4985
- if (!isRecord3(decoded)) {
5743
+ if (!isRecord2(decoded)) {
4986
5744
  throwInvalid(CONFIG_INVALID, "Invalid CountResultPayload: expected an object");
4987
5745
  }
4988
5746
  if (!Array.isArray(decoded.partitions)) {
@@ -4990,7 +5748,7 @@ function validateCountResultPayload(decoded) {
4990
5748
  }
4991
5749
  for (let i = 0; i < decoded.partitions.length; i++) {
4992
5750
  const entry = decoded.partitions[i];
4993
- if (!isRecord3(entry)) {
5751
+ if (!isRecord2(entry)) {
4994
5752
  throwInvalid(CONFIG_INVALID, `Invalid CountResultPayload: "partitions[${i}]" must be an object`);
4995
5753
  }
4996
5754
  validateNonNegativeInteger(entry.partitionId, `partitions[${i}].partitionId`, MAX_COUNT_VALUE, CONFIG_INVALID);
@@ -5017,7 +5775,7 @@ function validateListSort(value) {
5017
5775
  }
5018
5776
  for (let i = 0; i < value.length; i++) {
5019
5777
  const entry = value[i];
5020
- if (!isRecord3(entry)) {
5778
+ if (!isRecord2(entry)) {
5021
5779
  throwInvalid(CONFIG_INVALID, `Invalid ListPayload: "sort[${i}]" must be an object`);
5022
5780
  }
5023
5781
  validateFieldName(entry.field, `sort[${i}].field`, SEARCH_INVALID_FIELD);
@@ -5027,7 +5785,7 @@ function validateListSort(value) {
5027
5785
  }
5028
5786
  }
5029
5787
  function validateListPayload(decoded) {
5030
- if (!isRecord3(decoded)) {
5788
+ if (!isRecord2(decoded)) {
5031
5789
  throwInvalid(CONFIG_INVALID, "Invalid ListPayload: expected an object");
5032
5790
  }
5033
5791
  validateIndexNameField3(decoded, "ListPayload");
@@ -5072,7 +5830,7 @@ function validateSortValues(value, fieldLabel) {
5072
5830
  }
5073
5831
  }
5074
5832
  function validateListResultPayload(decoded) {
5075
- if (!isRecord3(decoded)) {
5833
+ if (!isRecord2(decoded)) {
5076
5834
  throwInvalid(CONFIG_INVALID, "Invalid ListResultPayload: expected an object");
5077
5835
  }
5078
5836
  if (!Array.isArray(decoded.entries)) {
@@ -5083,11 +5841,11 @@ function validateListResultPayload(decoded) {
5083
5841
  }
5084
5842
  for (let i = 0; i < decoded.entries.length; i++) {
5085
5843
  const entry = decoded.entries[i];
5086
- if (!isRecord3(entry)) {
5844
+ if (!isRecord2(entry)) {
5087
5845
  throwInvalid(CONFIG_INVALID, `Invalid ListResultPayload: "entries[${i}]" must be an object`);
5088
5846
  }
5089
5847
  validateStringField(entry.docId, `entries[${i}].docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
5090
- if (!isRecord3(entry.document)) {
5848
+ if (!isRecord2(entry.document)) {
5091
5849
  throwInvalid(CONFIG_INVALID, `Invalid ListResultPayload: "entries[${i}].document" must be an object`);
5092
5850
  }
5093
5851
  if (entry.sortValues !== null) {
@@ -5101,7 +5859,7 @@ function validateListResultPayload(decoded) {
5101
5859
  return decoded;
5102
5860
  }
5103
5861
  function validateSuggestPayload(decoded) {
5104
- if (!isRecord3(decoded)) {
5862
+ if (!isRecord2(decoded)) {
5105
5863
  throwInvalid(CONFIG_INVALID, "Invalid SuggestPayload: expected an object");
5106
5864
  }
5107
5865
  validateIndexNameField3(decoded, "SuggestPayload");
@@ -5111,7 +5869,7 @@ function validateSuggestPayload(decoded) {
5111
5869
  return decoded;
5112
5870
  }
5113
5871
  function validateSuggestResultPayload(decoded) {
5114
- if (!isRecord3(decoded)) {
5872
+ if (!isRecord2(decoded)) {
5115
5873
  throwInvalid(CONFIG_INVALID, "Invalid SuggestResultPayload: expected an object");
5116
5874
  }
5117
5875
  if (!Array.isArray(decoded.terms)) {
@@ -5125,7 +5883,7 @@ function validateSuggestResultPayload(decoded) {
5125
5883
  }
5126
5884
  for (let i = 0; i < decoded.terms.length; i++) {
5127
5885
  const entry = decoded.terms[i];
5128
- if (!isRecord3(entry)) {
5886
+ if (!isRecord2(entry)) {
5129
5887
  throwInvalid(CONFIG_INVALID, `Invalid SuggestResultPayload: "terms[${i}]" must be an object`);
5130
5888
  }
5131
5889
  validateStringField(entry.term, `terms[${i}].term`, MAX_TERM_LENGTH, CONFIG_INVALID);
@@ -5142,7 +5900,7 @@ function validateSuggestResultPayload(decoded) {
5142
5900
  return decoded;
5143
5901
  }
5144
5902
  function validatePreflightPayload(decoded) {
5145
- if (!isRecord3(decoded)) {
5903
+ if (!isRecord2(decoded)) {
5146
5904
  throwInvalid(CONFIG_INVALID, "Invalid PreflightPayload: expected an object");
5147
5905
  }
5148
5906
  validateIndexNameField3(decoded, "PreflightPayload");
@@ -5151,7 +5909,7 @@ function validatePreflightPayload(decoded) {
5151
5909
  return decoded;
5152
5910
  }
5153
5911
  function validatePreflightResultPayload(decoded) {
5154
- if (!isRecord3(decoded)) {
5912
+ if (!isRecord2(decoded)) {
5155
5913
  throwInvalid(CONFIG_INVALID, "Invalid PreflightResultPayload: expected an object");
5156
5914
  }
5157
5915
  if (!isInteger(decoded.count) || decoded.count < 0) {
@@ -5170,7 +5928,7 @@ var MAX_FACET_FIELDS = 64;
5170
5928
  var MAX_FACET_BUCKETS = 1e4;
5171
5929
  var MAX_FACET_VALUE_LENGTH = 1024;
5172
5930
  function validateScoredEntry(value, fieldLabel) {
5173
- if (!isRecord3(value)) {
5931
+ if (!isRecord2(value)) {
5174
5932
  throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
5175
5933
  }
5176
5934
  if (typeof value.docId !== "string") {
@@ -5216,7 +5974,7 @@ function validateScoredEntry(value, fieldLabel) {
5216
5974
  }
5217
5975
  }
5218
5976
  function validatePartitionSearchResult(value, fieldLabel) {
5219
- if (!isRecord3(value)) {
5977
+ if (!isRecord2(value)) {
5220
5978
  throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
5221
5979
  }
5222
5980
  validatePartitionId(value.partitionId, `${fieldLabel}.partitionId`, CONFIG_INVALID);
@@ -5241,7 +5999,7 @@ function validatePartitionSearchResult(value, fieldLabel) {
5241
5999
  }
5242
6000
  }
5243
6001
  function validateFacetBucket(value, fieldLabel) {
5244
- if (!isRecord3(value)) {
6002
+ if (!isRecord2(value)) {
5245
6003
  throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
5246
6004
  }
5247
6005
  if (typeof value.value !== "string") {
@@ -5260,7 +6018,7 @@ function validateFacetBucket(value, fieldLabel) {
5260
6018
  return { value: value.value, count: value.count };
5261
6019
  }
5262
6020
  function validateFacets(value) {
5263
- if (!isRecord3(value)) {
6021
+ if (!isRecord2(value)) {
5264
6022
  throwInvalid(CONFIG_INVALID, 'Invalid SearchResultPayload: "facets" must be an object or null');
5265
6023
  }
5266
6024
  const entries = Object.entries(value);
@@ -5291,7 +6049,7 @@ function validateFacets(value) {
5291
6049
  }
5292
6050
  }
5293
6051
  function validateFacetErrorBounds(value) {
5294
- if (!isRecord3(value)) {
6052
+ if (!isRecord2(value)) {
5295
6053
  throwInvalid(CONFIG_INVALID, 'Invalid SearchResultPayload: "facetErrorBounds" must be an object or null');
5296
6054
  }
5297
6055
  const entries = Object.entries(value);
@@ -5312,7 +6070,7 @@ function validateFacetErrorBounds(value) {
5312
6070
  }
5313
6071
  }
5314
6072
  function validateSearchResultPayload(decoded) {
5315
- if (!isRecord3(decoded)) {
6073
+ if (!isRecord2(decoded)) {
5316
6074
  throwInvalid(CONFIG_INVALID, "Invalid SearchResultPayload: expected an object");
5317
6075
  }
5318
6076
  if (!Array.isArray(decoded.results)) {
@@ -5339,7 +6097,7 @@ function validateIndexNameField4(value, fieldLabel) {
5339
6097
  return value;
5340
6098
  }
5341
6099
  function validateNumericRecord(value, fieldLabel) {
5342
- if (!isRecord3(value)) {
6100
+ if (!isRecord2(value)) {
5343
6101
  throwInvalid(CONFIG_INVALID, `Invalid payload: "${fieldLabel}" must be an object`);
5344
6102
  }
5345
6103
  for (const [key, entry] of Object.entries(value)) {
@@ -5352,7 +6110,7 @@ function validateNumericRecord(value, fieldLabel) {
5352
6110
  }
5353
6111
  }
5354
6112
  function validateStatsPayload(decoded) {
5355
- if (!isRecord3(decoded)) {
6113
+ if (!isRecord2(decoded)) {
5356
6114
  throwInvalid(CONFIG_INVALID, "Invalid StatsPayload: expected an object");
5357
6115
  }
5358
6116
  validateIndexNameField4(decoded.indexName, "indexName");
@@ -5361,7 +6119,7 @@ function validateStatsPayload(decoded) {
5361
6119
  return decoded;
5362
6120
  }
5363
6121
  function validateStatsResultPayload(decoded) {
5364
- if (!isRecord3(decoded)) {
6122
+ if (!isRecord2(decoded)) {
5365
6123
  throwInvalid(CONFIG_INVALID, "Invalid StatsResultPayload: expected an object");
5366
6124
  }
5367
6125
  if (!isInteger(decoded.totalDocuments) || decoded.totalDocuments < 0) {
@@ -5761,10 +6519,10 @@ function toWireSortValue2(value) {
5761
6519
  function decideSyncTier(log, replicaLastSeqNo) {
5762
6520
  const oldest = log.oldestSeqNo;
5763
6521
  if (oldest === void 0) {
5764
- if (log.committedSeqNo === 0) {
6522
+ if (log.localLogEnd === 0) {
5765
6523
  return replicaLastSeqNo === 0 ? "incremental" : "snapshot";
5766
6524
  }
5767
- return replicaLastSeqNo >= log.committedSeqNo ? "incremental" : "snapshot";
6525
+ return replicaLastSeqNo >= log.localLogEnd ? "incremental" : "snapshot";
5768
6526
  }
5769
6527
  if (oldest <= replicaLastSeqNo + 1) {
5770
6528
  return "incremental";
@@ -5843,13 +6601,22 @@ async function handleSyncRequestMessage(message, respond, deps) {
5843
6601
  }
5844
6602
  );
5845
6603
  }
6604
+ if (message.sourceId !== deps.nodeId) {
6605
+ recordReplicaPosition(
6606
+ deps.writeDeps.catchUp,
6607
+ request.indexName,
6608
+ request.partitionId,
6609
+ message.sourceId,
6610
+ request.lastSeqNo
6611
+ );
6612
+ }
5846
6613
  const log = deps.writeDeps.getReplicationLog(request.indexName, request.partitionId);
5847
6614
  const tier = decideSyncTier(log, request.lastSeqNo);
5848
6615
  if (tier === "incremental") {
5849
6616
  await sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1));
5850
6617
  return;
5851
6618
  }
5852
- const snapshotSeqNo = log.committedSeqNo;
6619
+ const snapshotSeqNo = log.localLogEnd;
5853
6620
  const sink = createSingleResponseSink(respond);
5854
6621
  await streamValidatedSnapshotRequest(
5855
6622
  message,
@@ -5902,76 +6669,6 @@ var DEFAULT_CAPACITY = {
5902
6669
  cpuCores: 4,
5903
6670
  diskBytes: null
5904
6671
  };
5905
-
5906
- // src/distribution/cluster-node/write-routing/assignment.ts
5907
- function resolvePartitionId(docId, partitionCount) {
5908
- return fnv1a(docId) % partitionCount;
5909
- }
5910
- async function resolveNodeTargets(nodeId, deps) {
5911
- if (deps.resolveNodeTargets === void 0) {
5912
- return [nodeId];
5913
- }
5914
- const targets = await deps.resolveNodeTargets(nodeId);
5915
- return targets.length > 0 ? targets : [nodeId];
5916
- }
5917
- function requireAssignedPrimary(assignment, indexName, partitionId) {
5918
- if (assignment === void 0 || assignment.primary === null) {
5919
- throw new NarsilError(
5920
- ErrorCodes.PARTITION_UNASSIGNED,
5921
- `No primary assigned for partition ${partitionId} of index '${indexName}'`,
5922
- { indexName, partitionId }
5923
- );
5924
- }
5925
- return assignment;
5926
- }
5927
- async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrimary) {
5928
- const table = await deps.coordinator.getAllocation(indexName);
5929
- if (table === null || table.assignments.size === 0) {
5930
- if (requireLocalPrimary) {
5931
- throw new NarsilError(
5932
- ErrorCodes.QUERY_ROUTING_FAILED,
5933
- `No allocation table is available for forwarded write to index '${indexName}'`,
5934
- { indexName }
5935
- );
5936
- }
5937
- return null;
5938
- }
5939
- const partitionCount = table.assignments.size;
5940
- const partitionId = resolvePartitionId(docId, partitionCount);
5941
- const assignment = requireAssignedPrimary(table.assignments.get(partitionId), indexName, partitionId);
5942
- if (requireLocalPrimary && assignment.primary !== deps.nodeId) {
5943
- throw new NarsilError(
5944
- ErrorCodes.PARTITION_NOT_PRIMARY,
5945
- `Node '${deps.nodeId}' is not primary for partition ${partitionId} of index '${indexName}'`,
5946
- { indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
5947
- );
5948
- }
5949
- return { partitionId, assignment };
5950
- }
5951
- function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
5952
- const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
5953
- if (requiredActiveCopies <= 1) {
5954
- return;
5955
- }
5956
- const activeCopies = 1 + getInSyncReplicaTargets(assignment, deps.nodeId).length;
5957
- if (activeCopies < requiredActiveCopies) {
5958
- throw new NarsilError(
5959
- ErrorCodes.INSUFFICIENT_REPLICAS,
5960
- `Write to partition ${partitionId} of index '${indexName}' requires ${requiredActiveCopies} active copies but only ${activeCopies} are available`,
5961
- { indexName, partitionId, requiredActiveCopies, activeCopies }
5962
- );
5963
- }
5964
- }
5965
- function getInSyncReplicaTargets(assignment, localNodeId) {
5966
- const configuredReplicas = new Set(assignment.replicas);
5967
- const targets = [];
5968
- for (const nodeId of assignment.inSyncSet) {
5969
- if (nodeId !== localNodeId && configuredReplicas.has(nodeId)) {
5970
- targets.push(nodeId);
5971
- }
5972
- }
5973
- return targets;
5974
- }
5975
6672
  function assertForwardResponse(response, indexName, primaryNodeId) {
5976
6673
  const decoded = decode(response.payload);
5977
6674
  if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
@@ -6039,76 +6736,6 @@ async function sendToNode(nodeId, message, deps) {
6039
6736
  }
6040
6737
  throw lastError instanceof Error ? lastError : new Error(String(lastError));
6041
6738
  }
6042
- async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
6043
- const message = createEntryMessage(entry, sourceNodeId);
6044
- return sendAndCollectAcks(message, entry, inSyncReplicas, transport, resolveNodeTargets3);
6045
- }
6046
- async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
6047
- if (entries.length === 0) {
6048
- return { acknowledged: [], failed: [] };
6049
- }
6050
- if (entries.length === 1) {
6051
- return replicateToReplicas(entries[0], inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3);
6052
- }
6053
- const message = createEntryBatchMessage(entries, sourceNodeId);
6054
- const lastEntry = entries[entries.length - 1];
6055
- return sendAndCollectAcks(message, lastEntry, inSyncReplicas, transport, resolveNodeTargets3);
6056
- }
6057
- async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport, resolveNodeTargets3) {
6058
- const uniqueReplicas = [...new Set(inSyncReplicas)];
6059
- if (uniqueReplicas.length === 0) {
6060
- return { acknowledged: [], failed: [] };
6061
- }
6062
- const sendResults = await Promise.allSettled(
6063
- uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
6064
- );
6065
- const acknowledged = [];
6066
- const failed = [];
6067
- for (let i = 0; i < sendResults.length; i++) {
6068
- const result = sendResults[i];
6069
- const replicaNodeId = uniqueReplicas[i];
6070
- if (result.status === "rejected") {
6071
- if (result.reason instanceof TransportError) {
6072
- failed.push(replicaNodeId);
6073
- continue;
6074
- }
6075
- throw result.reason;
6076
- }
6077
- const response = result.value;
6078
- if (isMatchingAck(response, ackEntry)) {
6079
- acknowledged.push(replicaNodeId);
6080
- } else {
6081
- failed.push(replicaNodeId);
6082
- }
6083
- }
6084
- return { acknowledged, failed };
6085
- }
6086
- async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
6087
- const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
6088
- const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
6089
- let lastError;
6090
- for (const target of resolvedTargets) {
6091
- try {
6092
- return await transport.send(target, message);
6093
- } catch (error) {
6094
- lastError = error;
6095
- }
6096
- }
6097
- throw lastError instanceof Error ? lastError : new Error(String(lastError));
6098
- }
6099
- function isMatchingAck(response, entry) {
6100
- if (response.type !== ReplicationMessageTypes.ACK) {
6101
- return false;
6102
- }
6103
- try {
6104
- const payload = validateAckPayload(decode(response.payload));
6105
- return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
6106
- } catch {
6107
- return false;
6108
- }
6109
- }
6110
-
6111
- // src/distribution/cluster-node/write-routing/replication.ts
6112
6739
  async function assertPrimaryWriteAuthority(entry, deps) {
6113
6740
  const table = await deps.coordinator.getAllocation(entry.indexName);
6114
6741
  const currentAssignment = table?.assignments.get(entry.partitionId);
@@ -6204,17 +6831,32 @@ async function requestInsyncRemovalWithTargets(indexName, partitionId, replicaNo
6204
6831
  throw lastError instanceof Error ? lastError : new Error(String(lastError));
6205
6832
  }
6206
6833
  async function replicateEntry(entry, assignment, deps) {
6207
- const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId);
6834
+ const pending = getPendingAdmissions(deps.catchUp, entry.indexName, entry.partitionId);
6835
+ const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId, pending);
6208
6836
  const result = await replicateToReplicas(entry, replicaTargets, deps.transport, deps.nodeId, deps.resolveNodeTargets);
6209
- await removeFailedReplicasFromInsync(entry, result.failed, deps);
6837
+ const inSyncFailures = cancelAdmissionsForFailures(entry, result.failed, assignment, deps);
6838
+ await removeFailedReplicasFromInsync(entry, inSyncFailures, deps);
6210
6839
  await assertPrimaryWriteAuthority(entry, deps);
6840
+ deps.getReplicationLog(entry.indexName, entry.partitionId).advanceCommitPoint(entry.seqNo);
6841
+ }
6842
+ function cancelAdmissionsForFailures(entry, failedReplicas, assignment, deps) {
6843
+ const inSyncFailures = [];
6844
+ for (const replicaNodeId of failedReplicas) {
6845
+ if (assignment.inSyncSet.includes(replicaNodeId)) {
6846
+ inSyncFailures.push(replicaNodeId);
6847
+ continue;
6848
+ }
6849
+ clearPendingAdmission(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId);
6850
+ }
6851
+ return inSyncFailures;
6211
6852
  }
6212
6853
  async function replicateEntryBatch(entries, assignment, deps) {
6213
6854
  if (entries.length === 0) {
6214
6855
  return;
6215
6856
  }
6216
6857
  const lastEntry = entries[entries.length - 1];
6217
- const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId);
6858
+ const pending = getPendingAdmissions(deps.catchUp, lastEntry.indexName, lastEntry.partitionId);
6859
+ const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId, pending);
6218
6860
  const result = await replicateBatchToReplicas(
6219
6861
  entries,
6220
6862
  replicaTargets,
@@ -6222,8 +6864,10 @@ async function replicateEntryBatch(entries, assignment, deps) {
6222
6864
  deps.nodeId,
6223
6865
  deps.resolveNodeTargets
6224
6866
  );
6225
- await removeFailedReplicasFromInsync(lastEntry, result.failed, deps);
6867
+ const inSyncFailures = cancelAdmissionsForFailures(lastEntry, result.failed, assignment, deps);
6868
+ await removeFailedReplicasFromInsync(lastEntry, inSyncFailures, deps);
6226
6869
  await assertPrimaryWriteAuthority(lastEntry, deps);
6870
+ deps.getReplicationLog(lastEntry.indexName, lastEntry.partitionId).advanceCommitPoint(lastEntry.seqNo);
6227
6871
  }
6228
6872
  function chunkReplicationEntries(items) {
6229
6873
  return chunkByBudget(items, {
@@ -7174,17 +7818,17 @@ async function handleForwardBatch(message, respond, deps) {
7174
7818
  async function handleReplicationEntry(message, respond, deps) {
7175
7819
  const payload = validateEntryPayload(decode(message.payload));
7176
7820
  const { entry } = payload;
7177
- const assignment = await resolveValidatedAssignment(entry, message.sourceId, deps);
7178
- await applyEntryToLog(entry, assignment, deps);
7821
+ await resolveValidatedAssignment(entry, message.sourceId, deps);
7822
+ await applyEntryToLog(entry, deps);
7179
7823
  await respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
7180
7824
  }
7181
7825
  async function handleReplicationEntryBatch(message, respond, deps) {
7182
7826
  const payload = validateEntryBatchPayload(decode(message.payload));
7183
7827
  const firstEntry = payload.entries[0];
7184
7828
  const lastEntry = payload.entries[payload.entries.length - 1];
7185
- const assignment = await resolveValidatedAssignment(firstEntry, message.sourceId, deps);
7829
+ await resolveValidatedAssignment(firstEntry, message.sourceId, deps);
7186
7830
  for (const entry of payload.entries) {
7187
- await applyEntryToLog(entry, assignment, deps);
7831
+ await applyEntryToLog(entry, deps);
7188
7832
  }
7189
7833
  await respond(
7190
7834
  createAckMessage(lastEntry.seqNo, lastEntry.partitionId, lastEntry.indexName, deps.nodeId, message.requestId)
@@ -7219,23 +7863,23 @@ async function resolveValidatedAssignment(entry, sourceNodeId, deps) {
7219
7863
  }
7220
7864
  );
7221
7865
  }
7222
- if (assignment.primaryTerm !== entry.primaryTerm) {
7866
+ if (entry.primaryTerm > assignment.primaryTerm) {
7223
7867
  throw new NarsilError(
7224
7868
  ErrorCodes.REPLICATION_TERM_MISMATCH,
7225
- `Replication entry term ${entry.primaryTerm} does not match allocation term ${assignment.primaryTerm}`,
7869
+ `Replication entry term ${entry.primaryTerm} is newer than allocation term ${assignment.primaryTerm}`,
7226
7870
  { indexName: entry.indexName, partitionId: entry.partitionId }
7227
7871
  );
7228
7872
  }
7229
- if (!assignment.replicas.includes(deps.nodeId) || !assignment.inSyncSet.includes(deps.nodeId)) {
7873
+ if (!assignment.replicas.includes(deps.nodeId)) {
7230
7874
  throw new NarsilError(
7231
7875
  ErrorCodes.REPLICATION_ENTRY_INVALID,
7232
- `Node '${deps.nodeId}' is not an in-sync replica for index '${entry.indexName}' partition ${entry.partitionId}`,
7876
+ `Node '${deps.nodeId}' is not an assigned replica for index '${entry.indexName}' partition ${entry.partitionId}`,
7233
7877
  { indexName: entry.indexName, partitionId: entry.partitionId, nodeId: deps.nodeId }
7234
7878
  );
7235
7879
  }
7236
7880
  return assignment;
7237
7881
  }
7238
- async function applyEntryToLog(entry, assignment, deps) {
7882
+ async function applyEntryToLog(entry, deps) {
7239
7883
  let log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
7240
7884
  const existing = log.getEntry(entry.seqNo);
7241
7885
  if (existing !== void 0) {
@@ -7261,7 +7905,7 @@ async function applyEntryToLog(entry, assignment, deps) {
7261
7905
  deps.writeDeps.resetReplicationLog(entry.indexName, entry.partitionId, entry.seqNo, entry.primaryTerm);
7262
7906
  log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
7263
7907
  }
7264
- const validation = validateReplicationEntry(entry, assignment.primaryTerm, log);
7908
+ const validation = validateReplicationEntry(entry, log.localLogEndPrimaryTerm, log);
7265
7909
  if (!validation.valid) {
7266
7910
  throw new NarsilError(
7267
7911
  validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
@@ -8652,16 +9296,17 @@ function createClusterNodeOperations(deps) {
8652
9296
  var DEFAULT_LOG_RETENTION_BYTES = 268435456;
8653
9297
 
8654
9298
  // src/distribution/replication/log.ts
8655
- var ENTRY_FIXED_OVERHEAD_BYTES = 40;
9299
+ var ENTRY_FIXED_OVERHEAD_BYTES2 = 40;
8656
9300
  function estimateEntrySize(entry) {
8657
- return ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
9301
+ return ENTRY_FIXED_OVERHEAD_BYTES2 + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
8658
9302
  }
8659
9303
  var computeChecksum = computeEntryChecksum;
8660
9304
  function createReplicationLog(partitionId, config) {
8661
9305
  const retentionBytes = config?.logRetentionBytes ?? DEFAULT_LOG_RETENTION_BYTES;
8662
9306
  let nextSeqNo = config?.startSeqNo ?? 1;
8663
- let committedSeqNo = nextSeqNo - 1;
8664
- let committedPrimaryTerm = config?.lastPrimaryTerm ?? 0;
9307
+ let localLogEnd = nextSeqNo - 1;
9308
+ let localLogEndPrimaryTerm = config?.lastPrimaryTerm ?? 0;
9309
+ let commitPoint = nextSeqNo - 1;
8665
9310
  let entries = [];
8666
9311
  let headIndex = 0;
8667
9312
  let totalSizeBytes = 0;
@@ -8704,7 +9349,7 @@ function createReplicationLog(partitionId, config) {
8704
9349
  }
8705
9350
  }
8706
9351
  function assertEntryFitsRetention(entry) {
8707
- const entrySize = ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
9352
+ const entrySize = ENTRY_FIXED_OVERHEAD_BYTES2 + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
8708
9353
  if (entrySize > retentionBytes) {
8709
9354
  throw new NarsilError(
8710
9355
  ErrorCodes.REPLICATION_LOG_FULL,
@@ -8715,9 +9360,9 @@ function createReplicationLog(partitionId, config) {
8715
9360
  }
8716
9361
  function storeEntry(entry) {
8717
9362
  entries.push(entry);
8718
- if (entry.seqNo >= committedSeqNo) {
8719
- committedSeqNo = entry.seqNo;
8720
- committedPrimaryTerm = entry.primaryTerm;
9363
+ if (entry.seqNo >= localLogEnd) {
9364
+ localLogEnd = entry.seqNo;
9365
+ localLogEndPrimaryTerm = entry.primaryTerm;
8721
9366
  }
8722
9367
  totalSizeBytes += estimateEntrySize(entry);
8723
9368
  evictOldEntries();
@@ -8820,11 +9465,20 @@ function createReplicationLog(partitionId, config) {
8820
9465
  verifyChecksum(entry) {
8821
9466
  return verifyEntryChecksum(entry);
8822
9467
  },
8823
- get committedSeqNo() {
8824
- return committedSeqNo;
9468
+ get localLogEnd() {
9469
+ return localLogEnd;
9470
+ },
9471
+ get localLogEndPrimaryTerm() {
9472
+ return localLogEndPrimaryTerm;
8825
9473
  },
8826
- get committedPrimaryTerm() {
8827
- return committedPrimaryTerm;
9474
+ get commitPoint() {
9475
+ return commitPoint;
9476
+ },
9477
+ advanceCommitPoint(seqNo) {
9478
+ const bounded = Math.min(seqNo, localLogEnd);
9479
+ if (bounded > commitPoint) {
9480
+ commitPoint = bounded;
9481
+ }
8828
9482
  },
8829
9483
  get oldestSeqNo() {
8830
9484
  if (liveCount() === 0) return void 0;
@@ -8844,7 +9498,8 @@ function createReplicationLog(partitionId, config) {
8844
9498
  entries = [];
8845
9499
  headIndex = 0;
8846
9500
  totalSizeBytes = 0;
8847
- committedSeqNo = nextSeqNo - 1;
9501
+ localLogEnd = nextSeqNo - 1;
9502
+ commitPoint = nextSeqNo - 1;
8848
9503
  }
8849
9504
  };
8850
9505
  }
@@ -8875,6 +9530,7 @@ function seedReplicationLog(replicationLogs, indexName, partitionId, startSeqNo,
8875
9530
 
8876
9531
  // src/distribution/cluster-node/transport-listener.ts
8877
9532
  var CONTROLLER_MESSAGE_TYPES = /* @__PURE__ */ new Set([
9533
+ ReplicationMessageTypes.INSYNC_ADD,
8878
9534
  ReplicationMessageTypes.INSYNC_REMOVE,
8879
9535
  ClusterMessageTypes.BOOTSTRAP_COMPLETE
8880
9536
  ]);
@@ -9022,6 +9678,7 @@ async function createClusterNode(config) {
9022
9678
  getReplicationLog: getReplicationLog2,
9023
9679
  resetReplicationLog: seedReplicationLog2,
9024
9680
  partitionWriteQueues: createPartitionWriteQueues(),
9681
+ catchUp: createCatchUpState(),
9025
9682
  resolveNodeTargets: resolveNodeTargets3,
9026
9683
  waitForActiveReplicas: config.replication?.waitForActiveReplicas
9027
9684
  };
@@ -9092,6 +9749,7 @@ async function createClusterNode(config) {
9092
9749
  bootstrapRetryMaxMs: DEFAULT_NODE_LIFECYCLE_CONFIG.bootstrapRetryMaxMs,
9093
9750
  bootstrapMaxRetries: DEFAULT_NODE_LIFECYCLE_CONFIG.bootstrapMaxRetries,
9094
9751
  allocationDebounceMs: DEFAULT_NODE_LIFECYCLE_CONFIG.allocationDebounceMs,
9752
+ nodeHeartbeatIntervalMs: DEFAULT_NODE_LIFECYCLE_CONFIG.nodeHeartbeatIntervalMs,
9095
9753
  onBootstrapPartition: bootstrapPartition2,
9096
9754
  onRemovePartition: (indexName, partitionId) => {
9097
9755
  clearBootstrapSyncIndex(bootstrapSyncState, indexName, partitionId);
@@ -9203,9 +9861,11 @@ async function createClusterNode(config) {
9203
9861
  });
9204
9862
  const listener = controllerTransport !== null ? controllerTransport.createHandler(dataHandler) : dataHandler;
9205
9863
  unregisterHandler = await config.transport.listen(listener);
9864
+ startCatchUpPump(writeDeps.catchUp, writeDeps);
9206
9865
  }
9207
9866
  },
9208
9867
  async shutdown() {
9868
+ await stopCatchUpPump(writeDeps.catchUp);
9209
9869
  if (isShutdown) {
9210
9870
  return;
9211
9871
  }