@delali/narsil 0.2.3-next.32 → 0.2.3-next.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-C6VVLMRX.mjs → chunk-36ER4WCG.mjs} +1 -0
- package/dist/{chunk-TEAJO7CU.mjs → chunk-GN7KF675.mjs} +10 -2
- package/dist/{chunk-MFFAJ2NR.mjs → chunk-GODECMDT.mjs} +1 -1
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +94 -15
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/coordinator/in-memory.mjs +10 -3
- package/dist/distribution/transport/grpc.mjs +2 -2
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +2 -2
- package/dist/distribution.mjs +1821 -1161
- package/dist/index.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +2 -2
- package/dist/{types-Bfhhnh3G.d.ts → types-BhdINMtE.d.ts} +2 -0
- package/package.json +1 -1
package/dist/distribution.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, MAX_PARTITION_COUNT, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, MAX_INDEX_NAME_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum,
|
|
2
|
-
import { ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-
|
|
1
|
+
import { createEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, applyIndexEntry, applyDeleteEntry, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, now, validateIndexName, indexConfigKey, putIndexMetadata, getIndexMetadata, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, MAX_PARTITION_COUNT, clampLimit, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, MAX_SUGGEST_SCATTER_LIMIT, chunkByBudget, WIRE_BATCH_BUDGET, MAX_INDEX_NAME_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, createEntryMessage, computeEntryChecksum, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-GN7KF675.mjs';
|
|
2
|
+
import { ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, MAX_MESSAGE_SIZE_BYTES } from './chunk-36ER4WCG.mjs';
|
|
3
3
|
import { deserializePayloadV2, createPartitionIndex, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, encodePageCursor, applyProjection, readSortValues, getLanguage, DEFAULT_PAGE_SIZE, requireWithinResultWindow, fnv1a, FILTER_EXPRESSION_KEYS, MAX_CURSOR_LENGTH, mergePartitionStats, FIELD_FILTER_OPERATORS, MAX_NESTING_DEPTH } from './chunk-RO4EFWE6.mjs';
|
|
4
4
|
import './chunk-TGP5M5R7.mjs';
|
|
5
5
|
import './chunk-OBXS7OZM.mjs';
|
|
@@ -60,61 +60,6 @@ function startRenewalInterval(state, coordinator, nodeId, ttlMs, onLeaseLost) {
|
|
|
60
60
|
function scheduleStandbyRetry(state, retryMs, onRetry) {
|
|
61
61
|
state.standbyTimer = setTimeout(onRetry, retryMs);
|
|
62
62
|
}
|
|
63
|
-
async function requestInsyncRemoval(indexName, partitionId, replicaNodeId, primaryTerm, controllerNodeId, transport, sourceNodeId) {
|
|
64
|
-
const payload = {
|
|
65
|
-
indexName,
|
|
66
|
-
partitionId,
|
|
67
|
-
replicaNodeId,
|
|
68
|
-
primaryTerm
|
|
69
|
-
};
|
|
70
|
-
const message = createInsyncRemoveMessage(payload, sourceNodeId);
|
|
71
|
-
const response = await transport.send(controllerNodeId, message);
|
|
72
|
-
const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
|
|
73
|
-
return { accepted: confirmPayload.accepted };
|
|
74
|
-
}
|
|
75
|
-
async function handleInsyncRemoval(payload, coordinator) {
|
|
76
|
-
const table = await coordinator.getAllocation(payload.indexName);
|
|
77
|
-
if (table === null) {
|
|
78
|
-
return {
|
|
79
|
-
indexName: payload.indexName,
|
|
80
|
-
partitionId: payload.partitionId,
|
|
81
|
-
accepted: false
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
const assignment = table.assignments.get(payload.partitionId);
|
|
85
|
-
if (assignment === void 0) {
|
|
86
|
-
return {
|
|
87
|
-
indexName: payload.indexName,
|
|
88
|
-
partitionId: payload.partitionId,
|
|
89
|
-
accepted: false
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
if (payload.primaryTerm !== assignment.primaryTerm) {
|
|
93
|
-
return {
|
|
94
|
-
indexName: payload.indexName,
|
|
95
|
-
partitionId: payload.partitionId,
|
|
96
|
-
accepted: false
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
|
-
const updatedInSyncSet = assignment.inSyncSet.filter((nodeId) => nodeId !== payload.replicaNodeId);
|
|
100
|
-
const updatedAssignment = {
|
|
101
|
-
...assignment,
|
|
102
|
-
inSyncSet: updatedInSyncSet
|
|
103
|
-
};
|
|
104
|
-
const updatedAssignments = new Map(table.assignments);
|
|
105
|
-
updatedAssignments.set(payload.partitionId, updatedAssignment);
|
|
106
|
-
const updatedTable = {
|
|
107
|
-
...table,
|
|
108
|
-
version: table.version + 1,
|
|
109
|
-
assignments: updatedAssignments
|
|
110
|
-
};
|
|
111
|
-
const written = await coordinator.putAllocation(payload.indexName, updatedTable, table.version);
|
|
112
|
-
return {
|
|
113
|
-
indexName: payload.indexName,
|
|
114
|
-
partitionId: payload.partitionId,
|
|
115
|
-
accepted: written
|
|
116
|
-
};
|
|
117
|
-
}
|
|
118
63
|
|
|
119
64
|
// src/distribution/cluster/allocator/deciders/balance.ts
|
|
120
65
|
var balanceDecider = {
|
|
@@ -392,6 +337,7 @@ function initialAllocate(nodes, indexName, partitionCount, replicationFactor, co
|
|
|
392
337
|
primary: primaryNodeId,
|
|
393
338
|
replicas,
|
|
394
339
|
inSyncSet: [],
|
|
340
|
+
commitPoint: 0,
|
|
395
341
|
state: "INITIALISING",
|
|
396
342
|
primaryTerm: 1
|
|
397
343
|
};
|
|
@@ -442,6 +388,7 @@ function cloneAssignments(assignments) {
|
|
|
442
388
|
primary: assignment.primary,
|
|
443
389
|
replicas: [...assignment.replicas],
|
|
444
390
|
inSyncSet: [...assignment.inSyncSet],
|
|
391
|
+
commitPoint: assignment.commitPoint,
|
|
445
392
|
state: assignment.state,
|
|
446
393
|
primaryTerm: assignment.primaryTerm
|
|
447
394
|
});
|
|
@@ -613,203 +560,14 @@ function allocate(nodes, currentTable, indexName, partitionCount, replicationFac
|
|
|
613
560
|
}
|
|
614
561
|
return rebalanceAllocate(dataNodes, currentTable, constraints, deciders);
|
|
615
562
|
}
|
|
616
|
-
function isRecord(value) {
|
|
617
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
618
|
-
}
|
|
619
|
-
function isValidInteger(value) {
|
|
620
|
-
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
621
|
-
}
|
|
622
|
-
function validateBootstrapCompletePayload(decoded) {
|
|
623
|
-
if (!isRecord(decoded)) {
|
|
624
|
-
return null;
|
|
625
|
-
}
|
|
626
|
-
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
627
|
-
return null;
|
|
628
|
-
}
|
|
629
|
-
if (!isValidInteger(decoded.partitionId) || decoded.partitionId < 0) {
|
|
630
|
-
return null;
|
|
631
|
-
}
|
|
632
|
-
if (typeof decoded.nodeId !== "string") {
|
|
633
|
-
return null;
|
|
634
|
-
}
|
|
635
|
-
if (!isValidInteger(decoded.primaryTerm)) {
|
|
636
|
-
return null;
|
|
637
|
-
}
|
|
638
|
-
return {
|
|
639
|
-
indexName: decoded.indexName,
|
|
640
|
-
partitionId: decoded.partitionId,
|
|
641
|
-
nodeId: decoded.nodeId,
|
|
642
|
-
primaryTerm: decoded.primaryTerm
|
|
643
|
-
};
|
|
644
|
-
}
|
|
645
|
-
async function sendRejection(respond, controllerNodeId, requestId, indexName, partitionId) {
|
|
646
|
-
const resultPayload = {
|
|
647
|
-
indexName,
|
|
648
|
-
partitionId,
|
|
649
|
-
accepted: false
|
|
650
|
-
};
|
|
651
|
-
const response = {
|
|
652
|
-
type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
|
|
653
|
-
sourceId: controllerNodeId,
|
|
654
|
-
requestId,
|
|
655
|
-
payload: encode(resultPayload)
|
|
656
|
-
};
|
|
657
|
-
await respond(response);
|
|
658
|
-
}
|
|
659
|
-
function handleBootstrapCompleteMessage(message, respond, coordinator, controllerNodeId) {
|
|
660
|
-
let decoded;
|
|
661
|
-
try {
|
|
662
|
-
decoded = decode(message.payload);
|
|
663
|
-
} catch (_) {
|
|
664
|
-
void sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
665
|
-
return;
|
|
666
|
-
}
|
|
667
|
-
const payload = validateBootstrapCompletePayload(decoded);
|
|
668
|
-
if (payload === null) {
|
|
669
|
-
void sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
670
|
-
return;
|
|
671
|
-
}
|
|
672
|
-
if (message.sourceId !== payload.nodeId) {
|
|
673
|
-
void sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
|
|
674
|
-
return;
|
|
675
|
-
}
|
|
676
|
-
processBootstrapComplete(payload, coordinator).then(async (accepted) => {
|
|
677
|
-
const resultPayload = {
|
|
678
|
-
indexName: payload.indexName,
|
|
679
|
-
partitionId: payload.partitionId,
|
|
680
|
-
accepted
|
|
681
|
-
};
|
|
682
|
-
const response = {
|
|
683
|
-
type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
|
|
684
|
-
sourceId: controllerNodeId,
|
|
685
|
-
requestId: message.requestId,
|
|
686
|
-
payload: encode(resultPayload)
|
|
687
|
-
};
|
|
688
|
-
await respond(response);
|
|
689
|
-
}).catch(() => sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId));
|
|
690
|
-
}
|
|
691
|
-
function ensurePrimaryInSyncSet(assignment, inSyncSet) {
|
|
692
|
-
if (assignment.primary === null) {
|
|
693
|
-
return inSyncSet;
|
|
694
|
-
}
|
|
695
|
-
if (inSyncSet.includes(assignment.primary)) {
|
|
696
|
-
return inSyncSet;
|
|
697
|
-
}
|
|
698
|
-
return [assignment.primary, ...inSyncSet];
|
|
699
|
-
}
|
|
700
|
-
async function processBootstrapComplete(payload, coordinator) {
|
|
701
|
-
const table = await coordinator.getAllocation(payload.indexName);
|
|
702
|
-
if (table === null) {
|
|
703
|
-
return false;
|
|
704
|
-
}
|
|
705
|
-
const assignment = table.assignments.get(payload.partitionId);
|
|
706
|
-
if (assignment === void 0) {
|
|
707
|
-
return false;
|
|
708
|
-
}
|
|
709
|
-
if (payload.primaryTerm !== assignment.primaryTerm) {
|
|
710
|
-
return false;
|
|
711
|
-
}
|
|
712
|
-
const isAssigned = assignment.primary === payload.nodeId || assignment.replicas.includes(payload.nodeId);
|
|
713
|
-
if (!isAssigned) {
|
|
714
|
-
return false;
|
|
715
|
-
}
|
|
716
|
-
if (assignment.state === "ACTIVE") {
|
|
717
|
-
if (assignment.inSyncSet.includes(payload.nodeId)) {
|
|
718
|
-
return true;
|
|
719
|
-
}
|
|
720
|
-
const updatedInSyncSet2 = [...assignment.inSyncSet, payload.nodeId];
|
|
721
|
-
const updatedAssignment2 = {
|
|
722
|
-
...assignment,
|
|
723
|
-
inSyncSet: updatedInSyncSet2
|
|
724
|
-
};
|
|
725
|
-
const updatedAssignments2 = new Map(table.assignments);
|
|
726
|
-
updatedAssignments2.set(payload.partitionId, updatedAssignment2);
|
|
727
|
-
const updatedTable2 = {
|
|
728
|
-
...table,
|
|
729
|
-
version: table.version + 1,
|
|
730
|
-
assignments: updatedAssignments2
|
|
731
|
-
};
|
|
732
|
-
return coordinator.putAllocation(payload.indexName, updatedTable2, table.version);
|
|
733
|
-
}
|
|
734
|
-
if (assignment.state !== "INITIALISING") {
|
|
735
|
-
return false;
|
|
736
|
-
}
|
|
737
|
-
const baseInSyncSet = assignment.inSyncSet.includes(payload.nodeId) ? assignment.inSyncSet : [...assignment.inSyncSet, payload.nodeId];
|
|
738
|
-
const updatedInSyncSet = ensurePrimaryInSyncSet(assignment, baseInSyncSet);
|
|
739
|
-
const updatedAssignment = {
|
|
740
|
-
...assignment,
|
|
741
|
-
state: "ACTIVE",
|
|
742
|
-
inSyncSet: updatedInSyncSet
|
|
743
|
-
};
|
|
744
|
-
const updatedAssignments = new Map(table.assignments);
|
|
745
|
-
updatedAssignments.set(payload.partitionId, updatedAssignment);
|
|
746
|
-
const updatedTable = {
|
|
747
|
-
...table,
|
|
748
|
-
version: table.version + 1,
|
|
749
|
-
assignments: updatedAssignments
|
|
750
|
-
};
|
|
751
|
-
return coordinator.putAllocation(payload.indexName, updatedTable, table.version);
|
|
752
|
-
}
|
|
753
563
|
|
|
754
|
-
// src/distribution/cluster/controller/event-loop.ts
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
debounceTimer: null,
|
|
762
|
-
insyncQueue: Promise.resolve()
|
|
763
|
-
};
|
|
764
|
-
}
|
|
765
|
-
function clearEventLoopWatchers(state) {
|
|
766
|
-
if (state.unwatchNodes !== null) {
|
|
767
|
-
state.unwatchNodes();
|
|
768
|
-
state.unwatchNodes = null;
|
|
769
|
-
}
|
|
770
|
-
if (state.unwatchSchemas !== null) {
|
|
771
|
-
state.unwatchSchemas();
|
|
772
|
-
state.unwatchSchemas = null;
|
|
773
|
-
}
|
|
774
|
-
if (state.unwatchTransport !== null) {
|
|
775
|
-
state.unwatchTransport();
|
|
776
|
-
state.unwatchTransport = null;
|
|
777
|
-
}
|
|
778
|
-
if (state.debounceTimer !== null) {
|
|
779
|
-
clearTimeout(state.debounceTimer);
|
|
780
|
-
state.debounceTimer = null;
|
|
781
|
-
}
|
|
782
|
-
}
|
|
783
|
-
function isRecord2(value) {
|
|
784
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
785
|
-
}
|
|
786
|
-
function isValidInteger2(value) {
|
|
787
|
-
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
788
|
-
}
|
|
789
|
-
function validateInsyncRemovePayload(decoded) {
|
|
790
|
-
if (!isRecord2(decoded)) {
|
|
791
|
-
return null;
|
|
792
|
-
}
|
|
793
|
-
if (typeof decoded.indexName !== "string") {
|
|
794
|
-
return null;
|
|
795
|
-
}
|
|
796
|
-
if (!isValidInteger2(decoded.partitionId)) {
|
|
797
|
-
return null;
|
|
798
|
-
}
|
|
799
|
-
if (typeof decoded.replicaNodeId !== "string") {
|
|
800
|
-
return null;
|
|
801
|
-
}
|
|
802
|
-
if (!isValidInteger2(decoded.primaryTerm)) {
|
|
803
|
-
return null;
|
|
804
|
-
}
|
|
805
|
-
return {
|
|
806
|
-
indexName: decoded.indexName,
|
|
807
|
-
partitionId: decoded.partitionId,
|
|
808
|
-
replicaNodeId: decoded.replicaNodeId,
|
|
809
|
-
primaryTerm: decoded.primaryTerm
|
|
810
|
-
};
|
|
811
|
-
}
|
|
812
|
-
async function runAllocatorForIndex(coordinator, indexName, nodes, isActive) {
|
|
564
|
+
// src/distribution/cluster/controller/event-loop/allocation.ts
|
|
565
|
+
var ALLOCATION_CAS_ATTEMPTS = 5;
|
|
566
|
+
var ALLOCATION_RETRY_DELAY_MS = 1e3;
|
|
567
|
+
var TEARDOWN_CAS_ATTEMPTS = 5;
|
|
568
|
+
var TEARDOWN_RETRY_ROUNDS = 3;
|
|
569
|
+
var TEARDOWN_RETRY_DELAY_MS = 500;
|
|
570
|
+
async function runAllocatorForIndex(coordinator, indexName, nodes, isActive, attempt = 0) {
|
|
813
571
|
if (!isActive()) {
|
|
814
572
|
return;
|
|
815
573
|
}
|
|
@@ -832,9 +590,20 @@ async function runAllocatorForIndex(coordinator, indexName, nodes, isActive) {
|
|
|
832
590
|
if (!isActive()) {
|
|
833
591
|
return;
|
|
834
592
|
}
|
|
835
|
-
await coordinator.putAllocation(indexName, result.table, currentTable.version)
|
|
593
|
+
if (await coordinator.putAllocation(indexName, result.table, currentTable.version)) {
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
if (attempt + 1 < ALLOCATION_CAS_ATTEMPTS) {
|
|
597
|
+
await runAllocatorForIndex(coordinator, indexName, nodes, isActive, attempt + 1);
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
throw new NarsilError(
|
|
601
|
+
ErrorCodes.ALLOCATION_FAILED,
|
|
602
|
+
`Allocation for index '${indexName}' lost ${ALLOCATION_CAS_ATTEMPTS} compare-and-set attempts`,
|
|
603
|
+
{ indexName, attempts: ALLOCATION_CAS_ATTEMPTS }
|
|
604
|
+
);
|
|
836
605
|
}
|
|
837
|
-
async function runAllocatorForAllIndexes(
|
|
606
|
+
async function runAllocatorForAllIndexes(state, coordinator, isActive, onError) {
|
|
838
607
|
if (!isActive()) {
|
|
839
608
|
return;
|
|
840
609
|
}
|
|
@@ -842,18 +611,23 @@ async function runAllocatorForAllIndexes(coordinator, knownIndexes, isActive, on
|
|
|
842
611
|
if (nodes.length === 0 || !isActive()) {
|
|
843
612
|
return;
|
|
844
613
|
}
|
|
845
|
-
|
|
614
|
+
let anyIndexFailed = false;
|
|
615
|
+
for (const indexName of state.knownIndexes) {
|
|
846
616
|
if (!isActive()) {
|
|
847
617
|
return;
|
|
848
618
|
}
|
|
849
619
|
try {
|
|
850
620
|
await runAllocatorForIndex(coordinator, indexName, nodes, isActive);
|
|
851
621
|
} catch (error) {
|
|
622
|
+
anyIndexFailed = true;
|
|
852
623
|
if (onError !== void 0) {
|
|
853
624
|
onError(indexName, error);
|
|
854
625
|
}
|
|
855
626
|
}
|
|
856
627
|
}
|
|
628
|
+
if (anyIndexFailed && isActive()) {
|
|
629
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError, ALLOCATION_RETRY_DELAY_MS);
|
|
630
|
+
}
|
|
857
631
|
}
|
|
858
632
|
function scheduleDebouncedAllocation(state, coordinator, isActive, onError, debounceMs = 500) {
|
|
859
633
|
if (state.debounceTimer !== null) {
|
|
@@ -861,9 +635,13 @@ function scheduleDebouncedAllocation(state, coordinator, isActive, onError, debo
|
|
|
861
635
|
}
|
|
862
636
|
state.debounceTimer = setTimeout(() => {
|
|
863
637
|
state.debounceTimer = null;
|
|
864
|
-
runAllocatorForAllIndexes(
|
|
638
|
+
runAllocatorForAllIndexes(state, coordinator, isActive, onError).catch(() => {
|
|
639
|
+
if (isActive()) {
|
|
640
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError, ALLOCATION_RETRY_DELAY_MS);
|
|
641
|
+
}
|
|
865
642
|
});
|
|
866
643
|
}, debounceMs);
|
|
644
|
+
state.debounceTimer.unref?.();
|
|
867
645
|
}
|
|
868
646
|
async function handleSchemaCreated(indexName, coordinator, knownIndexes, isActive) {
|
|
869
647
|
knownIndexes.add(indexName);
|
|
@@ -895,11 +673,28 @@ async function handleSchemaCreated(indexName, coordinator, knownIndexes, isActiv
|
|
|
895
673
|
return;
|
|
896
674
|
}
|
|
897
675
|
const expectedVersion = storedTable !== null ? storedTable.version : null;
|
|
898
|
-
await coordinator.putAllocation(indexName, result.table, expectedVersion)
|
|
676
|
+
if (await coordinator.putAllocation(indexName, result.table, expectedVersion)) {
|
|
677
|
+
return;
|
|
678
|
+
}
|
|
679
|
+
await runAllocatorForIndex(coordinator, indexName, nodes, isActive);
|
|
899
680
|
}
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
681
|
+
function scheduleTeardownRetry(indexName, coordinator, state, isActive, round, onError) {
|
|
682
|
+
const existing = state.teardownTimers.get(indexName);
|
|
683
|
+
if (existing !== void 0) {
|
|
684
|
+
clearTimeout(existing);
|
|
685
|
+
}
|
|
686
|
+
const timer = setTimeout(() => {
|
|
687
|
+
state.teardownTimers.delete(indexName);
|
|
688
|
+
tearDownAllocation(indexName, coordinator, state, isActive, round, onError).catch((error) => {
|
|
689
|
+
if (onError !== void 0) {
|
|
690
|
+
onError(indexName, error);
|
|
691
|
+
}
|
|
692
|
+
});
|
|
693
|
+
}, TEARDOWN_RETRY_DELAY_MS);
|
|
694
|
+
timer.unref?.();
|
|
695
|
+
state.teardownTimers.set(indexName, timer);
|
|
696
|
+
}
|
|
697
|
+
async function tearDownAllocation(indexName, coordinator, state, isActive, round, onError) {
|
|
903
698
|
for (let attempt = 0; attempt < TEARDOWN_CAS_ATTEMPTS; attempt++) {
|
|
904
699
|
if (!isActive()) {
|
|
905
700
|
return;
|
|
@@ -929,461 +724,892 @@ async function handleSchemaDropped(indexName, coordinator, knownIndexes, isActiv
|
|
|
929
724
|
await coordinator.deleteAllocation(indexName);
|
|
930
725
|
return;
|
|
931
726
|
}
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
await handleSchemaCreated(event.indexName, coordinator, knownIndexes, isActive);
|
|
936
|
-
} else if (event.type === "schema_dropped") {
|
|
937
|
-
await handleSchemaDropped(event.indexName, coordinator, knownIndexes, isActive);
|
|
727
|
+
if (round + 1 < TEARDOWN_RETRY_ROUNDS && isActive()) {
|
|
728
|
+
scheduleTeardownRetry(indexName, coordinator, state, isActive, round + 1, onError);
|
|
729
|
+
return;
|
|
938
730
|
}
|
|
731
|
+
throw new NarsilError(
|
|
732
|
+
ErrorCodes.ALLOCATION_FAILED,
|
|
733
|
+
`Teardown of the allocation for dropped index '${indexName}' lost every compare-and-set attempt`,
|
|
734
|
+
{ indexName, attempts: TEARDOWN_CAS_ATTEMPTS * TEARDOWN_RETRY_ROUNDS }
|
|
735
|
+
);
|
|
939
736
|
}
|
|
940
|
-
async function
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
944
|
-
});
|
|
945
|
-
state.unwatchNodes = unwatchNodes;
|
|
946
|
-
const unwatchSchemas = await coordinator.watchSchemas((event) => {
|
|
947
|
-
handleSchemaEvent(event, coordinator, state.knownIndexes, isActive).catch(() => {
|
|
948
|
-
});
|
|
949
|
-
});
|
|
950
|
-
state.unwatchSchemas = unwatchSchemas;
|
|
951
|
-
const unwatchTransport = await transport.listen((message, respond) => {
|
|
952
|
-
if (!isActive()) {
|
|
953
|
-
return;
|
|
954
|
-
}
|
|
955
|
-
if (message.type === ReplicationMessageTypes.INSYNC_REMOVE) {
|
|
956
|
-
handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId);
|
|
957
|
-
} else if (message.type === ClusterMessageTypes.BOOTSTRAP_COMPLETE) {
|
|
958
|
-
handleBootstrapCompleteMessage(message, respond, coordinator, nodeId);
|
|
959
|
-
}
|
|
960
|
-
});
|
|
961
|
-
state.unwatchTransport = unwatchTransport;
|
|
962
|
-
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
963
|
-
}
|
|
964
|
-
function handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId) {
|
|
965
|
-
let decoded;
|
|
966
|
-
try {
|
|
967
|
-
decoded = decode(message.payload);
|
|
968
|
-
} catch (_) {
|
|
737
|
+
async function handleSchemaEvent(event, coordinator, state, isActive, onError) {
|
|
738
|
+
if (event.type === "schema_created") {
|
|
739
|
+
await handleSchemaCreated(event.indexName, coordinator, state.knownIndexes, isActive);
|
|
969
740
|
return;
|
|
970
741
|
}
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
742
|
+
if (event.type === "schema_dropped") {
|
|
743
|
+
state.knownIndexes.delete(event.indexName);
|
|
744
|
+
await tearDownAllocation(event.indexName, coordinator, state, isActive, 0, onError);
|
|
974
745
|
}
|
|
975
|
-
|
|
976
|
-
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
// src/distribution/payload-guards.ts
|
|
749
|
+
function isRecord(value) {
|
|
750
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
751
|
+
}
|
|
752
|
+
function isValidInteger(value) {
|
|
753
|
+
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
754
|
+
}
|
|
755
|
+
var INSYNC_CAS_ATTEMPTS = 5;
|
|
756
|
+
var REFUSE = { outcome: "refuse" };
|
|
757
|
+
var ACCEPT = { outcome: "accept" };
|
|
758
|
+
async function updateInsyncSet(request, coordinator, decide) {
|
|
759
|
+
const refused = {
|
|
760
|
+
indexName: request.indexName,
|
|
761
|
+
partitionId: request.partitionId,
|
|
762
|
+
accepted: false
|
|
763
|
+
};
|
|
764
|
+
for (let attempt = 0; attempt < INSYNC_CAS_ATTEMPTS; attempt++) {
|
|
765
|
+
const table = await coordinator.getAllocation(request.indexName);
|
|
977
766
|
if (table === null) {
|
|
978
|
-
return;
|
|
767
|
+
return refused;
|
|
979
768
|
}
|
|
980
|
-
const assignment = table.assignments.get(
|
|
769
|
+
const assignment = table.assignments.get(request.partitionId);
|
|
981
770
|
if (assignment === void 0) {
|
|
982
|
-
return;
|
|
771
|
+
return refused;
|
|
983
772
|
}
|
|
984
|
-
if (
|
|
985
|
-
return;
|
|
773
|
+
if (request.primaryTerm !== assignment.primaryTerm) {
|
|
774
|
+
return refused;
|
|
986
775
|
}
|
|
987
|
-
const
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
}).catch(() => {
|
|
991
|
-
});
|
|
992
|
-
}
|
|
993
|
-
|
|
994
|
-
// src/distribution/cluster/controller/index.ts
|
|
995
|
-
function createController(config) {
|
|
996
|
-
const { nodeId, coordinator, transport, leaseTtlMs, standbyRetryMs, knownIndexNames, onError } = config;
|
|
997
|
-
let electionState = createElectionState();
|
|
998
|
-
let eventLoopState = createEventLoopState(knownIndexNames);
|
|
999
|
-
let stopped = false;
|
|
1000
|
-
function isActive() {
|
|
1001
|
-
return electionState.active;
|
|
1002
|
-
}
|
|
1003
|
-
function stepDown() {
|
|
1004
|
-
if (!electionState.active) {
|
|
1005
|
-
return;
|
|
776
|
+
const decision = decide(assignment);
|
|
777
|
+
if (decision.outcome === "refuse") {
|
|
778
|
+
return refused;
|
|
1006
779
|
}
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
}
|
|
780
|
+
if (decision.outcome === "accept") {
|
|
781
|
+
return { ...refused, accepted: true };
|
|
782
|
+
}
|
|
783
|
+
const updatedAssignments = new Map(table.assignments);
|
|
784
|
+
updatedAssignments.set(request.partitionId, decision.assignment);
|
|
785
|
+
const written = await coordinator.putAllocation(
|
|
786
|
+
request.indexName,
|
|
787
|
+
{ ...table, version: table.version + 1, assignments: updatedAssignments },
|
|
788
|
+
table.version
|
|
789
|
+
);
|
|
790
|
+
if (written) {
|
|
791
|
+
return { ...refused, accepted: true };
|
|
1015
792
|
}
|
|
1016
793
|
}
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
794
|
+
return refused;
|
|
795
|
+
}
|
|
796
|
+
async function requestInsyncRemoval(indexName, partitionId, replicaNodeId, primaryTerm, controllerNodeId, transport, sourceNodeId) {
|
|
797
|
+
const payload = {
|
|
798
|
+
indexName,
|
|
799
|
+
partitionId,
|
|
800
|
+
replicaNodeId,
|
|
801
|
+
primaryTerm
|
|
802
|
+
};
|
|
803
|
+
const message = createInsyncRemoveMessage(payload, sourceNodeId);
|
|
804
|
+
const response = await transport.send(controllerNodeId, message);
|
|
805
|
+
const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
|
|
806
|
+
return { accepted: confirmPayload.accepted };
|
|
807
|
+
}
|
|
808
|
+
function handleInsyncRemoval(payload, coordinator) {
|
|
809
|
+
return updateInsyncSet(payload, coordinator, (assignment) => {
|
|
810
|
+
if (!assignment.inSyncSet.includes(payload.replicaNodeId)) {
|
|
811
|
+
return ACCEPT;
|
|
1020
812
|
}
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
813
|
+
return {
|
|
814
|
+
outcome: "write",
|
|
815
|
+
assignment: {
|
|
816
|
+
...assignment,
|
|
817
|
+
inSyncSet: assignment.inSyncSet.filter((nodeId) => nodeId !== payload.replicaNodeId)
|
|
1026
818
|
}
|
|
1027
|
-
}
|
|
819
|
+
};
|
|
820
|
+
});
|
|
821
|
+
}
|
|
822
|
+
async function requestInsyncAdmission(payload, controllerNodeId, transport, sourceNodeId) {
|
|
823
|
+
const message = createInsyncAddMessage(payload, sourceNodeId);
|
|
824
|
+
const response = await transport.send(controllerNodeId, message);
|
|
825
|
+
const confirmPayload = validateInsyncConfirmPayload(decode(response.payload));
|
|
826
|
+
return { accepted: confirmPayload.accepted };
|
|
827
|
+
}
|
|
828
|
+
function handleInsyncAdmission(payload, coordinator) {
|
|
829
|
+
return updateInsyncSet(payload, coordinator, (assignment) => {
|
|
830
|
+
if (assignment.state !== "ACTIVE") {
|
|
831
|
+
return REFUSE;
|
|
1028
832
|
}
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
async function tryElection() {
|
|
1032
|
-
if (stopped) {
|
|
1033
|
-
return;
|
|
833
|
+
if (!assignment.replicas.includes(payload.replicaNodeId)) {
|
|
834
|
+
return REFUSE;
|
|
1034
835
|
}
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
await becomeActive();
|
|
1038
|
-
} else {
|
|
1039
|
-
scheduleStandbyRetry(electionState, standbyRetryMs, () => {
|
|
1040
|
-
tryElection().catch(() => {
|
|
1041
|
-
});
|
|
1042
|
-
});
|
|
836
|
+
if (payload.appliedSeqNo < assignment.commitPoint || payload.appliedSeqNo < payload.commitPoint) {
|
|
837
|
+
return REFUSE;
|
|
1043
838
|
}
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
return
|
|
1048
|
-
},
|
|
1049
|
-
async start() {
|
|
1050
|
-
stopped = false;
|
|
1051
|
-
await tryElection();
|
|
1052
|
-
},
|
|
1053
|
-
async stop() {
|
|
1054
|
-
stopped = true;
|
|
1055
|
-
clearElectionTimers(electionState);
|
|
1056
|
-
clearEventLoopWatchers(eventLoopState);
|
|
1057
|
-
if (electionState.active) {
|
|
1058
|
-
electionState.active = false;
|
|
1059
|
-
try {
|
|
1060
|
-
await releaseLease(coordinator);
|
|
1061
|
-
} catch (_) {
|
|
1062
|
-
}
|
|
1063
|
-
}
|
|
1064
|
-
},
|
|
1065
|
-
async shutdown() {
|
|
1066
|
-
await controller.stop();
|
|
1067
|
-
eventLoopState = createEventLoopState([]);
|
|
1068
|
-
electionState = createElectionState();
|
|
839
|
+
const alreadyInSync = assignment.inSyncSet.includes(payload.replicaNodeId);
|
|
840
|
+
const commitPoint = Math.max(assignment.commitPoint, payload.commitPoint);
|
|
841
|
+
if (alreadyInSync && commitPoint === assignment.commitPoint) {
|
|
842
|
+
return ACCEPT;
|
|
1069
843
|
}
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
844
|
+
return {
|
|
845
|
+
outcome: "write",
|
|
846
|
+
assignment: {
|
|
847
|
+
...assignment,
|
|
848
|
+
inSyncSet: alreadyInSync ? assignment.inSyncSet : [...assignment.inSyncSet, payload.replicaNodeId],
|
|
849
|
+
commitPoint
|
|
850
|
+
}
|
|
851
|
+
};
|
|
852
|
+
});
|
|
1078
853
|
}
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
if (table === null) {
|
|
854
|
+
function validateBootstrapCompletePayload(decoded) {
|
|
855
|
+
if (!isRecord(decoded)) {
|
|
1082
856
|
return null;
|
|
1083
857
|
}
|
|
1084
|
-
|
|
1085
|
-
if (assignment === void 0) {
|
|
858
|
+
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
1086
859
|
return null;
|
|
1087
860
|
}
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
async function resolveControllerTargets(coordinator, controllerNodeId) {
|
|
1091
|
-
const targets = [controllerNodeId];
|
|
1092
|
-
try {
|
|
1093
|
-
const nodes = await coordinator.listNodes();
|
|
1094
|
-
const registration = nodes.find((entry) => entry.nodeId === controllerNodeId);
|
|
1095
|
-
if (registration !== void 0 && registration.address.length > 0 && registration.address !== controllerNodeId) {
|
|
1096
|
-
targets.push(registration.address);
|
|
1097
|
-
}
|
|
1098
|
-
} catch (_) {
|
|
1099
|
-
return targets;
|
|
861
|
+
if (!isValidInteger(decoded.partitionId) || decoded.partitionId < 0) {
|
|
862
|
+
return null;
|
|
1100
863
|
}
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
async function reportBootstrapComplete(indexName, partitionId, nodeId, coordinator, transport) {
|
|
1104
|
-
const controllerNodeId = await coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
1105
|
-
if (controllerNodeId === null) {
|
|
1106
|
-
return false;
|
|
864
|
+
if (typeof decoded.nodeId !== "string") {
|
|
865
|
+
return null;
|
|
1107
866
|
}
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
return false;
|
|
867
|
+
if (!isValidInteger(decoded.primaryTerm)) {
|
|
868
|
+
return null;
|
|
1111
869
|
}
|
|
1112
|
-
|
|
870
|
+
return {
|
|
871
|
+
indexName: decoded.indexName,
|
|
872
|
+
partitionId: decoded.partitionId,
|
|
873
|
+
nodeId: decoded.nodeId,
|
|
874
|
+
primaryTerm: decoded.primaryTerm
|
|
875
|
+
};
|
|
876
|
+
}
|
|
877
|
+
async function sendRejection(respond, controllerNodeId, requestId, indexName, partitionId) {
|
|
878
|
+
const resultPayload = {
|
|
1113
879
|
indexName,
|
|
1114
880
|
partitionId,
|
|
1115
|
-
|
|
1116
|
-
primaryTerm
|
|
881
|
+
accepted: false
|
|
1117
882
|
};
|
|
1118
|
-
const
|
|
883
|
+
const response = {
|
|
1119
884
|
type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
|
|
1120
|
-
sourceId:
|
|
1121
|
-
requestId
|
|
1122
|
-
payload: encode(
|
|
885
|
+
sourceId: controllerNodeId,
|
|
886
|
+
requestId,
|
|
887
|
+
payload: encode(resultPayload)
|
|
1123
888
|
};
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
889
|
+
await respond(response);
|
|
890
|
+
}
|
|
891
|
+
async function rejectBootstrapComplete(message, respond, controllerNodeId) {
|
|
892
|
+
let decoded;
|
|
893
|
+
try {
|
|
894
|
+
decoded = decode(message.payload);
|
|
895
|
+
} catch (_) {
|
|
896
|
+
await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
897
|
+
return;
|
|
1130
898
|
}
|
|
1131
|
-
|
|
899
|
+
const payload = validateBootstrapCompletePayload(decoded);
|
|
900
|
+
if (payload === null) {
|
|
901
|
+
await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
902
|
+
return;
|
|
903
|
+
}
|
|
904
|
+
await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
|
|
1132
905
|
}
|
|
1133
|
-
function
|
|
906
|
+
async function handleBootstrapCompleteMessage(message, respond, coordinator, controllerNodeId) {
|
|
907
|
+
let decoded;
|
|
1134
908
|
try {
|
|
1135
|
-
|
|
1136
|
-
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
1137
|
-
return false;
|
|
1138
|
-
}
|
|
1139
|
-
return decoded.accepted === true;
|
|
909
|
+
decoded = decode(message.payload);
|
|
1140
910
|
} catch (_) {
|
|
1141
|
-
|
|
911
|
+
await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
912
|
+
return;
|
|
913
|
+
}
|
|
914
|
+
const payload = validateBootstrapCompletePayload(decoded);
|
|
915
|
+
if (payload === null) {
|
|
916
|
+
await sendRejection(respond, controllerNodeId, message.requestId, "", -1);
|
|
917
|
+
return;
|
|
918
|
+
}
|
|
919
|
+
if (message.sourceId !== payload.nodeId) {
|
|
920
|
+
await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
|
|
921
|
+
return;
|
|
922
|
+
}
|
|
923
|
+
let accepted;
|
|
924
|
+
try {
|
|
925
|
+
accepted = await processBootstrapComplete(payload, coordinator);
|
|
926
|
+
} catch (_) {
|
|
927
|
+
await sendRejection(respond, controllerNodeId, message.requestId, payload.indexName, payload.partitionId);
|
|
928
|
+
return;
|
|
1142
929
|
}
|
|
930
|
+
const resultPayload = {
|
|
931
|
+
indexName: payload.indexName,
|
|
932
|
+
partitionId: payload.partitionId,
|
|
933
|
+
accepted
|
|
934
|
+
};
|
|
935
|
+
await respond({
|
|
936
|
+
type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
|
|
937
|
+
sourceId: controllerNodeId,
|
|
938
|
+
requestId: message.requestId,
|
|
939
|
+
payload: encode(resultPayload)
|
|
940
|
+
});
|
|
1143
941
|
}
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
942
|
+
var BOOTSTRAP_CAS_ATTEMPTS = 5;
|
|
943
|
+
function ensurePrimaryInSyncSet(assignment, inSyncSet) {
|
|
944
|
+
if (assignment.primary === null) {
|
|
945
|
+
return inSyncSet;
|
|
946
|
+
}
|
|
947
|
+
if (inSyncSet.includes(assignment.primary)) {
|
|
948
|
+
return inSyncSet;
|
|
949
|
+
}
|
|
950
|
+
return [assignment.primary, ...inSyncSet];
|
|
951
|
+
}
|
|
952
|
+
async function processBootstrapComplete(payload, coordinator) {
|
|
953
|
+
for (let attempt = 0; attempt < BOOTSTRAP_CAS_ATTEMPTS; attempt++) {
|
|
954
|
+
const table = await coordinator.getAllocation(payload.indexName);
|
|
955
|
+
if (table === null) {
|
|
1147
956
|
return false;
|
|
1148
957
|
}
|
|
1149
|
-
const
|
|
1150
|
-
if (
|
|
958
|
+
const assignment = table.assignments.get(payload.partitionId);
|
|
959
|
+
if (assignment === void 0) {
|
|
1151
960
|
return false;
|
|
1152
961
|
}
|
|
1153
|
-
if (
|
|
1154
|
-
return retryReportBootstrapComplete(
|
|
1155
|
-
state,
|
|
1156
|
-
coordinator,
|
|
1157
|
-
transport,
|
|
1158
|
-
nodeId,
|
|
1159
|
-
bootstrapRetryBaseMs,
|
|
1160
|
-
bootstrapRetryMaxMs,
|
|
1161
|
-
bootstrapMaxRetries,
|
|
1162
|
-
onError
|
|
1163
|
-
);
|
|
1164
|
-
}
|
|
1165
|
-
if (attempt < bootstrapMaxRetries) {
|
|
1166
|
-
const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
|
|
1167
|
-
await waitWithAbort(state, backoffMs);
|
|
1168
|
-
}
|
|
1169
|
-
}
|
|
1170
|
-
return false;
|
|
1171
|
-
}
|
|
1172
|
-
async function runBootstrapSyncAttempt(state, onBootstrapPartition, onError) {
|
|
1173
|
-
try {
|
|
1174
|
-
return await onBootstrapPartition(state.indexName, state.partitionId, state.primaryNodeId);
|
|
1175
|
-
} catch (error) {
|
|
1176
|
-
if (state.aborted) {
|
|
962
|
+
if (payload.primaryTerm !== assignment.primaryTerm) {
|
|
1177
963
|
return false;
|
|
1178
964
|
}
|
|
1179
|
-
if (
|
|
1180
|
-
|
|
1181
|
-
onError(
|
|
1182
|
-
new NarsilError(
|
|
1183
|
-
ErrorCodes.NODE_BOOTSTRAP_FAILED,
|
|
1184
|
-
`Bootstrap sync failed for ${state.indexName}:${state.partitionId}`,
|
|
1185
|
-
{ indexName: state.indexName, partitionId: state.partitionId, cause }
|
|
1186
|
-
)
|
|
1187
|
-
);
|
|
965
|
+
if (assignment.primary !== payload.nodeId && !assignment.replicas.includes(payload.nodeId)) {
|
|
966
|
+
return false;
|
|
1188
967
|
}
|
|
1189
|
-
|
|
1190
|
-
}
|
|
1191
|
-
}
|
|
1192
|
-
async function retryReportBootstrapComplete(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onError) {
|
|
1193
|
-
for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
|
|
1194
|
-
if (state.aborted) {
|
|
968
|
+
if (assignment.state !== "INITIALISING") {
|
|
1195
969
|
return false;
|
|
1196
970
|
}
|
|
1197
|
-
const
|
|
1198
|
-
|
|
971
|
+
const baseInSyncSet = assignment.inSyncSet.includes(payload.nodeId) ? assignment.inSyncSet : [...assignment.inSyncSet, payload.nodeId];
|
|
972
|
+
const updatedAssignments = new Map(table.assignments);
|
|
973
|
+
updatedAssignments.set(payload.partitionId, {
|
|
974
|
+
...assignment,
|
|
975
|
+
state: "ACTIVE",
|
|
976
|
+
inSyncSet: ensurePrimaryInSyncSet(assignment, baseInSyncSet)
|
|
977
|
+
});
|
|
978
|
+
const written = await coordinator.putAllocation(
|
|
979
|
+
payload.indexName,
|
|
980
|
+
{ ...table, version: table.version + 1, assignments: updatedAssignments },
|
|
981
|
+
table.version
|
|
982
|
+
);
|
|
983
|
+
if (written) {
|
|
1199
984
|
return true;
|
|
1200
985
|
}
|
|
1201
|
-
if (attempt < bootstrapMaxRetries && !state.aborted) {
|
|
1202
|
-
const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
|
|
1203
|
-
await waitWithAbort(state, backoffMs);
|
|
1204
|
-
}
|
|
1205
|
-
}
|
|
1206
|
-
if (onError !== void 0) {
|
|
1207
|
-
onError(
|
|
1208
|
-
new NarsilError(
|
|
1209
|
-
ErrorCodes.NODE_BOOTSTRAP_FAILED,
|
|
1210
|
-
`Failed to report bootstrap completion for ${state.indexName}:${state.partitionId} after ${bootstrapMaxRetries + 1} attempts`,
|
|
1211
|
-
{ indexName: state.indexName, partitionId: state.partitionId }
|
|
1212
|
-
)
|
|
1213
|
-
);
|
|
1214
986
|
}
|
|
1215
987
|
return false;
|
|
1216
988
|
}
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
state.abortResolve = resolve;
|
|
1224
|
-
state.retryTimer = setTimeout(() => {
|
|
1225
|
-
state.retryTimer = null;
|
|
1226
|
-
state.abortResolve = null;
|
|
1227
|
-
resolve();
|
|
1228
|
-
}, ms);
|
|
1229
|
-
});
|
|
1230
|
-
}
|
|
1231
|
-
function abortBootstrapState(state) {
|
|
1232
|
-
state.aborted = true;
|
|
1233
|
-
if (state.retryTimer !== null) {
|
|
1234
|
-
clearTimeout(state.retryTimer);
|
|
1235
|
-
state.retryTimer = null;
|
|
989
|
+
|
|
990
|
+
// src/distribution/cluster/controller/event-loop/insync-messages.ts
|
|
991
|
+
var UNKNOWN_PARTITION_ID = -1;
|
|
992
|
+
function validateInsyncRemovePayload(decoded) {
|
|
993
|
+
if (!isRecord(decoded)) {
|
|
994
|
+
return null;
|
|
1236
995
|
}
|
|
1237
|
-
if (
|
|
1238
|
-
|
|
1239
|
-
|
|
996
|
+
if (typeof decoded.indexName !== "string") {
|
|
997
|
+
return null;
|
|
998
|
+
}
|
|
999
|
+
if (!isValidInteger(decoded.partitionId)) {
|
|
1000
|
+
return null;
|
|
1001
|
+
}
|
|
1002
|
+
if (typeof decoded.replicaNodeId !== "string") {
|
|
1003
|
+
return null;
|
|
1004
|
+
}
|
|
1005
|
+
if (!isValidInteger(decoded.primaryTerm)) {
|
|
1006
|
+
return null;
|
|
1240
1007
|
}
|
|
1241
|
-
}
|
|
1242
|
-
function createBootstrapState(indexName, partitionId, primaryNodeId) {
|
|
1243
1008
|
return {
|
|
1244
|
-
indexName,
|
|
1245
|
-
partitionId,
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
retryTimer: null,
|
|
1249
|
-
aborted: false,
|
|
1250
|
-
abortResolve: null
|
|
1009
|
+
indexName: decoded.indexName,
|
|
1010
|
+
partitionId: decoded.partitionId,
|
|
1011
|
+
replicaNodeId: decoded.replicaNodeId,
|
|
1012
|
+
primaryTerm: decoded.primaryTerm
|
|
1251
1013
|
};
|
|
1252
1014
|
}
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1015
|
+
function validateInsyncAddPayload(decoded) {
|
|
1016
|
+
const base = validateInsyncRemovePayload(decoded);
|
|
1017
|
+
if (base === null || !isRecord(decoded)) {
|
|
1018
|
+
return null;
|
|
1019
|
+
}
|
|
1020
|
+
if (!isValidInteger(decoded.appliedSeqNo) || decoded.appliedSeqNo < 0) {
|
|
1021
|
+
return null;
|
|
1022
|
+
}
|
|
1023
|
+
if (!isValidInteger(decoded.commitPoint) || decoded.commitPoint < 0) {
|
|
1024
|
+
return null;
|
|
1025
|
+
}
|
|
1026
|
+
return { ...base, appliedSeqNo: decoded.appliedSeqNo, commitPoint: decoded.commitPoint };
|
|
1257
1027
|
}
|
|
1258
|
-
function
|
|
1028
|
+
function decodeMessage(message) {
|
|
1029
|
+
try {
|
|
1030
|
+
return decode(message.payload);
|
|
1031
|
+
} catch (_) {
|
|
1032
|
+
return null;
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
function describeRequest(decoded) {
|
|
1036
|
+
if (!isRecord(decoded)) {
|
|
1037
|
+
return { indexName: "", partitionId: UNKNOWN_PARTITION_ID };
|
|
1038
|
+
}
|
|
1259
1039
|
return {
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
activeBootstraps: /* @__PURE__ */ new Map(),
|
|
1263
|
-
trackedPartitions: /* @__PURE__ */ new Map(),
|
|
1264
|
-
pendingTables: /* @__PURE__ */ new Map(),
|
|
1265
|
-
stopped: false
|
|
1040
|
+
indexName: typeof decoded.indexName === "string" ? decoded.indexName : "",
|
|
1041
|
+
partitionId: isValidInteger(decoded.partitionId) ? decoded.partitionId : UNKNOWN_PARTITION_ID
|
|
1266
1042
|
};
|
|
1267
1043
|
}
|
|
1268
|
-
async function
|
|
1269
|
-
const
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1044
|
+
async function assertSenderLeadsPartition(coordinator, indexName, partitionId, sourceId) {
|
|
1045
|
+
const table = await coordinator.getAllocation(indexName);
|
|
1046
|
+
const assignment = table?.assignments.get(partitionId);
|
|
1047
|
+
if (assignment === void 0) {
|
|
1048
|
+
return false;
|
|
1049
|
+
}
|
|
1050
|
+
return assignment.primary === sourceId;
|
|
1051
|
+
}
|
|
1052
|
+
function enqueue(state, task) {
|
|
1053
|
+
state.insyncQueue = state.insyncQueue.then(task).catch(() => {
|
|
1274
1054
|
});
|
|
1275
|
-
state.unwatchAllocation = unwatchAllocation;
|
|
1276
1055
|
}
|
|
1277
|
-
function
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1056
|
+
async function respondRefusal(respond, request, nodeId, requestId) {
|
|
1057
|
+
const payload = {
|
|
1058
|
+
indexName: request.indexName,
|
|
1059
|
+
partitionId: request.partitionId,
|
|
1060
|
+
accepted: false
|
|
1061
|
+
};
|
|
1062
|
+
try {
|
|
1063
|
+
await respond(createInsyncConfirmMessage(payload, nodeId, requestId));
|
|
1064
|
+
} catch (_) {
|
|
1281
1065
|
}
|
|
1282
|
-
state.debounceTimer = setTimeout(() => {
|
|
1283
|
-
state.debounceTimer = null;
|
|
1284
|
-
const tablesToProcess = Array.from(state.pendingTables.values());
|
|
1285
|
-
state.pendingTables.clear();
|
|
1286
|
-
for (const pending of tablesToProcess) {
|
|
1287
|
-
processAllocationChange(state, config, pending);
|
|
1288
|
-
}
|
|
1289
|
-
}, config.allocationDebounceMs);
|
|
1290
1066
|
}
|
|
1291
|
-
function
|
|
1292
|
-
|
|
1067
|
+
function handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId, isActive) {
|
|
1068
|
+
const decoded = decodeMessage(message);
|
|
1069
|
+
const payload = validateInsyncRemovePayload(decoded);
|
|
1070
|
+
if (payload === null) {
|
|
1071
|
+
void respondRefusal(respond, describeRequest(decoded), nodeId, message.requestId);
|
|
1293
1072
|
return;
|
|
1294
1073
|
}
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
if (existing === void 0) {
|
|
1306
|
-
handleNewAssignment(state, config, table.indexName, partitionId, assignment, nodeId);
|
|
1307
|
-
} else {
|
|
1308
|
-
handleExistingAssignment(state, config, existing, assignment, nodeId);
|
|
1074
|
+
enqueue(state, async () => {
|
|
1075
|
+
const leadsPartition = await assertSenderLeadsPartition(
|
|
1076
|
+
coordinator,
|
|
1077
|
+
payload.indexName,
|
|
1078
|
+
payload.partitionId,
|
|
1079
|
+
message.sourceId
|
|
1080
|
+
);
|
|
1081
|
+
if (!leadsPartition || !isActive()) {
|
|
1082
|
+
await respondRefusal(respond, payload, nodeId, message.requestId);
|
|
1083
|
+
return;
|
|
1309
1084
|
}
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
isPrimary,
|
|
1314
|
-
primaryNodeId: assignment.primary,
|
|
1315
|
-
primaryTerm: assignment.primaryTerm
|
|
1316
|
-
});
|
|
1317
|
-
}
|
|
1318
|
-
removeUnassignedPartitions(state, config, table.indexName, currentKeys);
|
|
1319
|
-
}
|
|
1320
|
-
function isNodeAssigned(assignment, nodeId) {
|
|
1321
|
-
if (assignment.primary === nodeId) {
|
|
1322
|
-
return true;
|
|
1323
|
-
}
|
|
1324
|
-
return assignment.replicas.includes(nodeId);
|
|
1085
|
+
const confirmPayload = await handleInsyncRemoval(payload, coordinator);
|
|
1086
|
+
await respond(createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId));
|
|
1087
|
+
});
|
|
1325
1088
|
}
|
|
1326
|
-
function
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1089
|
+
function handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive) {
|
|
1090
|
+
const decoded = decodeMessage(message);
|
|
1091
|
+
const payload = validateInsyncAddPayload(decoded);
|
|
1092
|
+
if (payload === null) {
|
|
1093
|
+
void respondRefusal(respond, describeRequest(decoded), nodeId, message.requestId);
|
|
1331
1094
|
return;
|
|
1332
1095
|
}
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1096
|
+
enqueue(state, async () => {
|
|
1097
|
+
const leadsPartition = await assertSenderLeadsPartition(
|
|
1098
|
+
coordinator,
|
|
1099
|
+
payload.indexName,
|
|
1100
|
+
payload.partitionId,
|
|
1101
|
+
message.sourceId
|
|
1102
|
+
);
|
|
1103
|
+
if (!leadsPartition || !isActive()) {
|
|
1104
|
+
await respondRefusal(respond, payload, nodeId, message.requestId);
|
|
1105
|
+
return;
|
|
1336
1106
|
}
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1107
|
+
const confirmPayload = await handleInsyncAdmission(payload, coordinator);
|
|
1108
|
+
await respond(createInsyncConfirmMessage(confirmPayload, nodeId, message.requestId));
|
|
1109
|
+
});
|
|
1340
1110
|
}
|
|
1341
|
-
function
|
|
1342
|
-
|
|
1343
|
-
if (
|
|
1344
|
-
|
|
1345
|
-
config.onPrimaryDemotion(existing.indexName, existing.partitionId, assignment.primary);
|
|
1346
|
-
}
|
|
1347
|
-
startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
|
|
1111
|
+
function handleQueuedBootstrapComplete(state, message, respond, coordinator, nodeId, isActive) {
|
|
1112
|
+
enqueue(state, () => {
|
|
1113
|
+
if (!isActive()) {
|
|
1114
|
+
return rejectBootstrapComplete(message, respond, nodeId);
|
|
1348
1115
|
}
|
|
1349
|
-
return;
|
|
1116
|
+
return handleBootstrapCompleteMessage(message, respond, coordinator, nodeId);
|
|
1117
|
+
});
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
// src/distribution/cluster/controller/event-loop/state.ts
|
|
1121
|
+
function createEventLoopState(initialIndexNames) {
|
|
1122
|
+
return {
|
|
1123
|
+
unwatchNodes: null,
|
|
1124
|
+
unwatchSchemas: null,
|
|
1125
|
+
unwatchTransport: null,
|
|
1126
|
+
knownIndexes: new Set(initialIndexNames),
|
|
1127
|
+
debounceTimer: null,
|
|
1128
|
+
teardownTimers: /* @__PURE__ */ new Map(),
|
|
1129
|
+
insyncQueue: Promise.resolve()
|
|
1130
|
+
};
|
|
1131
|
+
}
|
|
1132
|
+
function clearEventLoopWatchers(state) {
|
|
1133
|
+
if (state.unwatchNodes !== null) {
|
|
1134
|
+
state.unwatchNodes();
|
|
1135
|
+
state.unwatchNodes = null;
|
|
1350
1136
|
}
|
|
1351
|
-
if (
|
|
1352
|
-
|
|
1137
|
+
if (state.unwatchSchemas !== null) {
|
|
1138
|
+
state.unwatchSchemas();
|
|
1139
|
+
state.unwatchSchemas = null;
|
|
1353
1140
|
}
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
return false;
|
|
1141
|
+
if (state.unwatchTransport !== null) {
|
|
1142
|
+
state.unwatchTransport();
|
|
1143
|
+
state.unwatchTransport = null;
|
|
1358
1144
|
}
|
|
1359
|
-
if (
|
|
1360
|
-
|
|
1145
|
+
if (state.debounceTimer !== null) {
|
|
1146
|
+
clearTimeout(state.debounceTimer);
|
|
1147
|
+
state.debounceTimer = null;
|
|
1361
1148
|
}
|
|
1362
|
-
|
|
1363
|
-
|
|
1149
|
+
for (const timer of state.teardownTimers.values()) {
|
|
1150
|
+
clearTimeout(timer);
|
|
1364
1151
|
}
|
|
1365
|
-
|
|
1366
|
-
|
|
1152
|
+
state.teardownTimers.clear();
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
// src/distribution/cluster/controller/event-loop/index.ts
|
|
1156
|
+
async function startEventLoop(state, coordinator, transport, nodeId, isActive, onError) {
|
|
1157
|
+
clearEventLoopWatchers(state);
|
|
1158
|
+
try {
|
|
1159
|
+
state.unwatchNodes = await coordinator.watchNodes((_event) => {
|
|
1160
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
1161
|
+
});
|
|
1162
|
+
state.unwatchSchemas = await coordinator.watchSchemas((event) => {
|
|
1163
|
+
handleSchemaEvent(event, coordinator, state, isActive, onError).catch((error) => {
|
|
1164
|
+
if (onError !== void 0) {
|
|
1165
|
+
onError(event.indexName, error);
|
|
1166
|
+
}
|
|
1167
|
+
});
|
|
1168
|
+
});
|
|
1169
|
+
state.unwatchTransport = await transport.listen((message, respond) => {
|
|
1170
|
+
if (!isActive()) {
|
|
1171
|
+
return;
|
|
1172
|
+
}
|
|
1173
|
+
if (message.type === ReplicationMessageTypes.INSYNC_REMOVE) {
|
|
1174
|
+
handleInsyncRemoveMessage(state, message, respond, coordinator, nodeId, isActive);
|
|
1175
|
+
} else if (message.type === ReplicationMessageTypes.INSYNC_ADD) {
|
|
1176
|
+
handleInsyncAddMessage(state, message, respond, coordinator, nodeId, isActive);
|
|
1177
|
+
} else if (message.type === ClusterMessageTypes.BOOTSTRAP_COMPLETE) {
|
|
1178
|
+
handleQueuedBootstrapComplete(state, message, respond, coordinator, nodeId, isActive);
|
|
1179
|
+
}
|
|
1180
|
+
});
|
|
1181
|
+
} catch (error) {
|
|
1182
|
+
clearEventLoopWatchers(state);
|
|
1183
|
+
throw error;
|
|
1367
1184
|
}
|
|
1368
|
-
|
|
1185
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
1369
1186
|
}
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1187
|
+
|
|
1188
|
+
// src/distribution/cluster/controller/index.ts
|
|
1189
|
+
function createController(config) {
|
|
1190
|
+
const { nodeId, coordinator, transport, leaseTtlMs, standbyRetryMs, knownIndexNames, onError } = config;
|
|
1191
|
+
let electionState = createElectionState();
|
|
1192
|
+
let eventLoopState = createEventLoopState(knownIndexNames);
|
|
1193
|
+
let stopped = false;
|
|
1194
|
+
function isActive() {
|
|
1195
|
+
return electionState.active;
|
|
1375
1196
|
}
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1197
|
+
function stepDown() {
|
|
1198
|
+
if (!electionState.active) {
|
|
1199
|
+
return;
|
|
1200
|
+
}
|
|
1201
|
+
electionState.active = false;
|
|
1202
|
+
clearElectionTimers(electionState);
|
|
1203
|
+
clearEventLoopWatchers(eventLoopState);
|
|
1204
|
+
if (!stopped) {
|
|
1205
|
+
scheduleStandbyRetry(electionState, standbyRetryMs, () => {
|
|
1206
|
+
tryElection().catch(() => {
|
|
1207
|
+
});
|
|
1208
|
+
});
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
async function becomeActive() {
|
|
1212
|
+
if (electionState.active) {
|
|
1213
|
+
return;
|
|
1214
|
+
}
|
|
1215
|
+
electionState.active = true;
|
|
1216
|
+
startRenewalInterval(electionState, coordinator, nodeId, leaseTtlMs, stepDown);
|
|
1217
|
+
try {
|
|
1218
|
+
for (const indexName of await coordinator.listSchemas()) {
|
|
1219
|
+
eventLoopState.knownIndexes.add(indexName);
|
|
1220
|
+
}
|
|
1221
|
+
} catch (_) {
|
|
1222
|
+
}
|
|
1223
|
+
try {
|
|
1224
|
+
await startEventLoop(eventLoopState, coordinator, transport, nodeId, isActive, onError);
|
|
1225
|
+
} catch (error) {
|
|
1226
|
+
electionState.active = false;
|
|
1227
|
+
clearElectionTimers(electionState);
|
|
1228
|
+
try {
|
|
1229
|
+
await releaseLease(coordinator);
|
|
1230
|
+
} catch (_) {
|
|
1231
|
+
}
|
|
1232
|
+
throw error;
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
async function tryElection() {
|
|
1236
|
+
if (stopped) {
|
|
1237
|
+
return;
|
|
1238
|
+
}
|
|
1239
|
+
const acquired = await tryAcquireLease(coordinator, nodeId, leaseTtlMs);
|
|
1240
|
+
if (acquired) {
|
|
1241
|
+
await becomeActive();
|
|
1242
|
+
} else {
|
|
1243
|
+
scheduleStandbyRetry(electionState, standbyRetryMs, () => {
|
|
1244
|
+
tryElection().catch(() => {
|
|
1245
|
+
});
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
const controller = {
|
|
1250
|
+
get isActive() {
|
|
1251
|
+
return electionState.active;
|
|
1252
|
+
},
|
|
1253
|
+
async start() {
|
|
1254
|
+
stopped = false;
|
|
1255
|
+
await tryElection();
|
|
1256
|
+
},
|
|
1257
|
+
async stop() {
|
|
1258
|
+
stopped = true;
|
|
1259
|
+
clearElectionTimers(electionState);
|
|
1260
|
+
clearEventLoopWatchers(eventLoopState);
|
|
1261
|
+
if (electionState.active) {
|
|
1262
|
+
electionState.active = false;
|
|
1263
|
+
try {
|
|
1264
|
+
await releaseLease(coordinator);
|
|
1265
|
+
} catch (_) {
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
},
|
|
1269
|
+
async shutdown() {
|
|
1270
|
+
await controller.stop();
|
|
1271
|
+
eventLoopState = createEventLoopState([]);
|
|
1272
|
+
electionState = createElectionState();
|
|
1273
|
+
}
|
|
1274
|
+
};
|
|
1275
|
+
return controller;
|
|
1276
|
+
}
|
|
1277
|
+
function computeBackoffMs(baseMs, maxMs, retryCount) {
|
|
1278
|
+
const exponential = baseMs * 2 ** retryCount;
|
|
1279
|
+
const capped = Math.min(exponential, maxMs);
|
|
1280
|
+
const jitter = capped * (0.5 + Math.random() * 0.5);
|
|
1281
|
+
return Math.floor(jitter);
|
|
1282
|
+
}
|
|
1283
|
+
async function resolvePrimaryTerm(indexName, partitionId, coordinator) {
|
|
1284
|
+
const table = await coordinator.getAllocation(indexName);
|
|
1285
|
+
if (table === null) {
|
|
1286
|
+
return null;
|
|
1287
|
+
}
|
|
1288
|
+
const assignment = table.assignments.get(partitionId);
|
|
1289
|
+
if (assignment === void 0) {
|
|
1290
|
+
return null;
|
|
1291
|
+
}
|
|
1292
|
+
return assignment.primaryTerm;
|
|
1293
|
+
}
|
|
1294
|
+
async function resolveControllerTargets(coordinator, controllerNodeId) {
|
|
1295
|
+
const targets = [controllerNodeId];
|
|
1296
|
+
try {
|
|
1297
|
+
const nodes = await coordinator.listNodes();
|
|
1298
|
+
const registration = nodes.find((entry) => entry.nodeId === controllerNodeId);
|
|
1299
|
+
if (registration !== void 0 && registration.address.length > 0 && registration.address !== controllerNodeId) {
|
|
1300
|
+
targets.push(registration.address);
|
|
1301
|
+
}
|
|
1302
|
+
} catch (_) {
|
|
1303
|
+
return targets;
|
|
1304
|
+
}
|
|
1305
|
+
return targets;
|
|
1306
|
+
}
|
|
1307
|
+
async function reportBootstrapComplete(indexName, partitionId, nodeId, coordinator, transport) {
|
|
1308
|
+
const controllerNodeId = await coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
1309
|
+
if (controllerNodeId === null) {
|
|
1310
|
+
return false;
|
|
1311
|
+
}
|
|
1312
|
+
const primaryTerm = await resolvePrimaryTerm(indexName, partitionId, coordinator);
|
|
1313
|
+
if (primaryTerm === null) {
|
|
1314
|
+
return false;
|
|
1315
|
+
}
|
|
1316
|
+
const payload = {
|
|
1317
|
+
indexName,
|
|
1318
|
+
partitionId,
|
|
1319
|
+
nodeId,
|
|
1320
|
+
primaryTerm
|
|
1321
|
+
};
|
|
1322
|
+
const message = {
|
|
1323
|
+
type: ClusterMessageTypes.BOOTSTRAP_COMPLETE,
|
|
1324
|
+
sourceId: nodeId,
|
|
1325
|
+
requestId: generateId(),
|
|
1326
|
+
payload: encode(payload)
|
|
1327
|
+
};
|
|
1328
|
+
for (const target of await resolveControllerTargets(coordinator, controllerNodeId)) {
|
|
1329
|
+
try {
|
|
1330
|
+
const response = await transport.send(target, message);
|
|
1331
|
+
return isAcceptedResponse(response);
|
|
1332
|
+
} catch (_) {
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
return false;
|
|
1336
|
+
}
|
|
1337
|
+
function isAcceptedResponse(response) {
|
|
1338
|
+
try {
|
|
1339
|
+
const decoded = decode(response.payload);
|
|
1340
|
+
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
1341
|
+
return false;
|
|
1342
|
+
}
|
|
1343
|
+
return decoded.accepted === true;
|
|
1344
|
+
} catch (_) {
|
|
1345
|
+
return false;
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
async function partitionAwaitsBootstrapReport(state, coordinator) {
|
|
1349
|
+
try {
|
|
1350
|
+
const table = await coordinator.getAllocation(state.indexName);
|
|
1351
|
+
const assignment = table?.assignments.get(state.partitionId);
|
|
1352
|
+
if (assignment === void 0) {
|
|
1353
|
+
return true;
|
|
1354
|
+
}
|
|
1355
|
+
return assignment.state === "INITIALISING";
|
|
1356
|
+
} catch (_) {
|
|
1357
|
+
return true;
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
async function bootstrapPartition(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onBootstrapPartition, onError) {
|
|
1361
|
+
for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
|
|
1362
|
+
if (state.aborted) {
|
|
1363
|
+
return false;
|
|
1364
|
+
}
|
|
1365
|
+
const synced = await runBootstrapSyncAttempt(state, onBootstrapPartition, onError);
|
|
1366
|
+
if (state.aborted) {
|
|
1367
|
+
return false;
|
|
1368
|
+
}
|
|
1369
|
+
if (synced) {
|
|
1370
|
+
if (!await partitionAwaitsBootstrapReport(state, coordinator)) {
|
|
1371
|
+
return true;
|
|
1372
|
+
}
|
|
1373
|
+
return retryReportBootstrapComplete(
|
|
1374
|
+
state,
|
|
1375
|
+
coordinator,
|
|
1376
|
+
transport,
|
|
1377
|
+
nodeId,
|
|
1378
|
+
bootstrapRetryBaseMs,
|
|
1379
|
+
bootstrapRetryMaxMs,
|
|
1380
|
+
bootstrapMaxRetries,
|
|
1381
|
+
onError
|
|
1382
|
+
);
|
|
1383
|
+
}
|
|
1384
|
+
if (attempt < bootstrapMaxRetries) {
|
|
1385
|
+
const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
|
|
1386
|
+
await waitWithAbort(state, backoffMs);
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
return false;
|
|
1390
|
+
}
|
|
1391
|
+
async function runBootstrapSyncAttempt(state, onBootstrapPartition, onError) {
|
|
1392
|
+
try {
|
|
1393
|
+
return await onBootstrapPartition(state.indexName, state.partitionId, state.primaryNodeId);
|
|
1394
|
+
} catch (error) {
|
|
1395
|
+
if (state.aborted) {
|
|
1396
|
+
return false;
|
|
1397
|
+
}
|
|
1398
|
+
if (onError !== void 0) {
|
|
1399
|
+
const cause = error instanceof Error ? error.message : String(error);
|
|
1400
|
+
onError(
|
|
1401
|
+
new NarsilError(
|
|
1402
|
+
ErrorCodes.NODE_BOOTSTRAP_FAILED,
|
|
1403
|
+
`Bootstrap sync failed for ${state.indexName}:${state.partitionId}`,
|
|
1404
|
+
{ indexName: state.indexName, partitionId: state.partitionId, cause }
|
|
1405
|
+
)
|
|
1406
|
+
);
|
|
1407
|
+
}
|
|
1408
|
+
return false;
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
async function retryReportBootstrapComplete(state, coordinator, transport, nodeId, bootstrapRetryBaseMs, bootstrapRetryMaxMs, bootstrapMaxRetries, onError) {
|
|
1412
|
+
for (let attempt = 0; attempt <= bootstrapMaxRetries; attempt++) {
|
|
1413
|
+
if (state.aborted) {
|
|
1414
|
+
return false;
|
|
1415
|
+
}
|
|
1416
|
+
if (attempt > 0 && !await partitionAwaitsBootstrapReport(state, coordinator)) {
|
|
1417
|
+
return true;
|
|
1418
|
+
}
|
|
1419
|
+
const accepted = await reportBootstrapComplete(state.indexName, state.partitionId, nodeId, coordinator, transport);
|
|
1420
|
+
if (accepted) {
|
|
1421
|
+
return true;
|
|
1422
|
+
}
|
|
1423
|
+
if (!await partitionAwaitsBootstrapReport(state, coordinator)) {
|
|
1424
|
+
return true;
|
|
1425
|
+
}
|
|
1426
|
+
if (attempt < bootstrapMaxRetries && !state.aborted) {
|
|
1427
|
+
const backoffMs = computeBackoffMs(bootstrapRetryBaseMs, bootstrapRetryMaxMs, attempt);
|
|
1428
|
+
await waitWithAbort(state, backoffMs);
|
|
1429
|
+
}
|
|
1430
|
+
}
|
|
1431
|
+
if (onError !== void 0) {
|
|
1432
|
+
onError(
|
|
1433
|
+
new NarsilError(
|
|
1434
|
+
ErrorCodes.NODE_BOOTSTRAP_FAILED,
|
|
1435
|
+
`Failed to report bootstrap completion for ${state.indexName}:${state.partitionId} after ${bootstrapMaxRetries + 1} attempts`,
|
|
1436
|
+
{ indexName: state.indexName, partitionId: state.partitionId }
|
|
1437
|
+
)
|
|
1438
|
+
);
|
|
1439
|
+
}
|
|
1440
|
+
return false;
|
|
1441
|
+
}
|
|
1442
|
+
function waitWithAbort(state, ms) {
|
|
1443
|
+
return new Promise((resolve) => {
|
|
1444
|
+
if (state.aborted) {
|
|
1445
|
+
resolve();
|
|
1446
|
+
return;
|
|
1447
|
+
}
|
|
1448
|
+
state.abortResolve = resolve;
|
|
1449
|
+
state.retryTimer = setTimeout(() => {
|
|
1450
|
+
state.retryTimer = null;
|
|
1451
|
+
state.abortResolve = null;
|
|
1452
|
+
resolve();
|
|
1453
|
+
}, ms);
|
|
1454
|
+
});
|
|
1455
|
+
}
|
|
1456
|
+
function abortBootstrapState(state) {
|
|
1457
|
+
state.aborted = true;
|
|
1458
|
+
if (state.retryTimer !== null) {
|
|
1459
|
+
clearTimeout(state.retryTimer);
|
|
1460
|
+
state.retryTimer = null;
|
|
1461
|
+
}
|
|
1462
|
+
if (state.abortResolve !== null) {
|
|
1463
|
+
state.abortResolve();
|
|
1464
|
+
state.abortResolve = null;
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
function createBootstrapState(indexName, partitionId, primaryNodeId) {
|
|
1468
|
+
return {
|
|
1469
|
+
indexName,
|
|
1470
|
+
partitionId,
|
|
1471
|
+
primaryNodeId,
|
|
1472
|
+
retryCount: 0,
|
|
1473
|
+
retryTimer: null,
|
|
1474
|
+
aborted: false,
|
|
1475
|
+
abortResolve: null
|
|
1476
|
+
};
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
// src/distribution/cluster/node-lifecycle/allocation-watcher.ts
|
|
1480
|
+
function partitionKey(indexName, partitionId) {
|
|
1481
|
+
return `${indexName}:${partitionId}`;
|
|
1482
|
+
}
|
|
1483
|
+
function createAllocationWatcherState() {
|
|
1484
|
+
return {
|
|
1485
|
+
unwatchAllocation: null,
|
|
1486
|
+
debounceTimer: null,
|
|
1487
|
+
activeBootstraps: /* @__PURE__ */ new Map(),
|
|
1488
|
+
trackedPartitions: /* @__PURE__ */ new Map(),
|
|
1489
|
+
pendingTables: /* @__PURE__ */ new Map(),
|
|
1490
|
+
restartWaiters: /* @__PURE__ */ new Map(),
|
|
1491
|
+
stopped: false
|
|
1492
|
+
};
|
|
1493
|
+
}
|
|
1494
|
+
async function startAllocationWatcher(state, config) {
|
|
1495
|
+
const unwatchAllocation = await config.coordinator.watchAllocation((event) => {
|
|
1496
|
+
if (state.stopped) {
|
|
1497
|
+
return;
|
|
1498
|
+
}
|
|
1499
|
+
scheduleProcessAllocation(state, config, event.table);
|
|
1500
|
+
});
|
|
1501
|
+
state.unwatchAllocation = unwatchAllocation;
|
|
1502
|
+
}
|
|
1503
|
+
function scheduleProcessAllocation(state, config, table) {
|
|
1504
|
+
state.pendingTables.set(table.indexName, table);
|
|
1505
|
+
if (state.debounceTimer !== null) {
|
|
1506
|
+
clearTimeout(state.debounceTimer);
|
|
1507
|
+
}
|
|
1508
|
+
state.debounceTimer = setTimeout(() => {
|
|
1509
|
+
state.debounceTimer = null;
|
|
1510
|
+
const tablesToProcess = Array.from(state.pendingTables.values());
|
|
1511
|
+
state.pendingTables.clear();
|
|
1512
|
+
for (const pending of tablesToProcess) {
|
|
1513
|
+
processAllocationChange(state, config, pending);
|
|
1514
|
+
}
|
|
1515
|
+
}, config.allocationDebounceMs);
|
|
1516
|
+
}
|
|
1517
|
+
function processAllocationChange(state, config, table) {
|
|
1518
|
+
if (state.stopped) {
|
|
1519
|
+
return;
|
|
1520
|
+
}
|
|
1521
|
+
const nodeId = config.registration.nodeId;
|
|
1522
|
+
const currentKeys = /* @__PURE__ */ new Set();
|
|
1523
|
+
for (const [partitionId, assignment] of table.assignments) {
|
|
1524
|
+
if (!isNodeAssigned(assignment, nodeId)) {
|
|
1525
|
+
continue;
|
|
1526
|
+
}
|
|
1527
|
+
const key = partitionKey(table.indexName, partitionId);
|
|
1528
|
+
currentKeys.add(key);
|
|
1529
|
+
const existing = state.trackedPartitions.get(key);
|
|
1530
|
+
const isPrimary = assignment.primary === nodeId;
|
|
1531
|
+
if (existing === void 0) {
|
|
1532
|
+
handleNewAssignment(state, config, table.indexName, partitionId, assignment, nodeId);
|
|
1533
|
+
} else {
|
|
1534
|
+
handleExistingAssignment(state, config, existing, assignment, nodeId);
|
|
1535
|
+
}
|
|
1536
|
+
state.trackedPartitions.set(key, {
|
|
1537
|
+
indexName: table.indexName,
|
|
1538
|
+
partitionId,
|
|
1539
|
+
isPrimary,
|
|
1540
|
+
primaryNodeId: assignment.primary,
|
|
1541
|
+
primaryTerm: assignment.primaryTerm
|
|
1542
|
+
});
|
|
1543
|
+
}
|
|
1544
|
+
removeUnassignedPartitions(state, config, table.indexName, currentKeys);
|
|
1545
|
+
}
|
|
1546
|
+
function isNodeAssigned(assignment, nodeId) {
|
|
1547
|
+
if (assignment.primary === nodeId) {
|
|
1548
|
+
return true;
|
|
1549
|
+
}
|
|
1550
|
+
return assignment.replicas.includes(nodeId);
|
|
1551
|
+
}
|
|
1552
|
+
function handleNewAssignment(state, config, indexName, partitionId, assignment, nodeId) {
|
|
1553
|
+
if (assignment.state === "ACTIVE" && assignment.primary === nodeId) {
|
|
1554
|
+
return;
|
|
1555
|
+
}
|
|
1556
|
+
if (assignment.primary === null) {
|
|
1557
|
+
return;
|
|
1558
|
+
}
|
|
1559
|
+
if (assignment.primary === nodeId) {
|
|
1560
|
+
if (assignment.state === "INITIALISING") {
|
|
1561
|
+
startBootstrap(state, config, indexName, partitionId, nodeId);
|
|
1562
|
+
}
|
|
1563
|
+
return;
|
|
1564
|
+
}
|
|
1565
|
+
startBootstrap(state, config, indexName, partitionId, assignment.primary);
|
|
1566
|
+
}
|
|
1567
|
+
function handleExistingAssignment(state, config, existing, assignment, nodeId) {
|
|
1568
|
+
if (existing.isPrimary && assignment.primary !== nodeId && assignment.primaryTerm > existing.primaryTerm) {
|
|
1569
|
+
if (assignment.primary !== null) {
|
|
1570
|
+
if (config.onPrimaryDemotion !== void 0) {
|
|
1571
|
+
config.onPrimaryDemotion(existing.indexName, existing.partitionId, assignment.primary);
|
|
1572
|
+
}
|
|
1573
|
+
startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
|
|
1574
|
+
}
|
|
1575
|
+
return;
|
|
1576
|
+
}
|
|
1577
|
+
if (assignment.primary !== null && replicaNeedsResync(state, existing, assignment, nodeId)) {
|
|
1578
|
+
startBootstrap(state, config, existing.indexName, existing.partitionId, assignment.primary);
|
|
1579
|
+
}
|
|
1580
|
+
}
|
|
1581
|
+
function replicaNeedsResync(state, existing, assignment, nodeId) {
|
|
1582
|
+
if (assignment.state !== "ACTIVE") {
|
|
1583
|
+
return false;
|
|
1584
|
+
}
|
|
1585
|
+
if (assignment.primary === null || assignment.primary === nodeId) {
|
|
1586
|
+
return false;
|
|
1587
|
+
}
|
|
1588
|
+
if (!assignment.replicas.includes(nodeId)) {
|
|
1589
|
+
return false;
|
|
1590
|
+
}
|
|
1591
|
+
if (assignment.inSyncSet.includes(nodeId)) {
|
|
1592
|
+
return false;
|
|
1593
|
+
}
|
|
1594
|
+
return !state.activeBootstraps.has(partitionKey(existing.indexName, existing.partitionId));
|
|
1595
|
+
}
|
|
1596
|
+
function startBootstrap(state, config, indexName, partitionId, primaryNodeId) {
|
|
1597
|
+
const key = partitionKey(indexName, partitionId);
|
|
1598
|
+
const existingBootstrap = state.activeBootstraps.get(key);
|
|
1599
|
+
if (existingBootstrap !== void 0) {
|
|
1600
|
+
abortBootstrapState(existingBootstrap);
|
|
1601
|
+
}
|
|
1602
|
+
const bootstrapState = createBootstrapState(indexName, partitionId, primaryNodeId);
|
|
1603
|
+
state.activeBootstraps.set(key, bootstrapState);
|
|
1604
|
+
bootstrapPartition(
|
|
1605
|
+
bootstrapState,
|
|
1606
|
+
config.coordinator,
|
|
1607
|
+
config.transport,
|
|
1608
|
+
config.registration.nodeId,
|
|
1609
|
+
config.bootstrapRetryBaseMs,
|
|
1610
|
+
config.bootstrapRetryMaxMs,
|
|
1611
|
+
config.bootstrapMaxRetries,
|
|
1612
|
+
config.onBootstrapPartition,
|
|
1387
1613
|
config.onError
|
|
1388
1614
|
).then((succeeded) => {
|
|
1389
1615
|
if (state.activeBootstraps.get(key) === bootstrapState) {
|
|
@@ -1391,12 +1617,14 @@ function startBootstrap(state, config, indexName, partitionId, primaryNodeId) {
|
|
|
1391
1617
|
}
|
|
1392
1618
|
if (!succeeded) {
|
|
1393
1619
|
state.trackedPartitions.delete(key);
|
|
1620
|
+
void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
|
|
1394
1621
|
}
|
|
1395
1622
|
}).catch(() => {
|
|
1396
1623
|
if (state.activeBootstraps.get(key) === bootstrapState) {
|
|
1397
1624
|
state.activeBootstraps.delete(key);
|
|
1398
1625
|
}
|
|
1399
1626
|
state.trackedPartitions.delete(key);
|
|
1627
|
+
void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
|
|
1400
1628
|
});
|
|
1401
1629
|
}
|
|
1402
1630
|
function removeUnassignedPartitions(state, config, indexName, currentKeys) {
|
|
@@ -1413,37 +1641,122 @@ function removeUnassignedPartitions(state, config, indexName, currentKeys) {
|
|
|
1413
1641
|
abortBootstrapState(activeBootstrap);
|
|
1414
1642
|
state.activeBootstraps.delete(key);
|
|
1415
1643
|
}
|
|
1416
|
-
if (config.onRemovePartition !== void 0) {
|
|
1417
|
-
config.onRemovePartition(tracked.indexName, tracked.partitionId);
|
|
1644
|
+
if (config.onRemovePartition !== void 0) {
|
|
1645
|
+
config.onRemovePartition(tracked.indexName, tracked.partitionId);
|
|
1646
|
+
}
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
function stopAllocationWatcher(state) {
|
|
1650
|
+
state.stopped = true;
|
|
1651
|
+
if (state.unwatchAllocation !== null) {
|
|
1652
|
+
state.unwatchAllocation();
|
|
1653
|
+
state.unwatchAllocation = null;
|
|
1654
|
+
}
|
|
1655
|
+
if (state.debounceTimer !== null) {
|
|
1656
|
+
clearTimeout(state.debounceTimer);
|
|
1657
|
+
state.debounceTimer = null;
|
|
1658
|
+
}
|
|
1659
|
+
for (const [timer, resolve] of state.restartWaiters) {
|
|
1660
|
+
clearTimeout(timer);
|
|
1661
|
+
resolve();
|
|
1662
|
+
}
|
|
1663
|
+
state.restartWaiters.clear();
|
|
1664
|
+
for (const bootstrapState of state.activeBootstraps.values()) {
|
|
1665
|
+
abortBootstrapState(bootstrapState);
|
|
1666
|
+
}
|
|
1667
|
+
state.activeBootstraps.clear();
|
|
1668
|
+
state.trackedPartitions.clear();
|
|
1669
|
+
state.pendingTables.clear();
|
|
1670
|
+
}
|
|
1671
|
+
function processInitialAllocations(state, config, tables) {
|
|
1672
|
+
for (const table of tables) {
|
|
1673
|
+
processAllocationChange(state, config, table);
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
function waitBeforeBootstrapRestart(state, config) {
|
|
1677
|
+
const delayMs = Math.min(config.bootstrapRetryMaxMs, config.bootstrapRetryBaseMs);
|
|
1678
|
+
if (delayMs <= 0) {
|
|
1679
|
+
return Promise.resolve();
|
|
1680
|
+
}
|
|
1681
|
+
return new Promise((resolve) => {
|
|
1682
|
+
const timer = setTimeout(() => {
|
|
1683
|
+
state.restartWaiters.delete(timer);
|
|
1684
|
+
resolve();
|
|
1685
|
+
}, delayMs);
|
|
1686
|
+
timer.unref?.();
|
|
1687
|
+
state.restartWaiters.set(timer, resolve);
|
|
1688
|
+
});
|
|
1689
|
+
}
|
|
1690
|
+
async function restartBootstrapWhileOutOfSync(state, config, indexName, partitionId) {
|
|
1691
|
+
if (state.stopped) {
|
|
1692
|
+
return;
|
|
1693
|
+
}
|
|
1694
|
+
await waitBeforeBootstrapRestart(state, config);
|
|
1695
|
+
if (state.stopped) {
|
|
1696
|
+
return;
|
|
1697
|
+
}
|
|
1698
|
+
const nodeId = config.registration.nodeId;
|
|
1699
|
+
let table;
|
|
1700
|
+
try {
|
|
1701
|
+
table = await config.coordinator.getAllocation(indexName);
|
|
1702
|
+
} catch (error) {
|
|
1703
|
+
if (config.onError !== void 0) {
|
|
1704
|
+
config.onError(error);
|
|
1705
|
+
}
|
|
1706
|
+
if (!state.stopped) {
|
|
1707
|
+
void restartBootstrapWhileOutOfSync(state, config, indexName, partitionId);
|
|
1418
1708
|
}
|
|
1709
|
+
return;
|
|
1419
1710
|
}
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
state.stopped = true;
|
|
1423
|
-
if (state.unwatchAllocation !== null) {
|
|
1424
|
-
state.unwatchAllocation();
|
|
1425
|
-
state.unwatchAllocation = null;
|
|
1711
|
+
if (state.stopped || table === null) {
|
|
1712
|
+
return;
|
|
1426
1713
|
}
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1714
|
+
const assignment = table.assignments.get(partitionId);
|
|
1715
|
+
if (assignment === void 0 || assignment.primary === null || assignment.primary === nodeId) {
|
|
1716
|
+
return;
|
|
1430
1717
|
}
|
|
1431
|
-
|
|
1432
|
-
|
|
1718
|
+
if (!assignment.replicas.includes(nodeId) || assignment.inSyncSet.includes(nodeId)) {
|
|
1719
|
+
return;
|
|
1433
1720
|
}
|
|
1434
|
-
state.
|
|
1435
|
-
|
|
1436
|
-
state.pendingTables.clear();
|
|
1437
|
-
}
|
|
1438
|
-
function processInitialAllocations(state, config, tables) {
|
|
1439
|
-
for (const table of tables) {
|
|
1440
|
-
processAllocationChange(state, config, table);
|
|
1721
|
+
if (assignment.state !== "ACTIVE" && assignment.state !== "INITIALISING" && assignment.state !== "MIGRATING") {
|
|
1722
|
+
return;
|
|
1441
1723
|
}
|
|
1724
|
+
if (state.activeBootstraps.has(partitionKey(indexName, partitionId))) {
|
|
1725
|
+
return;
|
|
1726
|
+
}
|
|
1727
|
+
startBootstrap(state, config, indexName, partitionId, assignment.primary);
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1730
|
+
// src/distribution/cluster/node-lifecycle/heartbeat.ts
|
|
1731
|
+
function createRegistrationHeartbeatState() {
|
|
1732
|
+
return { timer: null, inFlight: Promise.resolve() };
|
|
1733
|
+
}
|
|
1734
|
+
function clearHeartbeatTimer(state) {
|
|
1735
|
+
if (state.timer !== null) {
|
|
1736
|
+
clearInterval(state.timer);
|
|
1737
|
+
state.timer = null;
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
function startRegistrationHeartbeat(state, config) {
|
|
1741
|
+
clearHeartbeatTimer(state);
|
|
1742
|
+
state.timer = setInterval(() => {
|
|
1743
|
+
state.inFlight = state.inFlight.then(() => config.coordinator.registerNode(config.registration)).catch((error) => {
|
|
1744
|
+
if (config.onError !== void 0) {
|
|
1745
|
+
config.onError(error);
|
|
1746
|
+
}
|
|
1747
|
+
});
|
|
1748
|
+
}, config.nodeHeartbeatIntervalMs);
|
|
1749
|
+
state.timer.unref?.();
|
|
1750
|
+
}
|
|
1751
|
+
async function stopRegistrationHeartbeat(state) {
|
|
1752
|
+
clearHeartbeatTimer(state);
|
|
1753
|
+
await state.inFlight;
|
|
1442
1754
|
}
|
|
1443
1755
|
|
|
1444
1756
|
// src/distribution/cluster/node-lifecycle/join.ts
|
|
1445
|
-
async function joinCluster(config, watcherState) {
|
|
1757
|
+
async function joinCluster(config, watcherState, heartbeatState) {
|
|
1446
1758
|
await config.coordinator.registerNode(config.registration);
|
|
1759
|
+
startRegistrationHeartbeat(heartbeatState, config);
|
|
1447
1760
|
const initialTables = await loadInitialAllocations(config);
|
|
1448
1761
|
processInitialAllocations(watcherState, config, initialTables);
|
|
1449
1762
|
await startAllocationWatcher(watcherState, config);
|
|
@@ -1465,15 +1778,17 @@ async function leaveCluster(config) {
|
|
|
1465
1778
|
// src/distribution/cluster/node-lifecycle/types.ts
|
|
1466
1779
|
var DEFAULT_NODE_LIFECYCLE_CONFIG = {
|
|
1467
1780
|
bootstrapRetryBaseMs: 1e3,
|
|
1468
|
-
bootstrapRetryMaxMs:
|
|
1781
|
+
bootstrapRetryMaxMs: 5e3,
|
|
1469
1782
|
bootstrapMaxRetries: 10,
|
|
1470
|
-
allocationDebounceMs: 250
|
|
1783
|
+
allocationDebounceMs: 250,
|
|
1784
|
+
nodeHeartbeatIntervalMs: 1e4
|
|
1471
1785
|
};
|
|
1472
1786
|
|
|
1473
1787
|
// src/distribution/cluster/node-lifecycle/index.ts
|
|
1474
1788
|
function createDataNodeLifecycle(config) {
|
|
1475
1789
|
let status = "stopped";
|
|
1476
1790
|
let watcherState = createAllocationWatcherState();
|
|
1791
|
+
const heartbeatState = createRegistrationHeartbeatState();
|
|
1477
1792
|
let operationLock = Promise.resolve();
|
|
1478
1793
|
function withLock(fn) {
|
|
1479
1794
|
const prev = operationLock;
|
|
@@ -1516,9 +1831,10 @@ function createDataNodeLifecycle(config) {
|
|
|
1516
1831
|
}
|
|
1517
1832
|
status = "joining";
|
|
1518
1833
|
try {
|
|
1519
|
-
await joinCluster(config, watcherState);
|
|
1834
|
+
await joinCluster(config, watcherState, heartbeatState);
|
|
1520
1835
|
status = "active";
|
|
1521
1836
|
} catch (error) {
|
|
1837
|
+
await stopRegistrationHeartbeat(heartbeatState);
|
|
1522
1838
|
status = "stopped";
|
|
1523
1839
|
throw error;
|
|
1524
1840
|
}
|
|
@@ -1530,6 +1846,7 @@ function createDataNodeLifecycle(config) {
|
|
|
1530
1846
|
return;
|
|
1531
1847
|
}
|
|
1532
1848
|
status = "leaving";
|
|
1849
|
+
await stopRegistrationHeartbeat(heartbeatState);
|
|
1533
1850
|
stopAllocationWatcher(watcherState);
|
|
1534
1851
|
try {
|
|
1535
1852
|
await leaveCluster(config);
|
|
@@ -1544,6 +1861,7 @@ function createDataNodeLifecycle(config) {
|
|
|
1544
1861
|
if (status === "shutdown") {
|
|
1545
1862
|
return;
|
|
1546
1863
|
}
|
|
1864
|
+
await stopRegistrationHeartbeat(heartbeatState);
|
|
1547
1865
|
stopAllocationWatcher(watcherState);
|
|
1548
1866
|
if (status === "active" || status === "joining") {
|
|
1549
1867
|
try {
|
|
@@ -2818,11 +3136,47 @@ function validateLocalPartitionCount(engine, indexName, primaryNodeId, partition
|
|
|
2818
3136
|
// src/distribution/cluster-node/bootstrap-sync/targets.ts
|
|
2819
3137
|
function readLocalLogState(indexName, partitionId, deps) {
|
|
2820
3138
|
const log = deps.getReplicationLog(indexName, partitionId);
|
|
2821
|
-
const lastSeqNo = log.
|
|
2822
|
-
const lastPrimaryTerm = log.
|
|
3139
|
+
const lastSeqNo = log.localLogEnd;
|
|
3140
|
+
const lastPrimaryTerm = log.localLogEndPrimaryTerm;
|
|
2823
3141
|
const newestEntry = log.getEntry(lastSeqNo);
|
|
2824
3142
|
return { lastSeqNo, lastPrimaryTerm: newestEntry?.primaryTerm ?? lastPrimaryTerm };
|
|
2825
3143
|
}
|
|
3144
|
+
var SYNCED_POSITION_REPORT_TIMEOUT_MS = 5e3;
|
|
3145
|
+
async function reportSyncedPosition(indexName, partitionId, target, knownSeqNo, deadline, deps) {
|
|
3146
|
+
const logState = readLocalLogState(indexName, partitionId, deps);
|
|
3147
|
+
if (logState.lastSeqNo <= knownSeqNo) {
|
|
3148
|
+
return;
|
|
3149
|
+
}
|
|
3150
|
+
const budgetMs = Math.min(SYNCED_POSITION_REPORT_TIMEOUT_MS, deadline - Date.now());
|
|
3151
|
+
if (budgetMs <= 0) {
|
|
3152
|
+
return;
|
|
3153
|
+
}
|
|
3154
|
+
const payload = {
|
|
3155
|
+
indexName,
|
|
3156
|
+
partitionId,
|
|
3157
|
+
lastSeqNo: logState.lastSeqNo,
|
|
3158
|
+
lastPrimaryTerm: logState.lastPrimaryTerm
|
|
3159
|
+
};
|
|
3160
|
+
const request = {
|
|
3161
|
+
type: ReplicationMessageTypes.SYNC_REQUEST,
|
|
3162
|
+
sourceId: deps.sourceNodeId,
|
|
3163
|
+
requestId: generateId(),
|
|
3164
|
+
payload: encode(payload)
|
|
3165
|
+
};
|
|
3166
|
+
try {
|
|
3167
|
+
await withDeadline(
|
|
3168
|
+
deps.transport.stream(target, request, () => {
|
|
3169
|
+
}),
|
|
3170
|
+
budgetMs,
|
|
3171
|
+
indexName,
|
|
3172
|
+
"synced-position-report"
|
|
3173
|
+
);
|
|
3174
|
+
} catch (error) {
|
|
3175
|
+
if (deps.onError !== void 0) {
|
|
3176
|
+
deps.onError(error);
|
|
3177
|
+
}
|
|
3178
|
+
}
|
|
3179
|
+
}
|
|
2826
3180
|
async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNodeId, targets, coordinatorSchema, partitionCount, deadline, deps, abortCheck) {
|
|
2827
3181
|
let lastError = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "no live sync attempts were made", {
|
|
2828
3182
|
indexName,
|
|
@@ -2856,6 +3210,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
|
|
|
2856
3210
|
)
|
|
2857
3211
|
};
|
|
2858
3212
|
}
|
|
3213
|
+
const positionBeforeAttempt = readLocalLogState(indexName, partitionId, deps).lastSeqNo;
|
|
2859
3214
|
const attempt = await syncFromTarget(
|
|
2860
3215
|
state,
|
|
2861
3216
|
entry,
|
|
@@ -2869,6 +3224,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
|
|
|
2869
3224
|
abortCheck
|
|
2870
3225
|
);
|
|
2871
3226
|
if (attempt.ok) {
|
|
3227
|
+
await reportSyncedPosition(indexName, partitionId, target, positionBeforeAttempt, deadline, deps);
|
|
2872
3228
|
return attempt;
|
|
2873
3229
|
}
|
|
2874
3230
|
lastError = new NarsilError(attempt.error.code, attempt.error.message, {
|
|
@@ -2974,94 +3330,288 @@ async function syncFromTarget(state, entry, indexName, partitionId, target, coor
|
|
|
2974
3330
|
};
|
|
2975
3331
|
}
|
|
2976
3332
|
|
|
2977
|
-
// src/distribution/cluster-node/bootstrap-sync/live.ts
|
|
2978
|
-
async function resolveLivePartitionCount(coordinator, indexName, partitionId, primaryNodeId, abortPromise) {
|
|
2979
|
-
try {
|
|
2980
|
-
const winner = await Promise.race([coordinator.getAllocation(indexName), abortPromise]);
|
|
2981
|
-
if (winner === ABORT_SENTINEL) {
|
|
2982
|
-
return "aborted";
|
|
3333
|
+
// src/distribution/cluster-node/bootstrap-sync/live.ts
|
|
3334
|
+
async function resolveLivePartitionCount(coordinator, indexName, partitionId, primaryNodeId, abortPromise) {
|
|
3335
|
+
try {
|
|
3336
|
+
const winner = await Promise.race([coordinator.getAllocation(indexName), abortPromise]);
|
|
3337
|
+
if (winner === ABORT_SENTINEL) {
|
|
3338
|
+
return "aborted";
|
|
3339
|
+
}
|
|
3340
|
+
const allocation = winner;
|
|
3341
|
+
if (allocation === null) {
|
|
3342
|
+
return {
|
|
3343
|
+
error: new NarsilError(
|
|
3344
|
+
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
3345
|
+
`coordinator has no allocation for index '${indexName}'`,
|
|
3346
|
+
{ indexName, primaryNodeId }
|
|
3347
|
+
)
|
|
3348
|
+
};
|
|
3349
|
+
}
|
|
3350
|
+
const assignment = allocation.assignments.get(partitionId);
|
|
3351
|
+
if (assignment === void 0) {
|
|
3352
|
+
return {
|
|
3353
|
+
error: new NarsilError(
|
|
3354
|
+
ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
3355
|
+
`No assignment exists for partition ${partitionId} of index '${indexName}'`,
|
|
3356
|
+
{ indexName, partitionId, primaryNodeId }
|
|
3357
|
+
)
|
|
3358
|
+
};
|
|
3359
|
+
}
|
|
3360
|
+
if (assignment.primary !== primaryNodeId) {
|
|
3361
|
+
return {
|
|
3362
|
+
error: new NarsilError(
|
|
3363
|
+
ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
3364
|
+
`Partition ${partitionId} of index '${indexName}' is no longer primary on '${primaryNodeId}'`,
|
|
3365
|
+
{ indexName, partitionId, primaryNodeId, currentPrimary: assignment.primary }
|
|
3366
|
+
)
|
|
3367
|
+
};
|
|
3368
|
+
}
|
|
3369
|
+
return { partitionCount: allocation.assignments.size };
|
|
3370
|
+
} catch (err) {
|
|
3371
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
3372
|
+
return {
|
|
3373
|
+
error: new NarsilError(
|
|
3374
|
+
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
3375
|
+
`coordinator.getAllocation failed: ${cause}`,
|
|
3376
|
+
{
|
|
3377
|
+
indexName,
|
|
3378
|
+
partitionId,
|
|
3379
|
+
primaryNodeId,
|
|
3380
|
+
cause
|
|
3381
|
+
}
|
|
3382
|
+
)
|
|
3383
|
+
};
|
|
3384
|
+
}
|
|
3385
|
+
}
|
|
3386
|
+
async function executeLiveBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps, deadlineMs, deadline, abortCheck) {
|
|
3387
|
+
const schemaResult = await loadCoordinatorSchema(deps.coordinator, indexName, primaryNodeId, entry.abortPromise);
|
|
3388
|
+
if (schemaResult === "aborted") {
|
|
3389
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3390
|
+
return false;
|
|
3391
|
+
}
|
|
3392
|
+
if ("error" in schemaResult) {
|
|
3393
|
+
surfaceError(deps, indexName, primaryNodeId, schemaResult.error);
|
|
3394
|
+
return false;
|
|
3395
|
+
}
|
|
3396
|
+
if (abortCheck()) {
|
|
3397
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3398
|
+
return false;
|
|
3399
|
+
}
|
|
3400
|
+
const allocationResult = await resolveLivePartitionCount(
|
|
3401
|
+
deps.coordinator,
|
|
3402
|
+
indexName,
|
|
3403
|
+
partitionId,
|
|
3404
|
+
primaryNodeId,
|
|
3405
|
+
entry.abortPromise
|
|
3406
|
+
);
|
|
3407
|
+
if (allocationResult === "aborted") {
|
|
3408
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3409
|
+
return false;
|
|
3410
|
+
}
|
|
3411
|
+
if ("error" in allocationResult) {
|
|
3412
|
+
surfaceError(deps, indexName, primaryNodeId, allocationResult.error);
|
|
3413
|
+
return false;
|
|
3414
|
+
}
|
|
3415
|
+
if (Date.now() >= deadline) {
|
|
3416
|
+
surfaceError(
|
|
3417
|
+
deps,
|
|
3418
|
+
indexName,
|
|
3419
|
+
primaryNodeId,
|
|
3420
|
+
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before starting live sync", {
|
|
3421
|
+
indexName,
|
|
3422
|
+
deadlineMs
|
|
3423
|
+
})
|
|
3424
|
+
);
|
|
3425
|
+
return false;
|
|
3426
|
+
}
|
|
3427
|
+
const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
|
|
3428
|
+
if (targetsResult instanceof NarsilError) {
|
|
3429
|
+
surfaceError(deps, indexName, primaryNodeId, targetsResult);
|
|
3430
|
+
return false;
|
|
3431
|
+
}
|
|
3432
|
+
const syncResult = await syncFromAnyTarget(
|
|
3433
|
+
state,
|
|
3434
|
+
entry,
|
|
3435
|
+
indexName,
|
|
3436
|
+
partitionId,
|
|
3437
|
+
primaryNodeId,
|
|
3438
|
+
targetsResult,
|
|
3439
|
+
schemaResult.schema,
|
|
3440
|
+
allocationResult.partitionCount,
|
|
3441
|
+
deadline,
|
|
3442
|
+
deps,
|
|
3443
|
+
abortCheck
|
|
3444
|
+
);
|
|
3445
|
+
if (!syncResult.ok) {
|
|
3446
|
+
if (syncResult.error.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
|
|
3447
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3448
|
+
return false;
|
|
3449
|
+
}
|
|
3450
|
+
if (syncResult.error.details.alreadySurfaced !== true) {
|
|
3451
|
+
surfaceError(deps, indexName, primaryNodeId, syncResult.error);
|
|
3452
|
+
}
|
|
3453
|
+
return false;
|
|
3454
|
+
}
|
|
3455
|
+
if (abortCheck()) {
|
|
3456
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3457
|
+
return false;
|
|
3458
|
+
}
|
|
3459
|
+
if (syncResult.snapshotHeader !== null) {
|
|
3460
|
+
deps.onSnapshotApplied?.(indexName, partitionId, syncResult.snapshotHeader);
|
|
3461
|
+
}
|
|
3462
|
+
state.completed.add(entryKey(indexName, partitionId));
|
|
3463
|
+
return true;
|
|
3464
|
+
}
|
|
3465
|
+
|
|
3466
|
+
// src/distribution/cluster-node/bootstrap-sync/snapshot-restore.ts
|
|
3467
|
+
async function fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps) {
|
|
3468
|
+
const dropError = await dropExistingIndex(deps.engine, indexName, primaryNodeId);
|
|
3469
|
+
if (dropError !== null) {
|
|
3470
|
+
surfaceError(deps, indexName, primaryNodeId, dropError);
|
|
3471
|
+
return null;
|
|
3472
|
+
}
|
|
3473
|
+
if (entry.aborted) {
|
|
3474
|
+
return "aborted";
|
|
3475
|
+
}
|
|
3476
|
+
const schemaResult = await loadCoordinatorSchema(deps.coordinator, indexName, primaryNodeId, entry.abortPromise);
|
|
3477
|
+
if (schemaResult === "aborted") {
|
|
3478
|
+
return "aborted";
|
|
3479
|
+
}
|
|
3480
|
+
if ("error" in schemaResult) {
|
|
3481
|
+
surfaceError(deps, indexName, primaryNodeId, schemaResult.error);
|
|
3482
|
+
return null;
|
|
3483
|
+
}
|
|
3484
|
+
return schemaResult.schema;
|
|
3485
|
+
}
|
|
3486
|
+
async function applyRestore(state, entry, indexName, primaryNodeId, bytes, coordinatorSchema, deadline, deps) {
|
|
3487
|
+
const remainingMs = deadline - Date.now();
|
|
3488
|
+
if (remainingMs <= 0) {
|
|
3489
|
+
surfaceError(
|
|
3490
|
+
deps,
|
|
3491
|
+
indexName,
|
|
3492
|
+
primaryNodeId,
|
|
3493
|
+
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before restore", {
|
|
3494
|
+
indexName
|
|
3495
|
+
})
|
|
3496
|
+
);
|
|
3497
|
+
return false;
|
|
3498
|
+
}
|
|
3499
|
+
const key = entryKey(indexName, entry.partitionId);
|
|
3500
|
+
const generationBeforeRestore = state.generations.get(key) ?? 0;
|
|
3501
|
+
if (generationBeforeRestore !== entry.generation || entry.aborted) {
|
|
3502
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3503
|
+
return false;
|
|
3504
|
+
}
|
|
3505
|
+
const restoreError = await executeEngineRestore(deps.engine, indexName, primaryNodeId, bytes, remainingMs);
|
|
3506
|
+
if (restoreError !== null) {
|
|
3507
|
+
surfaceError(deps, indexName, primaryNodeId, restoreError);
|
|
3508
|
+
return false;
|
|
3509
|
+
}
|
|
3510
|
+
const generationAfterRestore = state.generations.get(key) ?? 0;
|
|
3511
|
+
if (generationAfterRestore !== entry.generation || entry.aborted) {
|
|
3512
|
+
if (!anotherBootstrapOwnsKey(state, key, entry)) {
|
|
3513
|
+
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
3514
|
+
}
|
|
3515
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3516
|
+
return false;
|
|
3517
|
+
}
|
|
3518
|
+
const schemaError = validateRestoredSchema(deps.engine, indexName, primaryNodeId, coordinatorSchema);
|
|
3519
|
+
if (schemaError !== null) {
|
|
3520
|
+
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
3521
|
+
surfaceError(deps, indexName, primaryNodeId, schemaError);
|
|
3522
|
+
return false;
|
|
3523
|
+
}
|
|
3524
|
+
return true;
|
|
3525
|
+
}
|
|
3526
|
+
|
|
3527
|
+
// src/distribution/cluster-node/bootstrap-sync/index.ts
|
|
3528
|
+
async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
|
|
3529
|
+
const validationError = validateArguments(indexName, partitionId, primaryNodeId);
|
|
3530
|
+
if (validationError !== null) {
|
|
3531
|
+
surfaceError(deps, indexName, primaryNodeId, validationError);
|
|
3532
|
+
return false;
|
|
3533
|
+
}
|
|
3534
|
+
const key = entryKey(indexName, partitionId);
|
|
3535
|
+
if (state.completed.has(key)) {
|
|
3536
|
+
if (!await replicaResyncRequired(indexName, partitionId, deps)) {
|
|
3537
|
+
return true;
|
|
2983
3538
|
}
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
)
|
|
2992
|
-
};
|
|
3539
|
+
clearBootstrapSyncIndex(state, indexName, partitionId);
|
|
3540
|
+
}
|
|
3541
|
+
const existing = state.inFlight.get(key);
|
|
3542
|
+
if (existing !== void 0) {
|
|
3543
|
+
const result = await Promise.race([existing.promise, existing.abortPromise]);
|
|
3544
|
+
if (result === ABORT_SENTINEL) {
|
|
3545
|
+
return false;
|
|
2993
3546
|
}
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
)
|
|
3002
|
-
};
|
|
3547
|
+
return result;
|
|
3548
|
+
}
|
|
3549
|
+
const entry = createEntry(state, indexName, partitionId, key);
|
|
3550
|
+
entry.promise = executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps).finally(() => {
|
|
3551
|
+
const current = state.inFlight.get(key);
|
|
3552
|
+
if (current === entry) {
|
|
3553
|
+
state.inFlight.delete(key);
|
|
3003
3554
|
}
|
|
3004
|
-
if (
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
{ indexName, partitionId, primaryNodeId, currentPrimary: assignment.primary }
|
|
3010
|
-
)
|
|
3011
|
-
};
|
|
3555
|
+
if (!state.completed.has(key) && !state.inFlight.has(key)) {
|
|
3556
|
+
const currentGen = state.generations.get(key);
|
|
3557
|
+
if (currentGen !== void 0 && currentGen !== entry.generation) {
|
|
3558
|
+
state.generations.delete(key);
|
|
3559
|
+
}
|
|
3012
3560
|
}
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
partitionId,
|
|
3023
|
-
primaryNodeId,
|
|
3024
|
-
cause
|
|
3025
|
-
}
|
|
3026
|
-
)
|
|
3027
|
-
};
|
|
3561
|
+
});
|
|
3562
|
+
state.inFlight.set(key, entry);
|
|
3563
|
+
return entry.promise;
|
|
3564
|
+
}
|
|
3565
|
+
async function replicaResyncRequired(indexName, partitionId, deps) {
|
|
3566
|
+
const allocation = await deps.coordinator.getAllocation(indexName);
|
|
3567
|
+
const assignment = allocation?.assignments.get(partitionId);
|
|
3568
|
+
if (assignment === void 0) {
|
|
3569
|
+
return false;
|
|
3028
3570
|
}
|
|
3571
|
+
return assignment.state === "ACTIVE" && assignment.replicas.includes(deps.sourceNodeId) && !assignment.inSyncSet.includes(deps.sourceNodeId);
|
|
3029
3572
|
}
|
|
3030
|
-
async function
|
|
3031
|
-
const
|
|
3573
|
+
async function executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps) {
|
|
3574
|
+
const deadlineMs = deps.deadlineMs ?? DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS;
|
|
3575
|
+
const deadline = Date.now() + deadlineMs;
|
|
3576
|
+
const key = entryKey(indexName, partitionId);
|
|
3577
|
+
const abortCheck = () => {
|
|
3578
|
+
const currentGeneration = state.generations.get(key) ?? 0;
|
|
3579
|
+
if (currentGeneration !== entry.generation) {
|
|
3580
|
+
entry.aborted = true;
|
|
3581
|
+
}
|
|
3582
|
+
return entry.aborted;
|
|
3583
|
+
};
|
|
3584
|
+
if (hasLiveBootstrapSyncDeps(deps)) {
|
|
3585
|
+
return executeLiveBootstrapSync(
|
|
3586
|
+
state,
|
|
3587
|
+
entry,
|
|
3588
|
+
indexName,
|
|
3589
|
+
partitionId,
|
|
3590
|
+
primaryNodeId,
|
|
3591
|
+
deps,
|
|
3592
|
+
deadlineMs,
|
|
3593
|
+
deadline,
|
|
3594
|
+
abortCheck
|
|
3595
|
+
);
|
|
3596
|
+
}
|
|
3597
|
+
const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
|
|
3032
3598
|
if (schemaResult === "aborted") {
|
|
3033
3599
|
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3034
3600
|
return false;
|
|
3035
3601
|
}
|
|
3036
|
-
if (
|
|
3037
|
-
surfaceError(deps, indexName, primaryNodeId, schemaResult.error);
|
|
3602
|
+
if (schemaResult === null) {
|
|
3038
3603
|
return false;
|
|
3039
3604
|
}
|
|
3040
3605
|
if (abortCheck()) {
|
|
3041
3606
|
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3042
3607
|
return false;
|
|
3043
3608
|
}
|
|
3044
|
-
const allocationResult = await resolveLivePartitionCount(
|
|
3045
|
-
deps.coordinator,
|
|
3046
|
-
indexName,
|
|
3047
|
-
partitionId,
|
|
3048
|
-
primaryNodeId,
|
|
3049
|
-
entry.abortPromise
|
|
3050
|
-
);
|
|
3051
|
-
if (allocationResult === "aborted") {
|
|
3052
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3053
|
-
return false;
|
|
3054
|
-
}
|
|
3055
|
-
if ("error" in allocationResult) {
|
|
3056
|
-
surfaceError(deps, indexName, primaryNodeId, allocationResult.error);
|
|
3057
|
-
return false;
|
|
3058
|
-
}
|
|
3059
3609
|
if (Date.now() >= deadline) {
|
|
3060
3610
|
surfaceError(
|
|
3061
3611
|
deps,
|
|
3062
3612
|
indexName,
|
|
3063
3613
|
primaryNodeId,
|
|
3064
|
-
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before
|
|
3614
|
+
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
|
|
3065
3615
|
indexName,
|
|
3066
3616
|
deadlineMs
|
|
3067
3617
|
})
|
|
@@ -3073,254 +3623,475 @@ async function executeLiveBootstrapSync(state, entry, indexName, partitionId, pr
|
|
|
3073
3623
|
surfaceError(deps, indexName, primaryNodeId, targetsResult);
|
|
3074
3624
|
return false;
|
|
3075
3625
|
}
|
|
3076
|
-
const
|
|
3077
|
-
|
|
3078
|
-
entry,
|
|
3626
|
+
const fetchDeps = { transport: deps.transport, sourceNodeId: deps.sourceNodeId };
|
|
3627
|
+
const fetchResult = await fetchSnapshotFromAnyTarget(
|
|
3079
3628
|
indexName,
|
|
3080
|
-
partitionId,
|
|
3081
3629
|
primaryNodeId,
|
|
3082
3630
|
targetsResult,
|
|
3083
|
-
schemaResult.schema,
|
|
3084
|
-
allocationResult.partitionCount,
|
|
3085
3631
|
deadline,
|
|
3086
|
-
|
|
3087
|
-
abortCheck
|
|
3632
|
+
fetchDeps,
|
|
3633
|
+
abortCheck,
|
|
3634
|
+
partitionId
|
|
3088
3635
|
);
|
|
3089
|
-
if (!
|
|
3090
|
-
if (
|
|
3636
|
+
if (!fetchResult.ok) {
|
|
3637
|
+
if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
|
|
3091
3638
|
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3092
3639
|
return false;
|
|
3093
3640
|
}
|
|
3094
|
-
|
|
3095
|
-
|
|
3641
|
+
surfaceError(
|
|
3642
|
+
deps,
|
|
3643
|
+
indexName,
|
|
3644
|
+
primaryNodeId,
|
|
3645
|
+
new NarsilError(fetchResult.code, fetchResult.message, {
|
|
3646
|
+
indexName,
|
|
3647
|
+
primaryNodeId,
|
|
3648
|
+
...fetchResult.details
|
|
3649
|
+
})
|
|
3650
|
+
);
|
|
3651
|
+
return false;
|
|
3652
|
+
}
|
|
3653
|
+
if (abortCheck()) {
|
|
3654
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3655
|
+
return false;
|
|
3656
|
+
}
|
|
3657
|
+
const restoreSucceeded = await applyRestore(
|
|
3658
|
+
state,
|
|
3659
|
+
entry,
|
|
3660
|
+
indexName,
|
|
3661
|
+
primaryNodeId,
|
|
3662
|
+
fetchResult.bytes,
|
|
3663
|
+
schemaResult,
|
|
3664
|
+
deadline,
|
|
3665
|
+
deps
|
|
3666
|
+
);
|
|
3667
|
+
if (!restoreSucceeded) {
|
|
3668
|
+
return false;
|
|
3669
|
+
}
|
|
3670
|
+
if (abortCheck()) {
|
|
3671
|
+
if (!anotherBootstrapOwnsKey(state, key, entry)) {
|
|
3672
|
+
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
3673
|
+
}
|
|
3674
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3675
|
+
return false;
|
|
3676
|
+
}
|
|
3677
|
+
deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
|
|
3678
|
+
state.completed.add(key);
|
|
3679
|
+
return true;
|
|
3680
|
+
}
|
|
3681
|
+
|
|
3682
|
+
// src/distribution/cluster-node/write-routing/assignment.ts
|
|
3683
|
+
function resolvePartitionId(docId, partitionCount) {
|
|
3684
|
+
return fnv1a(docId) % partitionCount;
|
|
3685
|
+
}
|
|
3686
|
+
async function resolveNodeTargets(nodeId, deps) {
|
|
3687
|
+
if (deps.resolveNodeTargets === void 0) {
|
|
3688
|
+
return [nodeId];
|
|
3689
|
+
}
|
|
3690
|
+
const targets = await deps.resolveNodeTargets(nodeId);
|
|
3691
|
+
return targets.length > 0 ? targets : [nodeId];
|
|
3692
|
+
}
|
|
3693
|
+
function requireAssignedPrimary(assignment, indexName, partitionId) {
|
|
3694
|
+
if (assignment === void 0 || assignment.primary === null) {
|
|
3695
|
+
throw new NarsilError(
|
|
3696
|
+
ErrorCodes.PARTITION_UNASSIGNED,
|
|
3697
|
+
`No primary assigned for partition ${partitionId} of index '${indexName}'`,
|
|
3698
|
+
{ indexName, partitionId }
|
|
3699
|
+
);
|
|
3700
|
+
}
|
|
3701
|
+
return assignment;
|
|
3702
|
+
}
|
|
3703
|
+
async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrimary) {
|
|
3704
|
+
const table = await deps.coordinator.getAllocation(indexName);
|
|
3705
|
+
if (table === null || table.assignments.size === 0) {
|
|
3706
|
+
if (requireLocalPrimary) {
|
|
3707
|
+
throw new NarsilError(
|
|
3708
|
+
ErrorCodes.QUERY_ROUTING_FAILED,
|
|
3709
|
+
`No allocation table is available for forwarded write to index '${indexName}'`,
|
|
3710
|
+
{ indexName }
|
|
3711
|
+
);
|
|
3712
|
+
}
|
|
3713
|
+
return null;
|
|
3714
|
+
}
|
|
3715
|
+
const partitionCount = table.assignments.size;
|
|
3716
|
+
const partitionId = resolvePartitionId(docId, partitionCount);
|
|
3717
|
+
const assignment = requireAssignedPrimary(table.assignments.get(partitionId), indexName, partitionId);
|
|
3718
|
+
if (requireLocalPrimary && assignment.primary !== deps.nodeId) {
|
|
3719
|
+
throw new NarsilError(
|
|
3720
|
+
ErrorCodes.PARTITION_NOT_PRIMARY,
|
|
3721
|
+
`Node '${deps.nodeId}' is not primary for partition ${partitionId} of index '${indexName}'`,
|
|
3722
|
+
{ indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
|
|
3723
|
+
);
|
|
3724
|
+
}
|
|
3725
|
+
return { partitionId, assignment };
|
|
3726
|
+
}
|
|
3727
|
+
function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
|
|
3728
|
+
const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
|
|
3729
|
+
if (requiredActiveCopies <= 1) {
|
|
3730
|
+
return;
|
|
3731
|
+
}
|
|
3732
|
+
const activeCopies = 1 + getInSyncReplicaTargets(assignment, deps.nodeId).length;
|
|
3733
|
+
if (activeCopies < requiredActiveCopies) {
|
|
3734
|
+
throw new NarsilError(
|
|
3735
|
+
ErrorCodes.INSUFFICIENT_REPLICAS,
|
|
3736
|
+
`Write to partition ${partitionId} of index '${indexName}' requires ${requiredActiveCopies} active copies but only ${activeCopies} are available`,
|
|
3737
|
+
{ indexName, partitionId, requiredActiveCopies, activeCopies }
|
|
3738
|
+
);
|
|
3739
|
+
}
|
|
3740
|
+
}
|
|
3741
|
+
function getInSyncReplicaTargets(assignment, localNodeId, pendingAdmissions = []) {
|
|
3742
|
+
const configuredReplicas = new Set(assignment.replicas);
|
|
3743
|
+
const targets = [];
|
|
3744
|
+
for (const nodeId of [...assignment.inSyncSet, ...pendingAdmissions]) {
|
|
3745
|
+
if (nodeId !== localNodeId && configuredReplicas.has(nodeId) && !targets.includes(nodeId)) {
|
|
3746
|
+
targets.push(nodeId);
|
|
3096
3747
|
}
|
|
3097
|
-
return false;
|
|
3098
3748
|
}
|
|
3099
|
-
|
|
3100
|
-
|
|
3749
|
+
return targets;
|
|
3750
|
+
}
|
|
3751
|
+
|
|
3752
|
+
// src/distribution/cluster-node/catch-up/state.ts
|
|
3753
|
+
var CATCH_UP_TICK_MS = 1e3;
|
|
3754
|
+
var CATCH_UP_IN_FLIGHT_BYTE_CEILING = 67108864;
|
|
3755
|
+
function partitionKey2(indexName, partitionId) {
|
|
3756
|
+
return `${indexName}:${partitionId}`;
|
|
3757
|
+
}
|
|
3758
|
+
function createCatchUpState() {
|
|
3759
|
+
return {
|
|
3760
|
+
cursors: /* @__PURE__ */ new Map(),
|
|
3761
|
+
pendingAdmissions: /* @__PURE__ */ new Map(),
|
|
3762
|
+
inFlightBytes: 0,
|
|
3763
|
+
timer: null,
|
|
3764
|
+
activeTick: null,
|
|
3765
|
+
stopped: false
|
|
3766
|
+
};
|
|
3767
|
+
}
|
|
3768
|
+
function recordReplicaPosition(state, indexName, partitionId, replicaNodeId, appliedSeqNo) {
|
|
3769
|
+
const key = partitionKey2(indexName, partitionId);
|
|
3770
|
+
const partition = state.cursors.get(key) ?? /* @__PURE__ */ new Map();
|
|
3771
|
+
const existing = partition.get(replicaNodeId);
|
|
3772
|
+
if (existing === void 0) {
|
|
3773
|
+
partition.set(replicaNodeId, { appliedSeqNo, sending: false, syncEpoch: 0 });
|
|
3774
|
+
} else if (appliedSeqNo > existing.appliedSeqNo) {
|
|
3775
|
+
existing.appliedSeqNo = appliedSeqNo;
|
|
3776
|
+
} else if (appliedSeqNo < existing.appliedSeqNo) {
|
|
3777
|
+
existing.appliedSeqNo = appliedSeqNo;
|
|
3778
|
+
existing.syncEpoch += 1;
|
|
3779
|
+
}
|
|
3780
|
+
state.cursors.set(key, partition);
|
|
3781
|
+
}
|
|
3782
|
+
function forgetReplica(state, indexName, partitionId, replicaNodeId) {
|
|
3783
|
+
const key = partitionKey2(indexName, partitionId);
|
|
3784
|
+
const partition = state.cursors.get(key);
|
|
3785
|
+
if (partition !== void 0) {
|
|
3786
|
+
partition.delete(replicaNodeId);
|
|
3787
|
+
if (partition.size === 0) {
|
|
3788
|
+
state.cursors.delete(key);
|
|
3789
|
+
}
|
|
3790
|
+
}
|
|
3791
|
+
clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
|
|
3792
|
+
}
|
|
3793
|
+
function getPendingAdmissions(state, indexName, partitionId) {
|
|
3794
|
+
const pending = state.pendingAdmissions.get(partitionKey2(indexName, partitionId));
|
|
3795
|
+
return pending === void 0 ? [] : [...pending];
|
|
3796
|
+
}
|
|
3797
|
+
function markPendingAdmission(state, indexName, partitionId, replicaNodeId) {
|
|
3798
|
+
const key = partitionKey2(indexName, partitionId);
|
|
3799
|
+
const pending = state.pendingAdmissions.get(key) ?? /* @__PURE__ */ new Set();
|
|
3800
|
+
if (pending.has(replicaNodeId)) {
|
|
3101
3801
|
return false;
|
|
3102
3802
|
}
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
}
|
|
3106
|
-
state.completed.add(entryKey(indexName, partitionId));
|
|
3803
|
+
pending.add(replicaNodeId);
|
|
3804
|
+
state.pendingAdmissions.set(key, pending);
|
|
3107
3805
|
return true;
|
|
3108
3806
|
}
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
surfaceError(deps, indexName, primaryNodeId, dropError);
|
|
3115
|
-
return null;
|
|
3116
|
-
}
|
|
3117
|
-
if (entry.aborted) {
|
|
3118
|
-
return "aborted";
|
|
3807
|
+
function clearPendingAdmission(state, indexName, partitionId, replicaNodeId) {
|
|
3808
|
+
const key = partitionKey2(indexName, partitionId);
|
|
3809
|
+
const pending = state.pendingAdmissions.get(key);
|
|
3810
|
+
if (pending === void 0) {
|
|
3811
|
+
return;
|
|
3119
3812
|
}
|
|
3120
|
-
|
|
3121
|
-
if (
|
|
3122
|
-
|
|
3813
|
+
pending.delete(replicaNodeId);
|
|
3814
|
+
if (pending.size === 0) {
|
|
3815
|
+
state.pendingAdmissions.delete(key);
|
|
3123
3816
|
}
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3817
|
+
}
|
|
3818
|
+
|
|
3819
|
+
// src/distribution/cluster-node/catch-up/admission.ts
|
|
3820
|
+
var ADMISSION_TIMEOUT_MS = 1e4;
|
|
3821
|
+
async function sendToController(payload, controllerNodeId, deps) {
|
|
3822
|
+
const targets = await resolveNodeTargets(controllerNodeId, deps);
|
|
3823
|
+
for (const target of targets) {
|
|
3824
|
+
try {
|
|
3825
|
+
const result = await requestInsyncAdmission(payload, target, deps.transport, deps.nodeId);
|
|
3826
|
+
return result.accepted;
|
|
3827
|
+
} catch (_) {
|
|
3828
|
+
}
|
|
3127
3829
|
}
|
|
3128
|
-
return
|
|
3830
|
+
return false;
|
|
3129
3831
|
}
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
);
|
|
3141
|
-
|
|
3832
|
+
function withTimeout(work, timeoutMs, fallback) {
|
|
3833
|
+
return new Promise((resolve) => {
|
|
3834
|
+
const timer = setTimeout(() => resolve(fallback), timeoutMs);
|
|
3835
|
+
timer.unref?.();
|
|
3836
|
+
work.then((value) => {
|
|
3837
|
+
clearTimeout(timer);
|
|
3838
|
+
resolve(value);
|
|
3839
|
+
}).catch(() => {
|
|
3840
|
+
clearTimeout(timer);
|
|
3841
|
+
resolve(fallback);
|
|
3842
|
+
});
|
|
3843
|
+
});
|
|
3844
|
+
}
|
|
3845
|
+
async function withdrawAdmission(payload, controllerNodeId, deps) {
|
|
3846
|
+
const targets = await resolveNodeTargets(controllerNodeId, deps);
|
|
3847
|
+
for (const target of targets) {
|
|
3848
|
+
try {
|
|
3849
|
+
const result = await requestInsyncRemoval(
|
|
3850
|
+
payload.indexName,
|
|
3851
|
+
payload.partitionId,
|
|
3852
|
+
payload.replicaNodeId,
|
|
3853
|
+
payload.primaryTerm,
|
|
3854
|
+
target,
|
|
3855
|
+
deps.transport,
|
|
3856
|
+
deps.nodeId
|
|
3857
|
+
);
|
|
3858
|
+
if (result.accepted) {
|
|
3859
|
+
return;
|
|
3860
|
+
}
|
|
3861
|
+
} catch (_) {
|
|
3862
|
+
}
|
|
3142
3863
|
}
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
return
|
|
3864
|
+
}
|
|
3865
|
+
async function proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
|
|
3866
|
+
const log = deps.getReplicationLog(indexName, partitionId);
|
|
3867
|
+
if (cursor.appliedSeqNo < log.commitPoint) {
|
|
3868
|
+
return;
|
|
3148
3869
|
}
|
|
3149
|
-
|
|
3150
|
-
|
|
3151
|
-
surfaceError(deps, indexName, primaryNodeId, restoreError);
|
|
3152
|
-
return false;
|
|
3870
|
+
if (!markPendingAdmission(state, indexName, partitionId, replicaNodeId)) {
|
|
3871
|
+
return;
|
|
3153
3872
|
}
|
|
3154
|
-
const
|
|
3155
|
-
|
|
3156
|
-
|
|
3157
|
-
|
|
3873
|
+
const proposedEpoch = cursor.syncEpoch;
|
|
3874
|
+
try {
|
|
3875
|
+
const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
3876
|
+
if (controllerNodeId === null) {
|
|
3877
|
+
return;
|
|
3158
3878
|
}
|
|
3159
|
-
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3879
|
+
const payload = {
|
|
3880
|
+
indexName,
|
|
3881
|
+
partitionId,
|
|
3882
|
+
replicaNodeId,
|
|
3883
|
+
primaryTerm: assignment.primaryTerm,
|
|
3884
|
+
appliedSeqNo: cursor.appliedSeqNo,
|
|
3885
|
+
commitPoint: log.commitPoint
|
|
3886
|
+
};
|
|
3887
|
+
const accepted = await withTimeout(sendToController(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
|
|
3888
|
+
if (accepted && cursor.syncEpoch !== proposedEpoch) {
|
|
3889
|
+
await withTimeout(withdrawAdmission(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, void 0);
|
|
3890
|
+
}
|
|
3891
|
+
} finally {
|
|
3892
|
+
clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
|
|
3167
3893
|
}
|
|
3168
|
-
return true;
|
|
3169
3894
|
}
|
|
3170
|
-
|
|
3171
|
-
|
|
3172
|
-
|
|
3173
|
-
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
return
|
|
3895
|
+
async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
|
|
3896
|
+
const message = createEntryMessage(entry, sourceNodeId);
|
|
3897
|
+
return sendAndCollectAcks(message, entry, inSyncReplicas, transport, resolveNodeTargets3);
|
|
3898
|
+
}
|
|
3899
|
+
async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
|
|
3900
|
+
if (entries.length === 0) {
|
|
3901
|
+
return { acknowledged: [], failed: [] };
|
|
3177
3902
|
}
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
if (!await replicaResyncRequired(indexName, partitionId, deps)) {
|
|
3181
|
-
return true;
|
|
3182
|
-
}
|
|
3183
|
-
clearBootstrapSyncIndex(state, indexName, partitionId);
|
|
3903
|
+
if (entries.length === 1) {
|
|
3904
|
+
return replicateToReplicas(entries[0], inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3);
|
|
3184
3905
|
}
|
|
3185
|
-
const
|
|
3186
|
-
|
|
3187
|
-
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3906
|
+
const message = createEntryBatchMessage(entries, sourceNodeId);
|
|
3907
|
+
const lastEntry = entries[entries.length - 1];
|
|
3908
|
+
return sendAndCollectAcks(message, lastEntry, inSyncReplicas, transport, resolveNodeTargets3);
|
|
3909
|
+
}
|
|
3910
|
+
async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport, resolveNodeTargets3) {
|
|
3911
|
+
const uniqueReplicas = [...new Set(inSyncReplicas)];
|
|
3912
|
+
if (uniqueReplicas.length === 0) {
|
|
3913
|
+
return { acknowledged: [], failed: [] };
|
|
3192
3914
|
}
|
|
3193
|
-
const
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3915
|
+
const sendResults = await Promise.allSettled(
|
|
3916
|
+
uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
|
|
3917
|
+
);
|
|
3918
|
+
const acknowledged = [];
|
|
3919
|
+
const failed = [];
|
|
3920
|
+
for (let i = 0; i < sendResults.length; i++) {
|
|
3921
|
+
const result = sendResults[i];
|
|
3922
|
+
const replicaNodeId = uniqueReplicas[i];
|
|
3923
|
+
if (result.status === "rejected") {
|
|
3924
|
+
if (result.reason instanceof TransportError) {
|
|
3925
|
+
failed.push(replicaNodeId);
|
|
3926
|
+
continue;
|
|
3203
3927
|
}
|
|
3928
|
+
throw result.reason;
|
|
3929
|
+
}
|
|
3930
|
+
const response = result.value;
|
|
3931
|
+
if (isMatchingAck(response, ackEntry)) {
|
|
3932
|
+
acknowledged.push(replicaNodeId);
|
|
3933
|
+
} else {
|
|
3934
|
+
failed.push(replicaNodeId);
|
|
3204
3935
|
}
|
|
3205
|
-
});
|
|
3206
|
-
state.inFlight.set(key, entry);
|
|
3207
|
-
return entry.promise;
|
|
3208
|
-
}
|
|
3209
|
-
async function replicaResyncRequired(indexName, partitionId, deps) {
|
|
3210
|
-
const allocation = await deps.coordinator.getAllocation(indexName);
|
|
3211
|
-
const assignment = allocation?.assignments.get(partitionId);
|
|
3212
|
-
if (assignment === void 0) {
|
|
3213
|
-
return false;
|
|
3214
3936
|
}
|
|
3215
|
-
return
|
|
3937
|
+
return { acknowledged, failed };
|
|
3216
3938
|
}
|
|
3217
|
-
async function
|
|
3218
|
-
const
|
|
3219
|
-
const
|
|
3220
|
-
|
|
3221
|
-
const
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3939
|
+
async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
|
|
3940
|
+
const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
|
|
3941
|
+
const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
|
|
3942
|
+
let lastError;
|
|
3943
|
+
for (const target of resolvedTargets) {
|
|
3944
|
+
try {
|
|
3945
|
+
return await transport.send(target, message);
|
|
3946
|
+
} catch (error) {
|
|
3947
|
+
lastError = error;
|
|
3225
3948
|
}
|
|
3226
|
-
return entry.aborted;
|
|
3227
|
-
};
|
|
3228
|
-
if (hasLiveBootstrapSyncDeps(deps)) {
|
|
3229
|
-
return executeLiveBootstrapSync(
|
|
3230
|
-
state,
|
|
3231
|
-
entry,
|
|
3232
|
-
indexName,
|
|
3233
|
-
partitionId,
|
|
3234
|
-
primaryNodeId,
|
|
3235
|
-
deps,
|
|
3236
|
-
deadlineMs,
|
|
3237
|
-
deadline,
|
|
3238
|
-
abortCheck
|
|
3239
|
-
);
|
|
3240
3949
|
}
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3950
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
3951
|
+
}
|
|
3952
|
+
function isMatchingAck(response, entry) {
|
|
3953
|
+
if (response.type !== ReplicationMessageTypes.ACK) {
|
|
3244
3954
|
return false;
|
|
3245
3955
|
}
|
|
3246
|
-
|
|
3956
|
+
try {
|
|
3957
|
+
const payload = validateAckPayload(decode(response.payload));
|
|
3958
|
+
return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
|
|
3959
|
+
} catch {
|
|
3247
3960
|
return false;
|
|
3248
3961
|
}
|
|
3249
|
-
|
|
3250
|
-
|
|
3962
|
+
}
|
|
3963
|
+
|
|
3964
|
+
// src/distribution/cluster-node/catch-up/pump.ts
|
|
3965
|
+
var ENTRY_FIXED_OVERHEAD_BYTES = 40;
|
|
3966
|
+
function entryBytes(entry) {
|
|
3967
|
+
return ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
3968
|
+
}
|
|
3969
|
+
function batchBytes(entries) {
|
|
3970
|
+
let total = 0;
|
|
3971
|
+
for (const entry of entries) {
|
|
3972
|
+
total += entryBytes(entry);
|
|
3973
|
+
}
|
|
3974
|
+
return total;
|
|
3975
|
+
}
|
|
3976
|
+
function replicaHasFallenOutOfRetention(log, cursor) {
|
|
3977
|
+
const oldest = log.oldestSeqNo;
|
|
3978
|
+
if (oldest === void 0) {
|
|
3251
3979
|
return false;
|
|
3252
3980
|
}
|
|
3253
|
-
|
|
3254
|
-
|
|
3255
|
-
|
|
3256
|
-
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3981
|
+
return cursor.appliedSeqNo + 1 < oldest;
|
|
3982
|
+
}
|
|
3983
|
+
function nextWireBatch(state, pending) {
|
|
3984
|
+
const chunks = chunkByBudget(pending, {
|
|
3985
|
+
...WIRE_BATCH_BUDGET,
|
|
3986
|
+
payloadBytesOf: (entry) => entry.document?.byteLength ?? 0,
|
|
3987
|
+
breaksRun: (entry, previous) => entry.seqNo !== previous.seqNo + 1
|
|
3988
|
+
});
|
|
3989
|
+
const batch = chunks[0] ?? [];
|
|
3990
|
+
if (batch.length === 0) {
|
|
3991
|
+
return [];
|
|
3992
|
+
}
|
|
3993
|
+
const available = CATCH_UP_IN_FLIGHT_BYTE_CEILING - state.inFlightBytes;
|
|
3994
|
+
return batchBytes(batch) > available ? [] : batch;
|
|
3995
|
+
}
|
|
3996
|
+
async function pumpReplica(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
|
|
3997
|
+
if (cursor.sending) {
|
|
3998
|
+
return;
|
|
3999
|
+
}
|
|
4000
|
+
const log = deps.getReplicationLog(indexName, partitionId);
|
|
4001
|
+
if (replicaHasFallenOutOfRetention(log, cursor)) {
|
|
4002
|
+
forgetReplica(state, indexName, partitionId, replicaNodeId);
|
|
4003
|
+
return;
|
|
4004
|
+
}
|
|
4005
|
+
if (cursor.appliedSeqNo >= log.localLogEnd) {
|
|
4006
|
+
await proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
|
|
4007
|
+
return;
|
|
4008
|
+
}
|
|
4009
|
+
const batch = nextWireBatch(state, log.getEntriesFrom(cursor.appliedSeqNo + 1));
|
|
4010
|
+
if (batch.length === 0) {
|
|
4011
|
+
return;
|
|
4012
|
+
}
|
|
4013
|
+
const reservedBytes = batchBytes(batch);
|
|
4014
|
+
const sendingEpoch = cursor.syncEpoch;
|
|
4015
|
+
cursor.sending = true;
|
|
4016
|
+
state.inFlightBytes += reservedBytes;
|
|
4017
|
+
try {
|
|
4018
|
+
const result = await replicateBatchToReplicas(
|
|
4019
|
+
batch,
|
|
4020
|
+
[replicaNodeId],
|
|
4021
|
+
deps.transport,
|
|
4022
|
+
deps.nodeId,
|
|
4023
|
+
deps.resolveNodeTargets
|
|
3262
4024
|
);
|
|
3263
|
-
|
|
4025
|
+
if (result.acknowledged.includes(replicaNodeId) && cursor.syncEpoch === sendingEpoch) {
|
|
4026
|
+
cursor.appliedSeqNo = batch[batch.length - 1].seqNo;
|
|
4027
|
+
}
|
|
4028
|
+
} finally {
|
|
4029
|
+
state.inFlightBytes -= reservedBytes;
|
|
4030
|
+
cursor.sending = false;
|
|
3264
4031
|
}
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
4032
|
+
}
|
|
4033
|
+
async function pumpPartition(state, deps, key, replicas) {
|
|
4034
|
+
const separator = key.lastIndexOf(":");
|
|
4035
|
+
const indexName = key.slice(0, separator);
|
|
4036
|
+
const partitionId = Number(key.slice(separator + 1));
|
|
4037
|
+
const table = await deps.coordinator.getAllocation(indexName);
|
|
4038
|
+
const assignment = table?.assignments.get(partitionId);
|
|
4039
|
+
if (assignment === void 0 || assignment.primary !== deps.nodeId) {
|
|
4040
|
+
state.cursors.delete(key);
|
|
4041
|
+
state.pendingAdmissions.delete(key);
|
|
4042
|
+
return;
|
|
3269
4043
|
}
|
|
3270
|
-
const
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
3274
|
-
|
|
3275
|
-
|
|
3276
|
-
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
|
|
3282
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3283
|
-
return false;
|
|
4044
|
+
for (const [replicaNodeId, cursor] of [...replicas]) {
|
|
4045
|
+
if (state.stopped) {
|
|
4046
|
+
return;
|
|
4047
|
+
}
|
|
4048
|
+
if (!assignment.replicas.includes(replicaNodeId) || assignment.inSyncSet.includes(replicaNodeId)) {
|
|
4049
|
+
forgetReplica(state, indexName, partitionId, replicaNodeId);
|
|
4050
|
+
continue;
|
|
4051
|
+
}
|
|
4052
|
+
try {
|
|
4053
|
+
await pumpReplica(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
|
|
4054
|
+
} catch (_) {
|
|
3284
4055
|
}
|
|
3285
|
-
surfaceError(
|
|
3286
|
-
deps,
|
|
3287
|
-
indexName,
|
|
3288
|
-
primaryNodeId,
|
|
3289
|
-
new NarsilError(fetchResult.code, fetchResult.message, {
|
|
3290
|
-
indexName,
|
|
3291
|
-
primaryNodeId,
|
|
3292
|
-
...fetchResult.details
|
|
3293
|
-
})
|
|
3294
|
-
);
|
|
3295
|
-
return false;
|
|
3296
|
-
}
|
|
3297
|
-
if (abortCheck()) {
|
|
3298
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3299
|
-
return false;
|
|
3300
4056
|
}
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
|
|
3304
|
-
|
|
3305
|
-
primaryNodeId,
|
|
3306
|
-
fetchResult.bytes,
|
|
3307
|
-
schemaResult,
|
|
3308
|
-
deadline,
|
|
3309
|
-
deps
|
|
3310
|
-
);
|
|
3311
|
-
if (!restoreSucceeded) {
|
|
3312
|
-
return false;
|
|
4057
|
+
}
|
|
4058
|
+
function runCatchUpTick(state, deps) {
|
|
4059
|
+
if (state.stopped || state.activeTick !== null || state.cursors.size === 0) {
|
|
4060
|
+
return state.activeTick ?? Promise.resolve();
|
|
3313
4061
|
}
|
|
3314
|
-
|
|
3315
|
-
|
|
3316
|
-
|
|
4062
|
+
const tick = (async () => {
|
|
4063
|
+
for (const [key, replicas] of [...state.cursors]) {
|
|
4064
|
+
if (state.stopped) {
|
|
4065
|
+
return;
|
|
4066
|
+
}
|
|
4067
|
+
try {
|
|
4068
|
+
await pumpPartition(state, deps, key, replicas);
|
|
4069
|
+
} catch (_) {
|
|
4070
|
+
}
|
|
3317
4071
|
}
|
|
3318
|
-
|
|
3319
|
-
|
|
4072
|
+
})().finally(() => {
|
|
4073
|
+
state.activeTick = null;
|
|
4074
|
+
});
|
|
4075
|
+
state.activeTick = tick;
|
|
4076
|
+
return tick;
|
|
4077
|
+
}
|
|
4078
|
+
function startCatchUpPump(state, deps) {
|
|
4079
|
+
void stopCatchUpPump(state);
|
|
4080
|
+
state.stopped = false;
|
|
4081
|
+
state.timer = setInterval(() => {
|
|
4082
|
+
void runCatchUpTick(state, deps);
|
|
4083
|
+
}, CATCH_UP_TICK_MS);
|
|
4084
|
+
state.timer.unref?.();
|
|
4085
|
+
}
|
|
4086
|
+
async function stopCatchUpPump(state) {
|
|
4087
|
+
state.stopped = true;
|
|
4088
|
+
if (state.timer !== null) {
|
|
4089
|
+
clearInterval(state.timer);
|
|
4090
|
+
state.timer = null;
|
|
4091
|
+
}
|
|
4092
|
+
if (state.activeTick !== null) {
|
|
4093
|
+
await state.activeTick;
|
|
3320
4094
|
}
|
|
3321
|
-
deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
|
|
3322
|
-
state.completed.add(key);
|
|
3323
|
-
return true;
|
|
3324
4095
|
}
|
|
3325
4096
|
|
|
3326
4097
|
// src/distribution/cluster-node/local-engine.ts
|
|
@@ -3663,21 +4434,14 @@ async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
|
|
|
3663
4434
|
}
|
|
3664
4435
|
let sawAssignment = false;
|
|
3665
4436
|
for (const assignment of allocation.assignments.values()) {
|
|
3666
|
-
const
|
|
3667
|
-
|
|
3668
|
-
if (!isPrimary && !isReplica) {
|
|
4437
|
+
const isAssigned = assignment.primary === sourceId || assignment.replicas.includes(sourceId);
|
|
4438
|
+
if (!isAssigned) {
|
|
3669
4439
|
continue;
|
|
3670
4440
|
}
|
|
3671
4441
|
sawAssignment = true;
|
|
3672
4442
|
if (!BOOTSTRAPPABLE_STATES.has(assignment.state)) {
|
|
3673
4443
|
continue;
|
|
3674
4444
|
}
|
|
3675
|
-
if (isPrimary) {
|
|
3676
|
-
return { outcome: "authorized" };
|
|
3677
|
-
}
|
|
3678
|
-
if (!assignmentAllowsReplicaBootstrap(assignment, sourceId)) {
|
|
3679
|
-
continue;
|
|
3680
|
-
}
|
|
3681
4445
|
return { outcome: "authorized" };
|
|
3682
4446
|
}
|
|
3683
4447
|
if (sawAssignment) {
|
|
@@ -3693,12 +4457,6 @@ async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
|
|
|
3693
4457
|
reason: `node '${sourceId}' is not an assigned replica for index '${indexName}'`
|
|
3694
4458
|
};
|
|
3695
4459
|
}
|
|
3696
|
-
function assignmentAllowsReplicaBootstrap(assignment, sourceId) {
|
|
3697
|
-
if (assignment.state === "INITIALISING" || assignment.state === "MIGRATING") {
|
|
3698
|
-
return true;
|
|
3699
|
-
}
|
|
3700
|
-
return assignment.inSyncSet.includes(sourceId);
|
|
3701
|
-
}
|
|
3702
4460
|
|
|
3703
4461
|
// src/distribution/cluster-node/snapshot-cache.ts
|
|
3704
4462
|
var DEFAULT_MAX_CONCURRENT_SNAPSHOTS = 2;
|
|
@@ -4217,7 +4975,7 @@ var NULL_BYTE = String.fromCharCode(0);
|
|
|
4217
4975
|
function throwInvalid(code, message, details) {
|
|
4218
4976
|
throw new NarsilError(code, message, details);
|
|
4219
4977
|
}
|
|
4220
|
-
function
|
|
4978
|
+
function isRecord2(value) {
|
|
4221
4979
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
4222
4980
|
}
|
|
4223
4981
|
function isInteger(value) {
|
|
@@ -4335,14 +5093,14 @@ function validateIndexNameField(value, fieldLabel) {
|
|
|
4335
5093
|
return value;
|
|
4336
5094
|
}
|
|
4337
5095
|
function validateDocumentId(value, fieldLabel) {
|
|
4338
|
-
if (!
|
|
5096
|
+
if (!isRecord2(value)) {
|
|
4339
5097
|
throwInvalid(CONFIG_INVALID, `Invalid FetchPayload: "${fieldLabel}" must be an object`);
|
|
4340
5098
|
}
|
|
4341
5099
|
validateStringField(value.docId, `${fieldLabel}.docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
|
|
4342
5100
|
validatePartitionId(value.partitionId, `${fieldLabel}.partitionId`, CONFIG_INVALID);
|
|
4343
5101
|
}
|
|
4344
5102
|
function validateHighlightConfig(value) {
|
|
4345
|
-
if (!
|
|
5103
|
+
if (!isRecord2(value)) {
|
|
4346
5104
|
throwInvalid(CONFIG_INVALID, 'Invalid FetchPayload: "highlight" must be an object or null');
|
|
4347
5105
|
}
|
|
4348
5106
|
if (value.fields !== null) {
|
|
@@ -4368,7 +5126,7 @@ function validateHighlightConfig(value) {
|
|
|
4368
5126
|
}
|
|
4369
5127
|
}
|
|
4370
5128
|
function validateFetchPayload(decoded) {
|
|
4371
|
-
if (!
|
|
5129
|
+
if (!isRecord2(decoded)) {
|
|
4372
5130
|
throwInvalid(CONFIG_INVALID, "Invalid FetchPayload: expected an object");
|
|
4373
5131
|
}
|
|
4374
5132
|
validateIndexNameField(decoded.indexName, "indexName");
|
|
@@ -4394,7 +5152,7 @@ function validateFetchPayload(decoded) {
|
|
|
4394
5152
|
return decoded;
|
|
4395
5153
|
}
|
|
4396
5154
|
function validateFetchResultPayload(decoded) {
|
|
4397
|
-
if (!
|
|
5155
|
+
if (!isRecord2(decoded)) {
|
|
4398
5156
|
throwInvalid(CONFIG_INVALID, "Invalid FetchResultPayload: expected an object");
|
|
4399
5157
|
}
|
|
4400
5158
|
if (!Array.isArray(decoded.documents)) {
|
|
@@ -4402,14 +5160,14 @@ function validateFetchResultPayload(decoded) {
|
|
|
4402
5160
|
}
|
|
4403
5161
|
for (let i = 0; i < decoded.documents.length; i++) {
|
|
4404
5162
|
const doc = decoded.documents[i];
|
|
4405
|
-
if (!
|
|
5163
|
+
if (!isRecord2(doc)) {
|
|
4406
5164
|
throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}]" must be an object`);
|
|
4407
5165
|
}
|
|
4408
5166
|
validateStringField(doc.docId, `documents[${i}].docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
|
|
4409
|
-
if (!
|
|
5167
|
+
if (!isRecord2(doc.document)) {
|
|
4410
5168
|
throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}].document" must be an object`);
|
|
4411
5169
|
}
|
|
4412
|
-
if (doc.highlights !== null && !
|
|
5170
|
+
if (doc.highlights !== null && !isRecord2(doc.highlights)) {
|
|
4413
5171
|
throwInvalid(CONFIG_INVALID, `Invalid FetchResultPayload: "documents[${i}].highlights" must be an object or null`);
|
|
4414
5172
|
}
|
|
4415
5173
|
}
|
|
@@ -4506,7 +5264,7 @@ function isKnownLeafKey(key) {
|
|
|
4506
5264
|
return FIELD_FILTER_OPERATORS.includes(key);
|
|
4507
5265
|
}
|
|
4508
5266
|
function validateGeoRadius(value, fieldLabel, errorCode) {
|
|
4509
|
-
if (!
|
|
5267
|
+
if (!isRecord2(value)) {
|
|
4510
5268
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
4511
5269
|
}
|
|
4512
5270
|
validateNumericOperand(value.lat, `${fieldLabel}.lat`, errorCode);
|
|
@@ -4526,7 +5284,7 @@ function validateGeoRadius(value, fieldLabel, errorCode) {
|
|
|
4526
5284
|
}
|
|
4527
5285
|
}
|
|
4528
5286
|
function validateGeoPolygon(value, fieldLabel, errorCode) {
|
|
4529
|
-
if (!
|
|
5287
|
+
if (!isRecord2(value)) {
|
|
4530
5288
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
4531
5289
|
}
|
|
4532
5290
|
if (!Array.isArray(value.points)) {
|
|
@@ -4542,7 +5300,7 @@ function validateGeoPolygon(value, fieldLabel, errorCode) {
|
|
|
4542
5300
|
}
|
|
4543
5301
|
for (let i = 0; i < points.length; i++) {
|
|
4544
5302
|
const point = points[i];
|
|
4545
|
-
if (!
|
|
5303
|
+
if (!isRecord2(point)) {
|
|
4546
5304
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}.points[${i}]" must be an object`);
|
|
4547
5305
|
}
|
|
4548
5306
|
validateNumericOperand(point.lat, `${fieldLabel}.points[${i}].lat`, errorCode);
|
|
@@ -4583,7 +5341,7 @@ function validateLeafOperator(opKey, opValue, leafLabel, errorCode) {
|
|
|
4583
5341
|
return;
|
|
4584
5342
|
}
|
|
4585
5343
|
if (opKey === "size") {
|
|
4586
|
-
if (!
|
|
5344
|
+
if (!isRecord2(opValue)) {
|
|
4587
5345
|
throwInvalid(errorCode, `Invalid payload: "${opLabel}" must be an object`);
|
|
4588
5346
|
}
|
|
4589
5347
|
for (const [innerKey, innerValue] of Object.entries(opValue)) {
|
|
@@ -4605,7 +5363,7 @@ function validateLeafOperator(opKey, opValue, leafLabel, errorCode) {
|
|
|
4605
5363
|
throwInvalid(errorCode, `Invalid payload: "${leafLabel}" contains unsupported operator "${opKey}"`);
|
|
4606
5364
|
}
|
|
4607
5365
|
function validateFieldFilter(value, fieldLabel, errorCode) {
|
|
4608
|
-
if (!
|
|
5366
|
+
if (!isRecord2(value)) {
|
|
4609
5367
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
4610
5368
|
}
|
|
4611
5369
|
const entries = Object.entries(value);
|
|
@@ -4623,7 +5381,7 @@ function validateFieldFilter(value, fieldLabel, errorCode) {
|
|
|
4623
5381
|
}
|
|
4624
5382
|
}
|
|
4625
5383
|
function validateFieldsRecord(value, fieldLabel, errorCode) {
|
|
4626
|
-
if (!
|
|
5384
|
+
if (!isRecord2(value)) {
|
|
4627
5385
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
4628
5386
|
}
|
|
4629
5387
|
const entries = Object.entries(value);
|
|
@@ -4666,7 +5424,7 @@ function validateFilterExpression(value, fieldLabel, errorCode = SEARCH_INVALID_
|
|
|
4666
5424
|
limit: MAX_FILTER_DEPTH
|
|
4667
5425
|
});
|
|
4668
5426
|
}
|
|
4669
|
-
if (!
|
|
5427
|
+
if (!isRecord2(value)) {
|
|
4670
5428
|
throwInvalid(errorCode, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
4671
5429
|
}
|
|
4672
5430
|
const entries = Object.entries(value);
|
|
@@ -4704,7 +5462,7 @@ function validateIndexNameField2(value, fieldLabel) {
|
|
|
4704
5462
|
return value;
|
|
4705
5463
|
}
|
|
4706
5464
|
function validateVectorParams(value) {
|
|
4707
|
-
if (!
|
|
5465
|
+
if (!isRecord2(value)) {
|
|
4708
5466
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.vector" must be an object');
|
|
4709
5467
|
}
|
|
4710
5468
|
validateFieldName(value.field, "params.vector.field", SEARCH_INVALID_FIELD);
|
|
@@ -4749,7 +5507,7 @@ function validateVectorParams(value) {
|
|
|
4749
5507
|
}
|
|
4750
5508
|
}
|
|
4751
5509
|
function validateHybridParams(value) {
|
|
4752
|
-
if (!
|
|
5510
|
+
if (!isRecord2(value)) {
|
|
4753
5511
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.hybrid" must be an object');
|
|
4754
5512
|
}
|
|
4755
5513
|
if (!ALLOWED_HYBRID_STRATEGIES.includes(value.strategy)) {
|
|
@@ -4769,7 +5527,7 @@ function validateHybridParams(value) {
|
|
|
4769
5527
|
}
|
|
4770
5528
|
}
|
|
4771
5529
|
function validateGroupParams(value) {
|
|
4772
|
-
if (!
|
|
5530
|
+
if (!isRecord2(value)) {
|
|
4773
5531
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.group" must be an object');
|
|
4774
5532
|
}
|
|
4775
5533
|
validateFieldName(value.field, "params.group.field", SEARCH_INVALID_FIELD);
|
|
@@ -4796,7 +5554,7 @@ function validateSortParams(value) {
|
|
|
4796
5554
|
}
|
|
4797
5555
|
for (let i = 0; i < value.length; i++) {
|
|
4798
5556
|
const entry = value[i];
|
|
4799
|
-
if (!
|
|
5557
|
+
if (!isRecord2(entry)) {
|
|
4800
5558
|
throwInvalid(CONFIG_INVALID, `Invalid SearchPayload: "params.sort[${i}]" must be an object`);
|
|
4801
5559
|
}
|
|
4802
5560
|
validateFieldName(entry.field, `params.sort[${i}].field`, SEARCH_INVALID_FIELD);
|
|
@@ -4809,7 +5567,7 @@ function validateSortParams(value) {
|
|
|
4809
5567
|
}
|
|
4810
5568
|
}
|
|
4811
5569
|
function validateBoostParams(value) {
|
|
4812
|
-
if (!
|
|
5570
|
+
if (!isRecord2(value)) {
|
|
4813
5571
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params.boost" must be an object or null');
|
|
4814
5572
|
}
|
|
4815
5573
|
const entries = Object.entries(value);
|
|
@@ -4828,16 +5586,16 @@ function validateBoostParams(value) {
|
|
|
4828
5586
|
}
|
|
4829
5587
|
}
|
|
4830
5588
|
function validateGlobalStatsShape(value) {
|
|
4831
|
-
if (!
|
|
5589
|
+
if (!isRecord2(value)) {
|
|
4832
5590
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats" must be an object or null');
|
|
4833
5591
|
}
|
|
4834
5592
|
if (!isFiniteNumber(value.totalDocuments) || value.totalDocuments < 0) {
|
|
4835
5593
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.totalDocuments" must be a non-negative number');
|
|
4836
5594
|
}
|
|
4837
|
-
if (!
|
|
5595
|
+
if (!isRecord2(value.docFrequencies)) {
|
|
4838
5596
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.docFrequencies" must be an object');
|
|
4839
5597
|
}
|
|
4840
|
-
if (!
|
|
5598
|
+
if (!isRecord2(value.totalFieldLengths)) {
|
|
4841
5599
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "globalStats.totalFieldLengths" must be an object');
|
|
4842
5600
|
}
|
|
4843
5601
|
}
|
|
@@ -4927,18 +5685,18 @@ function validateParams(params) {
|
|
|
4927
5685
|
}
|
|
4928
5686
|
}
|
|
4929
5687
|
function validateWireParams(params) {
|
|
4930
|
-
if (!
|
|
5688
|
+
if (!isRecord2(params)) {
|
|
4931
5689
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params" must be an object');
|
|
4932
5690
|
}
|
|
4933
5691
|
validateParams(params);
|
|
4934
5692
|
}
|
|
4935
5693
|
function validateSearchPayload(decoded) {
|
|
4936
|
-
if (!
|
|
5694
|
+
if (!isRecord2(decoded)) {
|
|
4937
5695
|
throwInvalid(CONFIG_INVALID, "Invalid SearchPayload: expected an object");
|
|
4938
5696
|
}
|
|
4939
5697
|
validateIndexNameField2(decoded.indexName, "indexName");
|
|
4940
5698
|
validatePartitionIdsArray(decoded.partitionIds, "partitionIds", CONFIG_INVALID);
|
|
4941
|
-
if (!
|
|
5699
|
+
if (!isRecord2(decoded.params)) {
|
|
4942
5700
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchPayload: "params" must be an object');
|
|
4943
5701
|
}
|
|
4944
5702
|
validateParams(decoded.params);
|
|
@@ -4974,7 +5732,7 @@ function validateIndexNameField3(decoded, payloadLabel) {
|
|
|
4974
5732
|
validateStringField(decoded.indexName, `${payloadLabel}.indexName`, 255, CONFIG_INVALID);
|
|
4975
5733
|
}
|
|
4976
5734
|
function validateCountPayload(decoded) {
|
|
4977
|
-
if (!
|
|
5735
|
+
if (!isRecord2(decoded)) {
|
|
4978
5736
|
throwInvalid(CONFIG_INVALID, "Invalid CountPayload: expected an object");
|
|
4979
5737
|
}
|
|
4980
5738
|
validateIndexNameField3(decoded, "CountPayload");
|
|
@@ -4982,7 +5740,7 @@ function validateCountPayload(decoded) {
|
|
|
4982
5740
|
return decoded;
|
|
4983
5741
|
}
|
|
4984
5742
|
function validateCountResultPayload(decoded) {
|
|
4985
|
-
if (!
|
|
5743
|
+
if (!isRecord2(decoded)) {
|
|
4986
5744
|
throwInvalid(CONFIG_INVALID, "Invalid CountResultPayload: expected an object");
|
|
4987
5745
|
}
|
|
4988
5746
|
if (!Array.isArray(decoded.partitions)) {
|
|
@@ -4990,7 +5748,7 @@ function validateCountResultPayload(decoded) {
|
|
|
4990
5748
|
}
|
|
4991
5749
|
for (let i = 0; i < decoded.partitions.length; i++) {
|
|
4992
5750
|
const entry = decoded.partitions[i];
|
|
4993
|
-
if (!
|
|
5751
|
+
if (!isRecord2(entry)) {
|
|
4994
5752
|
throwInvalid(CONFIG_INVALID, `Invalid CountResultPayload: "partitions[${i}]" must be an object`);
|
|
4995
5753
|
}
|
|
4996
5754
|
validateNonNegativeInteger(entry.partitionId, `partitions[${i}].partitionId`, MAX_COUNT_VALUE, CONFIG_INVALID);
|
|
@@ -5017,7 +5775,7 @@ function validateListSort(value) {
|
|
|
5017
5775
|
}
|
|
5018
5776
|
for (let i = 0; i < value.length; i++) {
|
|
5019
5777
|
const entry = value[i];
|
|
5020
|
-
if (!
|
|
5778
|
+
if (!isRecord2(entry)) {
|
|
5021
5779
|
throwInvalid(CONFIG_INVALID, `Invalid ListPayload: "sort[${i}]" must be an object`);
|
|
5022
5780
|
}
|
|
5023
5781
|
validateFieldName(entry.field, `sort[${i}].field`, SEARCH_INVALID_FIELD);
|
|
@@ -5027,7 +5785,7 @@ function validateListSort(value) {
|
|
|
5027
5785
|
}
|
|
5028
5786
|
}
|
|
5029
5787
|
function validateListPayload(decoded) {
|
|
5030
|
-
if (!
|
|
5788
|
+
if (!isRecord2(decoded)) {
|
|
5031
5789
|
throwInvalid(CONFIG_INVALID, "Invalid ListPayload: expected an object");
|
|
5032
5790
|
}
|
|
5033
5791
|
validateIndexNameField3(decoded, "ListPayload");
|
|
@@ -5072,7 +5830,7 @@ function validateSortValues(value, fieldLabel) {
|
|
|
5072
5830
|
}
|
|
5073
5831
|
}
|
|
5074
5832
|
function validateListResultPayload(decoded) {
|
|
5075
|
-
if (!
|
|
5833
|
+
if (!isRecord2(decoded)) {
|
|
5076
5834
|
throwInvalid(CONFIG_INVALID, "Invalid ListResultPayload: expected an object");
|
|
5077
5835
|
}
|
|
5078
5836
|
if (!Array.isArray(decoded.entries)) {
|
|
@@ -5083,11 +5841,11 @@ function validateListResultPayload(decoded) {
|
|
|
5083
5841
|
}
|
|
5084
5842
|
for (let i = 0; i < decoded.entries.length; i++) {
|
|
5085
5843
|
const entry = decoded.entries[i];
|
|
5086
|
-
if (!
|
|
5844
|
+
if (!isRecord2(entry)) {
|
|
5087
5845
|
throwInvalid(CONFIG_INVALID, `Invalid ListResultPayload: "entries[${i}]" must be an object`);
|
|
5088
5846
|
}
|
|
5089
5847
|
validateStringField(entry.docId, `entries[${i}].docId`, MAX_DOC_ID_LENGTH, CONFIG_INVALID);
|
|
5090
|
-
if (!
|
|
5848
|
+
if (!isRecord2(entry.document)) {
|
|
5091
5849
|
throwInvalid(CONFIG_INVALID, `Invalid ListResultPayload: "entries[${i}].document" must be an object`);
|
|
5092
5850
|
}
|
|
5093
5851
|
if (entry.sortValues !== null) {
|
|
@@ -5101,7 +5859,7 @@ function validateListResultPayload(decoded) {
|
|
|
5101
5859
|
return decoded;
|
|
5102
5860
|
}
|
|
5103
5861
|
function validateSuggestPayload(decoded) {
|
|
5104
|
-
if (!
|
|
5862
|
+
if (!isRecord2(decoded)) {
|
|
5105
5863
|
throwInvalid(CONFIG_INVALID, "Invalid SuggestPayload: expected an object");
|
|
5106
5864
|
}
|
|
5107
5865
|
validateIndexNameField3(decoded, "SuggestPayload");
|
|
@@ -5111,7 +5869,7 @@ function validateSuggestPayload(decoded) {
|
|
|
5111
5869
|
return decoded;
|
|
5112
5870
|
}
|
|
5113
5871
|
function validateSuggestResultPayload(decoded) {
|
|
5114
|
-
if (!
|
|
5872
|
+
if (!isRecord2(decoded)) {
|
|
5115
5873
|
throwInvalid(CONFIG_INVALID, "Invalid SuggestResultPayload: expected an object");
|
|
5116
5874
|
}
|
|
5117
5875
|
if (!Array.isArray(decoded.terms)) {
|
|
@@ -5125,7 +5883,7 @@ function validateSuggestResultPayload(decoded) {
|
|
|
5125
5883
|
}
|
|
5126
5884
|
for (let i = 0; i < decoded.terms.length; i++) {
|
|
5127
5885
|
const entry = decoded.terms[i];
|
|
5128
|
-
if (!
|
|
5886
|
+
if (!isRecord2(entry)) {
|
|
5129
5887
|
throwInvalid(CONFIG_INVALID, `Invalid SuggestResultPayload: "terms[${i}]" must be an object`);
|
|
5130
5888
|
}
|
|
5131
5889
|
validateStringField(entry.term, `terms[${i}].term`, MAX_TERM_LENGTH, CONFIG_INVALID);
|
|
@@ -5142,7 +5900,7 @@ function validateSuggestResultPayload(decoded) {
|
|
|
5142
5900
|
return decoded;
|
|
5143
5901
|
}
|
|
5144
5902
|
function validatePreflightPayload(decoded) {
|
|
5145
|
-
if (!
|
|
5903
|
+
if (!isRecord2(decoded)) {
|
|
5146
5904
|
throwInvalid(CONFIG_INVALID, "Invalid PreflightPayload: expected an object");
|
|
5147
5905
|
}
|
|
5148
5906
|
validateIndexNameField3(decoded, "PreflightPayload");
|
|
@@ -5151,7 +5909,7 @@ function validatePreflightPayload(decoded) {
|
|
|
5151
5909
|
return decoded;
|
|
5152
5910
|
}
|
|
5153
5911
|
function validatePreflightResultPayload(decoded) {
|
|
5154
|
-
if (!
|
|
5912
|
+
if (!isRecord2(decoded)) {
|
|
5155
5913
|
throwInvalid(CONFIG_INVALID, "Invalid PreflightResultPayload: expected an object");
|
|
5156
5914
|
}
|
|
5157
5915
|
if (!isInteger(decoded.count) || decoded.count < 0) {
|
|
@@ -5170,7 +5928,7 @@ var MAX_FACET_FIELDS = 64;
|
|
|
5170
5928
|
var MAX_FACET_BUCKETS = 1e4;
|
|
5171
5929
|
var MAX_FACET_VALUE_LENGTH = 1024;
|
|
5172
5930
|
function validateScoredEntry(value, fieldLabel) {
|
|
5173
|
-
if (!
|
|
5931
|
+
if (!isRecord2(value)) {
|
|
5174
5932
|
throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
|
|
5175
5933
|
}
|
|
5176
5934
|
if (typeof value.docId !== "string") {
|
|
@@ -5216,7 +5974,7 @@ function validateScoredEntry(value, fieldLabel) {
|
|
|
5216
5974
|
}
|
|
5217
5975
|
}
|
|
5218
5976
|
function validatePartitionSearchResult(value, fieldLabel) {
|
|
5219
|
-
if (!
|
|
5977
|
+
if (!isRecord2(value)) {
|
|
5220
5978
|
throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
|
|
5221
5979
|
}
|
|
5222
5980
|
validatePartitionId(value.partitionId, `${fieldLabel}.partitionId`, CONFIG_INVALID);
|
|
@@ -5241,7 +5999,7 @@ function validatePartitionSearchResult(value, fieldLabel) {
|
|
|
5241
5999
|
}
|
|
5242
6000
|
}
|
|
5243
6001
|
function validateFacetBucket(value, fieldLabel) {
|
|
5244
|
-
if (!
|
|
6002
|
+
if (!isRecord2(value)) {
|
|
5245
6003
|
throwInvalid(CONFIG_INVALID, `Invalid SearchResultPayload: "${fieldLabel}" must be an object`);
|
|
5246
6004
|
}
|
|
5247
6005
|
if (typeof value.value !== "string") {
|
|
@@ -5260,7 +6018,7 @@ function validateFacetBucket(value, fieldLabel) {
|
|
|
5260
6018
|
return { value: value.value, count: value.count };
|
|
5261
6019
|
}
|
|
5262
6020
|
function validateFacets(value) {
|
|
5263
|
-
if (!
|
|
6021
|
+
if (!isRecord2(value)) {
|
|
5264
6022
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchResultPayload: "facets" must be an object or null');
|
|
5265
6023
|
}
|
|
5266
6024
|
const entries = Object.entries(value);
|
|
@@ -5291,7 +6049,7 @@ function validateFacets(value) {
|
|
|
5291
6049
|
}
|
|
5292
6050
|
}
|
|
5293
6051
|
function validateFacetErrorBounds(value) {
|
|
5294
|
-
if (!
|
|
6052
|
+
if (!isRecord2(value)) {
|
|
5295
6053
|
throwInvalid(CONFIG_INVALID, 'Invalid SearchResultPayload: "facetErrorBounds" must be an object or null');
|
|
5296
6054
|
}
|
|
5297
6055
|
const entries = Object.entries(value);
|
|
@@ -5312,7 +6070,7 @@ function validateFacetErrorBounds(value) {
|
|
|
5312
6070
|
}
|
|
5313
6071
|
}
|
|
5314
6072
|
function validateSearchResultPayload(decoded) {
|
|
5315
|
-
if (!
|
|
6073
|
+
if (!isRecord2(decoded)) {
|
|
5316
6074
|
throwInvalid(CONFIG_INVALID, "Invalid SearchResultPayload: expected an object");
|
|
5317
6075
|
}
|
|
5318
6076
|
if (!Array.isArray(decoded.results)) {
|
|
@@ -5339,7 +6097,7 @@ function validateIndexNameField4(value, fieldLabel) {
|
|
|
5339
6097
|
return value;
|
|
5340
6098
|
}
|
|
5341
6099
|
function validateNumericRecord(value, fieldLabel) {
|
|
5342
|
-
if (!
|
|
6100
|
+
if (!isRecord2(value)) {
|
|
5343
6101
|
throwInvalid(CONFIG_INVALID, `Invalid payload: "${fieldLabel}" must be an object`);
|
|
5344
6102
|
}
|
|
5345
6103
|
for (const [key, entry] of Object.entries(value)) {
|
|
@@ -5352,7 +6110,7 @@ function validateNumericRecord(value, fieldLabel) {
|
|
|
5352
6110
|
}
|
|
5353
6111
|
}
|
|
5354
6112
|
function validateStatsPayload(decoded) {
|
|
5355
|
-
if (!
|
|
6113
|
+
if (!isRecord2(decoded)) {
|
|
5356
6114
|
throwInvalid(CONFIG_INVALID, "Invalid StatsPayload: expected an object");
|
|
5357
6115
|
}
|
|
5358
6116
|
validateIndexNameField4(decoded.indexName, "indexName");
|
|
@@ -5361,7 +6119,7 @@ function validateStatsPayload(decoded) {
|
|
|
5361
6119
|
return decoded;
|
|
5362
6120
|
}
|
|
5363
6121
|
function validateStatsResultPayload(decoded) {
|
|
5364
|
-
if (!
|
|
6122
|
+
if (!isRecord2(decoded)) {
|
|
5365
6123
|
throwInvalid(CONFIG_INVALID, "Invalid StatsResultPayload: expected an object");
|
|
5366
6124
|
}
|
|
5367
6125
|
if (!isInteger(decoded.totalDocuments) || decoded.totalDocuments < 0) {
|
|
@@ -5761,10 +6519,10 @@ function toWireSortValue2(value) {
|
|
|
5761
6519
|
function decideSyncTier(log, replicaLastSeqNo) {
|
|
5762
6520
|
const oldest = log.oldestSeqNo;
|
|
5763
6521
|
if (oldest === void 0) {
|
|
5764
|
-
if (log.
|
|
6522
|
+
if (log.localLogEnd === 0) {
|
|
5765
6523
|
return replicaLastSeqNo === 0 ? "incremental" : "snapshot";
|
|
5766
6524
|
}
|
|
5767
|
-
return replicaLastSeqNo >= log.
|
|
6525
|
+
return replicaLastSeqNo >= log.localLogEnd ? "incremental" : "snapshot";
|
|
5768
6526
|
}
|
|
5769
6527
|
if (oldest <= replicaLastSeqNo + 1) {
|
|
5770
6528
|
return "incremental";
|
|
@@ -5843,13 +6601,22 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
5843
6601
|
}
|
|
5844
6602
|
);
|
|
5845
6603
|
}
|
|
6604
|
+
if (message.sourceId !== deps.nodeId) {
|
|
6605
|
+
recordReplicaPosition(
|
|
6606
|
+
deps.writeDeps.catchUp,
|
|
6607
|
+
request.indexName,
|
|
6608
|
+
request.partitionId,
|
|
6609
|
+
message.sourceId,
|
|
6610
|
+
request.lastSeqNo
|
|
6611
|
+
);
|
|
6612
|
+
}
|
|
5846
6613
|
const log = deps.writeDeps.getReplicationLog(request.indexName, request.partitionId);
|
|
5847
6614
|
const tier = decideSyncTier(log, request.lastSeqNo);
|
|
5848
6615
|
if (tier === "incremental") {
|
|
5849
6616
|
await sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1));
|
|
5850
6617
|
return;
|
|
5851
6618
|
}
|
|
5852
|
-
const snapshotSeqNo = log.
|
|
6619
|
+
const snapshotSeqNo = log.localLogEnd;
|
|
5853
6620
|
const sink = createSingleResponseSink(respond);
|
|
5854
6621
|
await streamValidatedSnapshotRequest(
|
|
5855
6622
|
message,
|
|
@@ -5902,76 +6669,6 @@ var DEFAULT_CAPACITY = {
|
|
|
5902
6669
|
cpuCores: 4,
|
|
5903
6670
|
diskBytes: null
|
|
5904
6671
|
};
|
|
5905
|
-
|
|
5906
|
-
// src/distribution/cluster-node/write-routing/assignment.ts
|
|
5907
|
-
function resolvePartitionId(docId, partitionCount) {
|
|
5908
|
-
return fnv1a(docId) % partitionCount;
|
|
5909
|
-
}
|
|
5910
|
-
async function resolveNodeTargets(nodeId, deps) {
|
|
5911
|
-
if (deps.resolveNodeTargets === void 0) {
|
|
5912
|
-
return [nodeId];
|
|
5913
|
-
}
|
|
5914
|
-
const targets = await deps.resolveNodeTargets(nodeId);
|
|
5915
|
-
return targets.length > 0 ? targets : [nodeId];
|
|
5916
|
-
}
|
|
5917
|
-
function requireAssignedPrimary(assignment, indexName, partitionId) {
|
|
5918
|
-
if (assignment === void 0 || assignment.primary === null) {
|
|
5919
|
-
throw new NarsilError(
|
|
5920
|
-
ErrorCodes.PARTITION_UNASSIGNED,
|
|
5921
|
-
`No primary assigned for partition ${partitionId} of index '${indexName}'`,
|
|
5922
|
-
{ indexName, partitionId }
|
|
5923
|
-
);
|
|
5924
|
-
}
|
|
5925
|
-
return assignment;
|
|
5926
|
-
}
|
|
5927
|
-
async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrimary) {
|
|
5928
|
-
const table = await deps.coordinator.getAllocation(indexName);
|
|
5929
|
-
if (table === null || table.assignments.size === 0) {
|
|
5930
|
-
if (requireLocalPrimary) {
|
|
5931
|
-
throw new NarsilError(
|
|
5932
|
-
ErrorCodes.QUERY_ROUTING_FAILED,
|
|
5933
|
-
`No allocation table is available for forwarded write to index '${indexName}'`,
|
|
5934
|
-
{ indexName }
|
|
5935
|
-
);
|
|
5936
|
-
}
|
|
5937
|
-
return null;
|
|
5938
|
-
}
|
|
5939
|
-
const partitionCount = table.assignments.size;
|
|
5940
|
-
const partitionId = resolvePartitionId(docId, partitionCount);
|
|
5941
|
-
const assignment = requireAssignedPrimary(table.assignments.get(partitionId), indexName, partitionId);
|
|
5942
|
-
if (requireLocalPrimary && assignment.primary !== deps.nodeId) {
|
|
5943
|
-
throw new NarsilError(
|
|
5944
|
-
ErrorCodes.PARTITION_NOT_PRIMARY,
|
|
5945
|
-
`Node '${deps.nodeId}' is not primary for partition ${partitionId} of index '${indexName}'`,
|
|
5946
|
-
{ indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
|
|
5947
|
-
);
|
|
5948
|
-
}
|
|
5949
|
-
return { partitionId, assignment };
|
|
5950
|
-
}
|
|
5951
|
-
function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
|
|
5952
|
-
const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
|
|
5953
|
-
if (requiredActiveCopies <= 1) {
|
|
5954
|
-
return;
|
|
5955
|
-
}
|
|
5956
|
-
const activeCopies = 1 + getInSyncReplicaTargets(assignment, deps.nodeId).length;
|
|
5957
|
-
if (activeCopies < requiredActiveCopies) {
|
|
5958
|
-
throw new NarsilError(
|
|
5959
|
-
ErrorCodes.INSUFFICIENT_REPLICAS,
|
|
5960
|
-
`Write to partition ${partitionId} of index '${indexName}' requires ${requiredActiveCopies} active copies but only ${activeCopies} are available`,
|
|
5961
|
-
{ indexName, partitionId, requiredActiveCopies, activeCopies }
|
|
5962
|
-
);
|
|
5963
|
-
}
|
|
5964
|
-
}
|
|
5965
|
-
function getInSyncReplicaTargets(assignment, localNodeId) {
|
|
5966
|
-
const configuredReplicas = new Set(assignment.replicas);
|
|
5967
|
-
const targets = [];
|
|
5968
|
-
for (const nodeId of assignment.inSyncSet) {
|
|
5969
|
-
if (nodeId !== localNodeId && configuredReplicas.has(nodeId)) {
|
|
5970
|
-
targets.push(nodeId);
|
|
5971
|
-
}
|
|
5972
|
-
}
|
|
5973
|
-
return targets;
|
|
5974
|
-
}
|
|
5975
6672
|
function assertForwardResponse(response, indexName, primaryNodeId) {
|
|
5976
6673
|
const decoded = decode(response.payload);
|
|
5977
6674
|
if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
|
|
@@ -6039,76 +6736,6 @@ async function sendToNode(nodeId, message, deps) {
|
|
|
6039
6736
|
}
|
|
6040
6737
|
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
6041
6738
|
}
|
|
6042
|
-
async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
|
|
6043
|
-
const message = createEntryMessage(entry, sourceNodeId);
|
|
6044
|
-
return sendAndCollectAcks(message, entry, inSyncReplicas, transport, resolveNodeTargets3);
|
|
6045
|
-
}
|
|
6046
|
-
async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
|
|
6047
|
-
if (entries.length === 0) {
|
|
6048
|
-
return { acknowledged: [], failed: [] };
|
|
6049
|
-
}
|
|
6050
|
-
if (entries.length === 1) {
|
|
6051
|
-
return replicateToReplicas(entries[0], inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3);
|
|
6052
|
-
}
|
|
6053
|
-
const message = createEntryBatchMessage(entries, sourceNodeId);
|
|
6054
|
-
const lastEntry = entries[entries.length - 1];
|
|
6055
|
-
return sendAndCollectAcks(message, lastEntry, inSyncReplicas, transport, resolveNodeTargets3);
|
|
6056
|
-
}
|
|
6057
|
-
async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport, resolveNodeTargets3) {
|
|
6058
|
-
const uniqueReplicas = [...new Set(inSyncReplicas)];
|
|
6059
|
-
if (uniqueReplicas.length === 0) {
|
|
6060
|
-
return { acknowledged: [], failed: [] };
|
|
6061
|
-
}
|
|
6062
|
-
const sendResults = await Promise.allSettled(
|
|
6063
|
-
uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
|
|
6064
|
-
);
|
|
6065
|
-
const acknowledged = [];
|
|
6066
|
-
const failed = [];
|
|
6067
|
-
for (let i = 0; i < sendResults.length; i++) {
|
|
6068
|
-
const result = sendResults[i];
|
|
6069
|
-
const replicaNodeId = uniqueReplicas[i];
|
|
6070
|
-
if (result.status === "rejected") {
|
|
6071
|
-
if (result.reason instanceof TransportError) {
|
|
6072
|
-
failed.push(replicaNodeId);
|
|
6073
|
-
continue;
|
|
6074
|
-
}
|
|
6075
|
-
throw result.reason;
|
|
6076
|
-
}
|
|
6077
|
-
const response = result.value;
|
|
6078
|
-
if (isMatchingAck(response, ackEntry)) {
|
|
6079
|
-
acknowledged.push(replicaNodeId);
|
|
6080
|
-
} else {
|
|
6081
|
-
failed.push(replicaNodeId);
|
|
6082
|
-
}
|
|
6083
|
-
}
|
|
6084
|
-
return { acknowledged, failed };
|
|
6085
|
-
}
|
|
6086
|
-
async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
|
|
6087
|
-
const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
|
|
6088
|
-
const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
|
|
6089
|
-
let lastError;
|
|
6090
|
-
for (const target of resolvedTargets) {
|
|
6091
|
-
try {
|
|
6092
|
-
return await transport.send(target, message);
|
|
6093
|
-
} catch (error) {
|
|
6094
|
-
lastError = error;
|
|
6095
|
-
}
|
|
6096
|
-
}
|
|
6097
|
-
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
6098
|
-
}
|
|
6099
|
-
function isMatchingAck(response, entry) {
|
|
6100
|
-
if (response.type !== ReplicationMessageTypes.ACK) {
|
|
6101
|
-
return false;
|
|
6102
|
-
}
|
|
6103
|
-
try {
|
|
6104
|
-
const payload = validateAckPayload(decode(response.payload));
|
|
6105
|
-
return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
|
|
6106
|
-
} catch {
|
|
6107
|
-
return false;
|
|
6108
|
-
}
|
|
6109
|
-
}
|
|
6110
|
-
|
|
6111
|
-
// src/distribution/cluster-node/write-routing/replication.ts
|
|
6112
6739
|
async function assertPrimaryWriteAuthority(entry, deps) {
|
|
6113
6740
|
const table = await deps.coordinator.getAllocation(entry.indexName);
|
|
6114
6741
|
const currentAssignment = table?.assignments.get(entry.partitionId);
|
|
@@ -6204,17 +6831,32 @@ async function requestInsyncRemovalWithTargets(indexName, partitionId, replicaNo
|
|
|
6204
6831
|
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
6205
6832
|
}
|
|
6206
6833
|
async function replicateEntry(entry, assignment, deps) {
|
|
6207
|
-
const
|
|
6834
|
+
const pending = getPendingAdmissions(deps.catchUp, entry.indexName, entry.partitionId);
|
|
6835
|
+
const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId, pending);
|
|
6208
6836
|
const result = await replicateToReplicas(entry, replicaTargets, deps.transport, deps.nodeId, deps.resolveNodeTargets);
|
|
6209
|
-
|
|
6837
|
+
const inSyncFailures = cancelAdmissionsForFailures(entry, result.failed, assignment, deps);
|
|
6838
|
+
await removeFailedReplicasFromInsync(entry, inSyncFailures, deps);
|
|
6210
6839
|
await assertPrimaryWriteAuthority(entry, deps);
|
|
6840
|
+
deps.getReplicationLog(entry.indexName, entry.partitionId).advanceCommitPoint(entry.seqNo);
|
|
6841
|
+
}
|
|
6842
|
+
function cancelAdmissionsForFailures(entry, failedReplicas, assignment, deps) {
|
|
6843
|
+
const inSyncFailures = [];
|
|
6844
|
+
for (const replicaNodeId of failedReplicas) {
|
|
6845
|
+
if (assignment.inSyncSet.includes(replicaNodeId)) {
|
|
6846
|
+
inSyncFailures.push(replicaNodeId);
|
|
6847
|
+
continue;
|
|
6848
|
+
}
|
|
6849
|
+
clearPendingAdmission(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId);
|
|
6850
|
+
}
|
|
6851
|
+
return inSyncFailures;
|
|
6211
6852
|
}
|
|
6212
6853
|
async function replicateEntryBatch(entries, assignment, deps) {
|
|
6213
6854
|
if (entries.length === 0) {
|
|
6214
6855
|
return;
|
|
6215
6856
|
}
|
|
6216
6857
|
const lastEntry = entries[entries.length - 1];
|
|
6217
|
-
const
|
|
6858
|
+
const pending = getPendingAdmissions(deps.catchUp, lastEntry.indexName, lastEntry.partitionId);
|
|
6859
|
+
const replicaTargets = getInSyncReplicaTargets(assignment, deps.nodeId, pending);
|
|
6218
6860
|
const result = await replicateBatchToReplicas(
|
|
6219
6861
|
entries,
|
|
6220
6862
|
replicaTargets,
|
|
@@ -6222,8 +6864,10 @@ async function replicateEntryBatch(entries, assignment, deps) {
|
|
|
6222
6864
|
deps.nodeId,
|
|
6223
6865
|
deps.resolveNodeTargets
|
|
6224
6866
|
);
|
|
6225
|
-
|
|
6867
|
+
const inSyncFailures = cancelAdmissionsForFailures(lastEntry, result.failed, assignment, deps);
|
|
6868
|
+
await removeFailedReplicasFromInsync(lastEntry, inSyncFailures, deps);
|
|
6226
6869
|
await assertPrimaryWriteAuthority(lastEntry, deps);
|
|
6870
|
+
deps.getReplicationLog(lastEntry.indexName, lastEntry.partitionId).advanceCommitPoint(lastEntry.seqNo);
|
|
6227
6871
|
}
|
|
6228
6872
|
function chunkReplicationEntries(items) {
|
|
6229
6873
|
return chunkByBudget(items, {
|
|
@@ -7174,17 +7818,17 @@ async function handleForwardBatch(message, respond, deps) {
|
|
|
7174
7818
|
async function handleReplicationEntry(message, respond, deps) {
|
|
7175
7819
|
const payload = validateEntryPayload(decode(message.payload));
|
|
7176
7820
|
const { entry } = payload;
|
|
7177
|
-
|
|
7178
|
-
await applyEntryToLog(entry,
|
|
7821
|
+
await resolveValidatedAssignment(entry, message.sourceId, deps);
|
|
7822
|
+
await applyEntryToLog(entry, deps);
|
|
7179
7823
|
await respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
|
|
7180
7824
|
}
|
|
7181
7825
|
async function handleReplicationEntryBatch(message, respond, deps) {
|
|
7182
7826
|
const payload = validateEntryBatchPayload(decode(message.payload));
|
|
7183
7827
|
const firstEntry = payload.entries[0];
|
|
7184
7828
|
const lastEntry = payload.entries[payload.entries.length - 1];
|
|
7185
|
-
|
|
7829
|
+
await resolveValidatedAssignment(firstEntry, message.sourceId, deps);
|
|
7186
7830
|
for (const entry of payload.entries) {
|
|
7187
|
-
await applyEntryToLog(entry,
|
|
7831
|
+
await applyEntryToLog(entry, deps);
|
|
7188
7832
|
}
|
|
7189
7833
|
await respond(
|
|
7190
7834
|
createAckMessage(lastEntry.seqNo, lastEntry.partitionId, lastEntry.indexName, deps.nodeId, message.requestId)
|
|
@@ -7219,23 +7863,23 @@ async function resolveValidatedAssignment(entry, sourceNodeId, deps) {
|
|
|
7219
7863
|
}
|
|
7220
7864
|
);
|
|
7221
7865
|
}
|
|
7222
|
-
if (
|
|
7866
|
+
if (entry.primaryTerm > assignment.primaryTerm) {
|
|
7223
7867
|
throw new NarsilError(
|
|
7224
7868
|
ErrorCodes.REPLICATION_TERM_MISMATCH,
|
|
7225
|
-
`Replication entry term ${entry.primaryTerm}
|
|
7869
|
+
`Replication entry term ${entry.primaryTerm} is newer than allocation term ${assignment.primaryTerm}`,
|
|
7226
7870
|
{ indexName: entry.indexName, partitionId: entry.partitionId }
|
|
7227
7871
|
);
|
|
7228
7872
|
}
|
|
7229
|
-
if (!assignment.replicas.includes(deps.nodeId)
|
|
7873
|
+
if (!assignment.replicas.includes(deps.nodeId)) {
|
|
7230
7874
|
throw new NarsilError(
|
|
7231
7875
|
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
7232
|
-
`Node '${deps.nodeId}' is not an
|
|
7876
|
+
`Node '${deps.nodeId}' is not an assigned replica for index '${entry.indexName}' partition ${entry.partitionId}`,
|
|
7233
7877
|
{ indexName: entry.indexName, partitionId: entry.partitionId, nodeId: deps.nodeId }
|
|
7234
7878
|
);
|
|
7235
7879
|
}
|
|
7236
7880
|
return assignment;
|
|
7237
7881
|
}
|
|
7238
|
-
async function applyEntryToLog(entry,
|
|
7882
|
+
async function applyEntryToLog(entry, deps) {
|
|
7239
7883
|
let log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
|
|
7240
7884
|
const existing = log.getEntry(entry.seqNo);
|
|
7241
7885
|
if (existing !== void 0) {
|
|
@@ -7261,7 +7905,7 @@ async function applyEntryToLog(entry, assignment, deps) {
|
|
|
7261
7905
|
deps.writeDeps.resetReplicationLog(entry.indexName, entry.partitionId, entry.seqNo, entry.primaryTerm);
|
|
7262
7906
|
log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
|
|
7263
7907
|
}
|
|
7264
|
-
const validation = validateReplicationEntry(entry,
|
|
7908
|
+
const validation = validateReplicationEntry(entry, log.localLogEndPrimaryTerm, log);
|
|
7265
7909
|
if (!validation.valid) {
|
|
7266
7910
|
throw new NarsilError(
|
|
7267
7911
|
validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
@@ -8652,16 +9296,17 @@ function createClusterNodeOperations(deps) {
|
|
|
8652
9296
|
var DEFAULT_LOG_RETENTION_BYTES = 268435456;
|
|
8653
9297
|
|
|
8654
9298
|
// src/distribution/replication/log.ts
|
|
8655
|
-
var
|
|
9299
|
+
var ENTRY_FIXED_OVERHEAD_BYTES2 = 40;
|
|
8656
9300
|
function estimateEntrySize(entry) {
|
|
8657
|
-
return
|
|
9301
|
+
return ENTRY_FIXED_OVERHEAD_BYTES2 + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
8658
9302
|
}
|
|
8659
9303
|
var computeChecksum = computeEntryChecksum;
|
|
8660
9304
|
function createReplicationLog(partitionId, config) {
|
|
8661
9305
|
const retentionBytes = config?.logRetentionBytes ?? DEFAULT_LOG_RETENTION_BYTES;
|
|
8662
9306
|
let nextSeqNo = config?.startSeqNo ?? 1;
|
|
8663
|
-
let
|
|
8664
|
-
let
|
|
9307
|
+
let localLogEnd = nextSeqNo - 1;
|
|
9308
|
+
let localLogEndPrimaryTerm = config?.lastPrimaryTerm ?? 0;
|
|
9309
|
+
let commitPoint = nextSeqNo - 1;
|
|
8665
9310
|
let entries = [];
|
|
8666
9311
|
let headIndex = 0;
|
|
8667
9312
|
let totalSizeBytes = 0;
|
|
@@ -8704,7 +9349,7 @@ function createReplicationLog(partitionId, config) {
|
|
|
8704
9349
|
}
|
|
8705
9350
|
}
|
|
8706
9351
|
function assertEntryFitsRetention(entry) {
|
|
8707
|
-
const entrySize =
|
|
9352
|
+
const entrySize = ENTRY_FIXED_OVERHEAD_BYTES2 + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
8708
9353
|
if (entrySize > retentionBytes) {
|
|
8709
9354
|
throw new NarsilError(
|
|
8710
9355
|
ErrorCodes.REPLICATION_LOG_FULL,
|
|
@@ -8715,9 +9360,9 @@ function createReplicationLog(partitionId, config) {
|
|
|
8715
9360
|
}
|
|
8716
9361
|
function storeEntry(entry) {
|
|
8717
9362
|
entries.push(entry);
|
|
8718
|
-
if (entry.seqNo >=
|
|
8719
|
-
|
|
8720
|
-
|
|
9363
|
+
if (entry.seqNo >= localLogEnd) {
|
|
9364
|
+
localLogEnd = entry.seqNo;
|
|
9365
|
+
localLogEndPrimaryTerm = entry.primaryTerm;
|
|
8721
9366
|
}
|
|
8722
9367
|
totalSizeBytes += estimateEntrySize(entry);
|
|
8723
9368
|
evictOldEntries();
|
|
@@ -8820,11 +9465,20 @@ function createReplicationLog(partitionId, config) {
|
|
|
8820
9465
|
verifyChecksum(entry) {
|
|
8821
9466
|
return verifyEntryChecksum(entry);
|
|
8822
9467
|
},
|
|
8823
|
-
get
|
|
8824
|
-
return
|
|
9468
|
+
get localLogEnd() {
|
|
9469
|
+
return localLogEnd;
|
|
9470
|
+
},
|
|
9471
|
+
get localLogEndPrimaryTerm() {
|
|
9472
|
+
return localLogEndPrimaryTerm;
|
|
8825
9473
|
},
|
|
8826
|
-
get
|
|
8827
|
-
return
|
|
9474
|
+
get commitPoint() {
|
|
9475
|
+
return commitPoint;
|
|
9476
|
+
},
|
|
9477
|
+
advanceCommitPoint(seqNo) {
|
|
9478
|
+
const bounded = Math.min(seqNo, localLogEnd);
|
|
9479
|
+
if (bounded > commitPoint) {
|
|
9480
|
+
commitPoint = bounded;
|
|
9481
|
+
}
|
|
8828
9482
|
},
|
|
8829
9483
|
get oldestSeqNo() {
|
|
8830
9484
|
if (liveCount() === 0) return void 0;
|
|
@@ -8844,7 +9498,8 @@ function createReplicationLog(partitionId, config) {
|
|
|
8844
9498
|
entries = [];
|
|
8845
9499
|
headIndex = 0;
|
|
8846
9500
|
totalSizeBytes = 0;
|
|
8847
|
-
|
|
9501
|
+
localLogEnd = nextSeqNo - 1;
|
|
9502
|
+
commitPoint = nextSeqNo - 1;
|
|
8848
9503
|
}
|
|
8849
9504
|
};
|
|
8850
9505
|
}
|
|
@@ -8875,6 +9530,7 @@ function seedReplicationLog(replicationLogs, indexName, partitionId, startSeqNo,
|
|
|
8875
9530
|
|
|
8876
9531
|
// src/distribution/cluster-node/transport-listener.ts
|
|
8877
9532
|
var CONTROLLER_MESSAGE_TYPES = /* @__PURE__ */ new Set([
|
|
9533
|
+
ReplicationMessageTypes.INSYNC_ADD,
|
|
8878
9534
|
ReplicationMessageTypes.INSYNC_REMOVE,
|
|
8879
9535
|
ClusterMessageTypes.BOOTSTRAP_COMPLETE
|
|
8880
9536
|
]);
|
|
@@ -9022,6 +9678,7 @@ async function createClusterNode(config) {
|
|
|
9022
9678
|
getReplicationLog: getReplicationLog2,
|
|
9023
9679
|
resetReplicationLog: seedReplicationLog2,
|
|
9024
9680
|
partitionWriteQueues: createPartitionWriteQueues(),
|
|
9681
|
+
catchUp: createCatchUpState(),
|
|
9025
9682
|
resolveNodeTargets: resolveNodeTargets3,
|
|
9026
9683
|
waitForActiveReplicas: config.replication?.waitForActiveReplicas
|
|
9027
9684
|
};
|
|
@@ -9092,6 +9749,7 @@ async function createClusterNode(config) {
|
|
|
9092
9749
|
bootstrapRetryMaxMs: DEFAULT_NODE_LIFECYCLE_CONFIG.bootstrapRetryMaxMs,
|
|
9093
9750
|
bootstrapMaxRetries: DEFAULT_NODE_LIFECYCLE_CONFIG.bootstrapMaxRetries,
|
|
9094
9751
|
allocationDebounceMs: DEFAULT_NODE_LIFECYCLE_CONFIG.allocationDebounceMs,
|
|
9752
|
+
nodeHeartbeatIntervalMs: DEFAULT_NODE_LIFECYCLE_CONFIG.nodeHeartbeatIntervalMs,
|
|
9095
9753
|
onBootstrapPartition: bootstrapPartition2,
|
|
9096
9754
|
onRemovePartition: (indexName, partitionId) => {
|
|
9097
9755
|
clearBootstrapSyncIndex(bootstrapSyncState, indexName, partitionId);
|
|
@@ -9203,9 +9861,11 @@ async function createClusterNode(config) {
|
|
|
9203
9861
|
});
|
|
9204
9862
|
const listener = controllerTransport !== null ? controllerTransport.createHandler(dataHandler) : dataHandler;
|
|
9205
9863
|
unregisterHandler = await config.transport.listen(listener);
|
|
9864
|
+
startCatchUpPump(writeDeps.catchUp, writeDeps);
|
|
9206
9865
|
}
|
|
9207
9866
|
},
|
|
9208
9867
|
async shutdown() {
|
|
9868
|
+
await stopCatchUpPump(writeDeps.catchUp);
|
|
9209
9869
|
if (isShutdown) {
|
|
9210
9870
|
return;
|
|
9211
9871
|
}
|