@delali/narsil 0.3.1-next.103 → 0.3.1-next.107
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-RHG4O77A.mjs → chunk-J3QB4LBD.mjs} +1 -0
- package/dist/browser/{chunk-6DHYBN7M.mjs → chunk-JJKVYLL3.mjs} +3 -2
- package/dist/browser/{chunk-CRNQAT2Q.mjs → chunk-NNQPZPTB.mjs} +5 -5
- package/dist/browser/{chunk-MM7WIDI5.mjs → chunk-OKZGAJEF.mjs} +111 -10
- package/dist/browser/{chunk-AQZDOKDT.mjs → chunk-OL7S2KLL.mjs} +2 -2
- package/dist/browser/{chunk-74OXU3R2.mjs → chunk-PA56PRWX.mjs} +2 -2
- package/dist/browser/{chunk-YMDH3JID.mjs → chunk-WZUSMF7A.mjs} +13 -13
- package/dist/browser/{durability-wiring-SGY7CZHD.mjs → durability-wiring-HFQULBSA.mjs} +6 -6
- package/dist/browser/{module-QJ55JCJ6.mjs → module-VROKZ7IZ.mjs} +20 -16
- package/dist/browser/{restore-queue-LPKB6QJR.mjs → restore-queue-EU77FRCV.mjs} +1 -1
- package/dist/browser/{runtime-7ZAHCHWR.mjs → runtime-JQYPNOK6.mjs} +1 -1
- package/dist/browser/{snapshot-R7GY6PAG.mjs → snapshot-G77LQBZX.mjs} +7 -7
- package/dist/browser/{write-GFFRUCJD.mjs → write-RUC64JIH.mjs} +5 -5
- package/dist/{capabilities-ByQRMOx-.d.ts → capabilities-ByKPha_q.d.ts} +2 -2
- package/dist/{chunk-PC2G76VW.mjs → chunk-2MNJ2HX7.mjs} +2 -2
- package/dist/{chunk-EMRMGYRE.mjs → chunk-3BRFOMGX.mjs} +2 -2
- package/dist/{chunk-LPQOEU5M.mjs → chunk-5FPMDU54.mjs} +38 -9
- package/dist/{chunk-BWJVNZPR.mjs → chunk-6JUOMMHQ.mjs} +1 -0
- package/dist/{chunk-LKOB5COA.mjs → chunk-77SO7G55.mjs} +1 -1
- package/dist/{chunk-JTNDFNDP.mjs → chunk-7IBX2LRA.mjs} +12 -28
- package/dist/{chunk-UTG5YNXV.mjs → chunk-AEF2O7B4.mjs} +1 -1
- package/dist/{chunk-JZVARCRZ.mjs → chunk-BRIOLMIX.mjs} +2 -2
- package/dist/{chunk-YVDYMXR2.mjs → chunk-CKNOGER5.mjs} +22 -22
- package/dist/{chunk-5WT7BKQZ.mjs → chunk-CYHQNM3T.mjs} +1 -1
- package/dist/{chunk-U4436DZG.mjs → chunk-DNKM67DF.mjs} +2 -2
- package/dist/chunk-DUWYU4D5.mjs +19 -0
- package/dist/{chunk-ZKWYWW4Y.mjs → chunk-FABI2JA7.mjs} +1 -1
- package/dist/{chunk-VI4R6W25.mjs → chunk-FEOLYA2Y.mjs} +6 -6
- package/dist/{chunk-BQ2JSVAG.mjs → chunk-G6YATIRT.mjs} +113 -10
- package/dist/chunk-HAQUJYMN.mjs +344 -0
- package/dist/{chunk-TZRWRIZ6.mjs → chunk-HNAQNAFS.mjs} +3 -3
- package/dist/{chunk-64MXRJZ7.mjs → chunk-IA3H54X4.mjs} +2 -2
- package/dist/{chunk-TXV375YK.mjs → chunk-IA6I5U36.mjs} +1 -1
- package/dist/{chunk-SDFIFGOL.mjs → chunk-IZLM2OOO.mjs} +4 -4
- package/dist/{chunk-2RIRGOUL.mjs → chunk-JSSGO4B5.mjs} +1 -1
- package/dist/{chunk-5JNA5SKS.mjs → chunk-KOCHOOS5.mjs} +177 -8
- package/dist/{chunk-XKLUOODW.mjs → chunk-M5EZVXIM.mjs} +2 -2
- package/dist/{chunk-YEEQHT4V.mjs → chunk-MJIJG4YT.mjs} +2 -2
- package/dist/{chunk-75XIEF6H.mjs → chunk-MZW4V4DS.mjs} +1 -1
- package/dist/{chunk-IQYUFGQ6.mjs → chunk-NRARRUKN.mjs} +7 -7
- package/dist/{chunk-S7FL6UAH.mjs → chunk-OKZPYJZX.mjs} +3 -3
- package/dist/{chunk-HMPYTBBN.mjs → chunk-QA6F5MML.mjs} +6 -6
- package/dist/{chunk-ZM7KU4YW.mjs → chunk-QALAFGF5.mjs} +1 -1
- package/dist/{chunk-46D6JQIG.mjs → chunk-RI5VNFAI.mjs} +2 -2
- package/dist/{chunk-PNEYIHA4.mjs → chunk-RPIE4QCL.mjs} +0 -1
- package/dist/{chunk-XRTFWD3E.mjs → chunk-UM6Z7AIP.mjs} +3 -3
- package/dist/{chunk-YYQIODMF.mjs → chunk-VHPVBJOI.mjs} +2 -2
- package/dist/{chunk-HMFNFVP7.mjs → chunk-VJS4JFBQ.mjs} +2 -1
- package/dist/{chunk-RFVUDM5E.mjs → chunk-VVDQNCOG.mjs} +2 -15
- package/dist/{chunk-OEYI3KRB.mjs → chunk-VVFKFHWE.mjs} +3 -3
- package/dist/{chunk-AHVWSZTY.mjs → chunk-VWFFCDOL.mjs} +6 -6
- package/dist/{chunk-SM7S3HAP.mjs → chunk-WCJJCBR4.mjs} +9 -9
- package/dist/{chunk-EWZLIYHV.mjs → chunk-WCNN2PLX.mjs} +1 -1
- package/dist/{chunk-OM362BRO.mjs → chunk-WXVTQ6NO.mjs} +3 -3
- package/dist/chunk-ZXZ26WSO.mjs +14 -0
- package/dist/{client-BOStAkc5.d.ts → client-DI-xIXgR.d.ts} +2 -2
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +1 -0
- package/dist/{config-H6SGzbPH.d.ts → config-BZ_e3X8w.d.ts} +8 -5
- package/dist/{config-Cjvzbs_j.d.ts → config-DL_bDDD8.d.ts} +1 -1
- package/dist/direct-executor-BPBD2PBK.mjs +25 -0
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +3 -1
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +5 -4
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +3 -2
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +4 -3
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +1818 -2425
- package/dist/{durability-wiring-2LOZIIJN.mjs → durability-wiring-AZL4EFLO.mjs} +27 -26
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-rC0vjjY-.d.ts → engine-D_SYDsId.d.ts} +1 -1
- package/dist/{events-DqOIg5xa.d.ts → events-C7_dboUr.d.ts} +1 -0
- package/dist/{index-Bkdt8nDw.d.ts → index-Cl5qQauj.d.ts} +6 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +19 -18
- package/dist/{module-H37P44S6.mjs → module-6EY7GW2W.mjs} +25 -21
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -22
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/{rerank-K7C4WZFG.mjs → rerank-WVFFZV76.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-KVCG36SI.mjs → restore-queue-DDXFDP7U.mjs} +1 -1
- package/dist/{runtime-6Y4ZYXOG.mjs → runtime-AVUUCJWQ.mjs} +5 -5
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +30 -17
- package/dist/{snapshot-YQ2CIFOZ.mjs → snapshot-IDAM65JO.mjs} +29 -28
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-CwPTz-tP.d.ts → tasks-BhjQZ8Qf.d.ts} +1 -1
- package/dist/{thread-5IYXJDWO.mjs → thread-LMSYLCPI.mjs} +16 -14
- package/dist/{types-2qoitTXg.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/{types-Cz5okLml.d.ts → types-DvncmvuQ.d.ts} +11 -0
- package/dist/vector/search-worker.d.ts +2 -0
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -6
- package/dist/{write-7UCWA3GX.mjs → write-3LIFIGOL.mjs} +21 -20
- package/package.json +7 -7
- package/dist/chunk-747OHOUA.mjs +0 -11
- package/dist/chunk-BOFBLJFZ.mjs +0 -209
- package/dist/direct-executor-QG7LEP6P.mjs +0 -24
package/dist/distribution.mjs
CHANGED
|
@@ -1,59 +1,60 @@
|
|
|
1
|
-
import { DEFAULT_NODE_LIFECYCLE_CONFIG, bindWriteBlockLookup, bindEngineWriteBlock, bindIndexReads, DEFAULT_CONTROLLER_CONFIG, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, readIndexState, readIndexMetadataSnapshot, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, indexConfigFromSettings, metadataWithSynonymSet, indexConfigKey, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_ROUNDS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-
|
|
2
|
-
import {
|
|
3
|
-
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-
|
|
4
|
-
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-
|
|
5
|
-
import { closePartitionWriter, replaySyncedWal } from './chunk-
|
|
6
|
-
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-
|
|
7
|
-
import './chunk-
|
|
8
|
-
import { indexMetadataOf, loadPartitionSegments, readSegmentManifest, readVectorFile } from './chunk-
|
|
9
|
-
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-
|
|
1
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, bindWriteBlockLookup, bindEngineWriteBlock, bindIndexReads, DEFAULT_CONTROLLER_CONFIG, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, readIndexState, readIndexMetadataSnapshot, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, indexConfigFromSettings, metadataWithSynonymSet, indexConfigKey, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_ROUNDS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-OKZPYJZX.mjs';
|
|
2
|
+
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, DEFAULT_RESUME_WRITES_BELOW, DEFAULT_BLOCK_WRITES_ABOVE, readMemoryFigures, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, onRefusalAfterCollection, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, refuseEncodeWithoutRoom, readRecentMemoryFigures, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, MAX_SYNONYM_REPLACEMENT_ATTEMPTS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, DECOMMISSION_RECHECK_INTERVAL_MS, withAdmittedWrite, estimateForwardedBytes, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, MAX_SOURCE_ID_LENGTH, reserveHeapBytes, reservedHeapBytes, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, ADMISSION_TIMEOUT_MS, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, COPY_FAILURE_RETRY_DELAY_MS, FETCH_GROUP_DOCUMENT_IDS, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, FULL_MEMORY_COLLECTION_INTERVAL_MS, MEMORY_SAMPLE_INTERVAL_MS, answerRefusalOf, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, PRIMARY_CHANGE_RETRY_WINDOW_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-HAQUJYMN.mjs';
|
|
3
|
+
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-HAQUJYMN.mjs';
|
|
4
|
+
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-WCNN2PLX.mjs';
|
|
5
|
+
import { closePartitionWriter, replaySyncedWal } from './chunk-UM6Z7AIP.mjs';
|
|
6
|
+
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-VWFFCDOL.mjs';
|
|
7
|
+
import './chunk-HNAQNAFS.mjs';
|
|
8
|
+
import { indexMetadataOf, loadPartitionSegments, readSegmentManifest, readVectorFile } from './chunk-QA6F5MML.mjs';
|
|
9
|
+
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-VHPVBJOI.mjs';
|
|
10
10
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
11
11
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-G6YATIRT.mjs';
|
|
13
13
|
import './chunk-DJHHF2L5.mjs';
|
|
14
14
|
import './chunk-37CD3OKS.mjs';
|
|
15
15
|
import { getPath, getFs } from './chunk-HLPRHOSO.mjs';
|
|
16
|
-
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, readListedFile, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, directoryCheckpointStore, checkpointLastSeqNo, checkpointPrimaryTerm,
|
|
17
|
-
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-
|
|
16
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, readListedFile, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, directoryCheckpointStore, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-WCJJCBR4.mjs';
|
|
17
|
+
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-IZLM2OOO.mjs';
|
|
18
18
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
19
|
-
import { TransportErrorCodes, ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-
|
|
20
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createByteReader, openChecksummedEnvelope, emptySerializablePartition, crc32Init, crc32Final, crc32Update, yieldToEventLoop } from './chunk-
|
|
21
|
-
import { runtimeHost, requireSurfaceFormsForPartialFields, queryBindingOf, now, listBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-
|
|
22
|
-
import { bindRerankWindowSource, rerankWindowCapOf, engineCoreOf, providedDocId } from './chunk-
|
|
23
|
-
import './chunk-
|
|
19
|
+
import { TransportErrorCodes, ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-RPIE4QCL.mjs';
|
|
20
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createByteReader, openChecksummedEnvelope, emptySerializablePartition, crc32Init, crc32Final, crc32Update, yieldToEventLoop } from './chunk-CKNOGER5.mjs';
|
|
21
|
+
import { runtimeHost, requireSurfaceFormsForPartialFields, queryBindingOf, now, listBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-NRARRUKN.mjs';
|
|
22
|
+
import { bindRerankWindowSource, rerankWindowCapOf, engineCoreOf, providedDocId } from './chunk-3BRFOMGX.mjs';
|
|
23
|
+
import './chunk-CYHQNM3T.mjs';
|
|
24
24
|
import './chunk-AV4IJKD2.mjs';
|
|
25
|
-
import { synonymModule } from './chunk-
|
|
26
|
-
import { oversampledShardSize, normalizeSort, sortModesOf, readSortValues, resolveProjection, applyProjection, fnv1a, FILTER_EXPRESSION_KEYS, filterTestCount, FIELD_FILTER_OPERATORS, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, encodePageCursor, requireWithinResultWindow, sortSignatureEntry, fnv1aUint32s, mergePartitionStats } from './chunk-
|
|
27
|
-
import { generateId, getLanguage } from './chunk-
|
|
28
|
-
import './chunk-
|
|
25
|
+
import { synonymModule } from './chunk-7IBX2LRA.mjs';
|
|
26
|
+
import { oversampledShardSize, normalizeSort, sortModesOf, readSortValues, resolveProjection, applyProjection, fnv1a, FILTER_EXPRESSION_KEYS, filterTestCount, FIELD_FILTER_OPERATORS, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, encodePageCursor, requireWithinResultWindow, sortSignatureEntry, fnv1aUint32s, mergePartitionStats } from './chunk-FEOLYA2Y.mjs';
|
|
27
|
+
import { generateId, getLanguage } from './chunk-QALAFGF5.mjs';
|
|
28
|
+
import './chunk-5FPMDU54.mjs';
|
|
29
29
|
import './chunk-HJSN7UYG.mjs';
|
|
30
|
-
import {
|
|
30
|
+
import { MAX_DOC_ID_LENGTH, BATCH_CHUNK_SIZE, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING } from './chunk-LXWDQLTW.mjs';
|
|
31
31
|
import './chunk-TGP5M5R7.mjs';
|
|
32
32
|
import './chunk-U2BKYEC7.mjs';
|
|
33
33
|
import './chunk-KOKM4PQN.mjs';
|
|
34
34
|
import { MIN_OVERSAMPLE } from './chunk-QSK35UC2.mjs';
|
|
35
|
-
import './chunk-
|
|
36
|
-
import { isPatternTolerance } from './chunk-
|
|
35
|
+
import './chunk-BRIOLMIX.mjs';
|
|
36
|
+
import { isPatternTolerance } from './chunk-2MNJ2HX7.mjs';
|
|
37
37
|
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
38
|
-
import { getNestedValue } from './chunk-
|
|
39
|
-
import { validateSchema, requireCreatableSchema, encodeDocumentReply, encodeDocument, schemaFieldsOf, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-
|
|
38
|
+
import { getNestedValue } from './chunk-WXVTQ6NO.mjs';
|
|
39
|
+
import { validateSchema, requireCreatableSchema, encodeDocumentReply, encodeDocument, schemaFieldsOf, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-VVFKFHWE.mjs';
|
|
40
40
|
import { MAX_DOCUMENT_ENCODING_DEPTH } from './chunk-U476IXKQ.mjs';
|
|
41
41
|
import { MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, DEFAULT_PAGE_SIZE, HYBRID_RESCORE_WINDOW } from './chunk-E74YLAIG.mjs';
|
|
42
|
-
import './chunk-
|
|
42
|
+
import './chunk-VJS4JFBQ.mjs';
|
|
43
43
|
import './chunk-GES7U2PM.mjs';
|
|
44
|
-
import './chunk-
|
|
45
|
-
import { resolveIndexAnalysis } from './chunk-
|
|
44
|
+
import './chunk-77SO7G55.mjs';
|
|
45
|
+
import { resolveIndexAnalysis } from './chunk-RI5VNFAI.mjs';
|
|
46
46
|
import './chunk-X5CV2E2B.mjs';
|
|
47
47
|
import './chunk-7X5JST5L.mjs';
|
|
48
|
-
import './chunk-
|
|
48
|
+
import './chunk-AEF2O7B4.mjs';
|
|
49
49
|
import { isSortMode, SORT_MODES, toComparableSortValue, compareComparableKeys, compareSortValues } from './chunk-R6A773WK.mjs';
|
|
50
50
|
import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
51
|
-
import {
|
|
51
|
+
import { collectAndReturnPages } from './chunk-VVDQNCOG.mjs';
|
|
52
52
|
import './chunk-CQUMNIIG.mjs';
|
|
53
|
-
import './chunk-
|
|
53
|
+
import './chunk-IA6I5U36.mjs';
|
|
54
|
+
import { HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE } from './chunk-ZXZ26WSO.mjs';
|
|
54
55
|
import './chunk-RFBLQLLZ.mjs';
|
|
55
56
|
import { MAX_STORED_VECTOR_DIMENSION, MAX_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
56
|
-
import { ErrorCodes, NarsilError, describeError } from './chunk-
|
|
57
|
+
import { ErrorCodes, NarsilError, describeError } from './chunk-6JUOMMHQ.mjs';
|
|
57
58
|
import { encode, decode } from '@msgpack/msgpack';
|
|
58
59
|
import { randomUUID } from 'crypto';
|
|
59
60
|
import { openSync, writeSync, closeSync, rmSync, readSync } from 'fs';
|
|
@@ -1163,300 +1164,534 @@ async function nodeHasOtherPartitions(indexName, removedPartitionId, deps) {
|
|
|
1163
1164
|
return false;
|
|
1164
1165
|
}
|
|
1165
1166
|
|
|
1166
|
-
// src/distribution/
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
}
|
|
1174
|
-
function releaseSpool(state) {
|
|
1175
|
-
const spool = state.spool;
|
|
1176
|
-
if (spool === null) return;
|
|
1177
|
-
state.spool = null;
|
|
1178
|
-
spool.discard();
|
|
1179
|
-
}
|
|
1180
|
-
function spoolFailure(state, err) {
|
|
1181
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1182
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `the snapshot could not be written to local storage: ${cause}`, {
|
|
1183
|
-
cause
|
|
1184
|
-
});
|
|
1185
|
-
}
|
|
1186
|
-
function applyErrorEnvelope(state, record) {
|
|
1187
|
-
const code = typeof record.code === "string" && record.code.length > 0 ? record.code : "UNKNOWN_ERROR";
|
|
1188
|
-
const message = typeof record.message === "string" && record.message.length > 0 ? record.message : "primary returned error envelope";
|
|
1189
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_PRIMARY_ERROR, `primary error: ${message}`, {
|
|
1190
|
-
primaryCode: code,
|
|
1191
|
-
primaryMessage: message
|
|
1192
|
-
});
|
|
1167
|
+
// src/distribution/cluster-node/schema-diff.ts
|
|
1168
|
+
var SCHEMA_DIFF_MAX_DEPTH = MAX_NESTING_DEPTH;
|
|
1169
|
+
var DEPTH_EXCEEDED_SENTINEL = "depth-exceeded";
|
|
1170
|
+
function diffSchemas(expected, actual) {
|
|
1171
|
+
const diffs = [];
|
|
1172
|
+
walk(expected, actual, "", diffs, 1);
|
|
1173
|
+
return diffs;
|
|
1193
1174
|
}
|
|
1194
|
-
function
|
|
1195
|
-
if (
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
const header = start.header;
|
|
1201
|
-
if (header.indexName !== state.expectation.indexName) {
|
|
1202
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "snapshot header indexName mismatch", {
|
|
1203
|
-
expected: state.expectation.indexName,
|
|
1204
|
-
received: header.indexName
|
|
1205
|
-
});
|
|
1206
|
-
return;
|
|
1207
|
-
}
|
|
1208
|
-
if (state.expectation.partitionId !== null && header.partitionId !== state.expectation.partitionId) {
|
|
1209
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "snapshot header partitionId mismatch", {
|
|
1210
|
-
expected: state.expectation.partitionId,
|
|
1211
|
-
received: header.partitionId
|
|
1175
|
+
function walk(expected, actual, prefix, diffs, depth) {
|
|
1176
|
+
if (depth > SCHEMA_DIFF_MAX_DEPTH) {
|
|
1177
|
+
diffs.push({
|
|
1178
|
+
path: prefix === "" ? DEPTH_EXCEEDED_SENTINEL : prefix,
|
|
1179
|
+
expected: DEPTH_EXCEEDED_SENTINEL,
|
|
1180
|
+
actual: DEPTH_EXCEEDED_SENTINEL
|
|
1212
1181
|
});
|
|
1213
1182
|
return;
|
|
1214
1183
|
}
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1184
|
+
const expectedKeys = Object.keys(expected).sort(compareCodePoints);
|
|
1185
|
+
const actualKeys = new Set(Object.keys(actual));
|
|
1186
|
+
for (const key of expectedKeys) {
|
|
1187
|
+
const path = prefix === "" ? key : `${prefix}.${key}`;
|
|
1188
|
+
const expectedValue = expected[key];
|
|
1189
|
+
const actualValue = actual[key];
|
|
1190
|
+
if (!actualKeys.has(key)) {
|
|
1191
|
+
diffs.push({ path, expected: describe(expectedValue), actual: "(absent)" });
|
|
1192
|
+
continue;
|
|
1193
|
+
}
|
|
1194
|
+
actualKeys.delete(key);
|
|
1195
|
+
const expectedIsObject = typeof expectedValue === "object" && expectedValue !== null;
|
|
1196
|
+
const actualIsObject = typeof actualValue === "object" && actualValue !== null;
|
|
1197
|
+
if (expectedIsObject !== actualIsObject) {
|
|
1198
|
+
diffs.push({ path, expected: describe(expectedValue), actual: describe(actualValue) });
|
|
1199
|
+
continue;
|
|
1200
|
+
}
|
|
1201
|
+
if (expectedIsObject && actualIsObject) {
|
|
1202
|
+
walk(expectedValue, actualValue, path, diffs, depth + 1);
|
|
1203
|
+
continue;
|
|
1204
|
+
}
|
|
1205
|
+
if (expectedValue !== actualValue) {
|
|
1206
|
+
diffs.push({ path, expected: String(expectedValue), actual: String(actualValue) });
|
|
1207
|
+
}
|
|
1234
1208
|
}
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
spoolFailure(state, err);
|
|
1239
|
-
return;
|
|
1209
|
+
for (const extraKey of actualKeys) {
|
|
1210
|
+
const path = prefix === "" ? extraKey : `${prefix}.${extraKey}`;
|
|
1211
|
+
diffs.push({ path, expected: "(absent)", actual: describe(actual[extraKey]) });
|
|
1240
1212
|
}
|
|
1241
|
-
state.header = header;
|
|
1242
|
-
state.totalBytes = start.totalBytes;
|
|
1243
1213
|
}
|
|
1244
|
-
function
|
|
1245
|
-
if (
|
|
1246
|
-
|
|
1247
|
-
return;
|
|
1248
|
-
}
|
|
1249
|
-
if (!isSafeNonNegativeInteger(payload.offset)) {
|
|
1250
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHUNK_OUT_OF_ORDER, "chunk offset must be a non-negative integer");
|
|
1251
|
-
return;
|
|
1214
|
+
function describe(value) {
|
|
1215
|
+
if (value === null) {
|
|
1216
|
+
return "null";
|
|
1252
1217
|
}
|
|
1253
|
-
if (
|
|
1254
|
-
|
|
1255
|
-
return;
|
|
1218
|
+
if (typeof value === "object") {
|
|
1219
|
+
return "object";
|
|
1256
1220
|
}
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1221
|
+
return String(value);
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
// src/distribution/cluster-node/bootstrap-restore.ts
|
|
1225
|
+
var ABORT_SENTINEL = /* @__PURE__ */ Symbol("bootstrap-sync-aborted");
|
|
1226
|
+
function validateArguments(indexName, partitionId, primaryNodeId) {
|
|
1227
|
+
if (typeof indexName !== "string" || indexName.length === 0) {
|
|
1228
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, "indexName must be a non-empty string", {
|
|
1229
|
+
indexName
|
|
1261
1230
|
});
|
|
1262
|
-
return;
|
|
1263
1231
|
}
|
|
1264
|
-
if (
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1232
|
+
if (!Number.isInteger(partitionId) || partitionId < 0) {
|
|
1233
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, "partitionId must be a non-negative integer", {
|
|
1234
|
+
indexName,
|
|
1235
|
+
partitionId
|
|
1268
1236
|
});
|
|
1269
|
-
return;
|
|
1270
1237
|
}
|
|
1271
|
-
if (
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
received: payload.offset
|
|
1238
|
+
if (typeof primaryNodeId !== "string" || primaryNodeId.length === 0) {
|
|
1239
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, "primaryNodeId must be a non-empty string", {
|
|
1240
|
+
indexName
|
|
1275
1241
|
});
|
|
1276
|
-
return;
|
|
1277
|
-
}
|
|
1278
|
-
if (!(payload.data instanceof Uint8Array)) {
|
|
1279
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "chunk data is not Uint8Array");
|
|
1280
|
-
return;
|
|
1281
1242
|
}
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1243
|
+
return null;
|
|
1244
|
+
}
|
|
1245
|
+
async function resolveTransportTargets(indexName, primaryNodeId, resolveNodeTargets3) {
|
|
1246
|
+
let targets;
|
|
1247
|
+
try {
|
|
1248
|
+
targets = await resolveNodeTargets3(primaryNodeId);
|
|
1249
|
+
} catch (err) {
|
|
1250
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
1251
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_NO_TARGETS, `resolveNodeTargets failed: ${cause}`, {
|
|
1252
|
+
indexName,
|
|
1253
|
+
primaryNodeId,
|
|
1254
|
+
cause
|
|
1288
1255
|
});
|
|
1289
|
-
return;
|
|
1290
1256
|
}
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
return;
|
|
1257
|
+
if (targets.length === 0) {
|
|
1258
|
+
return new NarsilError(
|
|
1259
|
+
ErrorCodes.SNAPSHOT_SYNC_NO_TARGETS,
|
|
1260
|
+
`no transport targets resolved for primary '${primaryNodeId}'`,
|
|
1261
|
+
{ indexName, primaryNodeId }
|
|
1262
|
+
);
|
|
1298
1263
|
}
|
|
1299
|
-
|
|
1300
|
-
|
|
1264
|
+
return targets;
|
|
1265
|
+
}
|
|
1266
|
+
function surfaceError(deps, indexName, primaryNodeId, error) {
|
|
1267
|
+
if (deps.onError === void 0) {
|
|
1301
1268
|
return;
|
|
1302
1269
|
}
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1270
|
+
const wrapped = new NarsilError(
|
|
1271
|
+
ErrorCodes.NODE_BOOTSTRAP_FAILED,
|
|
1272
|
+
`Bootstrap sync failed for index '${indexName}': ${error.message}`,
|
|
1273
|
+
{
|
|
1274
|
+
indexName,
|
|
1275
|
+
primaryNodeId,
|
|
1276
|
+
innerCode: error.code,
|
|
1277
|
+
...error.details
|
|
1278
|
+
}
|
|
1279
|
+
);
|
|
1280
|
+
deps.onError(wrapped);
|
|
1281
|
+
}
|
|
1282
|
+
function surfaceAborted(deps, indexName, primaryNodeId) {
|
|
1283
|
+
if (deps.onError === void 0) {
|
|
1307
1284
|
return;
|
|
1308
1285
|
}
|
|
1309
|
-
|
|
1310
|
-
|
|
1286
|
+
deps.onError(
|
|
1287
|
+
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, `bootstrap sync aborted for index '${indexName}'`, {
|
|
1288
|
+
indexName,
|
|
1289
|
+
primaryNodeId
|
|
1290
|
+
})
|
|
1291
|
+
);
|
|
1311
1292
|
}
|
|
1312
|
-
function
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1293
|
+
function validateRestoredSchema(engine, indexName, primaryNodeId, coordinatorSchema) {
|
|
1294
|
+
let restoredSchema;
|
|
1295
|
+
try {
|
|
1296
|
+
restoredSchema = engine.getStats(indexName).schema;
|
|
1297
|
+
} catch (err) {
|
|
1298
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
1299
|
+
return new NarsilError(
|
|
1300
|
+
ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
|
|
1301
|
+
`failed to read restored schema for '${indexName}': ${cause}`,
|
|
1302
|
+
{ indexName, primaryNodeId, reason: "schema lookup failed", cause }
|
|
1303
|
+
);
|
|
1316
1304
|
}
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
received: payload.indexName
|
|
1321
|
-
});
|
|
1322
|
-
return;
|
|
1305
|
+
const differences = diffSchemas(coordinatorSchema, restoredSchema);
|
|
1306
|
+
if (differences.length === 0) {
|
|
1307
|
+
return null;
|
|
1323
1308
|
}
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
received: payload.totalBytes
|
|
1335
|
-
});
|
|
1336
|
-
return;
|
|
1337
|
-
}
|
|
1338
|
-
if (!isSafeNonNegativeInteger(payload.checksum) || payload.checksum !== state.header.checksum) {
|
|
1339
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHECKSUM_MISMATCH, "end checksum does not match header", {
|
|
1340
|
-
expected: state.header.checksum,
|
|
1341
|
-
received: payload.checksum
|
|
1342
|
-
});
|
|
1343
|
-
return;
|
|
1344
|
-
}
|
|
1345
|
-
state.endPayload = payload;
|
|
1309
|
+
return new NarsilError(
|
|
1310
|
+
ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
|
|
1311
|
+
`restored snapshot schema disagrees with coordinator schema for '${indexName}'`,
|
|
1312
|
+
{
|
|
1313
|
+
indexName,
|
|
1314
|
+
primaryNodeId,
|
|
1315
|
+
reason: "schema mismatch",
|
|
1316
|
+
differences
|
|
1317
|
+
}
|
|
1318
|
+
);
|
|
1346
1319
|
}
|
|
1347
|
-
function
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1320
|
+
async function loadCoordinatorIndexConfig(coordinator, indexName, primaryNodeId, abortPromise) {
|
|
1321
|
+
let schema;
|
|
1322
|
+
try {
|
|
1323
|
+
const winner = await Promise.race([coordinator.getSchema(indexName), abortPromise]);
|
|
1324
|
+
if (winner === ABORT_SENTINEL) {
|
|
1325
|
+
return "aborted";
|
|
1326
|
+
}
|
|
1327
|
+
schema = winner;
|
|
1328
|
+
} catch (err) {
|
|
1329
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
1330
|
+
return {
|
|
1331
|
+
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE, `coordinator.getSchema failed: ${cause}`, {
|
|
1332
|
+
indexName,
|
|
1333
|
+
primaryNodeId,
|
|
1334
|
+
cause
|
|
1335
|
+
})
|
|
1336
|
+
};
|
|
1356
1337
|
}
|
|
1357
|
-
if (
|
|
1358
|
-
return
|
|
1338
|
+
if (schema === null) {
|
|
1339
|
+
return {
|
|
1340
|
+
error: new NarsilError(
|
|
1341
|
+
ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE,
|
|
1342
|
+
`coordinator has no schema for index '${indexName}'`,
|
|
1343
|
+
{ indexName, primaryNodeId }
|
|
1344
|
+
)
|
|
1345
|
+
};
|
|
1359
1346
|
}
|
|
1360
|
-
|
|
1361
|
-
|
|
1347
|
+
let validSchema;
|
|
1348
|
+
try {
|
|
1349
|
+
validSchema = validateSchema(schema);
|
|
1350
|
+
} catch (err) {
|
|
1351
|
+
if (!(err instanceof NarsilError)) throw err;
|
|
1352
|
+
return {
|
|
1353
|
+
error: new NarsilError(err.code, `coordinator schema for index '${indexName}' is invalid: ${err.message}`, {
|
|
1354
|
+
...err.details,
|
|
1355
|
+
indexName,
|
|
1356
|
+
primaryNodeId
|
|
1357
|
+
})
|
|
1358
|
+
};
|
|
1362
1359
|
}
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1360
|
+
try {
|
|
1361
|
+
const winner = await Promise.race([getClusterIndexConfig(coordinator, indexName, validSchema), abortPromise]);
|
|
1362
|
+
if (winner === ABORT_SENTINEL) {
|
|
1363
|
+
return "aborted";
|
|
1364
|
+
}
|
|
1365
|
+
return { config: winner };
|
|
1366
|
+
} catch (err) {
|
|
1367
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
1368
|
+
return {
|
|
1369
|
+
error: new NarsilError(
|
|
1370
|
+
err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE,
|
|
1371
|
+
`failed to read the coordinator index metadata for '${indexName}': ${cause}`,
|
|
1372
|
+
{ indexName, primaryNodeId, cause }
|
|
1373
|
+
)
|
|
1374
|
+
};
|
|
1368
1375
|
}
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
// src/distribution/cluster-node/bootstrap-sync/state.ts
|
|
1379
|
+
function createBootstrapSyncState() {
|
|
1380
|
+
return {
|
|
1381
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
1382
|
+
completed: /* @__PURE__ */ new Set(),
|
|
1383
|
+
generations: /* @__PURE__ */ new Map()
|
|
1384
|
+
};
|
|
1385
|
+
}
|
|
1386
|
+
function entryKey(indexName, partitionId) {
|
|
1387
|
+
return `${indexName}:${partitionId}`;
|
|
1388
|
+
}
|
|
1389
|
+
function clearBootstrapSyncIndex(state, indexName, partitionId) {
|
|
1390
|
+
const key = entryKey(indexName, partitionId);
|
|
1391
|
+
state.completed.delete(key);
|
|
1392
|
+
const inFlight = state.inFlight.get(key);
|
|
1393
|
+
if (inFlight === void 0) {
|
|
1394
|
+
state.generations.delete(key);
|
|
1395
|
+
return;
|
|
1372
1396
|
}
|
|
1373
|
-
const
|
|
1374
|
-
|
|
1397
|
+
const previous = state.generations.get(key) ?? 0;
|
|
1398
|
+
state.generations.set(key, previous + 1);
|
|
1399
|
+
inFlight.aborted = true;
|
|
1400
|
+
inFlight.abortResolve();
|
|
1401
|
+
state.inFlight.delete(key);
|
|
1375
1402
|
}
|
|
1376
|
-
function
|
|
1377
|
-
return
|
|
1403
|
+
function hasCompletedBootstrapSync(state, indexName, partitionId) {
|
|
1404
|
+
return state.completed.has(entryKey(indexName, partitionId));
|
|
1378
1405
|
}
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1406
|
+
function createEntry(state, indexName, partitionId, key) {
|
|
1407
|
+
const startGeneration = state.generations.get(key) ?? 0;
|
|
1408
|
+
let abortResolve = () => {
|
|
1409
|
+
};
|
|
1410
|
+
const abortPromise = new Promise((resolve) => {
|
|
1411
|
+
abortResolve = () => resolve(ABORT_SENTINEL);
|
|
1412
|
+
});
|
|
1382
1413
|
return {
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
failure: null,
|
|
1391
|
-
firstFrameSeen: false
|
|
1414
|
+
indexName,
|
|
1415
|
+
partitionId,
|
|
1416
|
+
generation: startGeneration,
|
|
1417
|
+
promise: Promise.resolve(false),
|
|
1418
|
+
aborted: false,
|
|
1419
|
+
abortResolve,
|
|
1420
|
+
abortPromise
|
|
1392
1421
|
};
|
|
1393
1422
|
}
|
|
1394
|
-
|
|
1423
|
+
|
|
1424
|
+
// src/distribution/replication/snapshot-stream-apply.ts
|
|
1425
|
+
function fail(state, code, message, details = {}) {
|
|
1395
1426
|
if (state.failure !== null) {
|
|
1396
1427
|
return;
|
|
1397
1428
|
}
|
|
1398
|
-
|
|
1399
|
-
|
|
1429
|
+
state.failure = { ok: false, code, message, details };
|
|
1430
|
+
releaseSpool(state);
|
|
1431
|
+
}
|
|
1432
|
+
function releaseSpool(state) {
|
|
1433
|
+
const spool = state.spool;
|
|
1434
|
+
if (spool === null) return;
|
|
1435
|
+
state.spool = null;
|
|
1436
|
+
spool.discard();
|
|
1437
|
+
}
|
|
1438
|
+
function spoolFailure(state, err) {
|
|
1439
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
1440
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `the snapshot could not be written to local storage: ${cause}`, {
|
|
1441
|
+
cause
|
|
1442
|
+
});
|
|
1443
|
+
}
|
|
1444
|
+
function applyErrorEnvelope(state, record) {
|
|
1445
|
+
const code = typeof record.code === "string" && record.code.length > 0 ? record.code : "UNKNOWN_ERROR";
|
|
1446
|
+
const message = typeof record.message === "string" && record.message.length > 0 ? record.message : "primary returned error envelope";
|
|
1447
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_PRIMARY_ERROR, `primary error: ${message}`, {
|
|
1448
|
+
primaryCode: code,
|
|
1449
|
+
primaryMessage: message
|
|
1450
|
+
});
|
|
1451
|
+
}
|
|
1452
|
+
function applyStartPayload(state, record) {
|
|
1453
|
+
if (!isValidStartPayload(record)) {
|
|
1454
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "expected SNAPSHOT_START as first frame");
|
|
1400
1455
|
return;
|
|
1401
1456
|
}
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1457
|
+
const start = record;
|
|
1458
|
+
const header = start.header;
|
|
1459
|
+
if (header.indexName !== state.expectation.indexName) {
|
|
1460
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "snapshot header indexName mismatch", {
|
|
1461
|
+
expected: state.expectation.indexName,
|
|
1462
|
+
received: header.indexName
|
|
1408
1463
|
});
|
|
1409
1464
|
return;
|
|
1410
1465
|
}
|
|
1411
|
-
if (
|
|
1412
|
-
fail(state, ErrorCodes.
|
|
1466
|
+
if (state.expectation.partitionId !== null && header.partitionId !== state.expectation.partitionId) {
|
|
1467
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "snapshot header partitionId mismatch", {
|
|
1468
|
+
expected: state.expectation.partitionId,
|
|
1469
|
+
received: header.partitionId
|
|
1470
|
+
});
|
|
1413
1471
|
return;
|
|
1414
1472
|
}
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
function handleIncomingSnapshotRecord(state, record) {
|
|
1418
|
-
if (state.failure !== null) {
|
|
1473
|
+
if (!isSafeNonNegativeInteger(header.checksum)) {
|
|
1474
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "header.checksum must be a non-negative integer");
|
|
1419
1475
|
return;
|
|
1420
1476
|
}
|
|
1421
|
-
if (
|
|
1422
|
-
|
|
1477
|
+
if (!isSafeNonNegativeInteger(header.partitionId)) {
|
|
1478
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "header.partitionId must be a non-negative integer");
|
|
1423
1479
|
return;
|
|
1424
1480
|
}
|
|
1425
|
-
if (!
|
|
1426
|
-
state.
|
|
1427
|
-
applyStartPayload(state, record);
|
|
1481
|
+
if (!isSafeNonNegativeInteger(header.lastSeqNo)) {
|
|
1482
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "header.lastSeqNo must be a non-negative integer");
|
|
1428
1483
|
return;
|
|
1429
1484
|
}
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
applyChunkPayload(state, record);
|
|
1485
|
+
if (!isSafeNonNegativeInteger(header.primaryTerm)) {
|
|
1486
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "header.primaryTerm must be a non-negative integer");
|
|
1433
1487
|
return;
|
|
1434
1488
|
}
|
|
1435
|
-
if (
|
|
1436
|
-
|
|
1489
|
+
if (!isSafeNonNegativeInteger(start.totalBytes)) {
|
|
1490
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "totalBytes must be a non-negative integer");
|
|
1437
1491
|
return;
|
|
1438
1492
|
}
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
}
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
if (!result.ok) releaseSpool(state);
|
|
1448
|
-
return result;
|
|
1493
|
+
try {
|
|
1494
|
+
state.spool = openSnapshotSpoolWriter(state.expectation.spoolDirectory);
|
|
1495
|
+
} catch (err) {
|
|
1496
|
+
spoolFailure(state, err);
|
|
1497
|
+
return;
|
|
1498
|
+
}
|
|
1499
|
+
state.header = header;
|
|
1500
|
+
state.totalBytes = start.totalBytes;
|
|
1449
1501
|
}
|
|
1450
|
-
function
|
|
1451
|
-
if (state.
|
|
1452
|
-
|
|
1502
|
+
function applyChunkPayload(state, payload) {
|
|
1503
|
+
if (state.totalBytes === null || state.header === null) {
|
|
1504
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received chunk before SNAPSHOT_START");
|
|
1505
|
+
return;
|
|
1453
1506
|
}
|
|
1454
|
-
if (
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1507
|
+
if (!isSafeNonNegativeInteger(payload.offset)) {
|
|
1508
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHUNK_OUT_OF_ORDER, "chunk offset must be a non-negative integer");
|
|
1509
|
+
return;
|
|
1510
|
+
}
|
|
1511
|
+
if (!isSafeNonNegativeInteger(payload.partitionId)) {
|
|
1512
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "chunk partitionId must be a non-negative integer");
|
|
1513
|
+
return;
|
|
1514
|
+
}
|
|
1515
|
+
if (payload.indexName !== state.header.indexName) {
|
|
1516
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "chunk indexName does not match header", {
|
|
1517
|
+
expected: state.header.indexName,
|
|
1518
|
+
received: payload.indexName
|
|
1519
|
+
});
|
|
1520
|
+
return;
|
|
1521
|
+
}
|
|
1522
|
+
if (payload.partitionId !== state.header.partitionId) {
|
|
1523
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "chunk partitionId does not match header", {
|
|
1524
|
+
expected: state.header.partitionId,
|
|
1525
|
+
received: payload.partitionId
|
|
1526
|
+
});
|
|
1527
|
+
return;
|
|
1528
|
+
}
|
|
1529
|
+
if (payload.offset !== state.expectedNextOffset) {
|
|
1530
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHUNK_OUT_OF_ORDER, "chunk arrived out of order", {
|
|
1531
|
+
expected: state.expectedNextOffset,
|
|
1532
|
+
received: payload.offset
|
|
1533
|
+
});
|
|
1534
|
+
return;
|
|
1535
|
+
}
|
|
1536
|
+
if (!(payload.data instanceof Uint8Array)) {
|
|
1537
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "chunk data is not Uint8Array");
|
|
1538
|
+
return;
|
|
1539
|
+
}
|
|
1540
|
+
const remainingBytes = state.totalBytes - state.accumulatedBytes;
|
|
1541
|
+
const perChunkLimit = Math.min(SNAPSHOT_CHUNK_SIZE, remainingBytes);
|
|
1542
|
+
if (payload.data.byteLength > perChunkLimit) {
|
|
1543
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHUNK_SIZE_EXCEEDED, "chunk exceeds the per-chunk size limit", {
|
|
1544
|
+
chunkBytes: payload.data.byteLength,
|
|
1545
|
+
limit: perChunkLimit
|
|
1546
|
+
});
|
|
1547
|
+
return;
|
|
1548
|
+
}
|
|
1549
|
+
const nextBytes = state.accumulatedBytes + payload.data.byteLength;
|
|
1550
|
+
if (nextBytes > state.totalBytes) {
|
|
1551
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHUNK_OVERFLOW, "chunk overflow beyond declared totalBytes", {
|
|
1552
|
+
nextBytes,
|
|
1553
|
+
totalBytes: state.totalBytes
|
|
1554
|
+
});
|
|
1555
|
+
return;
|
|
1556
|
+
}
|
|
1557
|
+
if (state.spool === null) {
|
|
1558
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received chunk after the snapshot stream closed");
|
|
1559
|
+
return;
|
|
1560
|
+
}
|
|
1561
|
+
try {
|
|
1562
|
+
state.spool.write(payload.data);
|
|
1563
|
+
} catch (err) {
|
|
1564
|
+
spoolFailure(state, err);
|
|
1565
|
+
return;
|
|
1566
|
+
}
|
|
1567
|
+
state.accumulatedBytes = nextBytes;
|
|
1568
|
+
state.expectedNextOffset = payload.offset + payload.data.byteLength;
|
|
1569
|
+
}
|
|
1570
|
+
function applyEndPayload(state, payload) {
|
|
1571
|
+
if (state.header === null || state.totalBytes === null) {
|
|
1572
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received SNAPSHOT_END before SNAPSHOT_START");
|
|
1573
|
+
return;
|
|
1574
|
+
}
|
|
1575
|
+
if (payload.indexName !== state.header.indexName) {
|
|
1576
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "end indexName does not match header", {
|
|
1577
|
+
expected: state.header.indexName,
|
|
1578
|
+
received: payload.indexName
|
|
1579
|
+
});
|
|
1580
|
+
return;
|
|
1581
|
+
}
|
|
1582
|
+
if (payload.partitionId !== state.header.partitionId) {
|
|
1583
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "end partitionId does not match header", {
|
|
1584
|
+
expected: state.header.partitionId,
|
|
1585
|
+
received: payload.partitionId
|
|
1586
|
+
});
|
|
1587
|
+
return;
|
|
1588
|
+
}
|
|
1589
|
+
if (!isSafeNonNegativeInteger(payload.totalBytes) || payload.totalBytes !== state.totalBytes) {
|
|
1590
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "end totalBytes does not match SNAPSHOT_START", {
|
|
1591
|
+
expected: state.totalBytes,
|
|
1592
|
+
received: payload.totalBytes
|
|
1593
|
+
});
|
|
1594
|
+
return;
|
|
1595
|
+
}
|
|
1596
|
+
if (!isSafeNonNegativeInteger(payload.checksum) || payload.checksum !== state.header.checksum) {
|
|
1597
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHECKSUM_MISMATCH, "end checksum does not match header", {
|
|
1598
|
+
expected: state.header.checksum,
|
|
1599
|
+
received: payload.checksum
|
|
1600
|
+
});
|
|
1601
|
+
return;
|
|
1602
|
+
}
|
|
1603
|
+
state.endPayload = payload;
|
|
1604
|
+
}
|
|
1605
|
+
function classifyFrameShape(record) {
|
|
1606
|
+
const hasOffset = typeof record.offset === "number";
|
|
1607
|
+
const hasData = record.data instanceof Uint8Array;
|
|
1608
|
+
const hasChecksum = typeof record.checksum === "number";
|
|
1609
|
+
const hasTotalBytes = typeof record.totalBytes === "number";
|
|
1610
|
+
const hasPartitionId = typeof record.partitionId === "number";
|
|
1611
|
+
const hasIndexName = typeof record.indexName === "string";
|
|
1612
|
+
if (!hasPartitionId || !hasIndexName) {
|
|
1613
|
+
return "unknown";
|
|
1614
|
+
}
|
|
1615
|
+
if (hasOffset && hasData && !hasChecksum && !hasTotalBytes) {
|
|
1616
|
+
return "chunk";
|
|
1617
|
+
}
|
|
1618
|
+
if (hasTotalBytes && hasChecksum && !hasOffset && !hasData) {
|
|
1619
|
+
return "end";
|
|
1620
|
+
}
|
|
1621
|
+
return "unknown";
|
|
1622
|
+
}
|
|
1623
|
+
function isValidStartPayload(record) {
|
|
1624
|
+
if (typeof record.totalBytes !== "number") {
|
|
1625
|
+
return false;
|
|
1626
|
+
}
|
|
1627
|
+
const header = record.header;
|
|
1628
|
+
if (header === null || typeof header !== "object" || Array.isArray(header)) {
|
|
1629
|
+
return false;
|
|
1630
|
+
}
|
|
1631
|
+
const headerRecord = header;
|
|
1632
|
+
return typeof headerRecord.indexName === "string" && typeof headerRecord.checksum === "number" && typeof headerRecord.partitionId === "number" && typeof headerRecord.lastSeqNo === "number" && typeof headerRecord.primaryTerm === "number";
|
|
1633
|
+
}
|
|
1634
|
+
function isSafeNonNegativeInteger(value) {
|
|
1635
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= Number.MAX_SAFE_INTEGER;
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
// src/distribution/replication/snapshot-stream-assembler.ts
|
|
1639
|
+
function createSnapshotStreamState(expectation) {
|
|
1640
|
+
return {
|
|
1641
|
+
expectation,
|
|
1642
|
+
header: null,
|
|
1643
|
+
totalBytes: null,
|
|
1644
|
+
spool: null,
|
|
1645
|
+
accumulatedBytes: 0,
|
|
1646
|
+
expectedNextOffset: 0,
|
|
1647
|
+
endPayload: null,
|
|
1648
|
+
failure: null,
|
|
1649
|
+
firstFrameSeen: false
|
|
1650
|
+
};
|
|
1651
|
+
}
|
|
1652
|
+
function handleIncomingSnapshotRecord(state, record) {
|
|
1653
|
+
if (state.failure !== null) {
|
|
1654
|
+
return;
|
|
1655
|
+
}
|
|
1656
|
+
if (record.error === true) {
|
|
1657
|
+
applyErrorEnvelope(state, record);
|
|
1658
|
+
return;
|
|
1659
|
+
}
|
|
1660
|
+
if (!state.firstFrameSeen) {
|
|
1661
|
+
state.firstFrameSeen = true;
|
|
1662
|
+
applyStartPayload(state, record);
|
|
1663
|
+
return;
|
|
1664
|
+
}
|
|
1665
|
+
const shape = classifyFrameShape(record);
|
|
1666
|
+
if (shape === "chunk") {
|
|
1667
|
+
applyChunkPayload(state, record);
|
|
1668
|
+
return;
|
|
1669
|
+
}
|
|
1670
|
+
if (shape === "end") {
|
|
1671
|
+
applyEndPayload(state, record);
|
|
1672
|
+
return;
|
|
1673
|
+
}
|
|
1674
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "unrecognised snapshot frame shape");
|
|
1675
|
+
}
|
|
1676
|
+
function abandonSnapshotStream(state) {
|
|
1677
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_ABORTED, "the snapshot stream was abandoned before it finished");
|
|
1678
|
+
releaseSpool(state);
|
|
1679
|
+
}
|
|
1680
|
+
function finalizeSnapshotStream(state) {
|
|
1681
|
+
const result = settleSnapshotStream(state);
|
|
1682
|
+
if (!result.ok) releaseSpool(state);
|
|
1683
|
+
return result;
|
|
1684
|
+
}
|
|
1685
|
+
function settleSnapshotStream(state) {
|
|
1686
|
+
if (state.failure !== null) {
|
|
1687
|
+
return state.failure;
|
|
1688
|
+
}
|
|
1689
|
+
if (state.header === null || state.totalBytes === null) {
|
|
1690
|
+
return {
|
|
1691
|
+
ok: false,
|
|
1692
|
+
code: ErrorCodes.SNAPSHOT_SYNC_CHUNK_MISSING,
|
|
1693
|
+
message: "stream ended before SNAPSHOT_START was received",
|
|
1694
|
+
details: { indexName: state.expectation.indexName }
|
|
1460
1695
|
};
|
|
1461
1696
|
}
|
|
1462
1697
|
if (state.endPayload === null) {
|
|
@@ -1509,32 +1744,13 @@ function settleSnapshotStream(state) {
|
|
|
1509
1744
|
end: state.endPayload
|
|
1510
1745
|
};
|
|
1511
1746
|
}
|
|
1512
|
-
function toStreamFailure(err, defaultCode, defaultMessage) {
|
|
1513
|
-
if (err instanceof NarsilError) {
|
|
1514
|
-
return {
|
|
1515
|
-
ok: false,
|
|
1516
|
-
code: err.code,
|
|
1517
|
-
message: err.message,
|
|
1518
|
-
details: { ...err.details }
|
|
1519
|
-
};
|
|
1520
|
-
}
|
|
1521
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
1522
|
-
return {
|
|
1523
|
-
ok: false,
|
|
1524
|
-
code: defaultCode,
|
|
1525
|
-
message: `${defaultMessage}: ${message}`,
|
|
1526
|
-
details: { cause: message }
|
|
1527
|
-
};
|
|
1528
|
-
}
|
|
1529
1747
|
|
|
1530
1748
|
// src/distribution/cluster-node/bootstrap-fetch.ts
|
|
1531
|
-
var
|
|
1749
|
+
var TRANSIENT_CODES = /* @__PURE__ */ new Set([
|
|
1532
1750
|
ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED,
|
|
1533
1751
|
ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
1534
1752
|
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
1535
|
-
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE
|
|
1536
|
-
]);
|
|
1537
|
-
var RETRY_DIFFERENT_TARGET_PROTOCOL_CODES = /* @__PURE__ */ new Set([
|
|
1753
|
+
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
1538
1754
|
ErrorCodes.SNAPSHOT_SYNC_DECODE_FAILED,
|
|
1539
1755
|
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
1540
1756
|
ErrorCodes.SNAPSHOT_SYNC_CHECKSUM_MISMATCH,
|
|
@@ -1548,890 +1764,408 @@ var TRANSIENT_PRIMARY_CODES = /* @__PURE__ */ new Set([
|
|
|
1548
1764
|
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
1549
1765
|
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE
|
|
1550
1766
|
]);
|
|
1551
|
-
function
|
|
1552
|
-
if (
|
|
1553
|
-
return
|
|
1767
|
+
function isTransientFailure(code, details) {
|
|
1768
|
+
if (TRANSIENT_CODES.has(code)) {
|
|
1769
|
+
return true;
|
|
1554
1770
|
}
|
|
1555
|
-
if (
|
|
1556
|
-
return
|
|
1771
|
+
if (code !== ErrorCodes.SNAPSHOT_SYNC_PRIMARY_ERROR) {
|
|
1772
|
+
return false;
|
|
1557
1773
|
}
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1774
|
+
return typeof details.primaryCode === "string" && TRANSIENT_PRIMARY_CODES.has(details.primaryCode);
|
|
1775
|
+
}
|
|
1776
|
+
async function withDeadline(work, ms, indexName, phase) {
|
|
1777
|
+
let timer = null;
|
|
1778
|
+
const deadlinePromise = new Promise((_, reject) => {
|
|
1779
|
+
timer = setTimeout(() => {
|
|
1780
|
+
reject(
|
|
1781
|
+
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, `bootstrap sync exceeded deadline during ${phase}`, {
|
|
1782
|
+
indexName,
|
|
1783
|
+
phase
|
|
1784
|
+
})
|
|
1785
|
+
);
|
|
1786
|
+
}, ms);
|
|
1787
|
+
});
|
|
1788
|
+
try {
|
|
1789
|
+
return await Promise.race([work, deadlinePromise]);
|
|
1790
|
+
} finally {
|
|
1791
|
+
if (timer !== null) {
|
|
1792
|
+
clearTimeout(timer);
|
|
1562
1793
|
}
|
|
1563
1794
|
}
|
|
1564
|
-
return "none";
|
|
1565
1795
|
}
|
|
1566
|
-
|
|
1567
|
-
|
|
1796
|
+
|
|
1797
|
+
// src/distribution/cluster-node/write-routing/partition-queue.ts
|
|
1798
|
+
function createPartitionWriteQueues() {
|
|
1799
|
+
return { chains: /* @__PURE__ */ new Map() };
|
|
1568
1800
|
}
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1801
|
+
function enqueuePartitionWrite(queues, indexName, partitionId, task) {
|
|
1802
|
+
const key = `${indexName}:${partitionId}`;
|
|
1803
|
+
const previous = queues.chains.get(key) ?? Promise.resolve();
|
|
1804
|
+
const holds = [];
|
|
1805
|
+
const hold = (settled) => {
|
|
1806
|
+
holds.push(settled.catch(() => void 0));
|
|
1575
1807
|
};
|
|
1576
|
-
const
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
details: { primaryNodeId, target }
|
|
1589
|
-
};
|
|
1590
|
-
}
|
|
1591
|
-
if (Date.now() >= deadline) {
|
|
1592
|
-
return {
|
|
1593
|
-
ok: false,
|
|
1594
|
-
code: ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
1595
|
-
message: "bootstrap sync exceeded deadline while iterating targets",
|
|
1596
|
-
details: { primaryNodeId, targets }
|
|
1597
|
-
};
|
|
1598
|
-
}
|
|
1599
|
-
const attempt = await fetchSnapshotFromTarget(indexName, partitionId, target, deadline, deps);
|
|
1600
|
-
if (attempt.ok) {
|
|
1601
|
-
return attempt;
|
|
1602
|
-
}
|
|
1603
|
-
lastFailure = { ...attempt, details: { ...attempt.details, target } };
|
|
1604
|
-
const kind = classifyFailure(attempt.code, attempt.details);
|
|
1605
|
-
if (kind === "none") {
|
|
1606
|
-
return lastFailure;
|
|
1808
|
+
const run = previous.then(
|
|
1809
|
+
() => task(hold),
|
|
1810
|
+
() => task(hold)
|
|
1811
|
+
);
|
|
1812
|
+
const chain = run.then(
|
|
1813
|
+
() => void 0,
|
|
1814
|
+
() => void 0
|
|
1815
|
+
).then(() => Promise.all(holds)).then(() => void 0);
|
|
1816
|
+
queues.chains.set(key, chain);
|
|
1817
|
+
chain.then(() => {
|
|
1818
|
+
if (queues.chains.get(key) === chain) {
|
|
1819
|
+
queues.chains.delete(key);
|
|
1607
1820
|
}
|
|
1608
|
-
if (kind === "protocol") {
|
|
1609
|
-
protocolErrorBudget -= 1;
|
|
1610
|
-
if (protocolErrorBudget <= 0) {
|
|
1611
|
-
return lastFailure;
|
|
1612
|
-
}
|
|
1613
|
-
}
|
|
1614
|
-
if (attempt.code === ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED) {
|
|
1615
|
-
const aborted = await jitteredBackoff(
|
|
1616
|
-
CAPACITY_EXHAUSTED_BACKOFF_BASE_MS,
|
|
1617
|
-
CAPACITY_EXHAUSTED_BACKOFF_MAX_MS,
|
|
1618
|
-
deadline,
|
|
1619
|
-
abortCheck
|
|
1620
|
-
);
|
|
1621
|
-
if (aborted) {
|
|
1622
|
-
return {
|
|
1623
|
-
ok: false,
|
|
1624
|
-
code: ErrorCodes.SNAPSHOT_SYNC_ABORTED,
|
|
1625
|
-
message: "bootstrap sync aborted while backing off after capacity exhaustion",
|
|
1626
|
-
details: { primaryNodeId, target }
|
|
1627
|
-
};
|
|
1628
|
-
}
|
|
1629
|
-
if (Date.now() >= deadline) {
|
|
1630
|
-
return {
|
|
1631
|
-
ok: false,
|
|
1632
|
-
code: ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
1633
|
-
message: "bootstrap sync exceeded deadline during capacity backoff",
|
|
1634
|
-
details: { primaryNodeId, targets }
|
|
1635
|
-
};
|
|
1636
|
-
}
|
|
1637
|
-
}
|
|
1638
|
-
}
|
|
1639
|
-
return lastFailure;
|
|
1640
|
-
}
|
|
1641
|
-
async function jitteredBackoff(baseMs, maxMs, deadline, abortCheck) {
|
|
1642
|
-
const plannedMs = Math.floor(baseMs + Math.random() * (maxMs - baseMs));
|
|
1643
|
-
const remainingMs = deadline - Date.now();
|
|
1644
|
-
if (remainingMs <= 0) {
|
|
1645
|
-
return false;
|
|
1646
|
-
}
|
|
1647
|
-
const sleepMs = Math.min(plannedMs, remainingMs);
|
|
1648
|
-
if (sleepMs <= 0) {
|
|
1649
|
-
return false;
|
|
1650
|
-
}
|
|
1651
|
-
const sleepUntil = Date.now() + sleepMs;
|
|
1652
|
-
if (abortCheck()) {
|
|
1653
|
-
return true;
|
|
1654
|
-
}
|
|
1655
|
-
while (Date.now() < sleepUntil) {
|
|
1656
|
-
if (abortCheck()) {
|
|
1657
|
-
return true;
|
|
1658
|
-
}
|
|
1659
|
-
if (Date.now() >= deadline) {
|
|
1660
|
-
return false;
|
|
1661
|
-
}
|
|
1662
|
-
const now3 = Date.now();
|
|
1663
|
-
const remaining = Math.min(sleepUntil - now3, deadline - now3);
|
|
1664
|
-
if (remaining <= 0) {
|
|
1665
|
-
return false;
|
|
1666
|
-
}
|
|
1667
|
-
if (remaining <= 2) {
|
|
1668
|
-
return false;
|
|
1669
|
-
}
|
|
1670
|
-
const tickMs = Math.min(remaining, JITTERED_BACKOFF_POLL_INTERVAL_MS);
|
|
1671
|
-
await sleepForMs(tickMs);
|
|
1672
|
-
}
|
|
1673
|
-
return abortCheck();
|
|
1674
|
-
}
|
|
1675
|
-
function sleepForMs(ms) {
|
|
1676
|
-
return new Promise((resolve) => {
|
|
1677
|
-
const timer = setTimeout(() => {
|
|
1678
|
-
clearTimeout(timer);
|
|
1679
|
-
resolve();
|
|
1680
|
-
}, ms);
|
|
1681
1821
|
});
|
|
1822
|
+
return run;
|
|
1682
1823
|
}
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
payload: encode(requestPayload)
|
|
1690
|
-
};
|
|
1691
|
-
const streamState = createSnapshotStreamState({
|
|
1692
|
-
indexName,
|
|
1693
|
-
partitionId: null,
|
|
1694
|
-
spoolDirectory: deps.spoolDirectory ?? defaultSpoolDirectory()
|
|
1695
|
-
});
|
|
1696
|
-
const remainingMs = deadline - Date.now();
|
|
1697
|
-
if (remainingMs <= 0) {
|
|
1698
|
-
return {
|
|
1699
|
-
ok: false,
|
|
1700
|
-
code: ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
1701
|
-
message: "deadline expired before streaming snapshot",
|
|
1702
|
-
details: { target }
|
|
1703
|
-
};
|
|
1824
|
+
|
|
1825
|
+
// src/distribution/cluster-node/bootstrap-sync/entries.ts
|
|
1826
|
+
async function applySyncEntries(indexName, partitionId, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, deps) {
|
|
1827
|
+
const terminalError = validateSyncEntryBatchTermination(entriesPayloads);
|
|
1828
|
+
if (terminalError !== null) {
|
|
1829
|
+
return { ok: false, error: terminalError };
|
|
1704
1830
|
}
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
return toStreamFailure(err, ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "snapshot transport failed");
|
|
1831
|
+
const log = deps.getReplicationLog(indexName, partitionId);
|
|
1832
|
+
let expectedSeqNo = expectedFirstSeqNo;
|
|
1833
|
+
let entriesApplied = 0;
|
|
1834
|
+
for (const payload of entriesPayloads) {
|
|
1835
|
+
const run = [];
|
|
1836
|
+
let entryError = null;
|
|
1837
|
+
for (const entry of payload.entries) {
|
|
1838
|
+
entryError = validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log);
|
|
1839
|
+
if (entryError !== null) break;
|
|
1840
|
+
run.push(entry);
|
|
1841
|
+
expectedSeqNo = entry.seqNo + 1;
|
|
1717
1842
|
}
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
indexName,
|
|
1729
|
-
phase
|
|
1730
|
-
})
|
|
1731
|
-
);
|
|
1732
|
-
}, ms);
|
|
1733
|
-
});
|
|
1734
|
-
try {
|
|
1735
|
-
return await Promise.race([work, deadlinePromise]);
|
|
1736
|
-
} finally {
|
|
1737
|
-
if (timer !== null) {
|
|
1738
|
-
clearTimeout(timer);
|
|
1843
|
+
const outcome = await enqueuePartitionWrite(
|
|
1844
|
+
deps.partitionWriteQueues,
|
|
1845
|
+
indexName,
|
|
1846
|
+
partitionId,
|
|
1847
|
+
() => applySyncRunOnce(run, log, deps)
|
|
1848
|
+
);
|
|
1849
|
+
entriesApplied += outcome.processed;
|
|
1850
|
+
const error = outcome.error ?? entryError;
|
|
1851
|
+
if (error !== null) {
|
|
1852
|
+
return { ok: false, error };
|
|
1739
1853
|
}
|
|
1740
1854
|
}
|
|
1855
|
+
return {
|
|
1856
|
+
ok: true,
|
|
1857
|
+
entriesApplied,
|
|
1858
|
+
newSeqNo: expectedSeqNo - 1
|
|
1859
|
+
};
|
|
1741
1860
|
}
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
const
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
const expectedKeys = Object.keys(expected).sort(compareCodePoints);
|
|
1761
|
-
const actualKeys = new Set(Object.keys(actual));
|
|
1762
|
-
for (const key of expectedKeys) {
|
|
1763
|
-
const path = prefix === "" ? key : `${prefix}.${key}`;
|
|
1764
|
-
const expectedValue = expected[key];
|
|
1765
|
-
const actualValue = actual[key];
|
|
1766
|
-
if (!actualKeys.has(key)) {
|
|
1767
|
-
diffs.push({ path, expected: describe(expectedValue), actual: "(absent)" });
|
|
1768
|
-
continue;
|
|
1769
|
-
}
|
|
1770
|
-
actualKeys.delete(key);
|
|
1771
|
-
const expectedIsObject = typeof expectedValue === "object" && expectedValue !== null;
|
|
1772
|
-
const actualIsObject = typeof actualValue === "object" && actualValue !== null;
|
|
1773
|
-
if (expectedIsObject !== actualIsObject) {
|
|
1774
|
-
diffs.push({ path, expected: describe(expectedValue), actual: describe(actualValue) });
|
|
1775
|
-
continue;
|
|
1776
|
-
}
|
|
1777
|
-
if (expectedIsObject && actualIsObject) {
|
|
1778
|
-
walk(expectedValue, actualValue, path, diffs, depth + 1);
|
|
1861
|
+
async function applySyncRunOnce(run, log, deps) {
|
|
1862
|
+
let processed = 0;
|
|
1863
|
+
let error = null;
|
|
1864
|
+
const pending = [];
|
|
1865
|
+
let tailSeqNo = log.localLogEnd;
|
|
1866
|
+
for (const entry of run) {
|
|
1867
|
+
const where = { indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo };
|
|
1868
|
+
const present = log.getEntry(entry.seqNo);
|
|
1869
|
+
if (present !== void 0) {
|
|
1870
|
+
if (present.checksum !== entry.checksum) {
|
|
1871
|
+
error = new NarsilError(
|
|
1872
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
1873
|
+
`Conflicting sync entry at sequence number ${entry.seqNo}`,
|
|
1874
|
+
where
|
|
1875
|
+
);
|
|
1876
|
+
break;
|
|
1877
|
+
}
|
|
1878
|
+
processed += 1;
|
|
1779
1879
|
continue;
|
|
1780
1880
|
}
|
|
1781
|
-
if (
|
|
1782
|
-
|
|
1881
|
+
if (entry.seqNo !== tailSeqNo + 1) {
|
|
1882
|
+
error = new NarsilError(
|
|
1883
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
1884
|
+
`Out-of-order sync entry ${entry.seqNo}; the local log ends at ${tailSeqNo}`,
|
|
1885
|
+
where
|
|
1886
|
+
);
|
|
1887
|
+
break;
|
|
1783
1888
|
}
|
|
1889
|
+
pending.push(entry);
|
|
1890
|
+
tailSeqNo = entry.seqNo;
|
|
1784
1891
|
}
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1892
|
+
if (pending.length === 0) return { processed, error };
|
|
1893
|
+
let applied;
|
|
1894
|
+
let applyError = null;
|
|
1895
|
+
try {
|
|
1896
|
+
const outcome = await deps.applyReplicationEntries(pending);
|
|
1897
|
+
applied = outcome.applied;
|
|
1898
|
+
applyError = outcome.error;
|
|
1899
|
+
} catch (err) {
|
|
1900
|
+
applied = 0;
|
|
1901
|
+
applyError = err;
|
|
1793
1902
|
}
|
|
1794
|
-
|
|
1795
|
-
|
|
1903
|
+
for (let position = 0; position < applied; position++) log.appendCommitted(pending[position]);
|
|
1904
|
+
processed += applied;
|
|
1905
|
+
if (applied < pending.length) {
|
|
1906
|
+
const failed2 = pending[applied];
|
|
1907
|
+
const cause = applyError instanceof Error ? applyError.message : String(applyError);
|
|
1908
|
+
return {
|
|
1909
|
+
processed,
|
|
1910
|
+
error: new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, `failed to apply sync entry: ${cause}`, {
|
|
1911
|
+
indexName: failed2.indexName,
|
|
1912
|
+
partitionId: failed2.partitionId,
|
|
1913
|
+
seqNo: failed2.seqNo,
|
|
1914
|
+
cause
|
|
1915
|
+
})
|
|
1916
|
+
};
|
|
1796
1917
|
}
|
|
1797
|
-
return
|
|
1918
|
+
return { processed, error };
|
|
1798
1919
|
}
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
function validateArguments(indexName, partitionId, primaryNodeId) {
|
|
1803
|
-
if (typeof indexName !== "string" || indexName.length === 0) {
|
|
1804
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, "indexName must be a non-empty string", {
|
|
1805
|
-
indexName
|
|
1806
|
-
});
|
|
1920
|
+
function validateSyncEntryBatchTermination(entriesPayloads) {
|
|
1921
|
+
if (entriesPayloads.length === 0) {
|
|
1922
|
+
return null;
|
|
1807
1923
|
}
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
});
|
|
1924
|
+
for (let i = 0; i < entriesPayloads.length - 1; i += 1) {
|
|
1925
|
+
if (entriesPayloads[i].isLast) {
|
|
1926
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "sync_entries marked isLast before final batch");
|
|
1927
|
+
}
|
|
1813
1928
|
}
|
|
1814
|
-
if (
|
|
1815
|
-
return new NarsilError(
|
|
1816
|
-
|
|
1817
|
-
|
|
1929
|
+
if (!entriesPayloads[entriesPayloads.length - 1].isLast) {
|
|
1930
|
+
return new NarsilError(
|
|
1931
|
+
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
1932
|
+
"live sync stream ended before final sync_entries batch"
|
|
1933
|
+
);
|
|
1818
1934
|
}
|
|
1819
1935
|
return null;
|
|
1820
1936
|
}
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
try {
|
|
1824
|
-
targets = await resolveNodeTargets3(primaryNodeId);
|
|
1825
|
-
} catch (err) {
|
|
1826
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1827
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_NO_TARGETS, `resolveNodeTargets failed: ${cause}`, {
|
|
1828
|
-
indexName,
|
|
1829
|
-
primaryNodeId,
|
|
1830
|
-
cause
|
|
1831
|
-
});
|
|
1832
|
-
}
|
|
1833
|
-
if (targets.length === 0) {
|
|
1937
|
+
function validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log) {
|
|
1938
|
+
if (entry.indexName !== indexName || entry.partitionId !== partitionId) {
|
|
1834
1939
|
return new NarsilError(
|
|
1835
|
-
ErrorCodes.
|
|
1836
|
-
|
|
1837
|
-
{
|
|
1940
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
1941
|
+
"replication entry scope does not match bootstrap target",
|
|
1942
|
+
{
|
|
1943
|
+
expectedIndexName: indexName,
|
|
1944
|
+
expectedPartitionId: partitionId,
|
|
1945
|
+
entryIndexName: entry.indexName,
|
|
1946
|
+
entryPartitionId: entry.partitionId
|
|
1947
|
+
}
|
|
1838
1948
|
);
|
|
1839
1949
|
}
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1950
|
+
if (entry.seqNo !== expectedSeqNo) {
|
|
1951
|
+
return new NarsilError(
|
|
1952
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
1953
|
+
`Out-of-order sync entry ${entry.seqNo}; expected ${expectedSeqNo}`,
|
|
1954
|
+
{ indexName, partitionId, seqNo: entry.seqNo, expectedSeqNo }
|
|
1955
|
+
);
|
|
1845
1956
|
}
|
|
1846
|
-
const
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
);
|
|
1856
|
-
deps.onError(wrapped);
|
|
1957
|
+
const validation = validateReplicationEntry(entry, localPrimaryTerm, log);
|
|
1958
|
+
if (!validation.valid) {
|
|
1959
|
+
return new NarsilError(
|
|
1960
|
+
validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
1961
|
+
`Invalid sync entry ${entry.seqNo}`,
|
|
1962
|
+
{ indexName, partitionId, seqNo: entry.seqNo }
|
|
1963
|
+
);
|
|
1964
|
+
}
|
|
1965
|
+
return null;
|
|
1857
1966
|
}
|
|
1858
|
-
function
|
|
1859
|
-
|
|
1967
|
+
function createLiveSyncFrameState(indexName, partitionId, spoolDirectory) {
|
|
1968
|
+
return {
|
|
1969
|
+
snapshotState: createSnapshotStreamState({ indexName, partitionId, spoolDirectory }),
|
|
1970
|
+
syncEntries: [],
|
|
1971
|
+
sawSnapshotFrame: false,
|
|
1972
|
+
error: null
|
|
1973
|
+
};
|
|
1974
|
+
}
|
|
1975
|
+
function handleLiveSyncFrame(state, frame) {
|
|
1976
|
+
if (state.error !== null || state.snapshotState.failure !== null) {
|
|
1860
1977
|
return;
|
|
1861
1978
|
}
|
|
1862
|
-
|
|
1863
|
-
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, `bootstrap sync aborted for index '${indexName}'`, {
|
|
1864
|
-
indexName,
|
|
1865
|
-
primaryNodeId
|
|
1866
|
-
})
|
|
1867
|
-
);
|
|
1868
|
-
}
|
|
1869
|
-
async function executeEngineRestore(engine, indexName, primaryNodeId, source, remainingMs) {
|
|
1979
|
+
let decoded;
|
|
1870
1980
|
try {
|
|
1871
|
-
|
|
1872
|
-
return null;
|
|
1981
|
+
decoded = decode(frame);
|
|
1873
1982
|
} catch (err) {
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
}
|
|
1877
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1878
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `engine.restore failed: ${cause}`, {
|
|
1879
|
-
indexName,
|
|
1880
|
-
primaryNodeId,
|
|
1881
|
-
cause
|
|
1983
|
+
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_DECODE_FAILED, "failed to decode live sync frame", {
|
|
1984
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
1882
1985
|
});
|
|
1986
|
+
return;
|
|
1883
1987
|
}
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
try {
|
|
1888
|
-
restoredSchema = engine.getStats(indexName).schema;
|
|
1889
|
-
} catch (err) {
|
|
1890
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1891
|
-
return new NarsilError(
|
|
1892
|
-
ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
|
|
1893
|
-
`failed to read restored schema for '${indexName}': ${cause}`,
|
|
1894
|
-
{ indexName, primaryNodeId, reason: "schema lookup failed", cause }
|
|
1895
|
-
);
|
|
1988
|
+
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
1989
|
+
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "live sync frame is not an object");
|
|
1990
|
+
return;
|
|
1896
1991
|
}
|
|
1897
|
-
const
|
|
1898
|
-
|
|
1899
|
-
|
|
1992
|
+
const record = decoded;
|
|
1993
|
+
const entriesPayload = decodeSyncEntriesRecord(record);
|
|
1994
|
+
if (entriesPayload instanceof NarsilError) {
|
|
1995
|
+
state.error = entriesPayload;
|
|
1996
|
+
return;
|
|
1900
1997
|
}
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
{
|
|
1905
|
-
indexName,
|
|
1906
|
-
primaryNodeId,
|
|
1907
|
-
reason: "schema mismatch",
|
|
1908
|
-
differences
|
|
1909
|
-
}
|
|
1910
|
-
);
|
|
1911
|
-
}
|
|
1912
|
-
async function dropRestoredIndexQuietly(engine, indexName, deps = {}) {
|
|
1913
|
-
try {
|
|
1914
|
-
const existing = engine.listIndexes().find((idx) => idx.name === indexName);
|
|
1915
|
-
if (existing === void 0) {
|
|
1916
|
-
return;
|
|
1917
|
-
}
|
|
1918
|
-
await engine.dropIndex(indexName);
|
|
1919
|
-
} catch (err) {
|
|
1920
|
-
if (localIndexIsGone(err)) {
|
|
1921
|
-
return;
|
|
1922
|
-
}
|
|
1923
|
-
if (deps.onError === void 0) {
|
|
1998
|
+
if (entriesPayload !== null) {
|
|
1999
|
+
if (state.sawSnapshotFrame && state.snapshotState.endPayload === null) {
|
|
2000
|
+
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received sync_entries before snapshot_end");
|
|
1924
2001
|
return;
|
|
1925
2002
|
}
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
new NarsilError(
|
|
1929
|
-
ErrorCodes.SNAPSHOT_SYNC_RESTORE_CLEANUP_FAILED,
|
|
1930
|
-
`failed to drop restored index '${indexName}' after bootstrap cleanup: ${cause}`,
|
|
1931
|
-
{ indexName, cause }
|
|
1932
|
-
)
|
|
1933
|
-
);
|
|
1934
|
-
}
|
|
1935
|
-
}
|
|
1936
|
-
async function loadCoordinatorIndexConfig(coordinator, indexName, primaryNodeId, abortPromise) {
|
|
1937
|
-
let schema;
|
|
1938
|
-
try {
|
|
1939
|
-
const winner = await Promise.race([coordinator.getSchema(indexName), abortPromise]);
|
|
1940
|
-
if (winner === ABORT_SENTINEL) {
|
|
1941
|
-
return "aborted";
|
|
1942
|
-
}
|
|
1943
|
-
schema = winner;
|
|
1944
|
-
} catch (err) {
|
|
1945
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1946
|
-
return {
|
|
1947
|
-
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE, `coordinator.getSchema failed: ${cause}`, {
|
|
1948
|
-
indexName,
|
|
1949
|
-
primaryNodeId,
|
|
1950
|
-
cause
|
|
1951
|
-
})
|
|
1952
|
-
};
|
|
1953
|
-
}
|
|
1954
|
-
if (schema === null) {
|
|
1955
|
-
return {
|
|
1956
|
-
error: new NarsilError(
|
|
1957
|
-
ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE,
|
|
1958
|
-
`coordinator has no schema for index '${indexName}'`,
|
|
1959
|
-
{ indexName, primaryNodeId }
|
|
1960
|
-
)
|
|
1961
|
-
};
|
|
2003
|
+
state.syncEntries.push(entriesPayload);
|
|
2004
|
+
return;
|
|
1962
2005
|
}
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
return
|
|
1969
|
-
error: new NarsilError(err.code, `coordinator schema for index '${indexName}' is invalid: ${err.message}`, {
|
|
1970
|
-
...err.details,
|
|
1971
|
-
indexName,
|
|
1972
|
-
primaryNodeId
|
|
1973
|
-
})
|
|
1974
|
-
};
|
|
2006
|
+
if (state.syncEntries.length > 0 && !state.sawSnapshotFrame) {
|
|
2007
|
+
state.error = new NarsilError(
|
|
2008
|
+
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
2009
|
+
"received snapshot frame after incremental sync_entries"
|
|
2010
|
+
);
|
|
2011
|
+
return;
|
|
1975
2012
|
}
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
}
|
|
1981
|
-
return { config: winner };
|
|
1982
|
-
} catch (err) {
|
|
1983
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1984
|
-
return {
|
|
1985
|
-
error: new NarsilError(
|
|
1986
|
-
err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE,
|
|
1987
|
-
`failed to read the coordinator index metadata for '${indexName}': ${cause}`,
|
|
1988
|
-
{ indexName, primaryNodeId, cause }
|
|
1989
|
-
)
|
|
1990
|
-
};
|
|
2013
|
+
state.sawSnapshotFrame = true;
|
|
2014
|
+
handleIncomingSnapshotRecord(state.snapshotState, record);
|
|
2015
|
+
if (state.snapshotState.failure !== null) {
|
|
2016
|
+
state.error = streamFailureToError(state.snapshotState.failure);
|
|
1991
2017
|
}
|
|
1992
2018
|
}
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
const existing = engine.listIndexes().find((idx) => idx.name === indexName);
|
|
1996
|
-
if (existing === void 0) {
|
|
1997
|
-
return null;
|
|
1998
|
-
}
|
|
1999
|
-
await engine.dropIndex(indexName);
|
|
2019
|
+
function decodeSyncEntriesRecord(record) {
|
|
2020
|
+
if (!("entries" in record) && !("isLast" in record)) {
|
|
2000
2021
|
return null;
|
|
2001
|
-
}
|
|
2002
|
-
|
|
2003
|
-
return null;
|
|
2004
|
-
}
|
|
2005
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
2022
|
+
}
|
|
2023
|
+
if (!Array.isArray(record.entries) || typeof record.isLast !== "boolean") {
|
|
2006
2024
|
return new NarsilError(
|
|
2007
|
-
ErrorCodes.
|
|
2008
|
-
|
|
2009
|
-
{ indexName, primaryNodeId, cause }
|
|
2025
|
+
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
2026
|
+
"Invalid SyncEntriesPayload: expected entries array and isLast boolean"
|
|
2010
2027
|
);
|
|
2011
2028
|
}
|
|
2029
|
+
const entries = [];
|
|
2030
|
+
for (const candidate of record.entries) {
|
|
2031
|
+
try {
|
|
2032
|
+
entries.push(validateEntryPayload({ entry: candidate }).entry);
|
|
2033
|
+
} catch (err) {
|
|
2034
|
+
return new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, err instanceof Error ? err.message : String(err));
|
|
2035
|
+
}
|
|
2036
|
+
}
|
|
2037
|
+
return { entries, isLast: record.isLast };
|
|
2038
|
+
}
|
|
2039
|
+
function streamFailureToError(failure) {
|
|
2040
|
+
return new NarsilError(failure.code, failure.message, failure.details);
|
|
2012
2041
|
}
|
|
2013
2042
|
|
|
2014
|
-
// src/distribution/cluster-node/bootstrap-sync/
|
|
2015
|
-
function
|
|
2043
|
+
// src/distribution/cluster-node/bootstrap-sync/apply.ts
|
|
2044
|
+
async function applyLiveIncrementalSync(indexName, partitionId, target, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, coordinatorConfig, partitionCount, deps) {
|
|
2045
|
+
const ensureResult = await ensureLocalIndexForIncremental(indexName, target, coordinatorConfig, partitionCount, deps);
|
|
2046
|
+
if (ensureResult instanceof NarsilError) {
|
|
2047
|
+
return { ok: false, error: ensureResult };
|
|
2048
|
+
}
|
|
2049
|
+
const applyResult = await applySyncEntries(
|
|
2050
|
+
indexName,
|
|
2051
|
+
partitionId,
|
|
2052
|
+
entriesPayloads,
|
|
2053
|
+
expectedFirstSeqNo,
|
|
2054
|
+
localPrimaryTerm,
|
|
2055
|
+
deps
|
|
2056
|
+
);
|
|
2057
|
+
if (!applyResult.ok) {
|
|
2058
|
+
return applyResult;
|
|
2059
|
+
}
|
|
2016
2060
|
return {
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2061
|
+
ok: true,
|
|
2062
|
+
tier: "incremental",
|
|
2063
|
+
entriesApplied: applyResult.entriesApplied,
|
|
2064
|
+
newSeqNo: applyResult.newSeqNo,
|
|
2065
|
+
snapshotHeader: null
|
|
2020
2066
|
};
|
|
2021
2067
|
}
|
|
2022
|
-
function
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
return deps.getReplicationLog !== void 0 && deps.resetReplicationLog !== void 0 && deps.applyReplicationEntries !== void 0 && deps.restoreReplicationPartition !== void 0;
|
|
2027
|
-
}
|
|
2028
|
-
function clearBootstrapSyncIndex(state, indexName, partitionId) {
|
|
2029
|
-
const key = entryKey(indexName, partitionId);
|
|
2030
|
-
state.completed.delete(key);
|
|
2031
|
-
const inFlight = state.inFlight.get(key);
|
|
2032
|
-
if (inFlight === void 0) {
|
|
2033
|
-
state.generations.delete(key);
|
|
2034
|
-
return;
|
|
2068
|
+
async function applyLiveSnapshotSync(state, entry, indexName, partitionId, target, frameState, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2069
|
+
const finalized = finalizeSnapshotStream(frameState.snapshotState);
|
|
2070
|
+
if (!finalized.ok) {
|
|
2071
|
+
return { ok: false, error: streamFailureToError({ ...finalized, details: { ...finalized.details, target } }) };
|
|
2035
2072
|
}
|
|
2036
|
-
const
|
|
2037
|
-
|
|
2038
|
-
inFlight.aborted = true;
|
|
2039
|
-
inFlight.abortResolve();
|
|
2040
|
-
state.inFlight.delete(key);
|
|
2041
|
-
}
|
|
2042
|
-
function hasCompletedBootstrapSync(state, indexName, partitionId) {
|
|
2043
|
-
return state.completed.has(entryKey(indexName, partitionId));
|
|
2044
|
-
}
|
|
2045
|
-
function createEntry(state, indexName, partitionId, key) {
|
|
2046
|
-
const startGeneration = state.generations.get(key) ?? 0;
|
|
2047
|
-
let abortResolve = () => {
|
|
2048
|
-
};
|
|
2049
|
-
const abortPromise = new Promise((resolve) => {
|
|
2050
|
-
abortResolve = () => resolve(ABORT_SENTINEL);
|
|
2051
|
-
});
|
|
2052
|
-
return {
|
|
2073
|
+
const restoreError = await enqueuePartitionWrite(
|
|
2074
|
+
deps.partitionWriteQueues,
|
|
2053
2075
|
indexName,
|
|
2054
2076
|
partitionId,
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
abortResolve,
|
|
2059
|
-
abortPromise
|
|
2060
|
-
};
|
|
2061
|
-
}
|
|
2062
|
-
function anotherBootstrapOwnsKey(state, key, self) {
|
|
2063
|
-
const current = state.inFlight.get(key);
|
|
2064
|
-
if (current === void 0) {
|
|
2065
|
-
return false;
|
|
2066
|
-
}
|
|
2067
|
-
return current !== self;
|
|
2068
|
-
}
|
|
2069
|
-
|
|
2070
|
-
// src/distribution/cluster-node/write-routing/partition-queue.ts
|
|
2071
|
-
function createPartitionWriteQueues() {
|
|
2072
|
-
return { chains: /* @__PURE__ */ new Map() };
|
|
2073
|
-
}
|
|
2074
|
-
function enqueuePartitionWrite(queues, indexName, partitionId, task) {
|
|
2075
|
-
const key = `${indexName}:${partitionId}`;
|
|
2076
|
-
const previous = queues.chains.get(key) ?? Promise.resolve();
|
|
2077
|
-
const run = previous.then(task, task);
|
|
2078
|
-
const chain = run.then(
|
|
2079
|
-
() => void 0,
|
|
2080
|
-
() => void 0
|
|
2081
|
-
);
|
|
2082
|
-
queues.chains.set(key, chain);
|
|
2083
|
-
chain.then(() => {
|
|
2084
|
-
if (queues.chains.get(key) === chain) {
|
|
2085
|
-
queues.chains.delete(key);
|
|
2086
|
-
}
|
|
2087
|
-
});
|
|
2088
|
-
return run;
|
|
2089
|
-
}
|
|
2090
|
-
|
|
2091
|
-
// src/distribution/cluster-node/bootstrap-sync/entries.ts
|
|
2092
|
-
async function applySyncEntries(indexName, partitionId, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, deps) {
|
|
2093
|
-
const terminalError = validateSyncEntryBatchTermination(entriesPayloads);
|
|
2094
|
-
if (terminalError !== null) {
|
|
2095
|
-
return { ok: false, error: terminalError };
|
|
2096
|
-
}
|
|
2097
|
-
const log = deps.getReplicationLog(indexName, partitionId);
|
|
2098
|
-
let expectedSeqNo = expectedFirstSeqNo;
|
|
2099
|
-
let entriesApplied = 0;
|
|
2100
|
-
for (const payload of entriesPayloads) {
|
|
2101
|
-
const run = [];
|
|
2102
|
-
let entryError = null;
|
|
2103
|
-
for (const entry of payload.entries) {
|
|
2104
|
-
entryError = validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log);
|
|
2105
|
-
if (entryError !== null) break;
|
|
2106
|
-
run.push(entry);
|
|
2107
|
-
expectedSeqNo = entry.seqNo + 1;
|
|
2108
|
-
}
|
|
2109
|
-
const outcome = await enqueuePartitionWrite(
|
|
2110
|
-
deps.partitionWriteQueues,
|
|
2077
|
+
() => restartFromSnapshot(
|
|
2078
|
+
state,
|
|
2079
|
+
entry,
|
|
2111
2080
|
indexName,
|
|
2112
2081
|
partitionId,
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2082
|
+
target,
|
|
2083
|
+
finalized,
|
|
2084
|
+
coordinatorConfig,
|
|
2085
|
+
partitionCount,
|
|
2086
|
+
deadline,
|
|
2087
|
+
deps
|
|
2088
|
+
)
|
|
2089
|
+
);
|
|
2090
|
+
if (restoreError !== null) {
|
|
2091
|
+
return { ok: false, error: restoreError };
|
|
2092
|
+
}
|
|
2093
|
+
const applyResult = await applySyncEntries(
|
|
2094
|
+
indexName,
|
|
2095
|
+
partitionId,
|
|
2096
|
+
frameState.syncEntries,
|
|
2097
|
+
finalized.header.lastSeqNo + 1,
|
|
2098
|
+
finalized.header.primaryTerm,
|
|
2099
|
+
deps
|
|
2100
|
+
);
|
|
2101
|
+
if (!applyResult.ok) {
|
|
2102
|
+
return applyResult;
|
|
2120
2103
|
}
|
|
2121
2104
|
return {
|
|
2122
2105
|
ok: true,
|
|
2123
|
-
|
|
2124
|
-
|
|
2106
|
+
tier: "snapshot",
|
|
2107
|
+
entriesApplied: applyResult.entriesApplied,
|
|
2108
|
+
newSeqNo: applyResult.newSeqNo,
|
|
2109
|
+
snapshotHeader: finalized.header
|
|
2125
2110
|
};
|
|
2126
2111
|
}
|
|
2127
|
-
async function
|
|
2128
|
-
let
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
processed += 1;
|
|
2145
|
-
continue;
|
|
2146
|
-
}
|
|
2147
|
-
if (entry.seqNo !== tailSeqNo + 1) {
|
|
2148
|
-
error = new NarsilError(
|
|
2149
|
-
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
2150
|
-
`Out-of-order sync entry ${entry.seqNo}; the local log ends at ${tailSeqNo}`,
|
|
2151
|
-
where
|
|
2152
|
-
);
|
|
2153
|
-
break;
|
|
2154
|
-
}
|
|
2155
|
-
pending.push(entry);
|
|
2156
|
-
tailSeqNo = entry.seqNo;
|
|
2112
|
+
async function restartFromSnapshot(state, entry, indexName, partitionId, target, finalized, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2113
|
+
let restoreError;
|
|
2114
|
+
try {
|
|
2115
|
+
restoreError = await restoreLiveSnapshotPartition(
|
|
2116
|
+
state,
|
|
2117
|
+
entry,
|
|
2118
|
+
indexName,
|
|
2119
|
+
partitionId,
|
|
2120
|
+
target,
|
|
2121
|
+
finalized.snapshot.read(),
|
|
2122
|
+
coordinatorConfig,
|
|
2123
|
+
partitionCount,
|
|
2124
|
+
deadline,
|
|
2125
|
+
deps
|
|
2126
|
+
);
|
|
2127
|
+
} finally {
|
|
2128
|
+
await finalized.snapshot.discard();
|
|
2157
2129
|
}
|
|
2158
|
-
if (
|
|
2159
|
-
let applied;
|
|
2160
|
-
let applyError = null;
|
|
2130
|
+
if (restoreError !== null) return restoreError;
|
|
2161
2131
|
try {
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2132
|
+
await deps.restartLocalLog?.(indexName, partitionId, {
|
|
2133
|
+
seqNo: finalized.header.lastSeqNo,
|
|
2134
|
+
primaryTerm: finalized.header.primaryTerm
|
|
2135
|
+
});
|
|
2165
2136
|
} catch (err) {
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
const cause = applyError instanceof Error ? applyError.message : String(applyError);
|
|
2174
|
-
return {
|
|
2175
|
-
processed,
|
|
2176
|
-
error: new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, `failed to apply sync entry: ${cause}`, {
|
|
2177
|
-
indexName: failed2.indexName,
|
|
2178
|
-
partitionId: failed2.partitionId,
|
|
2179
|
-
seqNo: failed2.seqNo,
|
|
2180
|
-
cause
|
|
2181
|
-
})
|
|
2182
|
-
};
|
|
2183
|
-
}
|
|
2184
|
-
return { processed, error };
|
|
2185
|
-
}
|
|
2186
|
-
function validateSyncEntryBatchTermination(entriesPayloads) {
|
|
2187
|
-
if (entriesPayloads.length === 0) {
|
|
2188
|
-
return null;
|
|
2189
|
-
}
|
|
2190
|
-
for (let i = 0; i < entriesPayloads.length - 1; i += 1) {
|
|
2191
|
-
if (entriesPayloads[i].isLast) {
|
|
2192
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "sync_entries marked isLast before final batch");
|
|
2193
|
-
}
|
|
2194
|
-
}
|
|
2195
|
-
if (!entriesPayloads[entriesPayloads.length - 1].isLast) {
|
|
2196
|
-
return new NarsilError(
|
|
2197
|
-
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
2198
|
-
"live sync stream ended before final sync_entries batch"
|
|
2199
|
-
);
|
|
2137
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
2138
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `local log restart failed: ${cause}`, {
|
|
2139
|
+
indexName,
|
|
2140
|
+
partitionId,
|
|
2141
|
+
primaryNodeId: target,
|
|
2142
|
+
cause
|
|
2143
|
+
});
|
|
2200
2144
|
}
|
|
2145
|
+
deps.resetReplicationLog(indexName, partitionId, finalized.header.lastSeqNo + 1, finalized.header.primaryTerm);
|
|
2201
2146
|
return null;
|
|
2202
2147
|
}
|
|
2203
|
-
function
|
|
2204
|
-
|
|
2148
|
+
async function restoreLiveSnapshotPartition(state, entry, indexName, partitionId, primaryNodeId, source, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2149
|
+
const remainingMs = deadline - Date.now();
|
|
2150
|
+
if (remainingMs <= 0) {
|
|
2205
2151
|
return new NarsilError(
|
|
2206
|
-
ErrorCodes.
|
|
2207
|
-
"
|
|
2152
|
+
ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
2153
|
+
"bootstrap sync exceeded deadline before partition restore",
|
|
2208
2154
|
{
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
entryPartitionId: entry.partitionId
|
|
2155
|
+
indexName,
|
|
2156
|
+
partitionId,
|
|
2157
|
+
primaryNodeId
|
|
2213
2158
|
}
|
|
2214
2159
|
);
|
|
2215
2160
|
}
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
if (!validation.valid) {
|
|
2225
|
-
return new NarsilError(
|
|
2226
|
-
validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
2227
|
-
`Invalid sync entry ${entry.seqNo}`,
|
|
2228
|
-
{ indexName, partitionId, seqNo: entry.seqNo }
|
|
2229
|
-
);
|
|
2230
|
-
}
|
|
2231
|
-
return null;
|
|
2232
|
-
}
|
|
2233
|
-
function createLiveSyncFrameState(indexName, partitionId, spoolDirectory) {
|
|
2234
|
-
return {
|
|
2235
|
-
snapshotState: createSnapshotStreamState({ indexName, partitionId, spoolDirectory }),
|
|
2236
|
-
syncEntries: [],
|
|
2237
|
-
sawSnapshotFrame: false,
|
|
2238
|
-
error: null
|
|
2239
|
-
};
|
|
2240
|
-
}
|
|
2241
|
-
function handleLiveSyncFrame(state, frame) {
|
|
2242
|
-
if (state.error !== null || state.snapshotState.failure !== null) {
|
|
2243
|
-
return;
|
|
2244
|
-
}
|
|
2245
|
-
let decoded;
|
|
2246
|
-
try {
|
|
2247
|
-
decoded = decode(frame);
|
|
2248
|
-
} catch (err) {
|
|
2249
|
-
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_DECODE_FAILED, "failed to decode live sync frame", {
|
|
2250
|
-
cause: err instanceof Error ? err.message : String(err)
|
|
2251
|
-
});
|
|
2252
|
-
return;
|
|
2253
|
-
}
|
|
2254
|
-
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
2255
|
-
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "live sync frame is not an object");
|
|
2256
|
-
return;
|
|
2257
|
-
}
|
|
2258
|
-
const record = decoded;
|
|
2259
|
-
const entriesPayload = decodeSyncEntriesRecord(record);
|
|
2260
|
-
if (entriesPayload instanceof NarsilError) {
|
|
2261
|
-
state.error = entriesPayload;
|
|
2262
|
-
return;
|
|
2263
|
-
}
|
|
2264
|
-
if (entriesPayload !== null) {
|
|
2265
|
-
if (state.sawSnapshotFrame && state.snapshotState.endPayload === null) {
|
|
2266
|
-
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received sync_entries before snapshot_end");
|
|
2267
|
-
return;
|
|
2268
|
-
}
|
|
2269
|
-
state.syncEntries.push(entriesPayload);
|
|
2270
|
-
return;
|
|
2271
|
-
}
|
|
2272
|
-
if (state.syncEntries.length > 0 && !state.sawSnapshotFrame) {
|
|
2273
|
-
state.error = new NarsilError(
|
|
2274
|
-
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
2275
|
-
"received snapshot frame after incremental sync_entries"
|
|
2276
|
-
);
|
|
2277
|
-
return;
|
|
2278
|
-
}
|
|
2279
|
-
state.sawSnapshotFrame = true;
|
|
2280
|
-
handleIncomingSnapshotRecord(state.snapshotState, record);
|
|
2281
|
-
if (state.snapshotState.failure !== null) {
|
|
2282
|
-
state.error = streamFailureToError(state.snapshotState.failure);
|
|
2283
|
-
}
|
|
2284
|
-
}
|
|
2285
|
-
function decodeSyncEntriesRecord(record) {
|
|
2286
|
-
if (!("entries" in record) && !("isLast" in record)) {
|
|
2287
|
-
return null;
|
|
2288
|
-
}
|
|
2289
|
-
if (!Array.isArray(record.entries) || typeof record.isLast !== "boolean") {
|
|
2290
|
-
return new NarsilError(
|
|
2291
|
-
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
2292
|
-
"Invalid SyncEntriesPayload: expected entries array and isLast boolean"
|
|
2293
|
-
);
|
|
2294
|
-
}
|
|
2295
|
-
const entries = [];
|
|
2296
|
-
for (const candidate of record.entries) {
|
|
2297
|
-
try {
|
|
2298
|
-
entries.push(validateEntryPayload({ entry: candidate }).entry);
|
|
2299
|
-
} catch (err) {
|
|
2300
|
-
return new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, err instanceof Error ? err.message : String(err));
|
|
2301
|
-
}
|
|
2302
|
-
}
|
|
2303
|
-
return { entries, isLast: record.isLast };
|
|
2304
|
-
}
|
|
2305
|
-
function streamFailureToError(failure) {
|
|
2306
|
-
return new NarsilError(failure.code, failure.message, failure.details);
|
|
2307
|
-
}
|
|
2308
|
-
|
|
2309
|
-
// src/distribution/cluster-node/bootstrap-sync/apply.ts
|
|
2310
|
-
async function applyLiveIncrementalSync(indexName, partitionId, target, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, coordinatorConfig, partitionCount, deps) {
|
|
2311
|
-
const ensureResult = await ensureLocalIndexForIncremental(indexName, target, coordinatorConfig, partitionCount, deps);
|
|
2312
|
-
if (ensureResult instanceof NarsilError) {
|
|
2313
|
-
return { ok: false, error: ensureResult };
|
|
2314
|
-
}
|
|
2315
|
-
const applyResult = await applySyncEntries(
|
|
2316
|
-
indexName,
|
|
2317
|
-
partitionId,
|
|
2318
|
-
entriesPayloads,
|
|
2319
|
-
expectedFirstSeqNo,
|
|
2320
|
-
localPrimaryTerm,
|
|
2321
|
-
deps
|
|
2322
|
-
);
|
|
2323
|
-
if (!applyResult.ok) {
|
|
2324
|
-
return applyResult;
|
|
2325
|
-
}
|
|
2326
|
-
return {
|
|
2327
|
-
ok: true,
|
|
2328
|
-
tier: "incremental",
|
|
2329
|
-
entriesApplied: applyResult.entriesApplied,
|
|
2330
|
-
newSeqNo: applyResult.newSeqNo,
|
|
2331
|
-
snapshotHeader: null
|
|
2332
|
-
};
|
|
2333
|
-
}
|
|
2334
|
-
async function applyLiveSnapshotSync(state, entry, indexName, partitionId, target, frameState, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2335
|
-
const finalized = finalizeSnapshotStream(frameState.snapshotState);
|
|
2336
|
-
if (!finalized.ok) {
|
|
2337
|
-
return { ok: false, error: streamFailureToError({ ...finalized, details: { ...finalized.details, target } }) };
|
|
2338
|
-
}
|
|
2339
|
-
const restoreError = await enqueuePartitionWrite(
|
|
2340
|
-
deps.partitionWriteQueues,
|
|
2341
|
-
indexName,
|
|
2342
|
-
partitionId,
|
|
2343
|
-
() => restartFromSnapshot(
|
|
2344
|
-
state,
|
|
2345
|
-
entry,
|
|
2346
|
-
indexName,
|
|
2347
|
-
partitionId,
|
|
2348
|
-
target,
|
|
2349
|
-
finalized,
|
|
2350
|
-
coordinatorConfig,
|
|
2351
|
-
partitionCount,
|
|
2352
|
-
deadline,
|
|
2353
|
-
deps
|
|
2354
|
-
)
|
|
2355
|
-
);
|
|
2356
|
-
if (restoreError !== null) {
|
|
2357
|
-
return { ok: false, error: restoreError };
|
|
2358
|
-
}
|
|
2359
|
-
const applyResult = await applySyncEntries(
|
|
2360
|
-
indexName,
|
|
2361
|
-
partitionId,
|
|
2362
|
-
frameState.syncEntries,
|
|
2363
|
-
finalized.header.lastSeqNo + 1,
|
|
2364
|
-
finalized.header.primaryTerm,
|
|
2365
|
-
deps
|
|
2366
|
-
);
|
|
2367
|
-
if (!applyResult.ok) {
|
|
2368
|
-
return applyResult;
|
|
2369
|
-
}
|
|
2370
|
-
return {
|
|
2371
|
-
ok: true,
|
|
2372
|
-
tier: "snapshot",
|
|
2373
|
-
entriesApplied: applyResult.entriesApplied,
|
|
2374
|
-
newSeqNo: applyResult.newSeqNo,
|
|
2375
|
-
snapshotHeader: finalized.header
|
|
2376
|
-
};
|
|
2377
|
-
}
|
|
2378
|
-
async function restartFromSnapshot(state, entry, indexName, partitionId, target, finalized, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2379
|
-
let restoreError;
|
|
2380
|
-
try {
|
|
2381
|
-
restoreError = await restoreLiveSnapshotPartition(
|
|
2382
|
-
state,
|
|
2383
|
-
entry,
|
|
2384
|
-
indexName,
|
|
2385
|
-
partitionId,
|
|
2386
|
-
target,
|
|
2387
|
-
finalized.snapshot.read(),
|
|
2388
|
-
coordinatorConfig,
|
|
2389
|
-
partitionCount,
|
|
2390
|
-
deadline,
|
|
2391
|
-
deps
|
|
2392
|
-
);
|
|
2393
|
-
} finally {
|
|
2394
|
-
await finalized.snapshot.discard();
|
|
2395
|
-
}
|
|
2396
|
-
if (restoreError !== null) return restoreError;
|
|
2397
|
-
try {
|
|
2398
|
-
await deps.restartLocalLog?.(indexName, partitionId, {
|
|
2399
|
-
seqNo: finalized.header.lastSeqNo,
|
|
2400
|
-
primaryTerm: finalized.header.primaryTerm
|
|
2401
|
-
});
|
|
2402
|
-
} catch (err) {
|
|
2403
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
2404
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `local log restart failed: ${cause}`, {
|
|
2405
|
-
indexName,
|
|
2406
|
-
partitionId,
|
|
2407
|
-
primaryNodeId: target,
|
|
2408
|
-
cause
|
|
2409
|
-
});
|
|
2410
|
-
}
|
|
2411
|
-
deps.resetReplicationLog(indexName, partitionId, finalized.header.lastSeqNo + 1, finalized.header.primaryTerm);
|
|
2412
|
-
return null;
|
|
2413
|
-
}
|
|
2414
|
-
async function restoreLiveSnapshotPartition(state, entry, indexName, partitionId, primaryNodeId, source, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2415
|
-
const remainingMs = deadline - Date.now();
|
|
2416
|
-
if (remainingMs <= 0) {
|
|
2417
|
-
return new NarsilError(
|
|
2418
|
-
ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
2419
|
-
"bootstrap sync exceeded deadline before partition restore",
|
|
2420
|
-
{
|
|
2421
|
-
indexName,
|
|
2422
|
-
partitionId,
|
|
2423
|
-
primaryNodeId
|
|
2424
|
-
}
|
|
2425
|
-
);
|
|
2426
|
-
}
|
|
2427
|
-
const key = entryKey(indexName, partitionId);
|
|
2428
|
-
const generationBeforeRestore = state.generations.get(key) ?? 0;
|
|
2429
|
-
if (generationBeforeRestore !== entry.generation || entry.aborted) {
|
|
2430
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, "bootstrap sync aborted before partition restore", {
|
|
2431
|
-
indexName,
|
|
2432
|
-
partitionId,
|
|
2433
|
-
primaryNodeId
|
|
2434
|
-
});
|
|
2161
|
+
const key = entryKey(indexName, partitionId);
|
|
2162
|
+
const generationBeforeRestore = state.generations.get(key) ?? 0;
|
|
2163
|
+
if (generationBeforeRestore !== entry.generation || entry.aborted) {
|
|
2164
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, "bootstrap sync aborted before partition restore", {
|
|
2165
|
+
indexName,
|
|
2166
|
+
partitionId,
|
|
2167
|
+
primaryNodeId
|
|
2168
|
+
});
|
|
2435
2169
|
}
|
|
2436
2170
|
try {
|
|
2437
2171
|
await withDeadline(
|
|
@@ -2682,6 +2416,9 @@ async function syncFrameStateFromTarget(state, entry, indexName, partitionId, ta
|
|
|
2682
2416
|
});
|
|
2683
2417
|
await withDeadline(streamWork, remainingMs, indexName, "live-sync");
|
|
2684
2418
|
} catch (err) {
|
|
2419
|
+
if (err instanceof NarsilError && err.code === ErrorCodes.SNAPSHOT_SYNC_TIMEOUT) {
|
|
2420
|
+
return { ok: false, error: err };
|
|
2421
|
+
}
|
|
2685
2422
|
const cause = err instanceof Error ? err.message : String(err);
|
|
2686
2423
|
return {
|
|
2687
2424
|
ok: false,
|
|
@@ -2876,67 +2613,6 @@ async function executeLiveBootstrapSync(state, entry, indexName, partitionId, pr
|
|
|
2876
2613
|
return true;
|
|
2877
2614
|
}
|
|
2878
2615
|
|
|
2879
|
-
// src/distribution/cluster-node/bootstrap-sync/snapshot-restore.ts
|
|
2880
|
-
async function fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps) {
|
|
2881
|
-
const dropError = await dropExistingIndex(deps.engine, indexName, primaryNodeId);
|
|
2882
|
-
if (dropError !== null) {
|
|
2883
|
-
surfaceError(deps, indexName, primaryNodeId, dropError);
|
|
2884
|
-
return null;
|
|
2885
|
-
}
|
|
2886
|
-
if (entry.aborted) {
|
|
2887
|
-
return "aborted";
|
|
2888
|
-
}
|
|
2889
|
-
const configResult = await loadCoordinatorIndexConfig(deps.coordinator, indexName, primaryNodeId, entry.abortPromise);
|
|
2890
|
-
if (configResult === "aborted") {
|
|
2891
|
-
return "aborted";
|
|
2892
|
-
}
|
|
2893
|
-
if ("error" in configResult) {
|
|
2894
|
-
surfaceError(deps, indexName, primaryNodeId, configResult.error);
|
|
2895
|
-
return null;
|
|
2896
|
-
}
|
|
2897
|
-
return configResult.config.schema;
|
|
2898
|
-
}
|
|
2899
|
-
async function applyRestore(state, entry, indexName, primaryNodeId, source, coordinatorSchema, deadline, deps) {
|
|
2900
|
-
const remainingMs = deadline - Date.now();
|
|
2901
|
-
if (remainingMs <= 0) {
|
|
2902
|
-
surfaceError(
|
|
2903
|
-
deps,
|
|
2904
|
-
indexName,
|
|
2905
|
-
primaryNodeId,
|
|
2906
|
-
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before restore", {
|
|
2907
|
-
indexName
|
|
2908
|
-
})
|
|
2909
|
-
);
|
|
2910
|
-
return false;
|
|
2911
|
-
}
|
|
2912
|
-
const key = entryKey(indexName, entry.partitionId);
|
|
2913
|
-
const generationBeforeRestore = state.generations.get(key) ?? 0;
|
|
2914
|
-
if (generationBeforeRestore !== entry.generation || entry.aborted) {
|
|
2915
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
2916
|
-
return false;
|
|
2917
|
-
}
|
|
2918
|
-
const restoreError = await executeEngineRestore(deps.engine, indexName, primaryNodeId, source, remainingMs);
|
|
2919
|
-
if (restoreError !== null) {
|
|
2920
|
-
surfaceError(deps, indexName, primaryNodeId, restoreError);
|
|
2921
|
-
return false;
|
|
2922
|
-
}
|
|
2923
|
-
const generationAfterRestore = state.generations.get(key) ?? 0;
|
|
2924
|
-
if (generationAfterRestore !== entry.generation || entry.aborted) {
|
|
2925
|
-
if (!anotherBootstrapOwnsKey(state, key, entry)) {
|
|
2926
|
-
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
2927
|
-
}
|
|
2928
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
2929
|
-
return false;
|
|
2930
|
-
}
|
|
2931
|
-
const schemaError = validateRestoredSchema(deps.engine, indexName, primaryNodeId, coordinatorSchema);
|
|
2932
|
-
if (schemaError !== null) {
|
|
2933
|
-
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
2934
|
-
surfaceError(deps, indexName, primaryNodeId, schemaError);
|
|
2935
|
-
return false;
|
|
2936
|
-
}
|
|
2937
|
-
return true;
|
|
2938
|
-
}
|
|
2939
|
-
|
|
2940
2616
|
// src/distribution/cluster-node/bootstrap-sync/index.ts
|
|
2941
2617
|
async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
|
|
2942
2618
|
const validationError = validateArguments(indexName, partitionId, primaryNodeId);
|
|
@@ -2994,113 +2670,17 @@ async function executeBootstrapSync(state, entry, indexName, partitionId, primar
|
|
|
2994
2670
|
}
|
|
2995
2671
|
return entry.aborted;
|
|
2996
2672
|
};
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
entry,
|
|
3001
|
-
indexName,
|
|
3002
|
-
partitionId,
|
|
3003
|
-
primaryNodeId,
|
|
3004
|
-
deps,
|
|
3005
|
-
deadlineMs,
|
|
3006
|
-
deadline,
|
|
3007
|
-
abortCheck
|
|
3008
|
-
);
|
|
3009
|
-
}
|
|
3010
|
-
const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
|
|
3011
|
-
if (schemaResult === "aborted") {
|
|
3012
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3013
|
-
return false;
|
|
3014
|
-
}
|
|
3015
|
-
if (schemaResult === null) {
|
|
3016
|
-
return false;
|
|
3017
|
-
}
|
|
3018
|
-
if (abortCheck()) {
|
|
3019
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3020
|
-
return false;
|
|
3021
|
-
}
|
|
3022
|
-
if (Date.now() >= deadline) {
|
|
3023
|
-
surfaceError(
|
|
3024
|
-
deps,
|
|
3025
|
-
indexName,
|
|
3026
|
-
primaryNodeId,
|
|
3027
|
-
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
|
|
3028
|
-
indexName,
|
|
3029
|
-
deadlineMs
|
|
3030
|
-
})
|
|
3031
|
-
);
|
|
3032
|
-
return false;
|
|
3033
|
-
}
|
|
3034
|
-
const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
|
|
3035
|
-
if (targetsResult instanceof NarsilError) {
|
|
3036
|
-
surfaceError(deps, indexName, primaryNodeId, targetsResult);
|
|
3037
|
-
return false;
|
|
3038
|
-
}
|
|
3039
|
-
const indexUuid = (await deps.coordinator.getAllocation(indexName))?.indexUuid;
|
|
3040
|
-
const fetchDeps = {
|
|
3041
|
-
transport: deps.transport,
|
|
3042
|
-
sourceNodeId: deps.sourceNodeId,
|
|
3043
|
-
spoolDirectory: deps.snapshotSpoolDirectory,
|
|
3044
|
-
...indexUuid === void 0 ? {} : { indexUuid }
|
|
3045
|
-
};
|
|
3046
|
-
const fetchResult = await fetchSnapshotFromAnyTarget(
|
|
2673
|
+
return executeLiveBootstrapSync(
|
|
2674
|
+
state,
|
|
2675
|
+
entry,
|
|
3047
2676
|
indexName,
|
|
2677
|
+
partitionId,
|
|
3048
2678
|
primaryNodeId,
|
|
3049
|
-
|
|
2679
|
+
deps,
|
|
2680
|
+
deadlineMs,
|
|
3050
2681
|
deadline,
|
|
3051
|
-
|
|
3052
|
-
abortCheck,
|
|
3053
|
-
partitionId
|
|
2682
|
+
abortCheck
|
|
3054
2683
|
);
|
|
3055
|
-
if (!fetchResult.ok) {
|
|
3056
|
-
if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
|
|
3057
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3058
|
-
return false;
|
|
3059
|
-
}
|
|
3060
|
-
surfaceError(
|
|
3061
|
-
deps,
|
|
3062
|
-
indexName,
|
|
3063
|
-
primaryNodeId,
|
|
3064
|
-
new NarsilError(fetchResult.code, fetchResult.message, {
|
|
3065
|
-
indexName,
|
|
3066
|
-
primaryNodeId,
|
|
3067
|
-
...fetchResult.details
|
|
3068
|
-
})
|
|
3069
|
-
);
|
|
3070
|
-
return false;
|
|
3071
|
-
}
|
|
3072
|
-
let restoreSucceeded;
|
|
3073
|
-
try {
|
|
3074
|
-
if (abortCheck()) {
|
|
3075
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3076
|
-
return false;
|
|
3077
|
-
}
|
|
3078
|
-
restoreSucceeded = await applyRestore(
|
|
3079
|
-
state,
|
|
3080
|
-
entry,
|
|
3081
|
-
indexName,
|
|
3082
|
-
primaryNodeId,
|
|
3083
|
-
fetchResult.snapshot.read(),
|
|
3084
|
-
schemaResult,
|
|
3085
|
-
deadline,
|
|
3086
|
-
deps
|
|
3087
|
-
);
|
|
3088
|
-
} finally {
|
|
3089
|
-
await fetchResult.snapshot.discard();
|
|
3090
|
-
}
|
|
3091
|
-
if (!restoreSucceeded) {
|
|
3092
|
-
return false;
|
|
3093
|
-
}
|
|
3094
|
-
if (abortCheck()) {
|
|
3095
|
-
if (!anotherBootstrapOwnsKey(state, key, entry)) {
|
|
3096
|
-
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
3097
|
-
}
|
|
3098
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3099
|
-
return false;
|
|
3100
|
-
}
|
|
3101
|
-
deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
|
|
3102
|
-
state.completed.add(key);
|
|
3103
|
-
return true;
|
|
3104
2684
|
}
|
|
3105
2685
|
var REFUSE = { outcome: "refuse" };
|
|
3106
2686
|
var ACCEPT = { outcome: "accept" };
|
|
@@ -7051,6 +6631,22 @@ function noteSupersedingPrimary(log, indexName, partitionId, primaryNodeId, prim
|
|
|
7051
6631
|
}
|
|
7052
6632
|
}
|
|
7053
6633
|
|
|
6634
|
+
// src/distribution/cluster-node/write-routing/deadline.ts
|
|
6635
|
+
function replicationTimeoutOf(deps) {
|
|
6636
|
+
return deps.transport.timeouts?.replicationTimeout ?? DEFAULT_TRANSPORT_CONFIG.replicationTimeout;
|
|
6637
|
+
}
|
|
6638
|
+
async function withinDeadline(work, timeoutMs, expired) {
|
|
6639
|
+
let timer;
|
|
6640
|
+
const timedOut = new Promise((_, reject) => {
|
|
6641
|
+
timer = setTimeout(() => reject(expired()), Math.max(0, timeoutMs));
|
|
6642
|
+
});
|
|
6643
|
+
try {
|
|
6644
|
+
return await Promise.race([work, timedOut]);
|
|
6645
|
+
} finally {
|
|
6646
|
+
clearTimeout(timer);
|
|
6647
|
+
}
|
|
6648
|
+
}
|
|
6649
|
+
|
|
7054
6650
|
// src/distribution/cluster-node/write-routing/replication.ts
|
|
7055
6651
|
async function assertPrimaryWriteAuthority(entry, deps) {
|
|
7056
6652
|
const table = await deps.coordinator.getAllocation(entry.indexName);
|
|
@@ -7099,6 +6695,18 @@ async function removeFailedReplicasFromInsync(entry, failedReplicas, deps) {
|
|
|
7099
6695
|
if (failedReplicas.length === 0) {
|
|
7100
6696
|
return;
|
|
7101
6697
|
}
|
|
6698
|
+
const timeoutMs = replicationTimeoutOf(deps);
|
|
6699
|
+
await withinDeadline(
|
|
6700
|
+
removeEachFailedReplica(entry, failedReplicas, deps),
|
|
6701
|
+
timeoutMs,
|
|
6702
|
+
() => new NarsilError(
|
|
6703
|
+
ErrorCodes.REPLICATION_INSYNC_REMOVAL_FAILED,
|
|
6704
|
+
`Removing failed replicas from the in-sync set of index '${entry.indexName}' partition ${entry.partitionId} did not finish within ${timeoutMs}ms`,
|
|
6705
|
+
{ indexName: entry.indexName, partitionId: entry.partitionId, failedReplicas, timeoutMs }
|
|
6706
|
+
)
|
|
6707
|
+
);
|
|
6708
|
+
}
|
|
6709
|
+
async function removeEachFailedReplica(entry, failedReplicas, deps) {
|
|
7102
6710
|
const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
7103
6711
|
if (controllerNodeId === null) {
|
|
7104
6712
|
await deps.refreshAllocation?.(entry.indexName);
|
|
@@ -10072,6 +9680,10 @@ function nodeRejectionOf(code, message, details) {
|
|
|
10072
9680
|
nodeRejections.add(rejection);
|
|
10073
9681
|
return rejection;
|
|
10074
9682
|
}
|
|
9683
|
+
function markNodeRejection(error) {
|
|
9684
|
+
nodeRejections.add(error);
|
|
9685
|
+
return error;
|
|
9686
|
+
}
|
|
10075
9687
|
function isNodeRejection(error) {
|
|
10076
9688
|
return error instanceof NarsilError && nodeRejections.has(error);
|
|
10077
9689
|
}
|
|
@@ -10220,6 +9832,44 @@ function selectReplicasForQuery(allocationTable, selector = randomSelector) {
|
|
|
10220
9832
|
}
|
|
10221
9833
|
return { nodeToPartitions, copiesByPartition, unavailablePartitions };
|
|
10222
9834
|
}
|
|
9835
|
+
function transportableDetails(error) {
|
|
9836
|
+
if (!(error instanceof NarsilError)) {
|
|
9837
|
+
return void 0;
|
|
9838
|
+
}
|
|
9839
|
+
try {
|
|
9840
|
+
encode(error.details);
|
|
9841
|
+
return error.details;
|
|
9842
|
+
} catch {
|
|
9843
|
+
return void 0;
|
|
9844
|
+
}
|
|
9845
|
+
}
|
|
9846
|
+
function encodeErrorReply(code, message, details) {
|
|
9847
|
+
if (details !== void 0) {
|
|
9848
|
+
try {
|
|
9849
|
+
return encode({ error: true, code, message, details });
|
|
9850
|
+
} catch {
|
|
9851
|
+
return encode({ error: true, code, message });
|
|
9852
|
+
}
|
|
9853
|
+
}
|
|
9854
|
+
return encode({ error: true, code, message });
|
|
9855
|
+
}
|
|
9856
|
+
function encodeErrorReplyFrom(error, fallbackCode) {
|
|
9857
|
+
return encodeErrorReply(
|
|
9858
|
+
error instanceof NarsilError ? error.code : fallbackCode,
|
|
9859
|
+
error instanceof Error ? error.message : String(error),
|
|
9860
|
+
error instanceof NarsilError ? error.details : void 0
|
|
9861
|
+
);
|
|
9862
|
+
}
|
|
9863
|
+
function isPlainRecord(value) {
|
|
9864
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
9865
|
+
return false;
|
|
9866
|
+
}
|
|
9867
|
+
const prototype = Object.getPrototypeOf(value);
|
|
9868
|
+
return prototype === Object.prototype || prototype === null;
|
|
9869
|
+
}
|
|
9870
|
+
function remoteErrorDetails(value) {
|
|
9871
|
+
return isPlainRecord(value) ? value : {};
|
|
9872
|
+
}
|
|
10223
9873
|
|
|
10224
9874
|
// src/distribution/cluster-node/vector-bytes.ts
|
|
10225
9875
|
var FLOAT32_BYTES = 4;
|
|
@@ -10372,141 +10022,6 @@ function requireWritableIndex(table, view) {
|
|
|
10372
10022
|
throw block;
|
|
10373
10023
|
}
|
|
10374
10024
|
}
|
|
10375
|
-
|
|
10376
|
-
// src/distribution/cluster-node/memory/write-pressure.ts
|
|
10377
|
-
var WRITE_BYTES_IN_FLIGHT_REASON = "write_bytes_in_flight";
|
|
10378
|
-
function createWritePressure(options) {
|
|
10379
|
-
let inFlight = 0;
|
|
10380
|
-
return {
|
|
10381
|
-
admit(indexName, bytes) {
|
|
10382
|
-
options.beforeAdmit();
|
|
10383
|
-
const requestBytes = Math.max(0, Math.ceil(bytes));
|
|
10384
|
-
if (inFlight > 0 && inFlight + requestBytes > options.limitBytes) {
|
|
10385
|
-
throw new NarsilError(
|
|
10386
|
-
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
10387
|
-
`Node '${options.nodeId}' refuses this write to index '${indexName}' because its ${requestBytes} bytes would take the writes in flight on the node from ${inFlight} bytes past the limit of ${options.limitBytes} bytes. Retry once the writes in flight finish, or send smaller batches.`,
|
|
10388
|
-
{
|
|
10389
|
-
reason: WRITE_BYTES_IN_FLIGHT_REASON,
|
|
10390
|
-
indexName,
|
|
10391
|
-
nodeId: options.nodeId,
|
|
10392
|
-
requestBytes,
|
|
10393
|
-
inFlightBytes: inFlight,
|
|
10394
|
-
limitBytes: options.limitBytes
|
|
10395
|
-
}
|
|
10396
|
-
);
|
|
10397
|
-
}
|
|
10398
|
-
inFlight += requestBytes;
|
|
10399
|
-
let released = false;
|
|
10400
|
-
return () => {
|
|
10401
|
-
if (released) {
|
|
10402
|
-
return;
|
|
10403
|
-
}
|
|
10404
|
-
released = true;
|
|
10405
|
-
inFlight -= requestBytes;
|
|
10406
|
-
};
|
|
10407
|
-
}
|
|
10408
|
-
};
|
|
10409
|
-
}
|
|
10410
|
-
async function withAdmittedWrite(pressure, indexName, bytes, work) {
|
|
10411
|
-
if (pressure === void 0) {
|
|
10412
|
-
return work();
|
|
10413
|
-
}
|
|
10414
|
-
const release = pressure.admit(indexName, bytes());
|
|
10415
|
-
try {
|
|
10416
|
-
return await work();
|
|
10417
|
-
} finally {
|
|
10418
|
-
release();
|
|
10419
|
-
}
|
|
10420
|
-
}
|
|
10421
|
-
function estimateDocumentBytes(document) {
|
|
10422
|
-
let bytes = 0;
|
|
10423
|
-
const pending = [document];
|
|
10424
|
-
const visited = /* @__PURE__ */ new Set();
|
|
10425
|
-
while (pending.length > 0) {
|
|
10426
|
-
const value = pending.pop();
|
|
10427
|
-
if (typeof value === "string") {
|
|
10428
|
-
bytes += value.length;
|
|
10429
|
-
continue;
|
|
10430
|
-
}
|
|
10431
|
-
if (typeof value === "number" || typeof value === "bigint") {
|
|
10432
|
-
bytes += DOCUMENT_NUMBER_BYTES;
|
|
10433
|
-
continue;
|
|
10434
|
-
}
|
|
10435
|
-
if (value === null || typeof value !== "object") {
|
|
10436
|
-
bytes += DOCUMENT_SCALAR_BYTES;
|
|
10437
|
-
continue;
|
|
10438
|
-
}
|
|
10439
|
-
if (visited.has(value)) {
|
|
10440
|
-
continue;
|
|
10441
|
-
}
|
|
10442
|
-
visited.add(value);
|
|
10443
|
-
if (ArrayBuffer.isView(value)) {
|
|
10444
|
-
bytes += value.byteLength;
|
|
10445
|
-
} else if (Array.isArray(value)) {
|
|
10446
|
-
bytes += DOCUMENT_ENTRY_OVERHEAD_BYTES;
|
|
10447
|
-
for (const item of value) {
|
|
10448
|
-
pending.push(item);
|
|
10449
|
-
}
|
|
10450
|
-
} else {
|
|
10451
|
-
for (const [key, item] of Object.entries(value)) {
|
|
10452
|
-
bytes += key.length + DOCUMENT_ENTRY_OVERHEAD_BYTES;
|
|
10453
|
-
pending.push(item);
|
|
10454
|
-
}
|
|
10455
|
-
}
|
|
10456
|
-
}
|
|
10457
|
-
return bytes;
|
|
10458
|
-
}
|
|
10459
|
-
function estimateDocumentsBytes(documents) {
|
|
10460
|
-
let bytes = 0;
|
|
10461
|
-
for (const document of documents) {
|
|
10462
|
-
bytes += estimateDocumentBytes(document);
|
|
10463
|
-
}
|
|
10464
|
-
return bytes;
|
|
10465
|
-
}
|
|
10466
|
-
function estimateForwardedBytes(operation) {
|
|
10467
|
-
const documentBytes = operation.document?.byteLength ?? 0;
|
|
10468
|
-
return operation.updateFields === null ? documentBytes : documentBytes + estimateDocumentBytes(operation.updateFields);
|
|
10469
|
-
}
|
|
10470
|
-
function transportableDetails(error) {
|
|
10471
|
-
if (!(error instanceof NarsilError)) {
|
|
10472
|
-
return void 0;
|
|
10473
|
-
}
|
|
10474
|
-
try {
|
|
10475
|
-
encode(error.details);
|
|
10476
|
-
return error.details;
|
|
10477
|
-
} catch {
|
|
10478
|
-
return void 0;
|
|
10479
|
-
}
|
|
10480
|
-
}
|
|
10481
|
-
function encodeErrorReply(code, message, details) {
|
|
10482
|
-
if (details !== void 0) {
|
|
10483
|
-
try {
|
|
10484
|
-
return encode({ error: true, code, message, details });
|
|
10485
|
-
} catch {
|
|
10486
|
-
return encode({ error: true, code, message });
|
|
10487
|
-
}
|
|
10488
|
-
}
|
|
10489
|
-
return encode({ error: true, code, message });
|
|
10490
|
-
}
|
|
10491
|
-
function encodeErrorReplyFrom(error, fallbackCode) {
|
|
10492
|
-
return encodeErrorReply(
|
|
10493
|
-
error instanceof NarsilError ? error.code : fallbackCode,
|
|
10494
|
-
error instanceof Error ? error.message : String(error),
|
|
10495
|
-
error instanceof NarsilError ? error.details : void 0
|
|
10496
|
-
);
|
|
10497
|
-
}
|
|
10498
|
-
function isPlainRecord(value) {
|
|
10499
|
-
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
10500
|
-
return false;
|
|
10501
|
-
}
|
|
10502
|
-
const prototype = Object.getPrototypeOf(value);
|
|
10503
|
-
return prototype === Object.prototype || prototype === null;
|
|
10504
|
-
}
|
|
10505
|
-
function remoteErrorDetails(value) {
|
|
10506
|
-
return isPlainRecord(value) ? value : {};
|
|
10507
|
-
}
|
|
10508
|
-
|
|
10509
|
-
// src/distribution/cluster-node/write-routing/forwarding.ts
|
|
10510
10025
|
function assertForwardResponse(response, indexName, primaryNodeId) {
|
|
10511
10026
|
const decoded = decode(response.payload);
|
|
10512
10027
|
if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
|
|
@@ -10573,6 +10088,112 @@ async function sendToNode(nodeId, message, deps) {
|
|
|
10573
10088
|
});
|
|
10574
10089
|
}
|
|
10575
10090
|
|
|
10091
|
+
// src/distribution/cluster-node/write-routing/rollback.ts
|
|
10092
|
+
function outcomeUnknown(scope, originalError, reason, because) {
|
|
10093
|
+
return new NarsilError(
|
|
10094
|
+
ErrorCodes.REPLICATION_OUTCOME_UNKNOWN,
|
|
10095
|
+
`A write to partition ${scope.partitionId} of index '${scope.indexName}' failed after the primary applied it, and ${because}, so the write may or may not have taken effect. Read the document before you retry the write.`,
|
|
10096
|
+
{
|
|
10097
|
+
indexName: scope.indexName,
|
|
10098
|
+
partitionId: scope.partitionId,
|
|
10099
|
+
primaryTerm: scope.assignment.primaryTerm,
|
|
10100
|
+
reason,
|
|
10101
|
+
originalError: describeError(originalError)
|
|
10102
|
+
}
|
|
10103
|
+
);
|
|
10104
|
+
}
|
|
10105
|
+
function isOutcomeUnknown(error) {
|
|
10106
|
+
return error instanceof NarsilError && error.code === ErrorCodes.REPLICATION_OUTCOME_UNKNOWN;
|
|
10107
|
+
}
|
|
10108
|
+
function abandonedWithTerm(error) {
|
|
10109
|
+
return isOutcomeUnknown(error) && error instanceof NarsilError && error.details.reason === "term_ended";
|
|
10110
|
+
}
|
|
10111
|
+
function lostTerm(error) {
|
|
10112
|
+
return error instanceof NarsilError && error.code === ErrorCodes.PARTITION_NOT_PRIMARY;
|
|
10113
|
+
}
|
|
10114
|
+
async function leadingAssignment(scope) {
|
|
10115
|
+
const { deps, indexName, partitionId } = scope;
|
|
10116
|
+
const table = deps.refreshAllocation !== void 0 ? await deps.refreshAllocation(indexName) : await deps.coordinator.getAllocation(indexName);
|
|
10117
|
+
const assignment = table?.assignments.get(partitionId);
|
|
10118
|
+
if (assignment !== void 0 && assignment.primary === deps.nodeId && assignment.primaryTerm === scope.assignment.primaryTerm && deps.copyFailed?.(indexName, partitionId) !== true) {
|
|
10119
|
+
return assignment;
|
|
10120
|
+
}
|
|
10121
|
+
throw new NarsilError(
|
|
10122
|
+
ErrorCodes.PARTITION_NOT_PRIMARY,
|
|
10123
|
+
`Node '${deps.nodeId}' no longer leads partition ${partitionId} of index '${indexName}' under term ${scope.assignment.primaryTerm}`,
|
|
10124
|
+
{
|
|
10125
|
+
indexName,
|
|
10126
|
+
partitionId,
|
|
10127
|
+
localNodeId: deps.nodeId,
|
|
10128
|
+
expectedPrimaryTerm: scope.assignment.primaryTerm,
|
|
10129
|
+
currentPrimaryNodeId: assignment?.primary ?? null,
|
|
10130
|
+
currentPrimaryTerm: assignment?.primaryTerm ?? null
|
|
10131
|
+
}
|
|
10132
|
+
);
|
|
10133
|
+
}
|
|
10134
|
+
async function sendRollback(scope, entries) {
|
|
10135
|
+
const assignment = await leadingAssignment(scope);
|
|
10136
|
+
for (const chunk of chunkReplicationEntries(entries.map((entry) => ({ entry })))) {
|
|
10137
|
+
await replicateEntryBatch(
|
|
10138
|
+
chunk.map((item) => item.entry),
|
|
10139
|
+
assignment,
|
|
10140
|
+
scope.deps,
|
|
10141
|
+
scope.indexUuid
|
|
10142
|
+
);
|
|
10143
|
+
}
|
|
10144
|
+
}
|
|
10145
|
+
async function tryRollback(scope, entries, timeoutMs, delayMs) {
|
|
10146
|
+
try {
|
|
10147
|
+
await withinDeadline(
|
|
10148
|
+
sendRollback(scope, entries),
|
|
10149
|
+
timeoutMs,
|
|
10150
|
+
() => new Error(`A try to complete a rollback did not finish within ${timeoutMs}ms`)
|
|
10151
|
+
);
|
|
10152
|
+
return true;
|
|
10153
|
+
} catch (error) {
|
|
10154
|
+
if (lostTerm(error)) throw error;
|
|
10155
|
+
}
|
|
10156
|
+
await pause(delayMs);
|
|
10157
|
+
return false;
|
|
10158
|
+
}
|
|
10159
|
+
async function completeRollback(scope, fromSeqNo, timeoutMs) {
|
|
10160
|
+
const log = scope.deps.getReplicationLog(scope.indexName, scope.partitionId);
|
|
10161
|
+
let delayMs = ROLLBACK_RETRY_INITIAL_DELAY_MS;
|
|
10162
|
+
for (; ; ) {
|
|
10163
|
+
if (scope.deps.stopped?.() === true) return "stopped";
|
|
10164
|
+
const entries = log.getEntriesFrom(fromSeqNo);
|
|
10165
|
+
const last = entries.at(-1);
|
|
10166
|
+
if (last === void 0) return "superseded";
|
|
10167
|
+
try {
|
|
10168
|
+
if (await untilSuperseded(log, last, tryRollback(scope, entries, timeoutMs, delayMs))) return "complete";
|
|
10169
|
+
} catch (error) {
|
|
10170
|
+
if (lostTerm(error)) return "superseded";
|
|
10171
|
+
}
|
|
10172
|
+
delayMs = Math.min(delayMs * 2, ROLLBACK_RETRY_DELAY_CAP_MS);
|
|
10173
|
+
}
|
|
10174
|
+
}
|
|
10175
|
+
async function settleRollback(scope, fromSeqNo, originalError, failedAt) {
|
|
10176
|
+
const timeoutMs = replicationTimeoutOf(scope.deps);
|
|
10177
|
+
const rollback = completeRollback(scope, fromSeqNo, timeoutMs);
|
|
10178
|
+
scope.holdQueue?.(rollback);
|
|
10179
|
+
let timer;
|
|
10180
|
+
const stillOpen = new Promise((resolve) => {
|
|
10181
|
+
timer = setTimeout(() => resolve("open"), Math.max(0, failedAt + timeoutMs - Date.now()));
|
|
10182
|
+
});
|
|
10183
|
+
const end = await Promise.race([rollback, stillOpen]).finally(() => clearTimeout(timer));
|
|
10184
|
+
if (end === "complete") return originalError;
|
|
10185
|
+
if (end === "superseded") {
|
|
10186
|
+
return outcomeUnknown(
|
|
10187
|
+
scope,
|
|
10188
|
+
originalError,
|
|
10189
|
+
"term_ended",
|
|
10190
|
+
"the primary lost its term before every in-sync copy dropped the write"
|
|
10191
|
+
);
|
|
10192
|
+
}
|
|
10193
|
+
const because = end === "stopped" ? "the primary shut down before every in-sync copy dropped the write" : `the primary could not confirm within ${timeoutMs}ms that every in-sync copy dropped it`;
|
|
10194
|
+
return outcomeUnknown(scope, originalError, "rollback_open", because);
|
|
10195
|
+
}
|
|
10196
|
+
|
|
10576
10197
|
// src/distribution/cluster-node/write-routing/sent-fields.ts
|
|
10577
10198
|
function isRecord3(value) {
|
|
10578
10199
|
return typeof value === "object" && value !== null && !Array.isArray(value) && !ArrayBuffer.isView(value);
|
|
@@ -10627,8 +10248,9 @@ function storedCopyMatchesSent(sent, stored, derivedFields) {
|
|
|
10627
10248
|
}
|
|
10628
10249
|
|
|
10629
10250
|
// src/distribution/cluster-node/write-routing/primary-change.ts
|
|
10630
|
-
function refusedByPrimaryChange(error) {
|
|
10631
|
-
|
|
10251
|
+
function refusedByPrimaryChange(error, retriesEndedTerm = false) {
|
|
10252
|
+
if (error instanceof NarsilError && error.code === ErrorCodes.PARTITION_NOT_PRIMARY) return true;
|
|
10253
|
+
return retriesEndedTerm && abandonedWithTerm(error);
|
|
10632
10254
|
}
|
|
10633
10255
|
function refusedForAnotherIndex(error) {
|
|
10634
10256
|
return refusedByPrimaryChange(error) && error instanceof NarsilError && typeof error.details.heldIndexUuid === "string";
|
|
@@ -10685,7 +10307,7 @@ async function awaitNewPrimary(deps, indexName, refusing, deadline) {
|
|
|
10685
10307
|
}
|
|
10686
10308
|
return false;
|
|
10687
10309
|
}
|
|
10688
|
-
async function retryOnPrimaryChange(deps, indexName, docId, attempt, settleRetried, settledValue) {
|
|
10310
|
+
async function retryOnPrimaryChange(deps, indexName, docId, attempt, settleRetried, settledValue, retriesEndedTerm = false) {
|
|
10689
10311
|
const deadline = Date.now() + PRIMARY_CHANGE_RETRY_WINDOW_MS;
|
|
10690
10312
|
let retryDelayMs = FORWARD_RETRY_INITIAL_DELAY_MS;
|
|
10691
10313
|
let retried = false;
|
|
@@ -10695,7 +10317,9 @@ async function retryOnPrimaryChange(deps, indexName, docId, attempt, settleRetri
|
|
|
10695
10317
|
return await attempt();
|
|
10696
10318
|
} catch (error) {
|
|
10697
10319
|
if (retried && (await settleRetried([{ docId, error }])).has(docId)) return settledValue();
|
|
10698
|
-
if (!(refusedByPrimaryChange(error) || forwardUndelivered(error)) || refusing === null)
|
|
10320
|
+
if (!(refusedByPrimaryChange(error, retriesEndedTerm) || forwardUndelivered(error)) || refusing === null) {
|
|
10321
|
+
throw error;
|
|
10322
|
+
}
|
|
10699
10323
|
let retryable;
|
|
10700
10324
|
if (retriedAfterDelay(error)) {
|
|
10701
10325
|
retryable = await pauseAndRereadTable(deps, indexName, retryDelayMs, deadline);
|
|
@@ -10708,7 +10332,7 @@ async function retryOnPrimaryChange(deps, indexName, docId, attempt, settleRetri
|
|
|
10708
10332
|
}
|
|
10709
10333
|
}
|
|
10710
10334
|
}
|
|
10711
|
-
async function retryBatchOnPrimaryChange(deps, indexName, items, docIdOf, attempt, settleRetried, deadline) {
|
|
10335
|
+
async function retryBatchOnPrimaryChange(deps, indexName, items, docIdOf, attempt, settleRetried, deadline, retriesEndedTerm) {
|
|
10712
10336
|
const succeeded2 = [];
|
|
10713
10337
|
const failed2 = [];
|
|
10714
10338
|
let pending = [...items];
|
|
@@ -10728,7 +10352,7 @@ async function retryBatchOnPrimaryChange(deps, indexName, items, docIdOf, attemp
|
|
|
10728
10352
|
}
|
|
10729
10353
|
const item = pendingById.get(failure.docId);
|
|
10730
10354
|
const refusing = primaryFor(table, failure.docId);
|
|
10731
|
-
if (item === void 0 || refusing === null || !(refusedByPrimaryChange(failure.error) || forwardUndelivered(failure.error))) {
|
|
10355
|
+
if (item === void 0 || refusing === null || !(refusedByPrimaryChange(failure.error, retriesEndedTerm) || forwardUndelivered(failure.error))) {
|
|
10732
10356
|
failed2.push(failure);
|
|
10733
10357
|
continue;
|
|
10734
10358
|
}
|
|
@@ -10783,12 +10407,12 @@ async function settleEachBounded(inputs, limit, run) {
|
|
|
10783
10407
|
await Promise.all(Array.from({ length: Math.min(limit, inputs.length) }, work));
|
|
10784
10408
|
return outcomes;
|
|
10785
10409
|
}
|
|
10786
|
-
async function retryEachPartition(deps, indexName, table, items, docIdOf, attempt, settleRetried) {
|
|
10410
|
+
async function retryEachPartition(deps, indexName, table, items, docIdOf, attempt, settleRetried, retriesEndedTerm = false) {
|
|
10787
10411
|
const deadline = Date.now() + PRIMARY_CHANGE_RETRY_WINDOW_MS;
|
|
10788
10412
|
const outcomes = await settleEachBounded(
|
|
10789
10413
|
groupByPartition(table, items, docIdOf),
|
|
10790
10414
|
PARTITION_WRITE_CONCURRENCY,
|
|
10791
|
-
(group) => retryBatchOnPrimaryChange(deps, indexName, group, docIdOf, attempt, settleRetried, deadline)
|
|
10415
|
+
(group) => retryBatchOnPrimaryChange(deps, indexName, group, docIdOf, attempt, settleRetried, deadline, retriesEndedTerm)
|
|
10792
10416
|
);
|
|
10793
10417
|
const merged = { succeeded: [], failed: [] };
|
|
10794
10418
|
for (const outcome of outcomes) {
|
|
@@ -10858,7 +10482,7 @@ function createGroupCommit(scope, appended, failed2) {
|
|
|
10858
10482
|
}
|
|
10859
10483
|
};
|
|
10860
10484
|
}
|
|
10861
|
-
async function replicateAppendedWrites(scope, appended, failed2,
|
|
10485
|
+
async function replicateAppendedWrites(scope, appended, failed2, compensate, deps) {
|
|
10862
10486
|
const succeeded2 = [];
|
|
10863
10487
|
const chunks = chunkReplicationEntries(appended);
|
|
10864
10488
|
const abandoned = [];
|
|
@@ -10880,14 +10504,23 @@ async function replicateAppendedWrites(scope, appended, failed2, rollback, deps)
|
|
|
10880
10504
|
}
|
|
10881
10505
|
abandoned.push(...chunk);
|
|
10882
10506
|
}
|
|
10507
|
+
const first = abandoned[0];
|
|
10508
|
+
if (first === void 0) return succeeded2;
|
|
10509
|
+
const failedAt = Date.now();
|
|
10510
|
+
const compensated = [];
|
|
10883
10511
|
for (let index = abandoned.length - 1; index >= 0; index--) {
|
|
10884
10512
|
const item = abandoned[index];
|
|
10885
10513
|
try {
|
|
10886
|
-
await
|
|
10514
|
+
await compensate(item, abortError);
|
|
10515
|
+
compensated.push(item);
|
|
10887
10516
|
} catch (err) {
|
|
10888
10517
|
failed2.push({ docId: item.docId, error: asWriteError(err, ErrorCodes.DOC_VALIDATION_FAILED, item.docId) });
|
|
10889
10518
|
}
|
|
10890
10519
|
}
|
|
10520
|
+
const answer = await settleRollback(scope, first.entry.seqNo, abortError, failedAt);
|
|
10521
|
+
for (const item of compensated) {
|
|
10522
|
+
failed2.push({ docId: item.docId, error: asWriteError(answer, ErrorCodes.DOC_VALIDATION_FAILED, item.docId) });
|
|
10523
|
+
}
|
|
10891
10524
|
return succeeded2;
|
|
10892
10525
|
}
|
|
10893
10526
|
|
|
@@ -10971,7 +10604,7 @@ function createRollbackFailure(operation, scope, documentId, originalError, roll
|
|
|
10971
10604
|
}
|
|
10972
10605
|
);
|
|
10973
10606
|
}
|
|
10974
|
-
async function
|
|
10607
|
+
async function compensateInsert(scope, documentId, originalError) {
|
|
10975
10608
|
try {
|
|
10976
10609
|
await numberedWrite(
|
|
10977
10610
|
scope,
|
|
@@ -10983,9 +10616,8 @@ async function rollbackPrimaryInsert(scope, documentId, originalError) {
|
|
|
10983
10616
|
throw createRollbackFailure("insert", scope, documentId, originalError, rollbackError);
|
|
10984
10617
|
}
|
|
10985
10618
|
}
|
|
10986
|
-
throwWriteFailure(originalError);
|
|
10987
10619
|
}
|
|
10988
|
-
async function
|
|
10620
|
+
async function compensateUpdate(scope, documentId, previousDocument, originalError) {
|
|
10989
10621
|
if (previousDocument === void 0) {
|
|
10990
10622
|
throw createRollbackFailure(
|
|
10991
10623
|
"update",
|
|
@@ -11004,9 +10636,8 @@ async function rollbackPrimaryUpdate(scope, documentId, previousDocument, origin
|
|
|
11004
10636
|
} catch (rollbackError) {
|
|
11005
10637
|
throw createRollbackFailure("update", scope, documentId, originalError, rollbackError);
|
|
11006
10638
|
}
|
|
11007
|
-
throwWriteFailure(originalError);
|
|
11008
10639
|
}
|
|
11009
|
-
async function
|
|
10640
|
+
async function compensateRemove(scope, documentId, previousDocument, originalError) {
|
|
11010
10641
|
if (previousDocument === void 0) {
|
|
11011
10642
|
throw createRollbackFailure(
|
|
11012
10643
|
"remove",
|
|
@@ -11025,7 +10656,6 @@ async function rollbackPrimaryRemove(scope, documentId, previousDocument, origin
|
|
|
11025
10656
|
} catch (rollbackError) {
|
|
11026
10657
|
throw createRollbackFailure("remove", scope, documentId, originalError, rollbackError);
|
|
11027
10658
|
}
|
|
11028
|
-
throwWriteFailure(originalError);
|
|
11029
10659
|
}
|
|
11030
10660
|
|
|
11031
10661
|
// src/distribution/cluster-node/reshard/state.ts
|
|
@@ -11157,6 +10787,9 @@ async function leadershipUnconfirmed(scope) {
|
|
|
11157
10787
|
if (assignment === void 0) return true;
|
|
11158
10788
|
return assignment.primary !== deps.nodeId || assignment.primaryTerm > scope.assignment.primaryTerm;
|
|
11159
10789
|
}
|
|
10790
|
+
function logEndOf(scope) {
|
|
10791
|
+
return scope.deps.getReplicationLog(scope.indexName, scope.partitionId).localLogEnd;
|
|
10792
|
+
}
|
|
11160
10793
|
function copyFailedIn(scope) {
|
|
11161
10794
|
return scope.deps.copyFailed?.(scope.indexName, scope.partitionId) === true;
|
|
11162
10795
|
}
|
|
@@ -11203,24 +10836,36 @@ async function currentScope(scope) {
|
|
|
11203
10836
|
}
|
|
11204
10837
|
async function inLeadingPartitionQueue(scope, task, failed2) {
|
|
11205
10838
|
let current = scope;
|
|
10839
|
+
let logEndBefore;
|
|
11206
10840
|
let result;
|
|
11207
10841
|
try {
|
|
11208
10842
|
result = await enqueuePartitionWrite(
|
|
11209
10843
|
scope.deps.partitionWriteQueues,
|
|
11210
10844
|
scope.indexName,
|
|
11211
10845
|
scope.partitionId,
|
|
11212
|
-
async () => {
|
|
11213
|
-
current = await currentScope(scope);
|
|
10846
|
+
async (hold) => {
|
|
10847
|
+
current = { ...await currentScope(scope), holdQueue: hold };
|
|
10848
|
+
logEndBefore = logEndOf(current);
|
|
11214
10849
|
return task(current);
|
|
11215
10850
|
}
|
|
11216
10851
|
);
|
|
11217
10852
|
} catch (error) {
|
|
10853
|
+
if (isOutcomeUnknown(error)) throw error;
|
|
11218
10854
|
if (copyFailedIn(current)) throw notPrimary(current);
|
|
11219
10855
|
if (refusedFromLocalCopy(error) && await leadershipUnconfirmed(current)) throw notPrimary(current);
|
|
11220
10856
|
throw error;
|
|
11221
10857
|
}
|
|
11222
|
-
if (await registrationLeaseLapsed(current.deps))
|
|
11223
|
-
|
|
10858
|
+
if (await registrationLeaseLapsed(current.deps)) {
|
|
10859
|
+
const refusal = notPrimary(current);
|
|
10860
|
+
if (logEndBefore === void 0 || logEndOf(current) === logEndBefore) throw refusal;
|
|
10861
|
+
throw outcomeUnknown(
|
|
10862
|
+
current,
|
|
10863
|
+
refusal,
|
|
10864
|
+
"lease_lapsed",
|
|
10865
|
+
"its registration lease lapsed before it could acknowledge the write"
|
|
10866
|
+
);
|
|
10867
|
+
}
|
|
10868
|
+
if (failed2 !== void 0 && copyFailedIn(current)) refuseFromFailedCopy(current, failed2);
|
|
11224
10869
|
if (failed2 !== void 0) await confirmBatchRefusals(current, failed2);
|
|
11225
10870
|
return result;
|
|
11226
10871
|
}
|
|
@@ -11509,7 +11154,9 @@ async function applyPrimaryInsert(indexName, document, docId, partitionId, assig
|
|
|
11509
11154
|
try {
|
|
11510
11155
|
await replicateEntry(inserted.entry, scope.assignment, deps, scope.indexUuid);
|
|
11511
11156
|
} catch (error) {
|
|
11512
|
-
|
|
11157
|
+
const failedAt = Date.now();
|
|
11158
|
+
await compensateInsert(scope, inserted.value, error);
|
|
11159
|
+
throwWriteFailure(await settleRollback(scope, inserted.entry.seqNo, error, failedAt));
|
|
11513
11160
|
}
|
|
11514
11161
|
return inserted.value;
|
|
11515
11162
|
} finally {
|
|
@@ -11530,7 +11177,9 @@ async function applyPrimaryRemove(indexName, docId, partitionId, assignment, dep
|
|
|
11530
11177
|
try {
|
|
11531
11178
|
await replicateEntry(removed.entry, scope.assignment, deps, scope.indexUuid);
|
|
11532
11179
|
} catch (error) {
|
|
11533
|
-
|
|
11180
|
+
const failedAt = Date.now();
|
|
11181
|
+
await compensateRemove(scope, docId, previousDocument, error);
|
|
11182
|
+
throwWriteFailure(await settleRollback(scope, removed.entry.seqNo, error, failedAt));
|
|
11534
11183
|
}
|
|
11535
11184
|
});
|
|
11536
11185
|
}
|
|
@@ -11549,7 +11198,9 @@ async function applyPrimaryUpdate(indexName, docId, document, partitionId, assig
|
|
|
11549
11198
|
try {
|
|
11550
11199
|
await replicateEntry(updated.entry, scope.assignment, deps, scope.indexUuid);
|
|
11551
11200
|
} catch (error) {
|
|
11552
|
-
|
|
11201
|
+
const failedAt = Date.now();
|
|
11202
|
+
await compensateUpdate(scope, docId, previousDocument, error);
|
|
11203
|
+
throwWriteFailure(await settleRollback(scope, updated.entry.seqNo, error, failedAt));
|
|
11553
11204
|
}
|
|
11554
11205
|
} finally {
|
|
11555
11206
|
projected.release();
|
|
@@ -11590,7 +11241,7 @@ async function applyPrimaryInsertBatch(indexName, items, partitionId, assignment
|
|
|
11590
11241
|
scope,
|
|
11591
11242
|
appended,
|
|
11592
11243
|
failed2,
|
|
11593
|
-
(write, error) =>
|
|
11244
|
+
(write, error) => compensateInsert(scope, write.docId, error),
|
|
11594
11245
|
deps
|
|
11595
11246
|
);
|
|
11596
11247
|
},
|
|
@@ -11636,7 +11287,7 @@ async function applyPrimaryUpdateBatch(indexName, updates, partitionId, assignme
|
|
|
11636
11287
|
scope,
|
|
11637
11288
|
appended,
|
|
11638
11289
|
failed2,
|
|
11639
|
-
(write, error) =>
|
|
11290
|
+
(write, error) => compensateUpdate(scope, write.docId, write.previousDocument, error),
|
|
11640
11291
|
deps
|
|
11641
11292
|
);
|
|
11642
11293
|
},
|
|
@@ -11672,7 +11323,7 @@ async function applyPrimaryRemoveBatch(indexName, docIds, partitionId, assignmen
|
|
|
11672
11323
|
scope,
|
|
11673
11324
|
appended,
|
|
11674
11325
|
failed2,
|
|
11675
|
-
(write, error) =>
|
|
11326
|
+
(write, error) => compensateRemove(scope, write.docId, write.previousDocument, error),
|
|
11676
11327
|
deps
|
|
11677
11328
|
);
|
|
11678
11329
|
},
|
|
@@ -11939,7 +11590,8 @@ async function routeAdmittedInsertBatch(indexName, documents, deps, options) {
|
|
|
11939
11590
|
withIds,
|
|
11940
11591
|
(item) => item.docId,
|
|
11941
11592
|
async (pending) => routeInsertItems(await requireRoutableTable(deps, indexName), indexName, pending, deps, options),
|
|
11942
|
-
settleEqualInserts(deps, indexName, sentById)
|
|
11593
|
+
settleEqualInserts(deps, indexName, sentById),
|
|
11594
|
+
true
|
|
11943
11595
|
);
|
|
11944
11596
|
}
|
|
11945
11597
|
async function requireRoutableTable(deps, indexName) {
|
|
@@ -12434,7 +12086,8 @@ async function routeAdmittedInsert(indexName, document, docId, deps, options) {
|
|
|
12434
12086
|
resolvedDocId,
|
|
12435
12087
|
() => insertOnce(indexName, document, resolvedDocId, deps, options),
|
|
12436
12088
|
settleEqualInserts(deps, indexName, /* @__PURE__ */ new Map([[resolvedDocId, document]])),
|
|
12437
|
-
() => resolvedDocId
|
|
12089
|
+
() => resolvedDocId,
|
|
12090
|
+
true
|
|
12438
12091
|
);
|
|
12439
12092
|
}
|
|
12440
12093
|
async function insertOnce(indexName, document, resolvedDocId, deps, options) {
|
|
@@ -12508,11 +12161,14 @@ async function routeAdmittedUpdate(indexName, docId, document, deps) {
|
|
|
12508
12161
|
|
|
12509
12162
|
// src/distribution/cluster-node/node-messaging.ts
|
|
12510
12163
|
async function resolveNodeTargets2(config, targetNodeId) {
|
|
12164
|
+
const targets = await resolveRegisteredTargets(config, targetNodeId);
|
|
12165
|
+
return targets.length > 0 ? targets : [targetNodeId];
|
|
12166
|
+
}
|
|
12167
|
+
async function resolveRegisteredTargets(config, targetNodeId) {
|
|
12511
12168
|
const nodes = await config.coordinator.listNodes();
|
|
12512
12169
|
const registration = nodes.find((node) => node.nodeId === targetNodeId);
|
|
12513
|
-
if (registration === void 0
|
|
12514
|
-
|
|
12515
|
-
}
|
|
12170
|
+
if (registration === void 0) return [];
|
|
12171
|
+
if (registration.address.length === 0 || registration.address === targetNodeId) return [targetNodeId];
|
|
12516
12172
|
return [registration.address, targetNodeId];
|
|
12517
12173
|
}
|
|
12518
12174
|
async function sendToNode2(config, targetNodeId, message, failure) {
|
|
@@ -12556,6 +12212,18 @@ function routeDocumentIds(docIds, allocation) {
|
|
|
12556
12212
|
}
|
|
12557
12213
|
return { routing, idsByPartition };
|
|
12558
12214
|
}
|
|
12215
|
+
function readRejectionOf(decoded, targetNodeId, indexName) {
|
|
12216
|
+
const details = { ...remoteErrorDetails(decoded.details), indexName, targetNodeId };
|
|
12217
|
+
return nodeRejectionOf(decoded.code, decoded.message, details) ?? new NarsilError(
|
|
12218
|
+
typeof decoded.code === "string" ? decoded.code : ErrorCodes.QUERY_ROUTING_FAILED,
|
|
12219
|
+
typeof decoded.message === "string" ? decoded.message : `Node '${targetNodeId}' rejected a read request for index '${indexName}'`,
|
|
12220
|
+
details
|
|
12221
|
+
);
|
|
12222
|
+
}
|
|
12223
|
+
function refuseDecodeWithoutRoom(payload) {
|
|
12224
|
+
const refusal = answerRefusalOf(payload.byteLength * ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE);
|
|
12225
|
+
if (refusal !== null) throw markNodeRejection(refusal);
|
|
12226
|
+
}
|
|
12559
12227
|
async function readFromCopies(config, nodeId, engine, indexName, documentRouting, fields) {
|
|
12560
12228
|
const documents = /* @__PURE__ */ new Map();
|
|
12561
12229
|
await sendToEveryPartition(documentRouting.routing, async (targetNodeId, partitionIds) => {
|
|
@@ -12570,14 +12238,19 @@ async function readFromCopies(config, nodeId, engine, indexName, documentRouting
|
|
|
12570
12238
|
);
|
|
12571
12239
|
for (const [docId, document] of local) found.set(docId, document);
|
|
12572
12240
|
} else {
|
|
12573
|
-
for (let start = 0; start < documentIds.length; start +=
|
|
12574
|
-
const chunk = documentIds.slice(start, start +
|
|
12241
|
+
for (let start = 0; start < documentIds.length; start += FETCH_GROUP_DOCUMENT_IDS) {
|
|
12242
|
+
const chunk = documentIds.slice(start, start + FETCH_GROUP_DOCUMENT_IDS);
|
|
12575
12243
|
const fetchMessage = createFetchMessage({ indexName, documentIds: chunk, fields, highlight: null }, nodeId);
|
|
12576
12244
|
const response = await sendToNode2(config, targetNodeId, fetchMessage, {
|
|
12577
12245
|
message: `Node '${targetNodeId}' did not answer a document fetch for index '${indexName}'`,
|
|
12578
12246
|
details: { indexName }
|
|
12579
12247
|
});
|
|
12580
|
-
|
|
12248
|
+
refuseDecodeWithoutRoom(response.payload);
|
|
12249
|
+
const decoded = decode(response.payload);
|
|
12250
|
+
if (response.type.endsWith(".error") || decoded.error === true) {
|
|
12251
|
+
throw readRejectionOf(decoded, targetNodeId, indexName);
|
|
12252
|
+
}
|
|
12253
|
+
const payload = validateFetchResultPayload(decoded);
|
|
12581
12254
|
for (const fetched of payload.documents) {
|
|
12582
12255
|
found.set(fetched.docId, restoreVectorValues(fetched.document));
|
|
12583
12256
|
}
|
|
@@ -13033,7 +12706,7 @@ async function purgeGeneration(deps, job) {
|
|
|
13033
12706
|
}
|
|
13034
12707
|
|
|
13035
12708
|
// src/distribution/cluster-node/reshard/leadership.ts
|
|
13036
|
-
async function
|
|
12709
|
+
async function leadingAssignment2(deps, indexName, partitionId) {
|
|
13037
12710
|
let assignment = (await deps.coordinator.getAllocation(indexName))?.assignments.get(partitionId);
|
|
13038
12711
|
if (assignment?.primary !== deps.nodeId && deps.refreshAllocation !== void 0) {
|
|
13039
12712
|
assignment = (await deps.refreshAllocation(indexName))?.assignments.get(partitionId);
|
|
@@ -13229,7 +12902,7 @@ async function runCopy(deps, job, announce) {
|
|
|
13229
12902
|
}
|
|
13230
12903
|
async function startReshardCopy(deps, payload) {
|
|
13231
12904
|
const state = reshardState(deps);
|
|
13232
|
-
const assignment = await
|
|
12905
|
+
const assignment = await leadingAssignment2(deps.writeDeps, payload.sourceGeneration, payload.partitionId);
|
|
13233
12906
|
const key = jobKey(payload.jobId, payload.partitionId);
|
|
13234
12907
|
const existing = state.jobs.get(key);
|
|
13235
12908
|
if (existing !== void 0 && !existing.stopped && existing.primaryTerm === assignment.primaryTerm) {
|
|
@@ -13278,7 +12951,7 @@ async function applyReshardCutover(deps, payload) {
|
|
|
13278
12951
|
);
|
|
13279
12952
|
}
|
|
13280
12953
|
const source = record.sourceGeneration;
|
|
13281
|
-
const assignment = await
|
|
12954
|
+
const assignment = await leadingAssignment2(deps.writeDeps, source, payload.partitionId);
|
|
13282
12955
|
setReshardBlock(state, source, payload.jobId, payload.partitionId);
|
|
13283
12956
|
await enqueuePartitionWrite(deps.writeDeps.partitionWriteQueues, source, payload.partitionId, async () => void 0);
|
|
13284
12957
|
return forwardableEnd(deps.writeDeps.getReplicationLog(source, payload.partitionId), assignment.primaryTerm);
|
|
@@ -13301,7 +12974,7 @@ function failed(documentId, error) {
|
|
|
13301
12974
|
async function applyInPartition(deps, engine, indexName, partitionId, queued, results) {
|
|
13302
12975
|
try {
|
|
13303
12976
|
await enqueuePartitionWrite(deps.partitionWriteQueues, indexName, partitionId, async () => {
|
|
13304
|
-
const assignment = await
|
|
12977
|
+
const assignment = await leadingAssignment2(deps, indexName, partitionId);
|
|
13305
12978
|
const log = deps.getReplicationLog(indexName, partitionId);
|
|
13306
12979
|
noteLedTerm(log, assignment.primaryTerm);
|
|
13307
12980
|
const applied = [];
|
|
@@ -13531,1067 +13204,812 @@ async function handleReshardPurge(message, respond, deps) {
|
|
|
13531
13204
|
const result = { removed };
|
|
13532
13205
|
await respond(reply(deps, message, result));
|
|
13533
13206
|
}
|
|
13534
|
-
|
|
13535
|
-
|
|
13536
|
-
|
|
13537
|
-
|
|
13207
|
+
async function handleIndexSettings(message, respond, deps) {
|
|
13208
|
+
const payload = validateIndexSettingsPayload(decode(message.payload));
|
|
13209
|
+
const applied = payload !== null && await applyStoredSynonyms(deps.engine, deps.coordinator, payload.indexName, payload.indexUuid);
|
|
13210
|
+
const indexName = payload?.indexName ?? "";
|
|
13211
|
+
await respond(createIndexSettingsResultMessage({ indexName, applied }, deps.nodeId, message.requestId));
|
|
13538
13212
|
}
|
|
13539
13213
|
|
|
13540
|
-
// src/distribution/
|
|
13541
|
-
var
|
|
13542
|
-
|
|
13543
|
-
|
|
13544
|
-
|
|
13545
|
-
|
|
13546
|
-
|
|
13547
|
-
|
|
13548
|
-
|
|
13549
|
-
|
|
13550
|
-
|
|
13551
|
-
|
|
13552
|
-
|
|
13214
|
+
// src/distribution/query/types.ts
|
|
13215
|
+
var DEFAULT_QUERY_CONFIG = {
|
|
13216
|
+
allowPartialResults: true,
|
|
13217
|
+
partitionTimeout: 5e3,
|
|
13218
|
+
defaultFacetSize: 10
|
|
13219
|
+
};
|
|
13220
|
+
|
|
13221
|
+
// src/distribution/cluster-node/query-conversion.ts
|
|
13222
|
+
function wireParamsToLocal(wire, facetShardSize) {
|
|
13223
|
+
return {
|
|
13224
|
+
term: wire.term ?? void 0,
|
|
13225
|
+
fields: wire.fields ?? void 0,
|
|
13226
|
+
filters: wire.filters ?? void 0,
|
|
13227
|
+
boost: wire.boost ?? void 0,
|
|
13228
|
+
scoring: wire.scoring,
|
|
13229
|
+
tolerance: wire.tolerance ?? void 0,
|
|
13230
|
+
minScore: wire.threshold ?? void 0,
|
|
13231
|
+
includeScores: wire.includeScores ?? void 0,
|
|
13232
|
+
termMatch: wire.termMatch ?? void 0,
|
|
13233
|
+
prefixLength: wire.prefixLength ?? void 0,
|
|
13234
|
+
prefix: wire.prefix ?? void 0,
|
|
13235
|
+
partialWords: wire.partialWords ?? void 0,
|
|
13236
|
+
exact: wire.exact ?? void 0,
|
|
13237
|
+
pinned: wire.pinned !== null ? wire.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : void 0,
|
|
13238
|
+
mode: wire.mode ?? void 0,
|
|
13239
|
+
limit: wire.limit,
|
|
13240
|
+
offset: wire.offset,
|
|
13241
|
+
searchAfter: wire.searchAfter ?? void 0,
|
|
13242
|
+
sort: convertWireSortToLocal(wire.sort),
|
|
13243
|
+
group: convertWireGroupToLocal(wire.group),
|
|
13244
|
+
facets: convertWireFacetConfigToLocal(wire.facets, facetShardSize ?? wire.facetSize),
|
|
13245
|
+
vector: convertWireVectorToLocal(wire.vector),
|
|
13246
|
+
hybrid: convertWireHybridToLocal(wire.hybrid)
|
|
13247
|
+
};
|
|
13248
|
+
}
|
|
13249
|
+
function convertWireGroupToLocal(group) {
|
|
13250
|
+
if (group === null) {
|
|
13251
|
+
return void 0;
|
|
13553
13252
|
}
|
|
13554
|
-
|
|
13555
|
-
|
|
13556
|
-
|
|
13557
|
-
|
|
13558
|
-
|
|
13559
|
-
|
|
13253
|
+
const local = { fields: [...group.fields], maxPerGroup: group.maxPerGroup };
|
|
13254
|
+
if (group.limit !== null) local.limit = group.limit;
|
|
13255
|
+
return local;
|
|
13256
|
+
}
|
|
13257
|
+
function convertWireVectorToLocal(vector) {
|
|
13258
|
+
if (vector === null) {
|
|
13259
|
+
return void 0;
|
|
13560
13260
|
}
|
|
13561
|
-
|
|
13562
|
-
|
|
13563
|
-
|
|
13564
|
-
|
|
13565
|
-
|
|
13566
|
-
|
|
13567
|
-
|
|
13568
|
-
|
|
13569
|
-
|
|
13570
|
-
|
|
13571
|
-
|
|
13261
|
+
const local = { field: vector.field };
|
|
13262
|
+
if (vector.value !== null) local.value = vector.value;
|
|
13263
|
+
if (vector.text !== null) local.text = vector.text;
|
|
13264
|
+
if (vector.similarity !== null) local.similarity = vector.similarity;
|
|
13265
|
+
if (vector.metric !== null) local.metric = vector.metric;
|
|
13266
|
+
if (vector.efSearch !== null) local.efSearch = vector.efSearch;
|
|
13267
|
+
if (vector.oversample !== null) local.oversample = vector.oversample;
|
|
13268
|
+
return local;
|
|
13269
|
+
}
|
|
13270
|
+
function convertWireHybridToLocal(hybrid) {
|
|
13271
|
+
if (hybrid === null) {
|
|
13272
|
+
return void 0;
|
|
13572
13273
|
}
|
|
13573
|
-
|
|
13574
|
-
|
|
13575
|
-
|
|
13576
|
-
|
|
13577
|
-
|
|
13578
|
-
|
|
13274
|
+
const local = {};
|
|
13275
|
+
if (hybrid.strategy !== null) local.strategy = hybrid.strategy;
|
|
13276
|
+
if (hybrid.k !== null) local.k = hybrid.k;
|
|
13277
|
+
if (hybrid.alpha !== null) local.alpha = hybrid.alpha;
|
|
13278
|
+
if (hybrid.depth !== null) local.depth = hybrid.depth;
|
|
13279
|
+
return local;
|
|
13280
|
+
}
|
|
13281
|
+
function convertWireSortToLocal(wireSort) {
|
|
13282
|
+
if (wireSort === null || wireSort.length === 0) {
|
|
13283
|
+
return void 0;
|
|
13284
|
+
}
|
|
13285
|
+
return normalizeSort(wireSort);
|
|
13286
|
+
}
|
|
13287
|
+
function countsEveryValue(facet) {
|
|
13288
|
+
return facet.ranges !== null || facet.sort === "asc";
|
|
13289
|
+
}
|
|
13290
|
+
function convertWireFacetConfigToLocal(facets, limit) {
|
|
13291
|
+
if (facets === null || facets.length === 0) {
|
|
13292
|
+
return void 0;
|
|
13293
|
+
}
|
|
13294
|
+
const result = {};
|
|
13295
|
+
for (const facet of facets) {
|
|
13296
|
+
const options = {};
|
|
13297
|
+
if (countsEveryValue(facet)) options.limit = MAX_FACET_BUCKETS;
|
|
13298
|
+
else if (limit !== null) options.limit = limit;
|
|
13299
|
+
if (facet.sort !== null) options.sort = facet.sort;
|
|
13300
|
+
if (facet.ranges !== null) options.ranges = facet.ranges.map((range) => ({ from: range.from, to: range.to }));
|
|
13301
|
+
result[facet.field] = options;
|
|
13302
|
+
}
|
|
13303
|
+
return result;
|
|
13304
|
+
}
|
|
13305
|
+
function ascendingFacetFields(facets) {
|
|
13306
|
+
const ascending = /* @__PURE__ */ new Set();
|
|
13307
|
+
for (const facet of facets ?? []) {
|
|
13308
|
+
if (facet.sort === "asc") ascending.add(facet.field);
|
|
13579
13309
|
}
|
|
13310
|
+
return ascending;
|
|
13311
|
+
}
|
|
13312
|
+
function localParamsToWire(params) {
|
|
13580
13313
|
return {
|
|
13581
|
-
|
|
13582
|
-
|
|
13583
|
-
|
|
13314
|
+
term: params.term ?? null,
|
|
13315
|
+
filters: params.filters ?? null,
|
|
13316
|
+
sort: convertLocalSortToWire(params.sort),
|
|
13317
|
+
group: convertLocalGroupToWire(params.group),
|
|
13318
|
+
facets: convertLocalFacetsToWire(params.facets),
|
|
13319
|
+
facetSize: wireFacetSize(params.facets),
|
|
13320
|
+
limit: clampRowCount(params.limit, DEFAULT_PAGE_SIZE),
|
|
13321
|
+
offset: clampRowCount(params.offset, 0),
|
|
13322
|
+
searchAfter: params.searchAfter ?? null,
|
|
13323
|
+
fields: params.fields ?? null,
|
|
13324
|
+
boost: params.boost ?? null,
|
|
13325
|
+
tolerance: params.tolerance ?? null,
|
|
13326
|
+
threshold: params.minScore ?? null,
|
|
13327
|
+
includeScores: params.includeScores ?? null,
|
|
13328
|
+
scoring: params.scoring ?? "local",
|
|
13329
|
+
termMatch: params.termMatch ?? null,
|
|
13330
|
+
prefixLength: params.prefixLength ?? null,
|
|
13331
|
+
prefix: params.prefix ?? null,
|
|
13332
|
+
partialWords: params.partialWords ?? null,
|
|
13333
|
+
exact: params.exact ?? null,
|
|
13334
|
+
pinned: params.pinned !== void 0 ? params.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : null,
|
|
13335
|
+
mode: params.mode ?? null,
|
|
13336
|
+
vector: convertLocalVectorToWire(params.vector),
|
|
13337
|
+
hybrid: convertLocalHybridToWire(params.hybrid)
|
|
13584
13338
|
};
|
|
13585
13339
|
}
|
|
13586
|
-
|
|
13587
|
-
|
|
13588
|
-
|
|
13589
|
-
|
|
13340
|
+
function countIsExactFor(params) {
|
|
13341
|
+
const vector = params.vector;
|
|
13342
|
+
if (vector === void 0 || vector.value === void 0 && vector.text === void 0) return true;
|
|
13343
|
+
const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
|
|
13344
|
+
if (params.mode === "hybrid" || hasTerm) return false;
|
|
13345
|
+
return vector.similarity === void 0;
|
|
13590
13346
|
}
|
|
13591
|
-
function
|
|
13592
|
-
|
|
13347
|
+
function distributedCountIsExact(params, coverage) {
|
|
13348
|
+
if (coverage.queriedPartitions < coverage.totalPartitions) return false;
|
|
13349
|
+
if (coverage.timedOutPartitions > 0 || coverage.failedPartitions > 0) return false;
|
|
13350
|
+
return countIsExactFor(params);
|
|
13351
|
+
}
|
|
13352
|
+
function distributedResultToLocal(result, countExact, documents = /* @__PURE__ */ new Map(), requestedFacets) {
|
|
13593
13353
|
return {
|
|
13594
|
-
|
|
13595
|
-
|
|
13596
|
-
|
|
13597
|
-
|
|
13598
|
-
|
|
13599
|
-
|
|
13600
|
-
|
|
13354
|
+
hits: result.scored.map((entry) => ({
|
|
13355
|
+
id: entry.docId,
|
|
13356
|
+
score: entry.score ?? void 0,
|
|
13357
|
+
document: documents.get(entry.docId) ?? {}
|
|
13358
|
+
})),
|
|
13359
|
+
count: result.totalHits,
|
|
13360
|
+
countExact,
|
|
13361
|
+
elapsed: 0,
|
|
13362
|
+
cursor: result.cursor ?? void 0,
|
|
13363
|
+
facets: result.facets !== null ? convertWireFacetsToLocal(result, result.facets, requestedFacets) : void 0,
|
|
13364
|
+
coverage: result.coverage
|
|
13601
13365
|
};
|
|
13602
13366
|
}
|
|
13603
|
-
|
|
13604
|
-
|
|
13605
|
-
return
|
|
13367
|
+
function convertLocalSortToWire(sort) {
|
|
13368
|
+
const fields = normalizeSort(sort);
|
|
13369
|
+
return fields.length === 0 ? null : fields;
|
|
13606
13370
|
}
|
|
13607
|
-
function
|
|
13608
|
-
if (
|
|
13609
|
-
|
|
13610
|
-
maxConcurrent
|
|
13611
|
-
});
|
|
13612
|
-
}
|
|
13613
|
-
if (!Number.isInteger(maxPerSource) || maxPerSource <= 0) {
|
|
13614
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "maxPerSource must be a positive integer", {
|
|
13615
|
-
maxPerSource
|
|
13616
|
-
});
|
|
13617
|
-
}
|
|
13618
|
-
if (!Number.isInteger(maxStreamsPerIndex) || maxStreamsPerIndex <= 0) {
|
|
13619
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "maxStreamsPerIndex must be a positive integer", {
|
|
13620
|
-
maxStreamsPerIndex
|
|
13621
|
-
});
|
|
13371
|
+
function convertLocalGroupToWire(group) {
|
|
13372
|
+
if (group === void 0) {
|
|
13373
|
+
return null;
|
|
13622
13374
|
}
|
|
13623
13375
|
return {
|
|
13624
|
-
|
|
13625
|
-
|
|
13626
|
-
|
|
13627
|
-
streamActive: /* @__PURE__ */ new Map(),
|
|
13628
|
-
maxConcurrent,
|
|
13629
|
-
maxPerSource,
|
|
13630
|
-
maxStreamsPerIndex
|
|
13376
|
+
fields: [...group.fields],
|
|
13377
|
+
maxPerGroup: group.reduce !== void 0 ? MAX_LIMIT : hitsKeptPerGroup(group.maxPerGroup),
|
|
13378
|
+
limit: group.limit ?? null
|
|
13631
13379
|
};
|
|
13632
13380
|
}
|
|
13633
|
-
function
|
|
13634
|
-
return
|
|
13381
|
+
function wireFacetOrder(sort) {
|
|
13382
|
+
return sort === "asc" || sort === "desc" ? sort : null;
|
|
13635
13383
|
}
|
|
13636
|
-
function
|
|
13637
|
-
|
|
13638
|
-
|
|
13639
|
-
if (current >= state.maxPerSource) {
|
|
13640
|
-
throw new NarsilError(
|
|
13641
|
-
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
13642
|
-
`snapshot producer per-source cap reached for '${sourceId}' index '${indexName}'`,
|
|
13643
|
-
{ sourceId, indexName, cap: state.maxPerSource }
|
|
13644
|
-
);
|
|
13384
|
+
function convertLocalFacetsToWire(facets) {
|
|
13385
|
+
if (facets === void 0) {
|
|
13386
|
+
return null;
|
|
13645
13387
|
}
|
|
13646
|
-
|
|
13647
|
-
|
|
13388
|
+
return Object.entries(facets).map(([field, options]) => ({
|
|
13389
|
+
field,
|
|
13390
|
+
sort: wireFacetOrder(options?.sort),
|
|
13391
|
+
ranges: options?.ranges?.map((range) => ({ from: range.from, to: range.to })) ?? null
|
|
13392
|
+
}));
|
|
13648
13393
|
}
|
|
13649
|
-
function
|
|
13650
|
-
if (
|
|
13651
|
-
return;
|
|
13394
|
+
function convertLocalVectorToWire(vector) {
|
|
13395
|
+
if (vector === void 0) {
|
|
13396
|
+
return null;
|
|
13652
13397
|
}
|
|
13653
|
-
|
|
13654
|
-
|
|
13655
|
-
|
|
13656
|
-
|
|
13657
|
-
|
|
13658
|
-
|
|
13659
|
-
|
|
13660
|
-
|
|
13661
|
-
|
|
13662
|
-
|
|
13398
|
+
return {
|
|
13399
|
+
field: vector.field,
|
|
13400
|
+
value: vector.value === void 0 ? null : Array.from(vector.value),
|
|
13401
|
+
text: vector.text ?? null,
|
|
13402
|
+
similarity: vector.similarity ?? null,
|
|
13403
|
+
metric: vector.metric ?? null,
|
|
13404
|
+
efSearch: vector.efSearch ?? null,
|
|
13405
|
+
oversample: vector.oversample ?? null,
|
|
13406
|
+
rescoreWindow: null
|
|
13407
|
+
};
|
|
13663
13408
|
}
|
|
13664
|
-
function
|
|
13665
|
-
|
|
13666
|
-
|
|
13667
|
-
throw new NarsilError(
|
|
13668
|
-
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
13669
|
-
`snapshot stream fan-out cap reached for index '${indexName}'`,
|
|
13670
|
-
{ indexName, cap: state.maxStreamsPerIndex }
|
|
13671
|
-
);
|
|
13409
|
+
function convertLocalHybridToWire(hybrid) {
|
|
13410
|
+
if (hybrid === void 0) {
|
|
13411
|
+
return null;
|
|
13672
13412
|
}
|
|
13673
|
-
|
|
13674
|
-
|
|
13413
|
+
return {
|
|
13414
|
+
strategy: hybrid.strategy ?? null,
|
|
13415
|
+
k: hybrid.k ?? null,
|
|
13416
|
+
alpha: hybrid.alpha ?? null,
|
|
13417
|
+
depth: hybrid.depth ?? null
|
|
13418
|
+
};
|
|
13675
13419
|
}
|
|
13676
|
-
function
|
|
13677
|
-
|
|
13678
|
-
|
|
13679
|
-
|
|
13680
|
-
|
|
13681
|
-
const current = state.streamActive.get(handle.indexName);
|
|
13682
|
-
if (current === void 0) {
|
|
13683
|
-
return;
|
|
13420
|
+
function facetLimitOf(options) {
|
|
13421
|
+
const limit = options?.limit;
|
|
13422
|
+
const ranges = options?.ranges;
|
|
13423
|
+
if ((limit === void 0 || !Number.isFinite(limit)) && Array.isArray(ranges)) {
|
|
13424
|
+
return Math.min(Math.max(ranges.length, 1), MAX_FACET_SIZE);
|
|
13684
13425
|
}
|
|
13685
|
-
if (
|
|
13686
|
-
|
|
13687
|
-
|
|
13426
|
+
if (limit === void 0 || !Number.isFinite(limit)) return DEFAULT_QUERY_CONFIG.defaultFacetSize;
|
|
13427
|
+
return Math.min(Math.max(Math.floor(limit), 1), MAX_FACET_SIZE);
|
|
13428
|
+
}
|
|
13429
|
+
function wireFacetSize(facets) {
|
|
13430
|
+
if (facets === void 0) return null;
|
|
13431
|
+
let size = 0;
|
|
13432
|
+
for (const options of Object.values(facets)) size = Math.max(size, facetLimitOf(options));
|
|
13433
|
+
return size === 0 ? null : size;
|
|
13434
|
+
}
|
|
13435
|
+
function convertWireFacetsToLocal(result, wireFacets, requested) {
|
|
13436
|
+
const converted = {};
|
|
13437
|
+
for (const [field, buckets] of Object.entries(wireFacets)) {
|
|
13438
|
+
const mergedBound = result.facetErrorBounds?.[field] ?? 0;
|
|
13439
|
+
const kept = requested === void 0 ? buckets.length : Math.min(buckets.length, facetLimitOf(requested[field]));
|
|
13440
|
+
const values = {};
|
|
13441
|
+
for (let index = 0; index < kept; index++) values[buckets[index].value] = buckets[index].count;
|
|
13442
|
+
let largestCut = 0;
|
|
13443
|
+
for (let index = kept; index < buckets.length; index++) largestCut = Math.max(largestCut, buckets[index].count);
|
|
13444
|
+
const errorBound = kept < buckets.length ? (result.facetUndercounts?.[field] ?? mergedBound) + largestCut : mergedBound;
|
|
13445
|
+
converted[field] = { values, count: kept, errorBound: Math.max(errorBound, mergedBound) };
|
|
13688
13446
|
}
|
|
13689
|
-
|
|
13447
|
+
return converted;
|
|
13690
13448
|
}
|
|
13691
|
-
|
|
13692
|
-
|
|
13693
|
-
|
|
13694
|
-
|
|
13449
|
+
|
|
13450
|
+
// src/distribution/cluster-node/message-handler/queries.ts
|
|
13451
|
+
async function handleSearch(message, respond, deps) {
|
|
13452
|
+
const decoded = decode(message.payload);
|
|
13453
|
+
const payload = validateSearchPayload(decoded);
|
|
13454
|
+
const queryParams = wireParamsToLocal(payload.params, payload.facetShardSize);
|
|
13455
|
+
if (queryParams.group?.limit !== void 0) {
|
|
13456
|
+
queryParams.group = { ...queryParams.group, limit: oversampledShardSize(queryParams.group.limit) };
|
|
13695
13457
|
}
|
|
13696
|
-
|
|
13697
|
-
|
|
13698
|
-
|
|
13699
|
-
"snapshot producer capacity exhausted; retry later",
|
|
13700
|
-
{ active: state.active, max: state.maxConcurrent }
|
|
13701
|
-
);
|
|
13458
|
+
const rescoreWindow = payload.params.vector?.rescoreWindow ?? void 0;
|
|
13459
|
+
if (rescoreWindow !== void 0 && queryParams.sort === void 0 && queryParams.group === void 0) {
|
|
13460
|
+
queryParams.document = false;
|
|
13702
13461
|
}
|
|
13703
|
-
|
|
13704
|
-
|
|
13705
|
-
|
|
13706
|
-
|
|
13707
|
-
|
|
13708
|
-
|
|
13709
|
-
settled = true;
|
|
13710
|
-
const current = state.cache.get(indexName);
|
|
13711
|
-
if (current === currentEntry) {
|
|
13712
|
-
state.cache.delete(indexName);
|
|
13713
|
-
}
|
|
13714
|
-
state.active = Math.max(0, state.active - 1);
|
|
13715
|
-
};
|
|
13716
|
-
const rawBuild = (async () => builder())();
|
|
13717
|
-
const entry = { promise: rawBuild, holders: 0 };
|
|
13718
|
-
entry.promise = rawBuild.then(
|
|
13719
|
-
(result) => {
|
|
13720
|
-
clearEntry(entry);
|
|
13721
|
-
return result;
|
|
13722
|
-
},
|
|
13723
|
-
(err) => {
|
|
13724
|
-
clearEntry(entry);
|
|
13725
|
-
throw err;
|
|
13726
|
-
}
|
|
13462
|
+
const queryResult = await deps.engine.queryPartitions(
|
|
13463
|
+
payload.indexName,
|
|
13464
|
+
queryParams,
|
|
13465
|
+
payload.partitionIds,
|
|
13466
|
+
payload.globalStats ?? void 0,
|
|
13467
|
+
rescoreWindow
|
|
13727
13468
|
);
|
|
13728
|
-
|
|
13729
|
-
|
|
13730
|
-
|
|
13731
|
-
|
|
13732
|
-
|
|
13733
|
-
|
|
13734
|
-
|
|
13735
|
-
|
|
13736
|
-
|
|
13737
|
-
|
|
13469
|
+
const normalizedSort = queryParams.sort !== void 0 ? normalizeSort(queryParams.sort) : null;
|
|
13470
|
+
const sortFields = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
13471
|
+
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
13472
|
+
const scored = queryResult.hits.map((hit) => ({
|
|
13473
|
+
docId: hit.id,
|
|
13474
|
+
score: hit.score ?? null,
|
|
13475
|
+
sortValues: sortFields !== null ? readSortValues(hit.document, sortFields, sortModes).map(toWireSortValue) : null
|
|
13476
|
+
}));
|
|
13477
|
+
const results = payload.partitionIds.map((partitionId) => ({
|
|
13478
|
+
partitionId,
|
|
13479
|
+
scored: partitionId === payload.partitionIds[0] ? scored : [],
|
|
13480
|
+
totalHits: partitionId === payload.partitionIds[0] ? queryResult.count : 0
|
|
13481
|
+
}));
|
|
13482
|
+
const resultPayload = {
|
|
13483
|
+
results,
|
|
13484
|
+
facets: convertLocalFacetsToWire2(queryResult.facets),
|
|
13485
|
+
facetErrorBounds: convertLocalFacetBoundsToWire(queryResult.facets),
|
|
13486
|
+
groups: await convertLocalGroupsToWire(queryResult.groups, sortFields, sortModes, deps, payload.indexName)
|
|
13487
|
+
};
|
|
13488
|
+
await respond({
|
|
13489
|
+
type: QueryMessageTypes.SEARCH_RESULT,
|
|
13490
|
+
sourceId: deps.nodeId,
|
|
13491
|
+
requestId: message.requestId,
|
|
13492
|
+
payload: encodeDocumentReply(resultPayload)
|
|
13738
13493
|
});
|
|
13739
|
-
sink.closed = false;
|
|
13740
|
-
return sink;
|
|
13741
13494
|
}
|
|
13742
|
-
async function
|
|
13743
|
-
|
|
13744
|
-
|
|
13745
|
-
const header = {
|
|
13746
|
-
lastSeqNo: metadata.lastSeqNo,
|
|
13747
|
-
primaryTerm: metadata.primaryTerm,
|
|
13748
|
-
partitionId: metadata.partitionId,
|
|
13749
|
-
indexName,
|
|
13750
|
-
checksum
|
|
13751
|
-
};
|
|
13752
|
-
const startPayload = { header, totalBytes };
|
|
13753
|
-
const startBytes = encode(startPayload);
|
|
13754
|
-
assertMessageWithinLimit(startBytes, "SNAPSHOT_START");
|
|
13755
|
-
await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_START, nodeId, requestId, startBytes);
|
|
13756
|
-
await yieldToEventLoop2();
|
|
13757
|
-
let offset = 0;
|
|
13758
|
-
let lastYieldAt = now2();
|
|
13759
|
-
for await (const source of build.read()) {
|
|
13760
|
-
for (let from = 0; from < source.byteLength; from += SNAPSHOT_CHUNK_SIZE) {
|
|
13761
|
-
const chunk = source.subarray(from, Math.min(from + SNAPSHOT_CHUNK_SIZE, source.byteLength));
|
|
13762
|
-
const chunkPayload = {
|
|
13763
|
-
partitionId: metadata.partitionId,
|
|
13764
|
-
indexName,
|
|
13765
|
-
offset,
|
|
13766
|
-
data: chunk
|
|
13767
|
-
};
|
|
13768
|
-
const chunkBytes = encode(chunkPayload);
|
|
13769
|
-
assertMessageWithinLimit(chunkBytes, "SNAPSHOT_CHUNK");
|
|
13770
|
-
await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_CHUNK, nodeId, requestId, chunkBytes);
|
|
13771
|
-
offset += chunk.byteLength;
|
|
13772
|
-
if (offset < totalBytes && now2() - lastYieldAt >= SNAPSHOT_CHUNK_YIELD_INTERVAL_MS) {
|
|
13773
|
-
await yieldToEventLoop2();
|
|
13774
|
-
lastYieldAt = now2();
|
|
13775
|
-
}
|
|
13776
|
-
}
|
|
13495
|
+
async function convertLocalGroupsToWire(groups, sortFields, sortModes, deps, indexName) {
|
|
13496
|
+
if (groups === void 0) {
|
|
13497
|
+
return null;
|
|
13777
13498
|
}
|
|
13778
|
-
|
|
13779
|
-
const
|
|
13780
|
-
|
|
13781
|
-
|
|
13782
|
-
|
|
13783
|
-
|
|
13784
|
-
|
|
13785
|
-
|
|
13786
|
-
|
|
13787
|
-
|
|
13788
|
-
|
|
13789
|
-
sink.closed = true;
|
|
13499
|
+
const unreadIds = sortFields === null ? [] : groups.flatMap((group) => group.hits.filter((hit) => hit.document === void 0).map((hit) => hit.id));
|
|
13500
|
+
const readDocuments = unreadIds.length > 0 ? await deps.engine.getMultiple(indexName, unreadIds) : /* @__PURE__ */ new Map();
|
|
13501
|
+
const wireGroups = [];
|
|
13502
|
+
for (const group of groups) {
|
|
13503
|
+
const scored = group.hits.map((hit) => {
|
|
13504
|
+
if (sortFields === null) return { docId: hit.id, score: hit.score ?? null, sortValues: null };
|
|
13505
|
+
const document = hit.document ?? readDocuments.get(hit.id);
|
|
13506
|
+
const sortValues = readSortValues(document, sortFields, sortModes).map(toWireSortValue);
|
|
13507
|
+
return { docId: hit.id, score: hit.score ?? null, sortValues };
|
|
13508
|
+
});
|
|
13509
|
+
wireGroups.push({ values: group.values, scored });
|
|
13790
13510
|
}
|
|
13511
|
+
return wireGroups;
|
|
13791
13512
|
}
|
|
13792
|
-
function
|
|
13793
|
-
|
|
13794
|
-
|
|
13513
|
+
async function handleFetch(message, respond, deps) {
|
|
13514
|
+
const decoded = decode(message.payload);
|
|
13515
|
+
const payload = validateFetchPayload(decoded);
|
|
13516
|
+
const projection = resolveProjection(payload.fields === null ? void 0 : { include: payload.fields });
|
|
13517
|
+
const documents = [];
|
|
13518
|
+
for (const docRef of payload.documentIds) {
|
|
13519
|
+
const doc = await deps.engine.get(payload.indexName, docRef.docId);
|
|
13520
|
+
if (doc !== void 0) {
|
|
13521
|
+
documents.push({
|
|
13522
|
+
docId: docRef.docId,
|
|
13523
|
+
document: applyProjection(doc, projection),
|
|
13524
|
+
highlights: null
|
|
13525
|
+
});
|
|
13526
|
+
}
|
|
13795
13527
|
}
|
|
13796
|
-
|
|
13528
|
+
refuseEncodeWithoutRoom(documents.map((fetched) => fetched.document));
|
|
13529
|
+
const resultPayload = { documents };
|
|
13530
|
+
await respond({
|
|
13531
|
+
type: QueryMessageTypes.FETCH_RESULT,
|
|
13532
|
+
sourceId: deps.nodeId,
|
|
13533
|
+
requestId: message.requestId,
|
|
13534
|
+
payload: encodeDocumentReply(resultPayload)
|
|
13535
|
+
});
|
|
13797
13536
|
}
|
|
13798
|
-
function
|
|
13799
|
-
|
|
13800
|
-
|
|
13801
|
-
|
|
13802
|
-
|
|
13803
|
-
|
|
13804
|
-
|
|
13537
|
+
async function handleStats(message, respond, deps) {
|
|
13538
|
+
const decoded = decode(message.payload);
|
|
13539
|
+
const payload = validateStatsPayload(decoded);
|
|
13540
|
+
const resultPayload = await deps.engine.collectQueryStats(
|
|
13541
|
+
payload.indexName,
|
|
13542
|
+
payload.terms,
|
|
13543
|
+
payload.partitionIds,
|
|
13544
|
+
{ expansion: payload.expansion ?? void 0, tokens: payload.tokens ?? void 0 }
|
|
13545
|
+
);
|
|
13546
|
+
await respond({
|
|
13547
|
+
type: QueryMessageTypes.STATS_RESULT,
|
|
13548
|
+
sourceId: deps.nodeId,
|
|
13549
|
+
requestId: message.requestId,
|
|
13550
|
+
payload: encode(resultPayload)
|
|
13805
13551
|
});
|
|
13806
13552
|
}
|
|
13807
|
-
function
|
|
13808
|
-
|
|
13553
|
+
function toWireSortValue(value) {
|
|
13554
|
+
if (typeof value === "string") return value;
|
|
13555
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
13556
|
+
if (typeof value === "boolean") return value;
|
|
13557
|
+
return null;
|
|
13809
13558
|
}
|
|
13810
|
-
function
|
|
13811
|
-
if (
|
|
13812
|
-
|
|
13813
|
-
ErrorCodes.CONFIG_INVALID,
|
|
13814
|
-
`${label} payload (${bytes.byteLength} bytes) exceeds transport message limit (${MAX_MESSAGE_SIZE_BYTES})`
|
|
13815
|
-
);
|
|
13816
|
-
}
|
|
13817
|
-
}
|
|
13818
|
-
async function respondError(sink, sourceId, requestId, code, message, details) {
|
|
13819
|
-
const response = {
|
|
13820
|
-
type: SNAPSHOT_SYNC_ERROR_TYPE,
|
|
13821
|
-
sourceId,
|
|
13822
|
-
requestId,
|
|
13823
|
-
payload: encodeErrorReply(code, message, details)
|
|
13824
|
-
};
|
|
13825
|
-
await sink(response);
|
|
13826
|
-
sink.closed = true;
|
|
13827
|
-
}
|
|
13828
|
-
|
|
13829
|
-
// src/distribution/replication/snapshot-constants.ts
|
|
13830
|
-
var SNAPSHOT_HEADER_SENTINEL_SEQNO = Number.MAX_SAFE_INTEGER;
|
|
13831
|
-
var SNAPSHOT_HEADER_SENTINEL_PARTITION_ID = 4294967295;
|
|
13832
|
-
|
|
13833
|
-
// src/distribution/cluster-node/snapshot-sync-handler/metadata.ts
|
|
13834
|
-
async function defaultSnapshotHeaderMetadataProvider(coordinator, indexName) {
|
|
13835
|
-
try {
|
|
13836
|
-
const allocation = await coordinator.getAllocation(indexName);
|
|
13837
|
-
if (allocation !== null) {
|
|
13838
|
-
for (const assignment of allocation.assignments.values()) {
|
|
13839
|
-
if (Number.isInteger(assignment.primaryTerm) && assignment.primaryTerm >= 0) {
|
|
13840
|
-
return {
|
|
13841
|
-
partitionId: SNAPSHOT_HEADER_SENTINEL_PARTITION_ID,
|
|
13842
|
-
primaryTerm: assignment.primaryTerm,
|
|
13843
|
-
lastSeqNo: SNAPSHOT_HEADER_SENTINEL_SEQNO
|
|
13844
|
-
};
|
|
13845
|
-
}
|
|
13846
|
-
}
|
|
13847
|
-
}
|
|
13848
|
-
} catch (_) {
|
|
13849
|
-
}
|
|
13850
|
-
return {
|
|
13851
|
-
partitionId: SNAPSHOT_HEADER_SENTINEL_PARTITION_ID,
|
|
13852
|
-
primaryTerm: 0,
|
|
13853
|
-
lastSeqNo: SNAPSHOT_HEADER_SENTINEL_SEQNO
|
|
13854
|
-
};
|
|
13855
|
-
}
|
|
13856
|
-
async function nodeSnapshotHeaderMetadata(deps, indexName, partitionId) {
|
|
13857
|
-
const fallback = await defaultSnapshotHeaderMetadataProvider(deps.coordinator, indexName);
|
|
13858
|
-
if (partitionId === null) {
|
|
13859
|
-
return fallback;
|
|
13860
|
-
}
|
|
13861
|
-
const localIndex = deps.engine.listIndexes().find((index) => index.name === indexName);
|
|
13862
|
-
const release = localIndex !== void 0 ? await deps.engine.acquireIndexForReplication(indexName) : null;
|
|
13863
|
-
try {
|
|
13864
|
-
const allocation = await deps.coordinator.getAllocation(indexName);
|
|
13865
|
-
const assignment = allocation?.assignments.get(partitionId);
|
|
13866
|
-
const log = deps.getReplicationLog(indexName, partitionId);
|
|
13867
|
-
return {
|
|
13868
|
-
partitionId,
|
|
13869
|
-
primaryTerm: assignment?.primaryTerm ?? fallback.primaryTerm,
|
|
13870
|
-
lastSeqNo: log.newestSeqNo ?? 0
|
|
13871
|
-
};
|
|
13872
|
-
} finally {
|
|
13873
|
-
release?.();
|
|
13874
|
-
}
|
|
13875
|
-
}
|
|
13876
|
-
async function resolveHeaderMetadata(deps, indexName, partitionId) {
|
|
13877
|
-
const provider = deps.resolveHeaderMetadata;
|
|
13878
|
-
if (provider === void 0) {
|
|
13879
|
-
return defaultSnapshotHeaderMetadataProvider(deps.coordinator, indexName);
|
|
13559
|
+
function convertLocalFacetBoundsToWire(facets) {
|
|
13560
|
+
if (facets === void 0) {
|
|
13561
|
+
return null;
|
|
13880
13562
|
}
|
|
13881
|
-
|
|
13882
|
-
|
|
13883
|
-
|
|
13884
|
-
return {
|
|
13885
|
-
partitionId: SNAPSHOT_HEADER_SENTINEL_PARTITION_ID,
|
|
13886
|
-
primaryTerm: 0,
|
|
13887
|
-
lastSeqNo: SNAPSHOT_HEADER_SENTINEL_SEQNO
|
|
13888
|
-
};
|
|
13563
|
+
const result = {};
|
|
13564
|
+
for (const [field, facet] of Object.entries(facets)) {
|
|
13565
|
+
result[field] = facet.errorBound;
|
|
13889
13566
|
}
|
|
13567
|
+
return result;
|
|
13890
13568
|
}
|
|
13891
|
-
|
|
13892
|
-
|
|
13893
|
-
|
|
13894
|
-
const { indexName } = request;
|
|
13895
|
-
let sourceHandle;
|
|
13896
|
-
try {
|
|
13897
|
-
sourceHandle = acquireSourceSlot(deps.state, message.sourceId, indexName);
|
|
13898
|
-
} catch (err) {
|
|
13899
|
-
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
|
|
13900
|
-
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13901
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
13902
|
-
return;
|
|
13569
|
+
function convertLocalFacetsToWire2(facets) {
|
|
13570
|
+
if (facets === void 0) {
|
|
13571
|
+
return null;
|
|
13903
13572
|
}
|
|
13904
|
-
|
|
13905
|
-
|
|
13906
|
-
|
|
13907
|
-
const buildForRequest = async () => buildSnapshotForRequest(deps, indexName, options);
|
|
13908
|
-
lease = options.disableBuildCache === true ? soleSnapshotLease(await buildForRequest()) : await acquireSnapshotBuild(deps.state, indexName, buildForRequest);
|
|
13909
|
-
} catch (err) {
|
|
13910
|
-
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
|
|
13911
|
-
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13912
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
13913
|
-
return;
|
|
13914
|
-
}
|
|
13915
|
-
try {
|
|
13916
|
-
await streamLeasedSnapshot(message, sink, request, deps, options, lease.build);
|
|
13917
|
-
} finally {
|
|
13918
|
-
await lease.release();
|
|
13919
|
-
}
|
|
13920
|
-
} finally {
|
|
13921
|
-
releaseSourceSlot(deps.state, sourceHandle);
|
|
13573
|
+
const result = {};
|
|
13574
|
+
for (const [field, facet] of Object.entries(facets)) {
|
|
13575
|
+
result[field] = Object.entries(facet.values).map(([value, count2]) => ({ value, count: count2 }));
|
|
13922
13576
|
}
|
|
13577
|
+
return result;
|
|
13923
13578
|
}
|
|
13924
|
-
async function
|
|
13925
|
-
const
|
|
13926
|
-
|
|
13927
|
-
|
|
13928
|
-
|
|
13929
|
-
|
|
13930
|
-
|
|
13931
|
-
|
|
13932
|
-
|
|
13933
|
-
|
|
13934
|
-
}
|
|
13935
|
-
try {
|
|
13936
|
-
const resolved = options.metadata ?? await resolveHeaderMetadata(deps, indexName, request.partitionId ?? null);
|
|
13937
|
-
const metadata = {
|
|
13938
|
-
...resolved,
|
|
13939
|
-
lastSeqNo: build.lastSeqNo ?? resolved.lastSeqNo,
|
|
13940
|
-
primaryTerm: build.primaryTerm ?? resolved.primaryTerm
|
|
13941
|
-
};
|
|
13942
|
-
await streamSnapshotToReplica(sink, deps.nodeId, message.requestId, indexName, build, metadata, {
|
|
13943
|
-
closeOnEnd: options.closeOnEnd
|
|
13944
|
-
});
|
|
13945
|
-
await options.afterSnapshot?.(sink, build);
|
|
13946
|
-
if (options.closeOnEnd === false) {
|
|
13947
|
-
sink.closed = true;
|
|
13948
|
-
}
|
|
13949
|
-
} finally {
|
|
13950
|
-
releaseStreamSlot(deps.state, streamHandle);
|
|
13951
|
-
}
|
|
13579
|
+
async function handleCount(message, respond, deps) {
|
|
13580
|
+
const payload = validateCountPayload(decode(message.payload));
|
|
13581
|
+
const requested = new Set(payload.partitionIds);
|
|
13582
|
+
const partitions = (await deps.engine.partitionStatsForRead(payload.indexName)).filter((partition) => requested.has(partition.partitionId)).map((partition) => ({
|
|
13583
|
+
partitionId: partition.partitionId,
|
|
13584
|
+
documentCount: partition.documentCount,
|
|
13585
|
+
estimatedMemoryBytes: partition.estimatedMemoryBytes
|
|
13586
|
+
}));
|
|
13587
|
+
const language = deps.engine.getStats(payload.indexName).language;
|
|
13588
|
+
await respond(createCountResultMessage({ partitions, language }, deps.nodeId, message.requestId));
|
|
13952
13589
|
}
|
|
13953
|
-
async function
|
|
13954
|
-
|
|
13955
|
-
|
|
13956
|
-
|
|
13957
|
-
|
|
13958
|
-
|
|
13959
|
-
|
|
13960
|
-
|
|
13961
|
-
|
|
13962
|
-
|
|
13963
|
-
|
|
13964
|
-
|
|
13965
|
-
|
|
13966
|
-
|
|
13967
|
-
|
|
13968
|
-
|
|
13590
|
+
async function handleList(message, respond, deps) {
|
|
13591
|
+
const payload = validateListPayload(decode(message.payload));
|
|
13592
|
+
const listParams = {
|
|
13593
|
+
cursor: payload.cursor ?? void 0,
|
|
13594
|
+
limit: payload.limit,
|
|
13595
|
+
filters: payload.filters ?? void 0,
|
|
13596
|
+
sort: payload.sort ?? void 0,
|
|
13597
|
+
document: payload.fields === null ? void 0 : { include: payload.fields }
|
|
13598
|
+
};
|
|
13599
|
+
const result = await deps.engine.listPartitions(payload.indexName, listParams, payload.partitionIds);
|
|
13600
|
+
const normalizedSort = payload.sort === null ? null : normalizeSort(payload.sort);
|
|
13601
|
+
const sortFields = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
13602
|
+
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
13603
|
+
const entries = result.documents.map((listed) => ({
|
|
13604
|
+
docId: listed.id,
|
|
13605
|
+
document: listed.document,
|
|
13606
|
+
sortValues: sortFields === null ? null : readSortValues(listed.document, sortFields, sortModes).map(toWireSortValue2)
|
|
13607
|
+
}));
|
|
13608
|
+
refuseEncodeWithoutRoom(entries.map((entry) => entry.document));
|
|
13609
|
+
await respond(
|
|
13610
|
+
createListResultMessage(
|
|
13611
|
+
{ entries, total: result.total, hasMore: result.cursor !== null },
|
|
13612
|
+
deps.nodeId,
|
|
13613
|
+
message.requestId
|
|
13614
|
+
)
|
|
13615
|
+
);
|
|
13969
13616
|
}
|
|
13970
|
-
function
|
|
13971
|
-
|
|
13972
|
-
|
|
13973
|
-
|
|
13974
|
-
|
|
13617
|
+
async function handleSuggest(message, respond, deps) {
|
|
13618
|
+
const payload = validateSuggestPayload(decode(message.payload));
|
|
13619
|
+
const result = await deps.engine.suggestPartitions(
|
|
13620
|
+
payload.indexName,
|
|
13621
|
+
{ prefix: payload.prefix, limit: payload.limit },
|
|
13622
|
+
payload.partitionIds
|
|
13623
|
+
);
|
|
13624
|
+
await respond(
|
|
13625
|
+
createSuggestResultMessage(
|
|
13626
|
+
{ terms: result.terms, analysisStale: result.analysisStale === true },
|
|
13627
|
+
deps.nodeId,
|
|
13628
|
+
message.requestId
|
|
13629
|
+
)
|
|
13630
|
+
);
|
|
13975
13631
|
}
|
|
13976
|
-
function
|
|
13977
|
-
|
|
13978
|
-
|
|
13979
|
-
|
|
13980
|
-
|
|
13981
|
-
|
|
13982
|
-
|
|
13983
|
-
|
|
13984
|
-
|
|
13985
|
-
|
|
13986
|
-
|
|
13987
|
-
}
|
|
13988
|
-
}
|
|
13989
|
-
return false;
|
|
13632
|
+
async function handlePreflight(message, respond, deps) {
|
|
13633
|
+
const payload = validatePreflightPayload(decode(message.payload));
|
|
13634
|
+
const params = wireParamsToLocal(payload.params);
|
|
13635
|
+
const result = await deps.engine.preflightPartitions(payload.indexName, params, payload.partitionIds);
|
|
13636
|
+
await respond(
|
|
13637
|
+
createPreflightResultMessage(
|
|
13638
|
+
{ count: result.count, analysisStale: result.analysisStale === true },
|
|
13639
|
+
deps.nodeId,
|
|
13640
|
+
message.requestId
|
|
13641
|
+
)
|
|
13642
|
+
);
|
|
13990
13643
|
}
|
|
13991
|
-
|
|
13992
|
-
|
|
13993
|
-
|
|
13994
|
-
|
|
13995
|
-
|
|
13996
|
-
maxExtLength: 0,
|
|
13997
|
-
keyDecoder: null
|
|
13998
|
-
};
|
|
13999
|
-
async function decodeRequest(message, sink, deps) {
|
|
14000
|
-
try {
|
|
14001
|
-
const decoded = decode(message.payload, REQUEST_DECODE_OPTIONS);
|
|
14002
|
-
return validateSnapshotSyncRequestPayload(decoded);
|
|
14003
|
-
} catch (err) {
|
|
14004
|
-
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID;
|
|
14005
|
-
const errMessage = err instanceof Error ? err.message : String(err);
|
|
14006
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
14007
|
-
return null;
|
|
14008
|
-
}
|
|
13644
|
+
function toWireSortValue2(value) {
|
|
13645
|
+
if (typeof value === "string") return value;
|
|
13646
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
13647
|
+
if (typeof value === "boolean") return value;
|
|
13648
|
+
return null;
|
|
14009
13649
|
}
|
|
14010
|
-
|
|
14011
|
-
|
|
14012
|
-
|
|
14013
|
-
|
|
14014
|
-
|
|
14015
|
-
return `request sourceId exceeds ${MAX_SOURCE_ID_LENGTH} characters`;
|
|
14016
|
-
}
|
|
14017
|
-
if (containsControlCharacter(sourceId)) {
|
|
14018
|
-
return "request sourceId contains control characters";
|
|
13650
|
+
|
|
13651
|
+
// src/distribution/replication/sync-primary.ts
|
|
13652
|
+
function decideSyncTier(log, replicaLastSeqNo, replicaLastPrimaryTerm) {
|
|
13653
|
+
if (replicaLastSeqNo === 0) {
|
|
13654
|
+
return log.localLogEnd === 0 || log.oldestSeqNo === 1 ? "incremental" : "snapshot";
|
|
14019
13655
|
}
|
|
14020
|
-
|
|
13656
|
+
const primaryTerm = log.termAt(replicaLastSeqNo);
|
|
13657
|
+
return primaryTerm !== void 0 && primaryTerm === replicaLastPrimaryTerm ? "incremental" : "snapshot";
|
|
14021
13658
|
}
|
|
14022
|
-
function
|
|
13659
|
+
function validateSyncRequest(decoded) {
|
|
14023
13660
|
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
14024
|
-
throw new
|
|
14025
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14026
|
-
"Invalid SnapshotSyncRequestPayload: expected an object"
|
|
14027
|
-
);
|
|
13661
|
+
throw new Error("Invalid SyncRequestPayload: expected an object");
|
|
14028
13662
|
}
|
|
14029
13663
|
const record = decoded;
|
|
14030
|
-
|
|
14031
|
-
|
|
14032
|
-
throw new NarsilError(
|
|
14033
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14034
|
-
'Invalid SnapshotSyncRequestPayload: "indexName" must be a non-empty string'
|
|
14035
|
-
);
|
|
14036
|
-
}
|
|
14037
|
-
if (indexName.length > MAX_INDEX_NAME_LENGTH) {
|
|
14038
|
-
throw new NarsilError(
|
|
14039
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14040
|
-
`Invalid SnapshotSyncRequestPayload: "indexName" must be at most ${MAX_INDEX_NAME_LENGTH} characters`,
|
|
14041
|
-
{ length: indexName.length, limit: MAX_INDEX_NAME_LENGTH }
|
|
14042
|
-
);
|
|
13664
|
+
if (typeof record.indexName !== "string") {
|
|
13665
|
+
throw new Error('Invalid SyncRequestPayload: "indexName" must be a string');
|
|
14043
13666
|
}
|
|
14044
|
-
if (!
|
|
14045
|
-
throw new
|
|
14046
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14047
|
-
'Invalid SnapshotSyncRequestPayload: "indexName" contains invalid characters'
|
|
14048
|
-
);
|
|
13667
|
+
if (typeof record.partitionId !== "number" || !Number.isInteger(record.partitionId) || record.partitionId < 0) {
|
|
13668
|
+
throw new Error('Invalid SyncRequestPayload: "partitionId" must be a non-negative integer');
|
|
14049
13669
|
}
|
|
14050
|
-
|
|
14051
|
-
|
|
14052
|
-
throw new NarsilError(
|
|
14053
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14054
|
-
'Invalid SnapshotSyncRequestPayload: "indexUuid" must be a non-empty string when provided'
|
|
14055
|
-
);
|
|
13670
|
+
if (typeof record.lastSeqNo !== "number" || !Number.isInteger(record.lastSeqNo) || record.lastSeqNo < 0) {
|
|
13671
|
+
throw new Error('Invalid SyncRequestPayload: "lastSeqNo" must be a non-negative integer');
|
|
14056
13672
|
}
|
|
14057
|
-
|
|
14058
|
-
|
|
14059
|
-
if (partitionId !== void 0 && partitionId !== null) {
|
|
14060
|
-
if (typeof partitionId !== "number" || !Number.isInteger(partitionId) || partitionId < 0) {
|
|
14061
|
-
throw new NarsilError(
|
|
14062
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14063
|
-
'Invalid SnapshotSyncRequestPayload: "partitionId" must be a non-negative integer when provided'
|
|
14064
|
-
);
|
|
14065
|
-
}
|
|
14066
|
-
return { indexName, partitionId, ...identity };
|
|
13673
|
+
if (typeof record.lastPrimaryTerm !== "number" || !Number.isInteger(record.lastPrimaryTerm) || record.lastPrimaryTerm < 0) {
|
|
13674
|
+
throw new Error('Invalid SyncRequestPayload: "lastPrimaryTerm" must be a non-negative integer');
|
|
14067
13675
|
}
|
|
14068
|
-
|
|
13676
|
+
optionalIndexUuid(record.indexUuid, "SyncRequestPayload");
|
|
13677
|
+
return record;
|
|
14069
13678
|
}
|
|
14070
13679
|
|
|
14071
|
-
// src/distribution/cluster-node/
|
|
14072
|
-
function
|
|
14073
|
-
return
|
|
13680
|
+
// src/distribution/cluster-node/request-identity.ts
|
|
13681
|
+
function holdsAnotherIndex(localUuid, requestedUuid) {
|
|
13682
|
+
return typeof requestedUuid === "string" && typeof localUuid === "string" && localUuid !== requestedUuid;
|
|
14074
13683
|
}
|
|
14075
|
-
|
|
14076
|
-
|
|
13684
|
+
|
|
13685
|
+
// src/distribution/cluster-node/snapshot-auth.ts
|
|
13686
|
+
var BOOTSTRAPPABLE_STATES = /* @__PURE__ */ new Set(["ACTIVE", "INITIALISING", "MIGRATING"]);
|
|
13687
|
+
async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
|
|
13688
|
+
let allocation;
|
|
14077
13689
|
try {
|
|
14078
|
-
|
|
14079
|
-
await respondError(
|
|
14080
|
-
sink,
|
|
14081
|
-
deps.nodeId,
|
|
14082
|
-
message.requestId,
|
|
14083
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14084
|
-
`SNAPSHOT_SYNC_REQUEST payload (${message.payload.byteLength} bytes) exceeds the ${MAX_SNAPSHOT_SYNC_REQUEST_BYTES} byte limit`
|
|
14085
|
-
);
|
|
14086
|
-
return;
|
|
14087
|
-
}
|
|
14088
|
-
await runSnapshotSyncRequest(message, sink, deps);
|
|
13690
|
+
allocation = await coordinator.getAllocation(indexName);
|
|
14089
13691
|
} catch (err) {
|
|
14090
|
-
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
|
|
14091
13692
|
const errMessage = err instanceof Error ? err.message : String(err);
|
|
14092
|
-
|
|
14093
|
-
|
|
14094
|
-
|
|
14095
|
-
|
|
14096
|
-
|
|
14097
|
-
if (request === null) {
|
|
14098
|
-
return;
|
|
13693
|
+
return {
|
|
13694
|
+
outcome: "denied",
|
|
13695
|
+
code: ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
13696
|
+
reason: `allocation lookup failed: ${errMessage}`
|
|
13697
|
+
};
|
|
14099
13698
|
}
|
|
14100
|
-
|
|
14101
|
-
|
|
14102
|
-
|
|
14103
|
-
|
|
14104
|
-
|
|
14105
|
-
|
|
14106
|
-
return;
|
|
14107
|
-
}
|
|
14108
|
-
const authResult = await authorizeSnapshotRequest(deps.coordinator, request.indexName, message.sourceId);
|
|
14109
|
-
if (authResult.outcome === "denied") {
|
|
14110
|
-
await respondError(sink, deps.nodeId, message.requestId, authResult.code, authResult.reason);
|
|
14111
|
-
return;
|
|
14112
|
-
}
|
|
14113
|
-
const existingIndex = deps.engine.listIndexes().find((idx) => idx.name === request.indexName);
|
|
14114
|
-
if (existingIndex === void 0 || holdsAnotherIndex(deps.indexUuidOf?.(request.indexName), request.indexUuid)) {
|
|
14115
|
-
await respondError(
|
|
14116
|
-
sink,
|
|
14117
|
-
deps.nodeId,
|
|
14118
|
-
message.requestId,
|
|
14119
|
-
ErrorCodes.SNAPSHOT_SYNC_INDEX_NOT_FOUND,
|
|
14120
|
-
`Index '${request.indexName}' is not hosted on this node`
|
|
14121
|
-
);
|
|
14122
|
-
return;
|
|
14123
|
-
}
|
|
14124
|
-
await acquireAndStream(message, sink, request, deps, options);
|
|
14125
|
-
}
|
|
14126
|
-
async function handleIndexSettings(message, respond, deps) {
|
|
14127
|
-
const payload = validateIndexSettingsPayload(decode(message.payload));
|
|
14128
|
-
const applied = payload !== null && await applyStoredSynonyms(deps.engine, deps.coordinator, payload.indexName, payload.indexUuid);
|
|
14129
|
-
const indexName = payload?.indexName ?? "";
|
|
14130
|
-
await respond(createIndexSettingsResultMessage({ indexName, applied }, deps.nodeId, message.requestId));
|
|
14131
|
-
}
|
|
14132
|
-
|
|
14133
|
-
// src/distribution/query/types.ts
|
|
14134
|
-
var DEFAULT_QUERY_CONFIG = {
|
|
14135
|
-
allowPartialResults: true,
|
|
14136
|
-
partitionTimeout: 5e3,
|
|
14137
|
-
defaultFacetSize: 10
|
|
14138
|
-
};
|
|
14139
|
-
|
|
14140
|
-
// src/distribution/cluster-node/query-conversion.ts
|
|
14141
|
-
function wireParamsToLocal(wire, facetShardSize) {
|
|
14142
|
-
return {
|
|
14143
|
-
term: wire.term ?? void 0,
|
|
14144
|
-
fields: wire.fields ?? void 0,
|
|
14145
|
-
filters: wire.filters ?? void 0,
|
|
14146
|
-
boost: wire.boost ?? void 0,
|
|
14147
|
-
scoring: wire.scoring,
|
|
14148
|
-
tolerance: wire.tolerance ?? void 0,
|
|
14149
|
-
minScore: wire.threshold ?? void 0,
|
|
14150
|
-
includeScores: wire.includeScores ?? void 0,
|
|
14151
|
-
termMatch: wire.termMatch ?? void 0,
|
|
14152
|
-
prefixLength: wire.prefixLength ?? void 0,
|
|
14153
|
-
prefix: wire.prefix ?? void 0,
|
|
14154
|
-
partialWords: wire.partialWords ?? void 0,
|
|
14155
|
-
exact: wire.exact ?? void 0,
|
|
14156
|
-
pinned: wire.pinned !== null ? wire.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : void 0,
|
|
14157
|
-
mode: wire.mode ?? void 0,
|
|
14158
|
-
limit: wire.limit,
|
|
14159
|
-
offset: wire.offset,
|
|
14160
|
-
searchAfter: wire.searchAfter ?? void 0,
|
|
14161
|
-
sort: convertWireSortToLocal(wire.sort),
|
|
14162
|
-
group: convertWireGroupToLocal(wire.group),
|
|
14163
|
-
facets: convertWireFacetConfigToLocal(wire.facets, facetShardSize ?? wire.facetSize),
|
|
14164
|
-
vector: convertWireVectorToLocal(wire.vector),
|
|
14165
|
-
hybrid: convertWireHybridToLocal(wire.hybrid)
|
|
14166
|
-
};
|
|
14167
|
-
}
|
|
14168
|
-
function convertWireGroupToLocal(group) {
|
|
14169
|
-
if (group === null) {
|
|
14170
|
-
return void 0;
|
|
14171
|
-
}
|
|
14172
|
-
const local = { fields: [...group.fields], maxPerGroup: group.maxPerGroup };
|
|
14173
|
-
if (group.limit !== null) local.limit = group.limit;
|
|
14174
|
-
return local;
|
|
14175
|
-
}
|
|
14176
|
-
function convertWireVectorToLocal(vector) {
|
|
14177
|
-
if (vector === null) {
|
|
14178
|
-
return void 0;
|
|
14179
|
-
}
|
|
14180
|
-
const local = { field: vector.field };
|
|
14181
|
-
if (vector.value !== null) local.value = vector.value;
|
|
14182
|
-
if (vector.text !== null) local.text = vector.text;
|
|
14183
|
-
if (vector.similarity !== null) local.similarity = vector.similarity;
|
|
14184
|
-
if (vector.metric !== null) local.metric = vector.metric;
|
|
14185
|
-
if (vector.efSearch !== null) local.efSearch = vector.efSearch;
|
|
14186
|
-
if (vector.oversample !== null) local.oversample = vector.oversample;
|
|
14187
|
-
return local;
|
|
14188
|
-
}
|
|
14189
|
-
function convertWireHybridToLocal(hybrid) {
|
|
14190
|
-
if (hybrid === null) {
|
|
14191
|
-
return void 0;
|
|
14192
|
-
}
|
|
14193
|
-
const local = {};
|
|
14194
|
-
if (hybrid.strategy !== null) local.strategy = hybrid.strategy;
|
|
14195
|
-
if (hybrid.k !== null) local.k = hybrid.k;
|
|
14196
|
-
if (hybrid.alpha !== null) local.alpha = hybrid.alpha;
|
|
14197
|
-
if (hybrid.depth !== null) local.depth = hybrid.depth;
|
|
14198
|
-
return local;
|
|
14199
|
-
}
|
|
14200
|
-
function convertWireSortToLocal(wireSort) {
|
|
14201
|
-
if (wireSort === null || wireSort.length === 0) {
|
|
14202
|
-
return void 0;
|
|
14203
|
-
}
|
|
14204
|
-
return normalizeSort(wireSort);
|
|
14205
|
-
}
|
|
14206
|
-
function countsEveryValue(facet) {
|
|
14207
|
-
return facet.ranges !== null || facet.sort === "asc";
|
|
14208
|
-
}
|
|
14209
|
-
function convertWireFacetConfigToLocal(facets, limit) {
|
|
14210
|
-
if (facets === null || facets.length === 0) {
|
|
14211
|
-
return void 0;
|
|
13699
|
+
if (allocation === null) {
|
|
13700
|
+
return {
|
|
13701
|
+
outcome: "denied",
|
|
13702
|
+
code: ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
13703
|
+
reason: `index '${indexName}' has no allocation`
|
|
13704
|
+
};
|
|
14212
13705
|
}
|
|
14213
|
-
|
|
14214
|
-
for (const
|
|
14215
|
-
const
|
|
14216
|
-
if (
|
|
14217
|
-
|
|
14218
|
-
|
|
14219
|
-
|
|
14220
|
-
|
|
13706
|
+
let sawAssignment = false;
|
|
13707
|
+
for (const assignment of allocation.assignments.values()) {
|
|
13708
|
+
const isAssigned = assignment.primary === sourceId || assignment.replicas.includes(sourceId);
|
|
13709
|
+
if (!isAssigned) {
|
|
13710
|
+
continue;
|
|
13711
|
+
}
|
|
13712
|
+
sawAssignment = true;
|
|
13713
|
+
if (!BOOTSTRAPPABLE_STATES.has(assignment.state)) {
|
|
13714
|
+
continue;
|
|
13715
|
+
}
|
|
13716
|
+
return { outcome: "authorized" };
|
|
14221
13717
|
}
|
|
14222
|
-
|
|
14223
|
-
|
|
14224
|
-
|
|
14225
|
-
|
|
14226
|
-
|
|
14227
|
-
|
|
13718
|
+
if (sawAssignment) {
|
|
13719
|
+
return {
|
|
13720
|
+
outcome: "denied",
|
|
13721
|
+
code: ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
13722
|
+
reason: `node '${sourceId}' is listed for index '${indexName}' but not in a bootstrappable state`
|
|
13723
|
+
};
|
|
14228
13724
|
}
|
|
14229
|
-
return ascending;
|
|
14230
|
-
}
|
|
14231
|
-
function localParamsToWire(params) {
|
|
14232
|
-
return {
|
|
14233
|
-
term: params.term ?? null,
|
|
14234
|
-
filters: params.filters ?? null,
|
|
14235
|
-
sort: convertLocalSortToWire(params.sort),
|
|
14236
|
-
group: convertLocalGroupToWire(params.group),
|
|
14237
|
-
facets: convertLocalFacetsToWire(params.facets),
|
|
14238
|
-
facetSize: wireFacetSize(params.facets),
|
|
14239
|
-
limit: clampRowCount(params.limit, DEFAULT_PAGE_SIZE),
|
|
14240
|
-
offset: clampRowCount(params.offset, 0),
|
|
14241
|
-
searchAfter: params.searchAfter ?? null,
|
|
14242
|
-
fields: params.fields ?? null,
|
|
14243
|
-
boost: params.boost ?? null,
|
|
14244
|
-
tolerance: params.tolerance ?? null,
|
|
14245
|
-
threshold: params.minScore ?? null,
|
|
14246
|
-
includeScores: params.includeScores ?? null,
|
|
14247
|
-
scoring: params.scoring ?? "local",
|
|
14248
|
-
termMatch: params.termMatch ?? null,
|
|
14249
|
-
prefixLength: params.prefixLength ?? null,
|
|
14250
|
-
prefix: params.prefix ?? null,
|
|
14251
|
-
partialWords: params.partialWords ?? null,
|
|
14252
|
-
exact: params.exact ?? null,
|
|
14253
|
-
pinned: params.pinned !== void 0 ? params.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : null,
|
|
14254
|
-
mode: params.mode ?? null,
|
|
14255
|
-
vector: convertLocalVectorToWire(params.vector),
|
|
14256
|
-
hybrid: convertLocalHybridToWire(params.hybrid)
|
|
14257
|
-
};
|
|
14258
|
-
}
|
|
14259
|
-
function countIsExactFor(params) {
|
|
14260
|
-
const vector = params.vector;
|
|
14261
|
-
if (vector === void 0 || vector.value === void 0 && vector.text === void 0) return true;
|
|
14262
|
-
const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
|
|
14263
|
-
if (params.mode === "hybrid" || hasTerm) return false;
|
|
14264
|
-
return vector.similarity === void 0;
|
|
14265
|
-
}
|
|
14266
|
-
function distributedCountIsExact(params, coverage) {
|
|
14267
|
-
if (coverage.queriedPartitions < coverage.totalPartitions) return false;
|
|
14268
|
-
if (coverage.timedOutPartitions > 0 || coverage.failedPartitions > 0) return false;
|
|
14269
|
-
return countIsExactFor(params);
|
|
14270
|
-
}
|
|
14271
|
-
function distributedResultToLocal(result, countExact, documents = /* @__PURE__ */ new Map(), requestedFacets) {
|
|
14272
13725
|
return {
|
|
14273
|
-
|
|
14274
|
-
|
|
14275
|
-
|
|
14276
|
-
document: documents.get(entry.docId) ?? {}
|
|
14277
|
-
})),
|
|
14278
|
-
count: result.totalHits,
|
|
14279
|
-
countExact,
|
|
14280
|
-
elapsed: 0,
|
|
14281
|
-
cursor: result.cursor ?? void 0,
|
|
14282
|
-
facets: result.facets !== null ? convertWireFacetsToLocal(result, result.facets, requestedFacets) : void 0,
|
|
14283
|
-
coverage: result.coverage
|
|
13726
|
+
outcome: "denied",
|
|
13727
|
+
code: ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
13728
|
+
reason: `node '${sourceId}' is not an assigned replica for index '${indexName}'`
|
|
14284
13729
|
};
|
|
14285
13730
|
}
|
|
14286
|
-
function
|
|
14287
|
-
const
|
|
14288
|
-
|
|
13731
|
+
function createSingleResponseSink(respond) {
|
|
13732
|
+
const sink = ((response) => {
|
|
13733
|
+
if (sink.closed) {
|
|
13734
|
+
return Promise.resolve();
|
|
13735
|
+
}
|
|
13736
|
+
return respond(response);
|
|
13737
|
+
});
|
|
13738
|
+
sink.closed = false;
|
|
13739
|
+
return sink;
|
|
14289
13740
|
}
|
|
14290
|
-
function
|
|
14291
|
-
|
|
14292
|
-
|
|
14293
|
-
|
|
14294
|
-
|
|
14295
|
-
|
|
14296
|
-
|
|
14297
|
-
|
|
13741
|
+
async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build, metadata) {
|
|
13742
|
+
const totalBytes = build.totalBytes;
|
|
13743
|
+
const checksum = build.checksum;
|
|
13744
|
+
const header = {
|
|
13745
|
+
lastSeqNo: metadata.lastSeqNo,
|
|
13746
|
+
primaryTerm: metadata.primaryTerm,
|
|
13747
|
+
partitionId: metadata.partitionId,
|
|
13748
|
+
indexName,
|
|
13749
|
+
checksum
|
|
14298
13750
|
};
|
|
14299
|
-
}
|
|
14300
|
-
|
|
14301
|
-
|
|
14302
|
-
|
|
14303
|
-
|
|
14304
|
-
|
|
14305
|
-
|
|
14306
|
-
|
|
14307
|
-
|
|
14308
|
-
|
|
14309
|
-
|
|
14310
|
-
|
|
14311
|
-
|
|
14312
|
-
|
|
14313
|
-
|
|
14314
|
-
|
|
14315
|
-
|
|
13751
|
+
const startPayload = { header, totalBytes };
|
|
13752
|
+
const startBytes = encode(startPayload);
|
|
13753
|
+
assertMessageWithinLimit(startBytes, "SNAPSHOT_START");
|
|
13754
|
+
await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_START, nodeId, requestId, startBytes);
|
|
13755
|
+
await yieldToEventLoop2();
|
|
13756
|
+
let offset = 0;
|
|
13757
|
+
let lastYieldAt = now2();
|
|
13758
|
+
for await (const source of build.read()) {
|
|
13759
|
+
for (let from = 0; from < source.byteLength; from += SNAPSHOT_CHUNK_SIZE) {
|
|
13760
|
+
const chunk = source.subarray(from, Math.min(from + SNAPSHOT_CHUNK_SIZE, source.byteLength));
|
|
13761
|
+
const chunkPayload = {
|
|
13762
|
+
partitionId: metadata.partitionId,
|
|
13763
|
+
indexName,
|
|
13764
|
+
offset,
|
|
13765
|
+
data: chunk
|
|
13766
|
+
};
|
|
13767
|
+
const chunkBytes = encode(chunkPayload);
|
|
13768
|
+
assertMessageWithinLimit(chunkBytes, "SNAPSHOT_CHUNK");
|
|
13769
|
+
await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_CHUNK, nodeId, requestId, chunkBytes);
|
|
13770
|
+
offset += chunk.byteLength;
|
|
13771
|
+
if (offset < totalBytes && now2() - lastYieldAt >= SNAPSHOT_CHUNK_YIELD_INTERVAL_MS) {
|
|
13772
|
+
await yieldToEventLoop2();
|
|
13773
|
+
lastYieldAt = now2();
|
|
13774
|
+
}
|
|
13775
|
+
}
|
|
14316
13776
|
}
|
|
14317
|
-
|
|
14318
|
-
|
|
14319
|
-
|
|
14320
|
-
|
|
14321
|
-
|
|
14322
|
-
|
|
14323
|
-
efSearch: vector.efSearch ?? null,
|
|
14324
|
-
oversample: vector.oversample ?? null,
|
|
14325
|
-
rescoreWindow: null
|
|
13777
|
+
await yieldToEventLoop2();
|
|
13778
|
+
const endPayload = {
|
|
13779
|
+
partitionId: metadata.partitionId,
|
|
13780
|
+
indexName,
|
|
13781
|
+
totalBytes,
|
|
13782
|
+
checksum
|
|
14326
13783
|
};
|
|
13784
|
+
const endBytes = encode(endPayload);
|
|
13785
|
+
assertMessageWithinLimit(endBytes, "SNAPSHOT_END");
|
|
13786
|
+
await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_END, nodeId, requestId, endBytes);
|
|
14327
13787
|
}
|
|
14328
|
-
function
|
|
14329
|
-
if (
|
|
14330
|
-
return
|
|
13788
|
+
function now2() {
|
|
13789
|
+
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
13790
|
+
return performance.now();
|
|
14331
13791
|
}
|
|
14332
|
-
return
|
|
14333
|
-
strategy: hybrid.strategy ?? null,
|
|
14334
|
-
k: hybrid.k ?? null,
|
|
14335
|
-
alpha: hybrid.alpha ?? null,
|
|
14336
|
-
depth: hybrid.depth ?? null
|
|
14337
|
-
};
|
|
13792
|
+
return Date.now();
|
|
14338
13793
|
}
|
|
14339
|
-
function
|
|
14340
|
-
|
|
14341
|
-
|
|
14342
|
-
|
|
14343
|
-
|
|
14344
|
-
|
|
14345
|
-
|
|
14346
|
-
|
|
13794
|
+
function yieldToEventLoop2() {
|
|
13795
|
+
return new Promise((resolve) => {
|
|
13796
|
+
if (typeof setImmediate === "function") {
|
|
13797
|
+
setImmediate(resolve);
|
|
13798
|
+
return;
|
|
13799
|
+
}
|
|
13800
|
+
setTimeout(resolve, 0);
|
|
13801
|
+
});
|
|
14347
13802
|
}
|
|
14348
|
-
function
|
|
14349
|
-
|
|
14350
|
-
let size = 0;
|
|
14351
|
-
for (const options of Object.values(facets)) size = Math.max(size, facetLimitOf(options));
|
|
14352
|
-
return size === 0 ? null : size;
|
|
13803
|
+
function respondMessage(sink, type, sourceId, requestId, payload) {
|
|
13804
|
+
return sink({ type, sourceId, requestId, payload });
|
|
14353
13805
|
}
|
|
14354
|
-
function
|
|
14355
|
-
|
|
14356
|
-
|
|
14357
|
-
|
|
14358
|
-
|
|
14359
|
-
|
|
14360
|
-
for (let index = 0; index < kept; index++) values[buckets[index].value] = buckets[index].count;
|
|
14361
|
-
let largestCut = 0;
|
|
14362
|
-
for (let index = kept; index < buckets.length; index++) largestCut = Math.max(largestCut, buckets[index].count);
|
|
14363
|
-
const errorBound = kept < buckets.length ? (result.facetUndercounts?.[field] ?? mergedBound) + largestCut : mergedBound;
|
|
14364
|
-
converted[field] = { values, count: kept, errorBound: Math.max(errorBound, mergedBound) };
|
|
13806
|
+
function assertMessageWithinLimit(bytes, label) {
|
|
13807
|
+
if (bytes.byteLength > MAX_MESSAGE_SIZE_BYTES) {
|
|
13808
|
+
throw new NarsilError(
|
|
13809
|
+
ErrorCodes.CONFIG_INVALID,
|
|
13810
|
+
`${label} payload (${bytes.byteLength} bytes) exceeds transport message limit (${MAX_MESSAGE_SIZE_BYTES})`
|
|
13811
|
+
);
|
|
14365
13812
|
}
|
|
14366
|
-
return converted;
|
|
14367
13813
|
}
|
|
14368
|
-
|
|
14369
|
-
|
|
14370
|
-
|
|
14371
|
-
|
|
14372
|
-
|
|
14373
|
-
|
|
14374
|
-
if (queryParams.group?.limit !== void 0) {
|
|
14375
|
-
queryParams.group = { ...queryParams.group, limit: oversampledShardSize(queryParams.group.limit) };
|
|
14376
|
-
}
|
|
14377
|
-
const rescoreWindow = payload.params.vector?.rescoreWindow ?? void 0;
|
|
14378
|
-
if (rescoreWindow !== void 0 && queryParams.sort === void 0 && queryParams.group === void 0) {
|
|
14379
|
-
queryParams.document = false;
|
|
14380
|
-
}
|
|
14381
|
-
const queryResult = await deps.engine.queryPartitions(
|
|
14382
|
-
payload.indexName,
|
|
14383
|
-
queryParams,
|
|
14384
|
-
payload.partitionIds,
|
|
14385
|
-
payload.globalStats ?? void 0,
|
|
14386
|
-
rescoreWindow
|
|
14387
|
-
);
|
|
14388
|
-
const normalizedSort = queryParams.sort !== void 0 ? normalizeSort(queryParams.sort) : null;
|
|
14389
|
-
const sortFields = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
14390
|
-
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
14391
|
-
const scored = queryResult.hits.map((hit) => ({
|
|
14392
|
-
docId: hit.id,
|
|
14393
|
-
score: hit.score ?? null,
|
|
14394
|
-
sortValues: sortFields !== null ? readSortValues(hit.document, sortFields, sortModes).map(toWireSortValue) : null
|
|
14395
|
-
}));
|
|
14396
|
-
const results = payload.partitionIds.map((partitionId) => ({
|
|
14397
|
-
partitionId,
|
|
14398
|
-
scored: partitionId === payload.partitionIds[0] ? scored : [],
|
|
14399
|
-
totalHits: partitionId === payload.partitionIds[0] ? queryResult.count : 0
|
|
14400
|
-
}));
|
|
14401
|
-
const resultPayload = {
|
|
14402
|
-
results,
|
|
14403
|
-
facets: convertLocalFacetsToWire2(queryResult.facets),
|
|
14404
|
-
facetErrorBounds: convertLocalFacetBoundsToWire(queryResult.facets),
|
|
14405
|
-
groups: await convertLocalGroupsToWire(queryResult.groups, sortFields, sortModes, deps, payload.indexName)
|
|
13814
|
+
async function respondError(sink, sourceId, request, code, message, details) {
|
|
13815
|
+
const response = {
|
|
13816
|
+
type: `${request.type}.error`,
|
|
13817
|
+
sourceId,
|
|
13818
|
+
requestId: request.requestId,
|
|
13819
|
+
payload: encodeErrorReply(code, message, details)
|
|
14406
13820
|
};
|
|
14407
|
-
await
|
|
14408
|
-
|
|
14409
|
-
sourceId: deps.nodeId,
|
|
14410
|
-
requestId: message.requestId,
|
|
14411
|
-
payload: encodeDocumentReply(resultPayload)
|
|
14412
|
-
});
|
|
13821
|
+
await sink(response);
|
|
13822
|
+
sink.closed = true;
|
|
14413
13823
|
}
|
|
14414
|
-
|
|
14415
|
-
|
|
14416
|
-
|
|
14417
|
-
|
|
14418
|
-
|
|
14419
|
-
|
|
14420
|
-
const wireGroups = [];
|
|
14421
|
-
for (const group of groups) {
|
|
14422
|
-
const scored = group.hits.map((hit) => {
|
|
14423
|
-
if (sortFields === null) return { docId: hit.id, score: hit.score ?? null, sortValues: null };
|
|
14424
|
-
const document = hit.document ?? readDocuments.get(hit.id);
|
|
14425
|
-
const sortValues = readSortValues(document, sortFields, sortModes).map(toWireSortValue);
|
|
14426
|
-
return { docId: hit.id, score: hit.score ?? null, sortValues };
|
|
13824
|
+
|
|
13825
|
+
// src/distribution/cluster-node/snapshot-cache.ts
|
|
13826
|
+
function createSnapshotCacheState(maxPerSource = DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, maxStreamsPerIndex = DEFAULT_MAX_STREAMS_PER_INDEX) {
|
|
13827
|
+
if (!Number.isInteger(maxPerSource) || maxPerSource <= 0) {
|
|
13828
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "maxPerSource must be a positive integer", {
|
|
13829
|
+
maxPerSource
|
|
14427
13830
|
});
|
|
14428
|
-
wireGroups.push({ values: group.values, scored });
|
|
14429
|
-
}
|
|
14430
|
-
return wireGroups;
|
|
14431
|
-
}
|
|
14432
|
-
async function handleFetch(message, respond, deps) {
|
|
14433
|
-
const decoded = decode(message.payload);
|
|
14434
|
-
const payload = validateFetchPayload(decoded);
|
|
14435
|
-
const projection = resolveProjection(payload.fields === null ? void 0 : { include: payload.fields });
|
|
14436
|
-
const documents = [];
|
|
14437
|
-
for (const docRef of payload.documentIds) {
|
|
14438
|
-
const doc = await deps.engine.get(payload.indexName, docRef.docId);
|
|
14439
|
-
if (doc !== void 0) {
|
|
14440
|
-
documents.push({
|
|
14441
|
-
docId: docRef.docId,
|
|
14442
|
-
document: applyProjection(doc, projection),
|
|
14443
|
-
highlights: null
|
|
14444
|
-
});
|
|
14445
|
-
}
|
|
14446
13831
|
}
|
|
14447
|
-
|
|
14448
|
-
|
|
14449
|
-
|
|
14450
|
-
|
|
14451
|
-
|
|
14452
|
-
|
|
14453
|
-
|
|
14454
|
-
|
|
14455
|
-
|
|
14456
|
-
|
|
14457
|
-
|
|
14458
|
-
const resultPayload = await deps.engine.collectQueryStats(
|
|
14459
|
-
payload.indexName,
|
|
14460
|
-
payload.terms,
|
|
14461
|
-
payload.partitionIds,
|
|
14462
|
-
{ expansion: payload.expansion ?? void 0, tokens: payload.tokens ?? void 0 }
|
|
14463
|
-
);
|
|
14464
|
-
await respond({
|
|
14465
|
-
type: QueryMessageTypes.STATS_RESULT,
|
|
14466
|
-
sourceId: deps.nodeId,
|
|
14467
|
-
requestId: message.requestId,
|
|
14468
|
-
payload: encode(resultPayload)
|
|
14469
|
-
});
|
|
13832
|
+
if (!Number.isInteger(maxStreamsPerIndex) || maxStreamsPerIndex <= 0) {
|
|
13833
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "maxStreamsPerIndex must be a positive integer", {
|
|
13834
|
+
maxStreamsPerIndex
|
|
13835
|
+
});
|
|
13836
|
+
}
|
|
13837
|
+
return {
|
|
13838
|
+
perSource: /* @__PURE__ */ new Map(),
|
|
13839
|
+
streamActive: /* @__PURE__ */ new Map(),
|
|
13840
|
+
maxPerSource,
|
|
13841
|
+
maxStreamsPerIndex
|
|
13842
|
+
};
|
|
14470
13843
|
}
|
|
14471
|
-
function
|
|
14472
|
-
|
|
14473
|
-
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
14474
|
-
if (typeof value === "boolean") return value;
|
|
14475
|
-
return null;
|
|
13844
|
+
function sourceSlotKey(sourceId, indexName) {
|
|
13845
|
+
return `${sourceId}\0${indexName}`;
|
|
14476
13846
|
}
|
|
14477
|
-
function
|
|
14478
|
-
|
|
14479
|
-
|
|
14480
|
-
|
|
14481
|
-
|
|
14482
|
-
|
|
14483
|
-
|
|
13847
|
+
function acquireSourceSlot(state, sourceId, indexName) {
|
|
13848
|
+
const key = sourceSlotKey(sourceId, indexName);
|
|
13849
|
+
const current = state.perSource.get(key) ?? 0;
|
|
13850
|
+
if (current >= state.maxPerSource) {
|
|
13851
|
+
throw new NarsilError(
|
|
13852
|
+
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
13853
|
+
`snapshot producer per-source cap reached for '${sourceId}' index '${indexName}'`,
|
|
13854
|
+
{ sourceId, indexName, cap: state.maxPerSource }
|
|
13855
|
+
);
|
|
14484
13856
|
}
|
|
14485
|
-
|
|
13857
|
+
state.perSource.set(key, current + 1);
|
|
13858
|
+
return { key, released: false };
|
|
14486
13859
|
}
|
|
14487
|
-
function
|
|
14488
|
-
if (
|
|
14489
|
-
return
|
|
13860
|
+
function releaseSourceSlot(state, handle) {
|
|
13861
|
+
if (handle.released) {
|
|
13862
|
+
return;
|
|
14490
13863
|
}
|
|
14491
|
-
|
|
14492
|
-
|
|
14493
|
-
|
|
13864
|
+
handle.released = true;
|
|
13865
|
+
const current = state.perSource.get(handle.key);
|
|
13866
|
+
if (current === void 0) {
|
|
13867
|
+
return;
|
|
14494
13868
|
}
|
|
14495
|
-
|
|
14496
|
-
|
|
14497
|
-
|
|
14498
|
-
|
|
14499
|
-
|
|
14500
|
-
const partitions = (await deps.engine.partitionStatsForRead(payload.indexName)).filter((partition) => requested.has(partition.partitionId)).map((partition) => ({
|
|
14501
|
-
partitionId: partition.partitionId,
|
|
14502
|
-
documentCount: partition.documentCount,
|
|
14503
|
-
estimatedMemoryBytes: partition.estimatedMemoryBytes
|
|
14504
|
-
}));
|
|
14505
|
-
const language = deps.engine.getStats(payload.indexName).language;
|
|
14506
|
-
await respond(createCountResultMessage({ partitions, language }, deps.nodeId, message.requestId));
|
|
13869
|
+
if (current <= 1) {
|
|
13870
|
+
state.perSource.delete(handle.key);
|
|
13871
|
+
return;
|
|
13872
|
+
}
|
|
13873
|
+
state.perSource.set(handle.key, current - 1);
|
|
14507
13874
|
}
|
|
14508
|
-
|
|
14509
|
-
const
|
|
14510
|
-
|
|
14511
|
-
|
|
14512
|
-
|
|
14513
|
-
|
|
14514
|
-
|
|
14515
|
-
|
|
14516
|
-
}
|
|
14517
|
-
|
|
14518
|
-
|
|
14519
|
-
const sortFields = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
14520
|
-
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
14521
|
-
const entries = result.documents.map((listed) => ({
|
|
14522
|
-
docId: listed.id,
|
|
14523
|
-
document: listed.document,
|
|
14524
|
-
sortValues: sortFields === null ? null : readSortValues(listed.document, sortFields, sortModes).map(toWireSortValue2)
|
|
14525
|
-
}));
|
|
14526
|
-
await respond(
|
|
14527
|
-
createListResultMessage(
|
|
14528
|
-
{ entries, total: result.total, hasMore: result.cursor !== null },
|
|
14529
|
-
deps.nodeId,
|
|
14530
|
-
message.requestId
|
|
14531
|
-
)
|
|
14532
|
-
);
|
|
13875
|
+
function acquireStreamSlot(state, indexName) {
|
|
13876
|
+
const current = state.streamActive.get(indexName) ?? 0;
|
|
13877
|
+
if (current >= state.maxStreamsPerIndex) {
|
|
13878
|
+
throw new NarsilError(
|
|
13879
|
+
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
13880
|
+
`snapshot stream fan-out cap reached for index '${indexName}'`,
|
|
13881
|
+
{ indexName, cap: state.maxStreamsPerIndex }
|
|
13882
|
+
);
|
|
13883
|
+
}
|
|
13884
|
+
state.streamActive.set(indexName, current + 1);
|
|
13885
|
+
return { indexName, released: false };
|
|
14533
13886
|
}
|
|
14534
|
-
|
|
14535
|
-
|
|
14536
|
-
|
|
14537
|
-
|
|
14538
|
-
|
|
14539
|
-
|
|
14540
|
-
)
|
|
14541
|
-
|
|
14542
|
-
|
|
14543
|
-
|
|
14544
|
-
|
|
14545
|
-
|
|
14546
|
-
|
|
14547
|
-
);
|
|
13887
|
+
function releaseStreamSlot(state, handle) {
|
|
13888
|
+
if (handle.released) {
|
|
13889
|
+
return;
|
|
13890
|
+
}
|
|
13891
|
+
handle.released = true;
|
|
13892
|
+
const current = state.streamActive.get(handle.indexName);
|
|
13893
|
+
if (current === void 0) {
|
|
13894
|
+
return;
|
|
13895
|
+
}
|
|
13896
|
+
if (current <= 1) {
|
|
13897
|
+
state.streamActive.delete(handle.indexName);
|
|
13898
|
+
return;
|
|
13899
|
+
}
|
|
13900
|
+
state.streamActive.set(handle.indexName, current - 1);
|
|
14548
13901
|
}
|
|
14549
|
-
|
|
14550
|
-
|
|
14551
|
-
|
|
14552
|
-
|
|
14553
|
-
|
|
14554
|
-
|
|
14555
|
-
|
|
14556
|
-
|
|
14557
|
-
|
|
14558
|
-
)
|
|
14559
|
-
|
|
13902
|
+
|
|
13903
|
+
// src/distribution/cluster-node/snapshot-sync-handler/stream.ts
|
|
13904
|
+
async function acquireAndStream(message, sink, indexName, deps, options) {
|
|
13905
|
+
let sourceHandle;
|
|
13906
|
+
try {
|
|
13907
|
+
sourceHandle = acquireSourceSlot(deps.state, message.sourceId, indexName);
|
|
13908
|
+
} catch (err) {
|
|
13909
|
+
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
|
|
13910
|
+
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13911
|
+
await respondError(sink, deps.nodeId, message, code, errMessage, transportableDetails(err));
|
|
13912
|
+
return;
|
|
13913
|
+
}
|
|
13914
|
+
try {
|
|
13915
|
+
let build;
|
|
13916
|
+
try {
|
|
13917
|
+
build = await options.buildSnapshot(indexName);
|
|
13918
|
+
} catch (err) {
|
|
13919
|
+
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
|
|
13920
|
+
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13921
|
+
await respondError(sink, deps.nodeId, message, code, errMessage, transportableDetails(err));
|
|
13922
|
+
return;
|
|
13923
|
+
}
|
|
13924
|
+
try {
|
|
13925
|
+
await streamBuiltSnapshot(message, sink, indexName, deps, options, build);
|
|
13926
|
+
} finally {
|
|
13927
|
+
await build.discard();
|
|
13928
|
+
}
|
|
13929
|
+
} finally {
|
|
13930
|
+
releaseSourceSlot(deps.state, sourceHandle);
|
|
13931
|
+
}
|
|
14560
13932
|
}
|
|
14561
|
-
function
|
|
14562
|
-
|
|
14563
|
-
|
|
14564
|
-
|
|
14565
|
-
|
|
13933
|
+
async function streamBuiltSnapshot(message, sink, indexName, deps, options, build) {
|
|
13934
|
+
let streamHandle;
|
|
13935
|
+
try {
|
|
13936
|
+
streamHandle = acquireStreamSlot(deps.state, indexName);
|
|
13937
|
+
} catch (err) {
|
|
13938
|
+
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
|
|
13939
|
+
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13940
|
+
await respondError(sink, deps.nodeId, message, code, errMessage, transportableDetails(err));
|
|
13941
|
+
return;
|
|
13942
|
+
}
|
|
13943
|
+
try {
|
|
13944
|
+
const metadata = {
|
|
13945
|
+
...options.metadata,
|
|
13946
|
+
lastSeqNo: build.lastSeqNo ?? options.metadata.lastSeqNo,
|
|
13947
|
+
primaryTerm: build.primaryTerm ?? options.metadata.primaryTerm
|
|
13948
|
+
};
|
|
13949
|
+
await streamSnapshotToReplica(sink, deps.nodeId, message.requestId, indexName, build, metadata);
|
|
13950
|
+
await options.afterSnapshot(sink, build);
|
|
13951
|
+
sink.closed = true;
|
|
13952
|
+
} finally {
|
|
13953
|
+
releaseStreamSlot(deps.state, streamHandle);
|
|
13954
|
+
}
|
|
14566
13955
|
}
|
|
14567
13956
|
|
|
14568
|
-
// src/distribution/
|
|
14569
|
-
function
|
|
14570
|
-
|
|
14571
|
-
|
|
13957
|
+
// src/distribution/cluster-node/snapshot-sync-handler/validation.ts
|
|
13958
|
+
function containsControlCharacter(value) {
|
|
13959
|
+
for (let i = 0; i < value.length; i++) {
|
|
13960
|
+
const code = value.charCodeAt(i);
|
|
13961
|
+
if (code < 32) {
|
|
13962
|
+
return true;
|
|
13963
|
+
}
|
|
13964
|
+
if (code >= 127 && code <= 159) {
|
|
13965
|
+
return true;
|
|
13966
|
+
}
|
|
13967
|
+
if (code === 8232 || code === 8233 || code === 65279) {
|
|
13968
|
+
return true;
|
|
13969
|
+
}
|
|
14572
13970
|
}
|
|
14573
|
-
|
|
14574
|
-
return primaryTerm !== void 0 && primaryTerm === replicaLastPrimaryTerm ? "incremental" : "snapshot";
|
|
13971
|
+
return false;
|
|
14575
13972
|
}
|
|
14576
|
-
function
|
|
14577
|
-
if (
|
|
14578
|
-
|
|
13973
|
+
function validateSourceId(sourceId) {
|
|
13974
|
+
if (typeof sourceId !== "string" || sourceId.length === 0) {
|
|
13975
|
+
return "request sourceId is missing";
|
|
14579
13976
|
}
|
|
14580
|
-
|
|
14581
|
-
|
|
14582
|
-
throw new Error('Invalid SyncRequestPayload: "indexName" must be a string');
|
|
13977
|
+
if (sourceId.length > MAX_SOURCE_ID_LENGTH) {
|
|
13978
|
+
return `request sourceId exceeds ${MAX_SOURCE_ID_LENGTH} characters`;
|
|
14583
13979
|
}
|
|
14584
|
-
if (
|
|
14585
|
-
|
|
13980
|
+
if (containsControlCharacter(sourceId)) {
|
|
13981
|
+
return "request sourceId contains control characters";
|
|
14586
13982
|
}
|
|
14587
|
-
|
|
14588
|
-
|
|
13983
|
+
return null;
|
|
13984
|
+
}
|
|
13985
|
+
|
|
13986
|
+
// src/distribution/cluster-node/snapshot-sync-handler/index.ts
|
|
13987
|
+
function createSnapshotSyncHandlerState() {
|
|
13988
|
+
return createSnapshotCacheState();
|
|
13989
|
+
}
|
|
13990
|
+
async function streamValidatedSnapshotRequest(message, sink, indexName, deps, options) {
|
|
13991
|
+
const sourceIdError = validateSourceId(message.sourceId);
|
|
13992
|
+
if (sourceIdError !== null) {
|
|
13993
|
+
await respondError(sink, deps.nodeId, message, ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, sourceIdError);
|
|
13994
|
+
return;
|
|
14589
13995
|
}
|
|
14590
|
-
|
|
14591
|
-
|
|
13996
|
+
const authResult = await authorizeSnapshotRequest(deps.coordinator, indexName, message.sourceId);
|
|
13997
|
+
if (authResult.outcome === "denied") {
|
|
13998
|
+
await respondError(sink, deps.nodeId, message, authResult.code, authResult.reason);
|
|
13999
|
+
return;
|
|
14592
14000
|
}
|
|
14593
|
-
|
|
14594
|
-
|
|
14001
|
+
const existingIndex = deps.engine.listIndexes().find((idx) => idx.name === indexName);
|
|
14002
|
+
if (existingIndex === void 0) {
|
|
14003
|
+
await respondError(
|
|
14004
|
+
sink,
|
|
14005
|
+
deps.nodeId,
|
|
14006
|
+
message,
|
|
14007
|
+
ErrorCodes.SNAPSHOT_SYNC_INDEX_NOT_FOUND,
|
|
14008
|
+
`Index '${indexName}' is not hosted on this node`
|
|
14009
|
+
);
|
|
14010
|
+
return;
|
|
14011
|
+
}
|
|
14012
|
+
await acquireAndStream(message, sink, indexName, deps, options);
|
|
14595
14013
|
}
|
|
14596
14014
|
|
|
14597
14015
|
// src/distribution/cluster-node/message-handler/sync.ts
|
|
@@ -14686,14 +14104,12 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
14686
14104
|
await streamValidatedSnapshotRequest(
|
|
14687
14105
|
message,
|
|
14688
14106
|
sink,
|
|
14689
|
-
|
|
14107
|
+
request.indexName,
|
|
14690
14108
|
{
|
|
14691
14109
|
nodeId: deps.nodeId,
|
|
14692
14110
|
engine: deps.engine,
|
|
14693
14111
|
coordinator: deps.coordinator,
|
|
14694
|
-
state: deps.snapshotSyncState
|
|
14695
|
-
resolveHeaderMetadata: deps.resolveHeaderMetadata,
|
|
14696
|
-
spoolDirectory: deps.snapshotSpoolDirectory
|
|
14112
|
+
state: deps.snapshotSyncState
|
|
14697
14113
|
},
|
|
14698
14114
|
{
|
|
14699
14115
|
metadata: {
|
|
@@ -14701,8 +14117,6 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
14701
14117
|
lastSeqNo: snapshotSeqNo,
|
|
14702
14118
|
primaryTerm: snapshotPrimaryTerm
|
|
14703
14119
|
},
|
|
14704
|
-
closeOnEnd: false,
|
|
14705
|
-
disableBuildCache: true,
|
|
14706
14120
|
buildSnapshot: async () => {
|
|
14707
14121
|
const archive = await deps.engine.archiveReplicationPartition(request.indexName, request.partitionId, () => {
|
|
14708
14122
|
snapshotPrimaryTerm = log.termAt(log.localLogEnd) ?? 0;
|
|
@@ -14967,18 +14381,6 @@ function validateForwardPayload(decoded) {
|
|
|
14967
14381
|
// src/distribution/cluster-node/message-handler/index.ts
|
|
14968
14382
|
function createDataNodeHandler(deps) {
|
|
14969
14383
|
return async (message, respond) => {
|
|
14970
|
-
if (message.type === ReplicationMessageTypes.SNAPSHOT_SYNC_REQUEST) {
|
|
14971
|
-
await handleSnapshotSyncRequest(message, respond, {
|
|
14972
|
-
nodeId: deps.nodeId,
|
|
14973
|
-
engine: deps.engine,
|
|
14974
|
-
coordinator: deps.coordinator,
|
|
14975
|
-
state: deps.snapshotSyncState,
|
|
14976
|
-
resolveHeaderMetadata: deps.resolveHeaderMetadata,
|
|
14977
|
-
spoolDirectory: deps.snapshotSpoolDirectory,
|
|
14978
|
-
indexUuidOf: (indexName) => deps.engine.indexUuidOf(indexName)
|
|
14979
|
-
});
|
|
14980
|
-
return;
|
|
14981
|
-
}
|
|
14982
14384
|
try {
|
|
14983
14385
|
switch (message.type) {
|
|
14984
14386
|
case ReplicationMessageTypes.SYNC_REQUEST:
|
|
@@ -16040,19 +15442,15 @@ async function gatherFromEveryPartition(allocation, indexName, read) {
|
|
|
16040
15442
|
const answered = await sendToEveryPartition(routing, (nodeId, partitionIds) => read({ nodeId, partitionIds }));
|
|
16041
15443
|
return answered.map((answer) => answer.result);
|
|
16042
15444
|
}
|
|
16043
|
-
async function sendReadRequest(deps, targetNodeId, message, indexName, validate) {
|
|
15445
|
+
async function sendReadRequest(deps, targetNodeId, message, indexName, validate, boundDecode = false) {
|
|
16044
15446
|
const response = await sendToNode2(deps.config, targetNodeId, message, {
|
|
16045
15447
|
message: `Node '${targetNodeId}' did not answer a read request for index '${indexName}'`,
|
|
16046
15448
|
details: { indexName }
|
|
16047
15449
|
});
|
|
15450
|
+
if (boundDecode) refuseDecodeWithoutRoom(response.payload);
|
|
16048
15451
|
const decoded = decode(response.payload);
|
|
16049
15452
|
if (response.type.endsWith(".error") || response.type === "error" || decoded.error === true) {
|
|
16050
|
-
|
|
16051
|
-
throw nodeRejectionOf(decoded.code, decoded.message, details) ?? new NarsilError(
|
|
16052
|
-
typeof decoded.code === "string" ? decoded.code : ErrorCodes.QUERY_ROUTING_FAILED,
|
|
16053
|
-
typeof decoded.message === "string" ? decoded.message : `Node '${targetNodeId}' rejected a read request for index '${indexName}'`,
|
|
16054
|
-
details
|
|
16055
|
-
);
|
|
15453
|
+
throw readRejectionOf(decoded, targetNodeId, indexName);
|
|
16056
15454
|
}
|
|
16057
15455
|
return validate(decoded);
|
|
16058
15456
|
}
|
|
@@ -16262,7 +15660,7 @@ async function listCluster(deps, indexName, params) {
|
|
|
16262
15660
|
);
|
|
16263
15661
|
}
|
|
16264
15662
|
const message = createListMessage({ ...payload, partitionIds: group.partitionIds }, deps.nodeId);
|
|
16265
|
-
return sendReadRequest(deps, group.nodeId, message, indexName, validateListResultPayload);
|
|
15663
|
+
return sendReadRequest(deps, group.nodeId, message, indexName, validateListResultPayload, true);
|
|
16266
15664
|
});
|
|
16267
15665
|
let total = 0;
|
|
16268
15666
|
let nodeReportsMore = false;
|
|
@@ -16421,7 +15819,7 @@ async function queryCluster(deps, indexName, params) {
|
|
|
16421
15819
|
return deps.engine.query(indexName, params);
|
|
16422
15820
|
}
|
|
16423
15821
|
if (params.rerank !== void 0) {
|
|
16424
|
-
const { rerankClusterQuery } = await import('./rerank-
|
|
15822
|
+
const { rerankClusterQuery } = await import('./rerank-WVFFZV76.mjs');
|
|
16425
15823
|
return rerankClusterQuery(
|
|
16426
15824
|
deps,
|
|
16427
15825
|
indexName,
|
|
@@ -17186,7 +16584,8 @@ function createNodeWiring(input) {
|
|
|
17186
16584
|
getReplicationLog: getReplicationLog2,
|
|
17187
16585
|
seedReplicationLog: seedReplicationLog2,
|
|
17188
16586
|
existingReplicationLog: (indexName, partitionId) => replicationLogs.get(replicationLogKey(indexName, partitionId)),
|
|
17189
|
-
resolveNodeTargets: resolveNodeTargets3
|
|
16587
|
+
resolveNodeTargets: resolveNodeTargets3,
|
|
16588
|
+
resolveRegisteredNodeTargets: (targetNodeId) => resolveRegisteredTargets(routingConfig2, targetNodeId)
|
|
17190
16589
|
};
|
|
17191
16590
|
}
|
|
17192
16591
|
|
|
@@ -17547,6 +16946,7 @@ function createNotReadyHandler(nodeId) {
|
|
|
17547
16946
|
function createMultiplexedControllerTransport(baseTransport, nodeId) {
|
|
17548
16947
|
let controllerHandler = null;
|
|
17549
16948
|
const transport = {
|
|
16949
|
+
timeouts: baseTransport.timeouts,
|
|
17550
16950
|
send(target, message) {
|
|
17551
16951
|
return baseTransport.send(target, message);
|
|
17552
16952
|
},
|
|
@@ -17798,6 +17198,7 @@ async function createClusterNode(callerConfig) {
|
|
|
17798
17198
|
});
|
|
17799
17199
|
writeDeps.copyFailed = copyFailures.isFailed;
|
|
17800
17200
|
writeDeps.copyUnlisted = copyFailures.isUnlisted;
|
|
17201
|
+
writeDeps.stopped = () => isShutdown;
|
|
17801
17202
|
engine.isolateCopyFailures(copyFailures.record);
|
|
17802
17203
|
const bootstrapPartition2 = createPartitionBootstrap({
|
|
17803
17204
|
nodeId,
|
|
@@ -17805,7 +17206,7 @@ async function createClusterNode(callerConfig) {
|
|
|
17805
17206
|
coordinator: config.coordinator,
|
|
17806
17207
|
transport: config.transport,
|
|
17807
17208
|
state: bootstrapSyncState,
|
|
17808
|
-
resolveNodeTargets:
|
|
17209
|
+
resolveNodeTargets: wiring.resolveRegisteredNodeTargets,
|
|
17809
17210
|
getReplicationLog: getReplicationLog2,
|
|
17810
17211
|
seedReplicationLog: seedReplicationLog2,
|
|
17811
17212
|
partitionWriteQueues: writeDeps.partitionWriteQueues,
|
|
@@ -17940,7 +17341,6 @@ async function createClusterNode(callerConfig) {
|
|
|
17940
17341
|
writeDeps,
|
|
17941
17342
|
snapshotSyncState: snapshotSyncHandlerState,
|
|
17942
17343
|
snapshotSpoolDirectory: snapshotSpool.path,
|
|
17943
|
-
resolveHeaderMetadata: resolveSnapshotHeaderMetadata,
|
|
17944
17344
|
isBootstrapSynced: (indexName, partitionId) => hasCompletedBootstrapSync(bootstrapSyncState, indexName, partitionId),
|
|
17945
17345
|
onError: forwardOnError
|
|
17946
17346
|
});
|
|
@@ -17991,13 +17391,6 @@ async function createClusterNode(callerConfig) {
|
|
|
17991
17391
|
});
|
|
17992
17392
|
bindRerankWindowSource(node, () => rerankWindowCapOf(engine));
|
|
17993
17393
|
return node;
|
|
17994
|
-
function resolveSnapshotHeaderMetadata(indexName, partitionId) {
|
|
17995
|
-
return nodeSnapshotHeaderMetadata(
|
|
17996
|
-
{ coordinator: config.coordinator, engine, getReplicationLog: getReplicationLog2 },
|
|
17997
|
-
indexName,
|
|
17998
|
-
partitionId
|
|
17999
|
-
);
|
|
18000
|
-
}
|
|
18001
17394
|
}
|
|
18002
17395
|
|
|
18003
17396
|
// src/distribution/cluster-node/server-engine.ts
|