@delali/narsil 0.3.1-next.102 → 0.3.1-next.104
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-KPJMDPHG.mjs → chunk-5YMT23EX.mjs} +13 -13
- package/dist/browser/{chunk-JOVZ6HMT.mjs → chunk-6E4SWUYZ.mjs} +65 -41
- package/dist/browser/{chunk-EJDUFWSO.mjs → chunk-CWWD2UHP.mjs} +2 -2
- package/dist/browser/{chunk-PGIM3HIM.mjs → chunk-ECSLQ3S5.mjs} +5 -5
- package/dist/browser/{chunk-RHG4O77A.mjs → chunk-J3QB4LBD.mjs} +1 -0
- package/dist/browser/{chunk-BDOSMEEP.mjs → chunk-PW7GJFPW.mjs} +3 -3
- package/dist/browser/{chunk-RIJSLY2I.mjs → chunk-SFXM7PLV.mjs} +2 -2
- package/dist/browser/{durability-wiring-SRUYT3VV.mjs → durability-wiring-HTBWDAOX.mjs} +6 -6
- package/dist/browser/{module-CBB2ETYP.mjs → module-6FVNK7DO.mjs} +5 -5
- package/dist/browser/{restore-queue-EEZQO6RF.mjs → restore-queue-CL43WZTZ.mjs} +1 -1
- package/dist/browser/{runtime-7ZAHCHWR.mjs → runtime-JQYPNOK6.mjs} +1 -1
- package/dist/browser/{snapshot-OAAA5UJ7.mjs → snapshot-NL4GSDUD.mjs} +7 -7
- package/dist/browser/{write-7D6VIK6O.mjs → write-YFOOPLAN.mjs} +5 -5
- package/dist/{capabilities-ByQRMOx-.d.ts → capabilities-DGaz7cb3.d.ts} +2 -2
- package/dist/{chunk-PC2G76VW.mjs → chunk-2MNJ2HX7.mjs} +2 -2
- package/dist/{chunk-SP3NM3ZG.mjs → chunk-35KEGAOS.mjs} +10 -10
- package/dist/{chunk-OEYI3KRB.mjs → chunk-3IQZYNW6.mjs} +2 -2
- package/dist/{chunk-S7GNDRPS.mjs → chunk-47PZE6CJ.mjs} +3 -3
- package/dist/{chunk-5JNA5SKS.mjs → chunk-5W2ODPPO.mjs} +4 -3
- package/dist/{chunk-KWTW6VTG.mjs → chunk-6GFVUXUH.mjs} +8 -8
- package/dist/{chunk-BWJVNZPR.mjs → chunk-6JUOMMHQ.mjs} +1 -0
- package/dist/{chunk-LKOB5COA.mjs → chunk-77SO7G55.mjs} +1 -1
- package/dist/{chunk-UTG5YNXV.mjs → chunk-AEF2O7B4.mjs} +1 -1
- package/dist/{chunk-BOFBLJFZ.mjs → chunk-B7BSQZ27.mjs} +3 -6
- package/dist/{chunk-LGAMKHA2.mjs → chunk-BLDJ5I6I.mjs} +65 -41
- package/dist/{chunk-OZDFA4WC.mjs → chunk-CYHQNM3T.mjs} +2 -2
- package/dist/{chunk-CVXLLEQ3.mjs → chunk-DQ5NJSBZ.mjs} +3 -3
- package/dist/chunk-DUWYU4D5.mjs +19 -0
- package/dist/{chunk-ZKWYWW4Y.mjs → chunk-FABI2JA7.mjs} +1 -1
- package/dist/{chunk-J5QP52HA.mjs → chunk-IGUTV3MX.mjs} +9 -9
- package/dist/{chunk-XADZPPZI.mjs → chunk-J2KRH2RJ.mjs} +4 -4
- package/dist/{chunk-2RIRGOUL.mjs → chunk-JSSGO4B5.mjs} +1 -1
- package/dist/{chunk-PNFKUCG3.mjs → chunk-KNRU5CJ4.mjs} +6 -6
- package/dist/{chunk-Q6XWVUZX.mjs → chunk-KSQS77DK.mjs} +1 -1
- package/dist/{chunk-YEEQHT4V.mjs → chunk-MJIJG4YT.mjs} +2 -2
- package/dist/{chunk-EMRMGYRE.mjs → chunk-MKPE3VZF.mjs} +2 -2
- package/dist/{chunk-75XIEF6H.mjs → chunk-MZW4V4DS.mjs} +1 -1
- package/dist/{chunk-WVCOCKBU.mjs → chunk-O372FZM4.mjs} +21 -21
- package/dist/{chunk-S7FL6UAH.mjs → chunk-OIN27OQH.mjs} +3 -3
- package/dist/{chunk-YYQIODMF.mjs → chunk-PAWVKEVS.mjs} +1 -1
- package/dist/{chunk-UA3AUGNV.mjs → chunk-PFQQ3LCQ.mjs} +6 -6
- package/dist/{chunk-JZVARCRZ.mjs → chunk-PQHR5FHC.mjs} +1 -1
- package/dist/{chunk-ZM7KU4YW.mjs → chunk-QALAFGF5.mjs} +1 -1
- package/dist/{chunk-GBEEG5AN.mjs → chunk-QSK35UC2.mjs} +2 -2
- package/dist/{chunk-46D6JQIG.mjs → chunk-RI5VNFAI.mjs} +2 -2
- package/dist/{chunk-PNEYIHA4.mjs → chunk-RPIE4QCL.mjs} +0 -1
- package/dist/{chunk-RF7PLG43.mjs → chunk-RWXFSGKJ.mjs} +2 -2
- package/dist/{chunk-OM362BRO.mjs → chunk-TDUBQ546.mjs} +3 -3
- package/dist/{chunk-R27EMH5M.mjs → chunk-VFEALJEA.mjs} +2 -2
- package/dist/{chunk-EWZLIYHV.mjs → chunk-WCNN2PLX.mjs} +1 -1
- package/dist/{chunk-ZI34TVOA.mjs → chunk-WU4P5J4G.mjs} +3 -3
- package/dist/{chunk-VI4R6W25.mjs → chunk-XWJGJ2SE.mjs} +6 -6
- package/dist/{client-BOStAkc5.d.ts → client-DI-xIXgR.d.ts} +2 -2
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +1 -0
- package/dist/{config-Cjvzbs_j.d.ts → config-DL_bDDD8.d.ts} +1 -1
- package/dist/{config-H6SGzbPH.d.ts → config-DWxtDsJ6.d.ts} +2 -2
- package/dist/{direct-executor-ZD64PPEZ.mjs → direct-executor-5NX2ZIAG.mjs} +11 -11
- package/dist/distribution/coordinator/etcd.mjs +3 -1
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +5 -4
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +3 -2
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +4 -3
- package/dist/distribution.d.ts +6 -6
- package/dist/distribution.mjs +1746 -2276
- package/dist/{durability-wiring-FEUBR5BT.mjs → durability-wiring-LD4GWPWD.mjs} +25 -25
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-rC0vjjY-.d.ts → engine-D_SYDsId.d.ts} +1 -1
- package/dist/{events-DqOIg5xa.d.ts → events-C7_dboUr.d.ts} +1 -0
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +17 -17
- package/dist/{module-3S65MVCM.mjs → module-QFWZ5OBF.mjs} +10 -10
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/{rerank-K7C4WZFG.mjs → rerank-WVFFZV76.mjs} +4 -4
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-KQHA7ZV2.mjs → restore-queue-7QIKM6PY.mjs} +1 -1
- package/dist/{runtime-6Y4ZYXOG.mjs → runtime-AVUUCJWQ.mjs} +5 -5
- package/dist/server.d.ts +9 -9
- package/dist/server.mjs +12 -12
- package/dist/{snapshot-QGZPODM5.mjs → snapshot-JYTT36WI.mjs} +26 -26
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-CwPTz-tP.d.ts → tasks-BhjQZ8Qf.d.ts} +1 -1
- package/dist/{thread-N3IUGUS3.mjs → thread-AHLISTRZ.mjs} +12 -12
- package/dist/{types-2qoitTXg.d.ts → types-Ck97nB3D.d.ts} +1 -1
- package/dist/{types-Cz5okLml.d.ts → types-DvncmvuQ.d.ts} +11 -0
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +5 -5
- package/dist/{write-U5RI7MAX.mjs → write-GPFM2EIX.mjs} +19 -19
- package/package.json +7 -7
- package/dist/chunk-747OHOUA.mjs +0 -11
package/dist/distribution.mjs
CHANGED
|
@@ -1,59 +1,59 @@
|
|
|
1
|
-
import { DEFAULT_NODE_LIFECYCLE_CONFIG, bindWriteBlockLookup, bindEngineWriteBlock, bindIndexReads, DEFAULT_CONTROLLER_CONFIG, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, readIndexState, readIndexMetadataSnapshot, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, indexConfigFromSettings, metadataWithSynonymSet, indexConfigKey, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_ROUNDS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-
|
|
2
|
-
import {
|
|
3
|
-
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-
|
|
4
|
-
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-
|
|
5
|
-
import { closePartitionWriter, replaySyncedWal } from './chunk-
|
|
6
|
-
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-
|
|
7
|
-
import './chunk-
|
|
8
|
-
import { indexMetadataOf, loadPartitionSegments, readSegmentManifest, readVectorFile } from './chunk-
|
|
9
|
-
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-
|
|
1
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, bindWriteBlockLookup, bindEngineWriteBlock, bindIndexReads, DEFAULT_CONTROLLER_CONFIG, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, readIndexState, readIndexMetadataSnapshot, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, indexConfigFromSettings, metadataWithSynonymSet, indexConfigKey, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, RESHARD_FORWARD_BATCH_DOCUMENTS, storedSynonymSetOf, RESHARD_RETRY_DELAY_MS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, TEARDOWN_RETRY_ROUNDS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS } from './chunk-OIN27OQH.mjs';
|
|
2
|
+
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, DEFAULT_RESUME_WRITES_BELOW, DEFAULT_BLOCK_WRITES_ABOVE, readMemoryFigures, DEFAULT_IN_FLIGHT_WRITE_FRACTION, onRefusalAfterCollection, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, readRecentMemoryFigures, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, MAX_SYNONYM_REPLACEMENT_ATTEMPTS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, DECOMMISSION_RECHECK_INTERVAL_MS, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, MAX_SOURCE_ID_LENGTH, reserveHeapBytes, reservedHeapBytes, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, DOCUMENT_NUMBER_BYTES, DOCUMENT_SCALAR_BYTES, DOCUMENT_ENTRY_OVERHEAD_BYTES, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, ADMISSION_TIMEOUT_MS, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, COPY_FAILURE_RETRY_DELAY_MS, collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, FULL_MEMORY_COLLECTION_INTERVAL_MS, MEMORY_SAMPLE_INTERVAL_MS, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, CLEAR_PAGE_SIZE, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, PRIMARY_CHANGE_RETRY_WINDOW_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-B7BSQZ27.mjs';
|
|
3
|
+
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-B7BSQZ27.mjs';
|
|
4
|
+
import { fixedHybridDepthOf, requireWithinHybridDepth, pageByPosition } from './chunk-WCNN2PLX.mjs';
|
|
5
|
+
import { closePartitionWriter, replaySyncedWal } from './chunk-WU4P5J4G.mjs';
|
|
6
|
+
import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-PFQQ3LCQ.mjs';
|
|
7
|
+
import './chunk-47PZE6CJ.mjs';
|
|
8
|
+
import { indexMetadataOf, loadPartitionSegments, readSegmentManifest, readVectorFile } from './chunk-KNRU5CJ4.mjs';
|
|
9
|
+
import { decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-PAWVKEVS.mjs';
|
|
10
10
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
11
11
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-BLDJ5I6I.mjs';
|
|
13
13
|
import './chunk-DJHHF2L5.mjs';
|
|
14
14
|
import './chunk-37CD3OKS.mjs';
|
|
15
15
|
import { getPath, getFs } from './chunk-HLPRHOSO.mjs';
|
|
16
|
-
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, readListedFile, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, directoryCheckpointStore, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardMessage, createForwardBatchMessage, validateForwardBatchResultPayload } from './chunk-
|
|
17
|
-
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-
|
|
16
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, readListedFile, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, directoryCheckpointStore, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardMessage, createForwardBatchMessage, validateForwardBatchResultPayload } from './chunk-IGUTV3MX.mjs';
|
|
17
|
+
import { applyIndexEntry, applyDeleteEntry, validateReplicationEntry } from './chunk-J2KRH2RJ.mjs';
|
|
18
18
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
19
|
-
import { TransportErrorCodes, ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-
|
|
20
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createByteReader, openChecksummedEnvelope, emptySerializablePartition, crc32Init, crc32Final, crc32Update, yieldToEventLoop } from './chunk-
|
|
21
|
-
import { runtimeHost, requireSurfaceFormsForPartialFields, queryBindingOf, now, listBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-
|
|
22
|
-
import { bindRerankWindowSource, rerankWindowCapOf, engineCoreOf, providedDocId } from './chunk-
|
|
23
|
-
import './chunk-
|
|
19
|
+
import { TransportErrorCodes, ReplicationMessageTypes, ClusterMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-RPIE4QCL.mjs';
|
|
20
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, encodeIndexConfiguration, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createByteReader, openChecksummedEnvelope, emptySerializablePartition, crc32Init, crc32Final, crc32Update, yieldToEventLoop } from './chunk-O372FZM4.mjs';
|
|
21
|
+
import { runtimeHost, requireSurfaceFormsForPartialFields, queryBindingOf, now, listBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-6GFVUXUH.mjs';
|
|
22
|
+
import { bindRerankWindowSource, rerankWindowCapOf, engineCoreOf, providedDocId } from './chunk-MKPE3VZF.mjs';
|
|
23
|
+
import './chunk-CYHQNM3T.mjs';
|
|
24
24
|
import './chunk-AV4IJKD2.mjs';
|
|
25
|
-
import { synonymModule } from './chunk-
|
|
26
|
-
import { oversampledShardSize, normalizeSort, sortModesOf, readSortValues, resolveProjection, applyProjection, fnv1a, FILTER_EXPRESSION_KEYS, filterTestCount, FIELD_FILTER_OPERATORS, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, encodePageCursor, requireWithinResultWindow, sortSignatureEntry, fnv1aUint32s, mergePartitionStats } from './chunk-
|
|
27
|
-
import { generateId, getLanguage } from './chunk-
|
|
25
|
+
import { synonymModule } from './chunk-35KEGAOS.mjs';
|
|
26
|
+
import { oversampledShardSize, normalizeSort, sortModesOf, readSortValues, resolveProjection, applyProjection, fnv1a, FILTER_EXPRESSION_KEYS, filterTestCount, FIELD_FILTER_OPERATORS, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, encodePageCursor, requireWithinResultWindow, sortSignatureEntry, fnv1aUint32s, mergePartitionStats } from './chunk-XWJGJ2SE.mjs';
|
|
27
|
+
import { generateId, getLanguage } from './chunk-QALAFGF5.mjs';
|
|
28
28
|
import './chunk-LPQOEU5M.mjs';
|
|
29
29
|
import './chunk-HJSN7UYG.mjs';
|
|
30
|
-
import {
|
|
30
|
+
import { MAX_DOC_ID_LENGTH, BATCH_CHUNK_SIZE, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING } from './chunk-LXWDQLTW.mjs';
|
|
31
31
|
import './chunk-TGP5M5R7.mjs';
|
|
32
32
|
import './chunk-U2BKYEC7.mjs';
|
|
33
33
|
import './chunk-KOKM4PQN.mjs';
|
|
34
|
-
import { MIN_OVERSAMPLE } from './chunk-
|
|
35
|
-
import './chunk-
|
|
36
|
-
import { isPatternTolerance } from './chunk-
|
|
34
|
+
import { MIN_OVERSAMPLE } from './chunk-QSK35UC2.mjs';
|
|
35
|
+
import './chunk-PQHR5FHC.mjs';
|
|
36
|
+
import { isPatternTolerance } from './chunk-2MNJ2HX7.mjs';
|
|
37
37
|
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
38
|
-
import { getNestedValue } from './chunk-
|
|
39
|
-
import { validateSchema, requireCreatableSchema, encodeDocumentReply, encodeDocument, schemaFieldsOf, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-
|
|
38
|
+
import { getNestedValue } from './chunk-TDUBQ546.mjs';
|
|
39
|
+
import { validateSchema, requireCreatableSchema, encodeDocumentReply, encodeDocument, schemaFieldsOf, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-3IQZYNW6.mjs';
|
|
40
40
|
import { MAX_DOCUMENT_ENCODING_DEPTH } from './chunk-U476IXKQ.mjs';
|
|
41
41
|
import { MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, DEFAULT_PAGE_SIZE, HYBRID_RESCORE_WINDOW } from './chunk-E74YLAIG.mjs';
|
|
42
42
|
import './chunk-HMFNFVP7.mjs';
|
|
43
43
|
import './chunk-GES7U2PM.mjs';
|
|
44
|
-
import './chunk-
|
|
45
|
-
import { resolveIndexAnalysis } from './chunk-
|
|
44
|
+
import './chunk-77SO7G55.mjs';
|
|
45
|
+
import { resolveIndexAnalysis } from './chunk-RI5VNFAI.mjs';
|
|
46
46
|
import './chunk-X5CV2E2B.mjs';
|
|
47
47
|
import './chunk-7X5JST5L.mjs';
|
|
48
|
-
import './chunk-
|
|
48
|
+
import './chunk-AEF2O7B4.mjs';
|
|
49
49
|
import { isSortMode, SORT_MODES, toComparableSortValue, compareComparableKeys, compareSortValues } from './chunk-R6A773WK.mjs';
|
|
50
50
|
import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
51
51
|
import { HEAP_RESUME_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, HEAP_BLOCK_FRACTION_BELOW_V8_INEFFECTIVE_MARK_COMPACT_LINE, collectAndReturnPages } from './chunk-RFVUDM5E.mjs';
|
|
52
52
|
import './chunk-CQUMNIIG.mjs';
|
|
53
|
-
import './chunk-
|
|
53
|
+
import './chunk-KSQS77DK.mjs';
|
|
54
54
|
import './chunk-RFBLQLLZ.mjs';
|
|
55
55
|
import { MAX_STORED_VECTOR_DIMENSION, MAX_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
|
|
56
|
-
import { ErrorCodes, NarsilError, describeError } from './chunk-
|
|
56
|
+
import { ErrorCodes, NarsilError, describeError } from './chunk-6JUOMMHQ.mjs';
|
|
57
57
|
import { encode, decode } from '@msgpack/msgpack';
|
|
58
58
|
import { randomUUID } from 'crypto';
|
|
59
59
|
import { openSync, writeSync, closeSync, rmSync, readSync } from 'fs';
|
|
@@ -1163,300 +1163,534 @@ async function nodeHasOtherPartitions(indexName, removedPartitionId, deps) {
|
|
|
1163
1163
|
return false;
|
|
1164
1164
|
}
|
|
1165
1165
|
|
|
1166
|
-
// src/distribution/
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
}
|
|
1174
|
-
function releaseSpool(state) {
|
|
1175
|
-
const spool = state.spool;
|
|
1176
|
-
if (spool === null) return;
|
|
1177
|
-
state.spool = null;
|
|
1178
|
-
spool.discard();
|
|
1179
|
-
}
|
|
1180
|
-
function spoolFailure(state, err) {
|
|
1181
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1182
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `the snapshot could not be written to local storage: ${cause}`, {
|
|
1183
|
-
cause
|
|
1184
|
-
});
|
|
1185
|
-
}
|
|
1186
|
-
function applyErrorEnvelope(state, record) {
|
|
1187
|
-
const code = typeof record.code === "string" && record.code.length > 0 ? record.code : "UNKNOWN_ERROR";
|
|
1188
|
-
const message = typeof record.message === "string" && record.message.length > 0 ? record.message : "primary returned error envelope";
|
|
1189
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_PRIMARY_ERROR, `primary error: ${message}`, {
|
|
1190
|
-
primaryCode: code,
|
|
1191
|
-
primaryMessage: message
|
|
1192
|
-
});
|
|
1166
|
+
// src/distribution/cluster-node/schema-diff.ts
|
|
1167
|
+
var SCHEMA_DIFF_MAX_DEPTH = MAX_NESTING_DEPTH;
|
|
1168
|
+
var DEPTH_EXCEEDED_SENTINEL = "depth-exceeded";
|
|
1169
|
+
function diffSchemas(expected, actual) {
|
|
1170
|
+
const diffs = [];
|
|
1171
|
+
walk(expected, actual, "", diffs, 1);
|
|
1172
|
+
return diffs;
|
|
1193
1173
|
}
|
|
1194
|
-
function
|
|
1195
|
-
if (
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
const header = start.header;
|
|
1201
|
-
if (header.indexName !== state.expectation.indexName) {
|
|
1202
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "snapshot header indexName mismatch", {
|
|
1203
|
-
expected: state.expectation.indexName,
|
|
1204
|
-
received: header.indexName
|
|
1205
|
-
});
|
|
1206
|
-
return;
|
|
1207
|
-
}
|
|
1208
|
-
if (state.expectation.partitionId !== null && header.partitionId !== state.expectation.partitionId) {
|
|
1209
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "snapshot header partitionId mismatch", {
|
|
1210
|
-
expected: state.expectation.partitionId,
|
|
1211
|
-
received: header.partitionId
|
|
1174
|
+
function walk(expected, actual, prefix, diffs, depth) {
|
|
1175
|
+
if (depth > SCHEMA_DIFF_MAX_DEPTH) {
|
|
1176
|
+
diffs.push({
|
|
1177
|
+
path: prefix === "" ? DEPTH_EXCEEDED_SENTINEL : prefix,
|
|
1178
|
+
expected: DEPTH_EXCEEDED_SENTINEL,
|
|
1179
|
+
actual: DEPTH_EXCEEDED_SENTINEL
|
|
1212
1180
|
});
|
|
1213
1181
|
return;
|
|
1214
1182
|
}
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1183
|
+
const expectedKeys = Object.keys(expected).sort(compareCodePoints);
|
|
1184
|
+
const actualKeys = new Set(Object.keys(actual));
|
|
1185
|
+
for (const key of expectedKeys) {
|
|
1186
|
+
const path = prefix === "" ? key : `${prefix}.${key}`;
|
|
1187
|
+
const expectedValue = expected[key];
|
|
1188
|
+
const actualValue = actual[key];
|
|
1189
|
+
if (!actualKeys.has(key)) {
|
|
1190
|
+
diffs.push({ path, expected: describe(expectedValue), actual: "(absent)" });
|
|
1191
|
+
continue;
|
|
1192
|
+
}
|
|
1193
|
+
actualKeys.delete(key);
|
|
1194
|
+
const expectedIsObject = typeof expectedValue === "object" && expectedValue !== null;
|
|
1195
|
+
const actualIsObject = typeof actualValue === "object" && actualValue !== null;
|
|
1196
|
+
if (expectedIsObject !== actualIsObject) {
|
|
1197
|
+
diffs.push({ path, expected: describe(expectedValue), actual: describe(actualValue) });
|
|
1198
|
+
continue;
|
|
1199
|
+
}
|
|
1200
|
+
if (expectedIsObject && actualIsObject) {
|
|
1201
|
+
walk(expectedValue, actualValue, path, diffs, depth + 1);
|
|
1202
|
+
continue;
|
|
1203
|
+
}
|
|
1204
|
+
if (expectedValue !== actualValue) {
|
|
1205
|
+
diffs.push({ path, expected: String(expectedValue), actual: String(actualValue) });
|
|
1206
|
+
}
|
|
1234
1207
|
}
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
spoolFailure(state, err);
|
|
1239
|
-
return;
|
|
1208
|
+
for (const extraKey of actualKeys) {
|
|
1209
|
+
const path = prefix === "" ? extraKey : `${prefix}.${extraKey}`;
|
|
1210
|
+
diffs.push({ path, expected: "(absent)", actual: describe(actual[extraKey]) });
|
|
1240
1211
|
}
|
|
1241
|
-
state.header = header;
|
|
1242
|
-
state.totalBytes = start.totalBytes;
|
|
1243
1212
|
}
|
|
1244
|
-
function
|
|
1245
|
-
if (
|
|
1246
|
-
|
|
1247
|
-
return;
|
|
1248
|
-
}
|
|
1249
|
-
if (!isSafeNonNegativeInteger(payload.offset)) {
|
|
1250
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHUNK_OUT_OF_ORDER, "chunk offset must be a non-negative integer");
|
|
1251
|
-
return;
|
|
1213
|
+
function describe(value) {
|
|
1214
|
+
if (value === null) {
|
|
1215
|
+
return "null";
|
|
1252
1216
|
}
|
|
1253
|
-
if (
|
|
1254
|
-
|
|
1255
|
-
return;
|
|
1217
|
+
if (typeof value === "object") {
|
|
1218
|
+
return "object";
|
|
1256
1219
|
}
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1220
|
+
return String(value);
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
// src/distribution/cluster-node/bootstrap-restore.ts
|
|
1224
|
+
var ABORT_SENTINEL = /* @__PURE__ */ Symbol("bootstrap-sync-aborted");
|
|
1225
|
+
function validateArguments(indexName, partitionId, primaryNodeId) {
|
|
1226
|
+
if (typeof indexName !== "string" || indexName.length === 0) {
|
|
1227
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, "indexName must be a non-empty string", {
|
|
1228
|
+
indexName
|
|
1261
1229
|
});
|
|
1262
|
-
return;
|
|
1263
1230
|
}
|
|
1264
|
-
if (
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1231
|
+
if (!Number.isInteger(partitionId) || partitionId < 0) {
|
|
1232
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, "partitionId must be a non-negative integer", {
|
|
1233
|
+
indexName,
|
|
1234
|
+
partitionId
|
|
1268
1235
|
});
|
|
1269
|
-
return;
|
|
1270
1236
|
}
|
|
1271
|
-
if (
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
received: payload.offset
|
|
1237
|
+
if (typeof primaryNodeId !== "string" || primaryNodeId.length === 0) {
|
|
1238
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, "primaryNodeId must be a non-empty string", {
|
|
1239
|
+
indexName
|
|
1275
1240
|
});
|
|
1276
|
-
return;
|
|
1277
|
-
}
|
|
1278
|
-
if (!(payload.data instanceof Uint8Array)) {
|
|
1279
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "chunk data is not Uint8Array");
|
|
1280
|
-
return;
|
|
1281
1241
|
}
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1242
|
+
return null;
|
|
1243
|
+
}
|
|
1244
|
+
async function resolveTransportTargets(indexName, primaryNodeId, resolveNodeTargets3) {
|
|
1245
|
+
let targets;
|
|
1246
|
+
try {
|
|
1247
|
+
targets = await resolveNodeTargets3(primaryNodeId);
|
|
1248
|
+
} catch (err) {
|
|
1249
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
1250
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_NO_TARGETS, `resolveNodeTargets failed: ${cause}`, {
|
|
1251
|
+
indexName,
|
|
1252
|
+
primaryNodeId,
|
|
1253
|
+
cause
|
|
1288
1254
|
});
|
|
1289
|
-
return;
|
|
1290
1255
|
}
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
return;
|
|
1256
|
+
if (targets.length === 0) {
|
|
1257
|
+
return new NarsilError(
|
|
1258
|
+
ErrorCodes.SNAPSHOT_SYNC_NO_TARGETS,
|
|
1259
|
+
`no transport targets resolved for primary '${primaryNodeId}'`,
|
|
1260
|
+
{ indexName, primaryNodeId }
|
|
1261
|
+
);
|
|
1298
1262
|
}
|
|
1299
|
-
|
|
1300
|
-
|
|
1263
|
+
return targets;
|
|
1264
|
+
}
|
|
1265
|
+
function surfaceError(deps, indexName, primaryNodeId, error) {
|
|
1266
|
+
if (deps.onError === void 0) {
|
|
1301
1267
|
return;
|
|
1302
1268
|
}
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1269
|
+
const wrapped = new NarsilError(
|
|
1270
|
+
ErrorCodes.NODE_BOOTSTRAP_FAILED,
|
|
1271
|
+
`Bootstrap sync failed for index '${indexName}': ${error.message}`,
|
|
1272
|
+
{
|
|
1273
|
+
indexName,
|
|
1274
|
+
primaryNodeId,
|
|
1275
|
+
innerCode: error.code,
|
|
1276
|
+
...error.details
|
|
1277
|
+
}
|
|
1278
|
+
);
|
|
1279
|
+
deps.onError(wrapped);
|
|
1280
|
+
}
|
|
1281
|
+
function surfaceAborted(deps, indexName, primaryNodeId) {
|
|
1282
|
+
if (deps.onError === void 0) {
|
|
1307
1283
|
return;
|
|
1308
1284
|
}
|
|
1309
|
-
|
|
1310
|
-
|
|
1285
|
+
deps.onError(
|
|
1286
|
+
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, `bootstrap sync aborted for index '${indexName}'`, {
|
|
1287
|
+
indexName,
|
|
1288
|
+
primaryNodeId
|
|
1289
|
+
})
|
|
1290
|
+
);
|
|
1311
1291
|
}
|
|
1312
|
-
function
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1292
|
+
function validateRestoredSchema(engine, indexName, primaryNodeId, coordinatorSchema) {
|
|
1293
|
+
let restoredSchema;
|
|
1294
|
+
try {
|
|
1295
|
+
restoredSchema = engine.getStats(indexName).schema;
|
|
1296
|
+
} catch (err) {
|
|
1297
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
1298
|
+
return new NarsilError(
|
|
1299
|
+
ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
|
|
1300
|
+
`failed to read restored schema for '${indexName}': ${cause}`,
|
|
1301
|
+
{ indexName, primaryNodeId, reason: "schema lookup failed", cause }
|
|
1302
|
+
);
|
|
1316
1303
|
}
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
received: payload.indexName
|
|
1321
|
-
});
|
|
1322
|
-
return;
|
|
1304
|
+
const differences = diffSchemas(coordinatorSchema, restoredSchema);
|
|
1305
|
+
if (differences.length === 0) {
|
|
1306
|
+
return null;
|
|
1323
1307
|
}
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
received: payload.totalBytes
|
|
1335
|
-
});
|
|
1336
|
-
return;
|
|
1337
|
-
}
|
|
1338
|
-
if (!isSafeNonNegativeInteger(payload.checksum) || payload.checksum !== state.header.checksum) {
|
|
1339
|
-
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHECKSUM_MISMATCH, "end checksum does not match header", {
|
|
1340
|
-
expected: state.header.checksum,
|
|
1341
|
-
received: payload.checksum
|
|
1342
|
-
});
|
|
1343
|
-
return;
|
|
1344
|
-
}
|
|
1345
|
-
state.endPayload = payload;
|
|
1308
|
+
return new NarsilError(
|
|
1309
|
+
ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
|
|
1310
|
+
`restored snapshot schema disagrees with coordinator schema for '${indexName}'`,
|
|
1311
|
+
{
|
|
1312
|
+
indexName,
|
|
1313
|
+
primaryNodeId,
|
|
1314
|
+
reason: "schema mismatch",
|
|
1315
|
+
differences
|
|
1316
|
+
}
|
|
1317
|
+
);
|
|
1346
1318
|
}
|
|
1347
|
-
function
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1319
|
+
async function loadCoordinatorIndexConfig(coordinator, indexName, primaryNodeId, abortPromise) {
|
|
1320
|
+
let schema;
|
|
1321
|
+
try {
|
|
1322
|
+
const winner = await Promise.race([coordinator.getSchema(indexName), abortPromise]);
|
|
1323
|
+
if (winner === ABORT_SENTINEL) {
|
|
1324
|
+
return "aborted";
|
|
1325
|
+
}
|
|
1326
|
+
schema = winner;
|
|
1327
|
+
} catch (err) {
|
|
1328
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
1329
|
+
return {
|
|
1330
|
+
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE, `coordinator.getSchema failed: ${cause}`, {
|
|
1331
|
+
indexName,
|
|
1332
|
+
primaryNodeId,
|
|
1333
|
+
cause
|
|
1334
|
+
})
|
|
1335
|
+
};
|
|
1356
1336
|
}
|
|
1357
|
-
if (
|
|
1358
|
-
return
|
|
1337
|
+
if (schema === null) {
|
|
1338
|
+
return {
|
|
1339
|
+
error: new NarsilError(
|
|
1340
|
+
ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE,
|
|
1341
|
+
`coordinator has no schema for index '${indexName}'`,
|
|
1342
|
+
{ indexName, primaryNodeId }
|
|
1343
|
+
)
|
|
1344
|
+
};
|
|
1359
1345
|
}
|
|
1360
|
-
|
|
1361
|
-
|
|
1346
|
+
let validSchema;
|
|
1347
|
+
try {
|
|
1348
|
+
validSchema = validateSchema(schema);
|
|
1349
|
+
} catch (err) {
|
|
1350
|
+
if (!(err instanceof NarsilError)) throw err;
|
|
1351
|
+
return {
|
|
1352
|
+
error: new NarsilError(err.code, `coordinator schema for index '${indexName}' is invalid: ${err.message}`, {
|
|
1353
|
+
...err.details,
|
|
1354
|
+
indexName,
|
|
1355
|
+
primaryNodeId
|
|
1356
|
+
})
|
|
1357
|
+
};
|
|
1362
1358
|
}
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1359
|
+
try {
|
|
1360
|
+
const winner = await Promise.race([getClusterIndexConfig(coordinator, indexName, validSchema), abortPromise]);
|
|
1361
|
+
if (winner === ABORT_SENTINEL) {
|
|
1362
|
+
return "aborted";
|
|
1363
|
+
}
|
|
1364
|
+
return { config: winner };
|
|
1365
|
+
} catch (err) {
|
|
1366
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
1367
|
+
return {
|
|
1368
|
+
error: new NarsilError(
|
|
1369
|
+
err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE,
|
|
1370
|
+
`failed to read the coordinator index metadata for '${indexName}': ${cause}`,
|
|
1371
|
+
{ indexName, primaryNodeId, cause }
|
|
1372
|
+
)
|
|
1373
|
+
};
|
|
1368
1374
|
}
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
// src/distribution/cluster-node/bootstrap-sync/state.ts
|
|
1378
|
+
function createBootstrapSyncState() {
|
|
1379
|
+
return {
|
|
1380
|
+
inFlight: /* @__PURE__ */ new Map(),
|
|
1381
|
+
completed: /* @__PURE__ */ new Set(),
|
|
1382
|
+
generations: /* @__PURE__ */ new Map()
|
|
1383
|
+
};
|
|
1384
|
+
}
|
|
1385
|
+
function entryKey(indexName, partitionId) {
|
|
1386
|
+
return `${indexName}:${partitionId}`;
|
|
1387
|
+
}
|
|
1388
|
+
function clearBootstrapSyncIndex(state, indexName, partitionId) {
|
|
1389
|
+
const key = entryKey(indexName, partitionId);
|
|
1390
|
+
state.completed.delete(key);
|
|
1391
|
+
const inFlight = state.inFlight.get(key);
|
|
1392
|
+
if (inFlight === void 0) {
|
|
1393
|
+
state.generations.delete(key);
|
|
1394
|
+
return;
|
|
1372
1395
|
}
|
|
1373
|
-
const
|
|
1374
|
-
|
|
1396
|
+
const previous = state.generations.get(key) ?? 0;
|
|
1397
|
+
state.generations.set(key, previous + 1);
|
|
1398
|
+
inFlight.aborted = true;
|
|
1399
|
+
inFlight.abortResolve();
|
|
1400
|
+
state.inFlight.delete(key);
|
|
1375
1401
|
}
|
|
1376
|
-
function
|
|
1377
|
-
return
|
|
1402
|
+
function hasCompletedBootstrapSync(state, indexName, partitionId) {
|
|
1403
|
+
return state.completed.has(entryKey(indexName, partitionId));
|
|
1378
1404
|
}
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1405
|
+
function createEntry(state, indexName, partitionId, key) {
|
|
1406
|
+
const startGeneration = state.generations.get(key) ?? 0;
|
|
1407
|
+
let abortResolve = () => {
|
|
1408
|
+
};
|
|
1409
|
+
const abortPromise = new Promise((resolve) => {
|
|
1410
|
+
abortResolve = () => resolve(ABORT_SENTINEL);
|
|
1411
|
+
});
|
|
1382
1412
|
return {
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
failure: null,
|
|
1391
|
-
firstFrameSeen: false
|
|
1413
|
+
indexName,
|
|
1414
|
+
partitionId,
|
|
1415
|
+
generation: startGeneration,
|
|
1416
|
+
promise: Promise.resolve(false),
|
|
1417
|
+
aborted: false,
|
|
1418
|
+
abortResolve,
|
|
1419
|
+
abortPromise
|
|
1392
1420
|
};
|
|
1393
1421
|
}
|
|
1394
|
-
|
|
1422
|
+
|
|
1423
|
+
// src/distribution/replication/snapshot-stream-apply.ts
|
|
1424
|
+
function fail(state, code, message, details = {}) {
|
|
1395
1425
|
if (state.failure !== null) {
|
|
1396
1426
|
return;
|
|
1397
1427
|
}
|
|
1398
|
-
|
|
1399
|
-
|
|
1428
|
+
state.failure = { ok: false, code, message, details };
|
|
1429
|
+
releaseSpool(state);
|
|
1430
|
+
}
|
|
1431
|
+
function releaseSpool(state) {
|
|
1432
|
+
const spool = state.spool;
|
|
1433
|
+
if (spool === null) return;
|
|
1434
|
+
state.spool = null;
|
|
1435
|
+
spool.discard();
|
|
1436
|
+
}
|
|
1437
|
+
function spoolFailure(state, err) {
|
|
1438
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
1439
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `the snapshot could not be written to local storage: ${cause}`, {
|
|
1440
|
+
cause
|
|
1441
|
+
});
|
|
1442
|
+
}
|
|
1443
|
+
function applyErrorEnvelope(state, record) {
|
|
1444
|
+
const code = typeof record.code === "string" && record.code.length > 0 ? record.code : "UNKNOWN_ERROR";
|
|
1445
|
+
const message = typeof record.message === "string" && record.message.length > 0 ? record.message : "primary returned error envelope";
|
|
1446
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_PRIMARY_ERROR, `primary error: ${message}`, {
|
|
1447
|
+
primaryCode: code,
|
|
1448
|
+
primaryMessage: message
|
|
1449
|
+
});
|
|
1450
|
+
}
|
|
1451
|
+
function applyStartPayload(state, record) {
|
|
1452
|
+
if (!isValidStartPayload(record)) {
|
|
1453
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "expected SNAPSHOT_START as first frame");
|
|
1400
1454
|
return;
|
|
1401
1455
|
}
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1456
|
+
const start = record;
|
|
1457
|
+
const header = start.header;
|
|
1458
|
+
if (header.indexName !== state.expectation.indexName) {
|
|
1459
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "snapshot header indexName mismatch", {
|
|
1460
|
+
expected: state.expectation.indexName,
|
|
1461
|
+
received: header.indexName
|
|
1408
1462
|
});
|
|
1409
1463
|
return;
|
|
1410
1464
|
}
|
|
1411
|
-
if (
|
|
1412
|
-
fail(state, ErrorCodes.
|
|
1465
|
+
if (state.expectation.partitionId !== null && header.partitionId !== state.expectation.partitionId) {
|
|
1466
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "snapshot header partitionId mismatch", {
|
|
1467
|
+
expected: state.expectation.partitionId,
|
|
1468
|
+
received: header.partitionId
|
|
1469
|
+
});
|
|
1413
1470
|
return;
|
|
1414
1471
|
}
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
function handleIncomingSnapshotRecord(state, record) {
|
|
1418
|
-
if (state.failure !== null) {
|
|
1472
|
+
if (!isSafeNonNegativeInteger(header.checksum)) {
|
|
1473
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "header.checksum must be a non-negative integer");
|
|
1419
1474
|
return;
|
|
1420
1475
|
}
|
|
1421
|
-
if (
|
|
1422
|
-
|
|
1476
|
+
if (!isSafeNonNegativeInteger(header.partitionId)) {
|
|
1477
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "header.partitionId must be a non-negative integer");
|
|
1423
1478
|
return;
|
|
1424
1479
|
}
|
|
1425
|
-
if (!
|
|
1426
|
-
state.
|
|
1427
|
-
applyStartPayload(state, record);
|
|
1480
|
+
if (!isSafeNonNegativeInteger(header.lastSeqNo)) {
|
|
1481
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "header.lastSeqNo must be a non-negative integer");
|
|
1428
1482
|
return;
|
|
1429
1483
|
}
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
applyChunkPayload(state, record);
|
|
1484
|
+
if (!isSafeNonNegativeInteger(header.primaryTerm)) {
|
|
1485
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "header.primaryTerm must be a non-negative integer");
|
|
1433
1486
|
return;
|
|
1434
1487
|
}
|
|
1435
|
-
if (
|
|
1436
|
-
|
|
1488
|
+
if (!isSafeNonNegativeInteger(start.totalBytes)) {
|
|
1489
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "totalBytes must be a non-negative integer");
|
|
1437
1490
|
return;
|
|
1438
1491
|
}
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
}
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
if (!result.ok) releaseSpool(state);
|
|
1448
|
-
return result;
|
|
1492
|
+
try {
|
|
1493
|
+
state.spool = openSnapshotSpoolWriter(state.expectation.spoolDirectory);
|
|
1494
|
+
} catch (err) {
|
|
1495
|
+
spoolFailure(state, err);
|
|
1496
|
+
return;
|
|
1497
|
+
}
|
|
1498
|
+
state.header = header;
|
|
1499
|
+
state.totalBytes = start.totalBytes;
|
|
1449
1500
|
}
|
|
1450
|
-
function
|
|
1451
|
-
if (state.
|
|
1452
|
-
|
|
1501
|
+
function applyChunkPayload(state, payload) {
|
|
1502
|
+
if (state.totalBytes === null || state.header === null) {
|
|
1503
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received chunk before SNAPSHOT_START");
|
|
1504
|
+
return;
|
|
1453
1505
|
}
|
|
1454
|
-
if (
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1506
|
+
if (!isSafeNonNegativeInteger(payload.offset)) {
|
|
1507
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHUNK_OUT_OF_ORDER, "chunk offset must be a non-negative integer");
|
|
1508
|
+
return;
|
|
1509
|
+
}
|
|
1510
|
+
if (!isSafeNonNegativeInteger(payload.partitionId)) {
|
|
1511
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "chunk partitionId must be a non-negative integer");
|
|
1512
|
+
return;
|
|
1513
|
+
}
|
|
1514
|
+
if (payload.indexName !== state.header.indexName) {
|
|
1515
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "chunk indexName does not match header", {
|
|
1516
|
+
expected: state.header.indexName,
|
|
1517
|
+
received: payload.indexName
|
|
1518
|
+
});
|
|
1519
|
+
return;
|
|
1520
|
+
}
|
|
1521
|
+
if (payload.partitionId !== state.header.partitionId) {
|
|
1522
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "chunk partitionId does not match header", {
|
|
1523
|
+
expected: state.header.partitionId,
|
|
1524
|
+
received: payload.partitionId
|
|
1525
|
+
});
|
|
1526
|
+
return;
|
|
1527
|
+
}
|
|
1528
|
+
if (payload.offset !== state.expectedNextOffset) {
|
|
1529
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHUNK_OUT_OF_ORDER, "chunk arrived out of order", {
|
|
1530
|
+
expected: state.expectedNextOffset,
|
|
1531
|
+
received: payload.offset
|
|
1532
|
+
});
|
|
1533
|
+
return;
|
|
1534
|
+
}
|
|
1535
|
+
if (!(payload.data instanceof Uint8Array)) {
|
|
1536
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "chunk data is not Uint8Array");
|
|
1537
|
+
return;
|
|
1538
|
+
}
|
|
1539
|
+
const remainingBytes = state.totalBytes - state.accumulatedBytes;
|
|
1540
|
+
const perChunkLimit = Math.min(SNAPSHOT_CHUNK_SIZE, remainingBytes);
|
|
1541
|
+
if (payload.data.byteLength > perChunkLimit) {
|
|
1542
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHUNK_SIZE_EXCEEDED, "chunk exceeds the per-chunk size limit", {
|
|
1543
|
+
chunkBytes: payload.data.byteLength,
|
|
1544
|
+
limit: perChunkLimit
|
|
1545
|
+
});
|
|
1546
|
+
return;
|
|
1547
|
+
}
|
|
1548
|
+
const nextBytes = state.accumulatedBytes + payload.data.byteLength;
|
|
1549
|
+
if (nextBytes > state.totalBytes) {
|
|
1550
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHUNK_OVERFLOW, "chunk overflow beyond declared totalBytes", {
|
|
1551
|
+
nextBytes,
|
|
1552
|
+
totalBytes: state.totalBytes
|
|
1553
|
+
});
|
|
1554
|
+
return;
|
|
1555
|
+
}
|
|
1556
|
+
if (state.spool === null) {
|
|
1557
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received chunk after the snapshot stream closed");
|
|
1558
|
+
return;
|
|
1559
|
+
}
|
|
1560
|
+
try {
|
|
1561
|
+
state.spool.write(payload.data);
|
|
1562
|
+
} catch (err) {
|
|
1563
|
+
spoolFailure(state, err);
|
|
1564
|
+
return;
|
|
1565
|
+
}
|
|
1566
|
+
state.accumulatedBytes = nextBytes;
|
|
1567
|
+
state.expectedNextOffset = payload.offset + payload.data.byteLength;
|
|
1568
|
+
}
|
|
1569
|
+
function applyEndPayload(state, payload) {
|
|
1570
|
+
if (state.header === null || state.totalBytes === null) {
|
|
1571
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received SNAPSHOT_END before SNAPSHOT_START");
|
|
1572
|
+
return;
|
|
1573
|
+
}
|
|
1574
|
+
if (payload.indexName !== state.header.indexName) {
|
|
1575
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "end indexName does not match header", {
|
|
1576
|
+
expected: state.header.indexName,
|
|
1577
|
+
received: payload.indexName
|
|
1578
|
+
});
|
|
1579
|
+
return;
|
|
1580
|
+
}
|
|
1581
|
+
if (payload.partitionId !== state.header.partitionId) {
|
|
1582
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "end partitionId does not match header", {
|
|
1583
|
+
expected: state.header.partitionId,
|
|
1584
|
+
received: payload.partitionId
|
|
1585
|
+
});
|
|
1586
|
+
return;
|
|
1587
|
+
}
|
|
1588
|
+
if (!isSafeNonNegativeInteger(payload.totalBytes) || payload.totalBytes !== state.totalBytes) {
|
|
1589
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_MISMATCH, "end totalBytes does not match SNAPSHOT_START", {
|
|
1590
|
+
expected: state.totalBytes,
|
|
1591
|
+
received: payload.totalBytes
|
|
1592
|
+
});
|
|
1593
|
+
return;
|
|
1594
|
+
}
|
|
1595
|
+
if (!isSafeNonNegativeInteger(payload.checksum) || payload.checksum !== state.header.checksum) {
|
|
1596
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_CHECKSUM_MISMATCH, "end checksum does not match header", {
|
|
1597
|
+
expected: state.header.checksum,
|
|
1598
|
+
received: payload.checksum
|
|
1599
|
+
});
|
|
1600
|
+
return;
|
|
1601
|
+
}
|
|
1602
|
+
state.endPayload = payload;
|
|
1603
|
+
}
|
|
1604
|
+
function classifyFrameShape(record) {
|
|
1605
|
+
const hasOffset = typeof record.offset === "number";
|
|
1606
|
+
const hasData = record.data instanceof Uint8Array;
|
|
1607
|
+
const hasChecksum = typeof record.checksum === "number";
|
|
1608
|
+
const hasTotalBytes = typeof record.totalBytes === "number";
|
|
1609
|
+
const hasPartitionId = typeof record.partitionId === "number";
|
|
1610
|
+
const hasIndexName = typeof record.indexName === "string";
|
|
1611
|
+
if (!hasPartitionId || !hasIndexName) {
|
|
1612
|
+
return "unknown";
|
|
1613
|
+
}
|
|
1614
|
+
if (hasOffset && hasData && !hasChecksum && !hasTotalBytes) {
|
|
1615
|
+
return "chunk";
|
|
1616
|
+
}
|
|
1617
|
+
if (hasTotalBytes && hasChecksum && !hasOffset && !hasData) {
|
|
1618
|
+
return "end";
|
|
1619
|
+
}
|
|
1620
|
+
return "unknown";
|
|
1621
|
+
}
|
|
1622
|
+
function isValidStartPayload(record) {
|
|
1623
|
+
if (typeof record.totalBytes !== "number") {
|
|
1624
|
+
return false;
|
|
1625
|
+
}
|
|
1626
|
+
const header = record.header;
|
|
1627
|
+
if (header === null || typeof header !== "object" || Array.isArray(header)) {
|
|
1628
|
+
return false;
|
|
1629
|
+
}
|
|
1630
|
+
const headerRecord = header;
|
|
1631
|
+
return typeof headerRecord.indexName === "string" && typeof headerRecord.checksum === "number" && typeof headerRecord.partitionId === "number" && typeof headerRecord.lastSeqNo === "number" && typeof headerRecord.primaryTerm === "number";
|
|
1632
|
+
}
|
|
1633
|
+
function isSafeNonNegativeInteger(value) {
|
|
1634
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= Number.MAX_SAFE_INTEGER;
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
// src/distribution/replication/snapshot-stream-assembler.ts
|
|
1638
|
+
function createSnapshotStreamState(expectation) {
|
|
1639
|
+
return {
|
|
1640
|
+
expectation,
|
|
1641
|
+
header: null,
|
|
1642
|
+
totalBytes: null,
|
|
1643
|
+
spool: null,
|
|
1644
|
+
accumulatedBytes: 0,
|
|
1645
|
+
expectedNextOffset: 0,
|
|
1646
|
+
endPayload: null,
|
|
1647
|
+
failure: null,
|
|
1648
|
+
firstFrameSeen: false
|
|
1649
|
+
};
|
|
1650
|
+
}
|
|
1651
|
+
function handleIncomingSnapshotRecord(state, record) {
|
|
1652
|
+
if (state.failure !== null) {
|
|
1653
|
+
return;
|
|
1654
|
+
}
|
|
1655
|
+
if (record.error === true) {
|
|
1656
|
+
applyErrorEnvelope(state, record);
|
|
1657
|
+
return;
|
|
1658
|
+
}
|
|
1659
|
+
if (!state.firstFrameSeen) {
|
|
1660
|
+
state.firstFrameSeen = true;
|
|
1661
|
+
applyStartPayload(state, record);
|
|
1662
|
+
return;
|
|
1663
|
+
}
|
|
1664
|
+
const shape = classifyFrameShape(record);
|
|
1665
|
+
if (shape === "chunk") {
|
|
1666
|
+
applyChunkPayload(state, record);
|
|
1667
|
+
return;
|
|
1668
|
+
}
|
|
1669
|
+
if (shape === "end") {
|
|
1670
|
+
applyEndPayload(state, record);
|
|
1671
|
+
return;
|
|
1672
|
+
}
|
|
1673
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "unrecognised snapshot frame shape");
|
|
1674
|
+
}
|
|
1675
|
+
function abandonSnapshotStream(state) {
|
|
1676
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_ABORTED, "the snapshot stream was abandoned before it finished");
|
|
1677
|
+
releaseSpool(state);
|
|
1678
|
+
}
|
|
1679
|
+
function finalizeSnapshotStream(state) {
|
|
1680
|
+
const result = settleSnapshotStream(state);
|
|
1681
|
+
if (!result.ok) releaseSpool(state);
|
|
1682
|
+
return result;
|
|
1683
|
+
}
|
|
1684
|
+
function settleSnapshotStream(state) {
|
|
1685
|
+
if (state.failure !== null) {
|
|
1686
|
+
return state.failure;
|
|
1687
|
+
}
|
|
1688
|
+
if (state.header === null || state.totalBytes === null) {
|
|
1689
|
+
return {
|
|
1690
|
+
ok: false,
|
|
1691
|
+
code: ErrorCodes.SNAPSHOT_SYNC_CHUNK_MISSING,
|
|
1692
|
+
message: "stream ended before SNAPSHOT_START was received",
|
|
1693
|
+
details: { indexName: state.expectation.indexName }
|
|
1460
1694
|
};
|
|
1461
1695
|
}
|
|
1462
1696
|
if (state.endPayload === null) {
|
|
@@ -1509,32 +1743,13 @@ function settleSnapshotStream(state) {
|
|
|
1509
1743
|
end: state.endPayload
|
|
1510
1744
|
};
|
|
1511
1745
|
}
|
|
1512
|
-
function toStreamFailure(err, defaultCode, defaultMessage) {
|
|
1513
|
-
if (err instanceof NarsilError) {
|
|
1514
|
-
return {
|
|
1515
|
-
ok: false,
|
|
1516
|
-
code: err.code,
|
|
1517
|
-
message: err.message,
|
|
1518
|
-
details: { ...err.details }
|
|
1519
|
-
};
|
|
1520
|
-
}
|
|
1521
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
1522
|
-
return {
|
|
1523
|
-
ok: false,
|
|
1524
|
-
code: defaultCode,
|
|
1525
|
-
message: `${defaultMessage}: ${message}`,
|
|
1526
|
-
details: { cause: message }
|
|
1527
|
-
};
|
|
1528
|
-
}
|
|
1529
1746
|
|
|
1530
1747
|
// src/distribution/cluster-node/bootstrap-fetch.ts
|
|
1531
|
-
var
|
|
1748
|
+
var TRANSIENT_CODES = /* @__PURE__ */ new Set([
|
|
1532
1749
|
ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED,
|
|
1533
1750
|
ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
1534
1751
|
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
1535
|
-
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE
|
|
1536
|
-
]);
|
|
1537
|
-
var RETRY_DIFFERENT_TARGET_PROTOCOL_CODES = /* @__PURE__ */ new Set([
|
|
1752
|
+
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
1538
1753
|
ErrorCodes.SNAPSHOT_SYNC_DECODE_FAILED,
|
|
1539
1754
|
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
1540
1755
|
ErrorCodes.SNAPSHOT_SYNC_CHECKSUM_MISMATCH,
|
|
@@ -1548,890 +1763,408 @@ var TRANSIENT_PRIMARY_CODES = /* @__PURE__ */ new Set([
|
|
|
1548
1763
|
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
1549
1764
|
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE
|
|
1550
1765
|
]);
|
|
1551
|
-
function
|
|
1552
|
-
if (
|
|
1553
|
-
return
|
|
1766
|
+
function isTransientFailure(code, details) {
|
|
1767
|
+
if (TRANSIENT_CODES.has(code)) {
|
|
1768
|
+
return true;
|
|
1554
1769
|
}
|
|
1555
|
-
if (
|
|
1556
|
-
return
|
|
1770
|
+
if (code !== ErrorCodes.SNAPSHOT_SYNC_PRIMARY_ERROR) {
|
|
1771
|
+
return false;
|
|
1557
1772
|
}
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1773
|
+
return typeof details.primaryCode === "string" && TRANSIENT_PRIMARY_CODES.has(details.primaryCode);
|
|
1774
|
+
}
|
|
1775
|
+
async function withDeadline(work, ms, indexName, phase) {
|
|
1776
|
+
let timer = null;
|
|
1777
|
+
const deadlinePromise = new Promise((_, reject) => {
|
|
1778
|
+
timer = setTimeout(() => {
|
|
1779
|
+
reject(
|
|
1780
|
+
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, `bootstrap sync exceeded deadline during ${phase}`, {
|
|
1781
|
+
indexName,
|
|
1782
|
+
phase
|
|
1783
|
+
})
|
|
1784
|
+
);
|
|
1785
|
+
}, ms);
|
|
1786
|
+
});
|
|
1787
|
+
try {
|
|
1788
|
+
return await Promise.race([work, deadlinePromise]);
|
|
1789
|
+
} finally {
|
|
1790
|
+
if (timer !== null) {
|
|
1791
|
+
clearTimeout(timer);
|
|
1562
1792
|
}
|
|
1563
1793
|
}
|
|
1564
|
-
return "none";
|
|
1565
1794
|
}
|
|
1566
|
-
|
|
1567
|
-
|
|
1795
|
+
|
|
1796
|
+
// src/distribution/cluster-node/write-routing/partition-queue.ts
|
|
1797
|
+
function createPartitionWriteQueues() {
|
|
1798
|
+
return { chains: /* @__PURE__ */ new Map() };
|
|
1568
1799
|
}
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1800
|
+
function enqueuePartitionWrite(queues, indexName, partitionId, task) {
|
|
1801
|
+
const key = `${indexName}:${partitionId}`;
|
|
1802
|
+
const previous = queues.chains.get(key) ?? Promise.resolve();
|
|
1803
|
+
const holds = [];
|
|
1804
|
+
const hold = (settled) => {
|
|
1805
|
+
holds.push(settled.catch(() => void 0));
|
|
1575
1806
|
};
|
|
1576
|
-
const
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
details: { primaryNodeId, target }
|
|
1589
|
-
};
|
|
1590
|
-
}
|
|
1591
|
-
if (Date.now() >= deadline) {
|
|
1592
|
-
return {
|
|
1593
|
-
ok: false,
|
|
1594
|
-
code: ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
1595
|
-
message: "bootstrap sync exceeded deadline while iterating targets",
|
|
1596
|
-
details: { primaryNodeId, targets }
|
|
1597
|
-
};
|
|
1598
|
-
}
|
|
1599
|
-
const attempt = await fetchSnapshotFromTarget(indexName, partitionId, target, deadline, deps);
|
|
1600
|
-
if (attempt.ok) {
|
|
1601
|
-
return attempt;
|
|
1602
|
-
}
|
|
1603
|
-
lastFailure = { ...attempt, details: { ...attempt.details, target } };
|
|
1604
|
-
const kind = classifyFailure(attempt.code, attempt.details);
|
|
1605
|
-
if (kind === "none") {
|
|
1606
|
-
return lastFailure;
|
|
1807
|
+
const run = previous.then(
|
|
1808
|
+
() => task(hold),
|
|
1809
|
+
() => task(hold)
|
|
1810
|
+
);
|
|
1811
|
+
const chain = run.then(
|
|
1812
|
+
() => void 0,
|
|
1813
|
+
() => void 0
|
|
1814
|
+
).then(() => Promise.all(holds)).then(() => void 0);
|
|
1815
|
+
queues.chains.set(key, chain);
|
|
1816
|
+
chain.then(() => {
|
|
1817
|
+
if (queues.chains.get(key) === chain) {
|
|
1818
|
+
queues.chains.delete(key);
|
|
1607
1819
|
}
|
|
1608
|
-
if (kind === "protocol") {
|
|
1609
|
-
protocolErrorBudget -= 1;
|
|
1610
|
-
if (protocolErrorBudget <= 0) {
|
|
1611
|
-
return lastFailure;
|
|
1612
|
-
}
|
|
1613
|
-
}
|
|
1614
|
-
if (attempt.code === ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED) {
|
|
1615
|
-
const aborted = await jitteredBackoff(
|
|
1616
|
-
CAPACITY_EXHAUSTED_BACKOFF_BASE_MS,
|
|
1617
|
-
CAPACITY_EXHAUSTED_BACKOFF_MAX_MS,
|
|
1618
|
-
deadline,
|
|
1619
|
-
abortCheck
|
|
1620
|
-
);
|
|
1621
|
-
if (aborted) {
|
|
1622
|
-
return {
|
|
1623
|
-
ok: false,
|
|
1624
|
-
code: ErrorCodes.SNAPSHOT_SYNC_ABORTED,
|
|
1625
|
-
message: "bootstrap sync aborted while backing off after capacity exhaustion",
|
|
1626
|
-
details: { primaryNodeId, target }
|
|
1627
|
-
};
|
|
1628
|
-
}
|
|
1629
|
-
if (Date.now() >= deadline) {
|
|
1630
|
-
return {
|
|
1631
|
-
ok: false,
|
|
1632
|
-
code: ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
1633
|
-
message: "bootstrap sync exceeded deadline during capacity backoff",
|
|
1634
|
-
details: { primaryNodeId, targets }
|
|
1635
|
-
};
|
|
1636
|
-
}
|
|
1637
|
-
}
|
|
1638
|
-
}
|
|
1639
|
-
return lastFailure;
|
|
1640
|
-
}
|
|
1641
|
-
async function jitteredBackoff(baseMs, maxMs, deadline, abortCheck) {
|
|
1642
|
-
const plannedMs = Math.floor(baseMs + Math.random() * (maxMs - baseMs));
|
|
1643
|
-
const remainingMs = deadline - Date.now();
|
|
1644
|
-
if (remainingMs <= 0) {
|
|
1645
|
-
return false;
|
|
1646
|
-
}
|
|
1647
|
-
const sleepMs = Math.min(plannedMs, remainingMs);
|
|
1648
|
-
if (sleepMs <= 0) {
|
|
1649
|
-
return false;
|
|
1650
|
-
}
|
|
1651
|
-
const sleepUntil = Date.now() + sleepMs;
|
|
1652
|
-
if (abortCheck()) {
|
|
1653
|
-
return true;
|
|
1654
|
-
}
|
|
1655
|
-
while (Date.now() < sleepUntil) {
|
|
1656
|
-
if (abortCheck()) {
|
|
1657
|
-
return true;
|
|
1658
|
-
}
|
|
1659
|
-
if (Date.now() >= deadline) {
|
|
1660
|
-
return false;
|
|
1661
|
-
}
|
|
1662
|
-
const now3 = Date.now();
|
|
1663
|
-
const remaining = Math.min(sleepUntil - now3, deadline - now3);
|
|
1664
|
-
if (remaining <= 0) {
|
|
1665
|
-
return false;
|
|
1666
|
-
}
|
|
1667
|
-
if (remaining <= 2) {
|
|
1668
|
-
return false;
|
|
1669
|
-
}
|
|
1670
|
-
const tickMs = Math.min(remaining, JITTERED_BACKOFF_POLL_INTERVAL_MS);
|
|
1671
|
-
await sleepForMs(tickMs);
|
|
1672
|
-
}
|
|
1673
|
-
return abortCheck();
|
|
1674
|
-
}
|
|
1675
|
-
function sleepForMs(ms) {
|
|
1676
|
-
return new Promise((resolve) => {
|
|
1677
|
-
const timer = setTimeout(() => {
|
|
1678
|
-
clearTimeout(timer);
|
|
1679
|
-
resolve();
|
|
1680
|
-
}, ms);
|
|
1681
1820
|
});
|
|
1821
|
+
return run;
|
|
1682
1822
|
}
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
payload: encode(requestPayload)
|
|
1690
|
-
};
|
|
1691
|
-
const streamState = createSnapshotStreamState({
|
|
1692
|
-
indexName,
|
|
1693
|
-
partitionId: null,
|
|
1694
|
-
spoolDirectory: deps.spoolDirectory ?? defaultSpoolDirectory()
|
|
1695
|
-
});
|
|
1696
|
-
const remainingMs = deadline - Date.now();
|
|
1697
|
-
if (remainingMs <= 0) {
|
|
1698
|
-
return {
|
|
1699
|
-
ok: false,
|
|
1700
|
-
code: ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
1701
|
-
message: "deadline expired before streaming snapshot",
|
|
1702
|
-
details: { target }
|
|
1703
|
-
};
|
|
1823
|
+
|
|
1824
|
+
// src/distribution/cluster-node/bootstrap-sync/entries.ts
|
|
1825
|
+
async function applySyncEntries(indexName, partitionId, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, deps) {
|
|
1826
|
+
const terminalError = validateSyncEntryBatchTermination(entriesPayloads);
|
|
1827
|
+
if (terminalError !== null) {
|
|
1828
|
+
return { ok: false, error: terminalError };
|
|
1704
1829
|
}
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
return toStreamFailure(err, ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "snapshot transport failed");
|
|
1830
|
+
const log = deps.getReplicationLog(indexName, partitionId);
|
|
1831
|
+
let expectedSeqNo = expectedFirstSeqNo;
|
|
1832
|
+
let entriesApplied = 0;
|
|
1833
|
+
for (const payload of entriesPayloads) {
|
|
1834
|
+
const run = [];
|
|
1835
|
+
let entryError = null;
|
|
1836
|
+
for (const entry of payload.entries) {
|
|
1837
|
+
entryError = validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log);
|
|
1838
|
+
if (entryError !== null) break;
|
|
1839
|
+
run.push(entry);
|
|
1840
|
+
expectedSeqNo = entry.seqNo + 1;
|
|
1717
1841
|
}
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
indexName,
|
|
1729
|
-
phase
|
|
1730
|
-
})
|
|
1731
|
-
);
|
|
1732
|
-
}, ms);
|
|
1733
|
-
});
|
|
1734
|
-
try {
|
|
1735
|
-
return await Promise.race([work, deadlinePromise]);
|
|
1736
|
-
} finally {
|
|
1737
|
-
if (timer !== null) {
|
|
1738
|
-
clearTimeout(timer);
|
|
1842
|
+
const outcome = await enqueuePartitionWrite(
|
|
1843
|
+
deps.partitionWriteQueues,
|
|
1844
|
+
indexName,
|
|
1845
|
+
partitionId,
|
|
1846
|
+
() => applySyncRunOnce(run, log, deps)
|
|
1847
|
+
);
|
|
1848
|
+
entriesApplied += outcome.processed;
|
|
1849
|
+
const error = outcome.error ?? entryError;
|
|
1850
|
+
if (error !== null) {
|
|
1851
|
+
return { ok: false, error };
|
|
1739
1852
|
}
|
|
1740
1853
|
}
|
|
1854
|
+
return {
|
|
1855
|
+
ok: true,
|
|
1856
|
+
entriesApplied,
|
|
1857
|
+
newSeqNo: expectedSeqNo - 1
|
|
1858
|
+
};
|
|
1741
1859
|
}
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
const
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
const expectedKeys = Object.keys(expected).sort(compareCodePoints);
|
|
1761
|
-
const actualKeys = new Set(Object.keys(actual));
|
|
1762
|
-
for (const key of expectedKeys) {
|
|
1763
|
-
const path = prefix === "" ? key : `${prefix}.${key}`;
|
|
1764
|
-
const expectedValue = expected[key];
|
|
1765
|
-
const actualValue = actual[key];
|
|
1766
|
-
if (!actualKeys.has(key)) {
|
|
1767
|
-
diffs.push({ path, expected: describe(expectedValue), actual: "(absent)" });
|
|
1768
|
-
continue;
|
|
1769
|
-
}
|
|
1770
|
-
actualKeys.delete(key);
|
|
1771
|
-
const expectedIsObject = typeof expectedValue === "object" && expectedValue !== null;
|
|
1772
|
-
const actualIsObject = typeof actualValue === "object" && actualValue !== null;
|
|
1773
|
-
if (expectedIsObject !== actualIsObject) {
|
|
1774
|
-
diffs.push({ path, expected: describe(expectedValue), actual: describe(actualValue) });
|
|
1775
|
-
continue;
|
|
1776
|
-
}
|
|
1777
|
-
if (expectedIsObject && actualIsObject) {
|
|
1778
|
-
walk(expectedValue, actualValue, path, diffs, depth + 1);
|
|
1860
|
+
async function applySyncRunOnce(run, log, deps) {
|
|
1861
|
+
let processed = 0;
|
|
1862
|
+
let error = null;
|
|
1863
|
+
const pending = [];
|
|
1864
|
+
let tailSeqNo = log.localLogEnd;
|
|
1865
|
+
for (const entry of run) {
|
|
1866
|
+
const where = { indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo };
|
|
1867
|
+
const present = log.getEntry(entry.seqNo);
|
|
1868
|
+
if (present !== void 0) {
|
|
1869
|
+
if (present.checksum !== entry.checksum) {
|
|
1870
|
+
error = new NarsilError(
|
|
1871
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
1872
|
+
`Conflicting sync entry at sequence number ${entry.seqNo}`,
|
|
1873
|
+
where
|
|
1874
|
+
);
|
|
1875
|
+
break;
|
|
1876
|
+
}
|
|
1877
|
+
processed += 1;
|
|
1779
1878
|
continue;
|
|
1780
1879
|
}
|
|
1781
|
-
if (
|
|
1782
|
-
|
|
1880
|
+
if (entry.seqNo !== tailSeqNo + 1) {
|
|
1881
|
+
error = new NarsilError(
|
|
1882
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
1883
|
+
`Out-of-order sync entry ${entry.seqNo}; the local log ends at ${tailSeqNo}`,
|
|
1884
|
+
where
|
|
1885
|
+
);
|
|
1886
|
+
break;
|
|
1783
1887
|
}
|
|
1888
|
+
pending.push(entry);
|
|
1889
|
+
tailSeqNo = entry.seqNo;
|
|
1784
1890
|
}
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1891
|
+
if (pending.length === 0) return { processed, error };
|
|
1892
|
+
let applied;
|
|
1893
|
+
let applyError = null;
|
|
1894
|
+
try {
|
|
1895
|
+
const outcome = await deps.applyReplicationEntries(pending);
|
|
1896
|
+
applied = outcome.applied;
|
|
1897
|
+
applyError = outcome.error;
|
|
1898
|
+
} catch (err) {
|
|
1899
|
+
applied = 0;
|
|
1900
|
+
applyError = err;
|
|
1793
1901
|
}
|
|
1794
|
-
|
|
1795
|
-
|
|
1902
|
+
for (let position = 0; position < applied; position++) log.appendCommitted(pending[position]);
|
|
1903
|
+
processed += applied;
|
|
1904
|
+
if (applied < pending.length) {
|
|
1905
|
+
const failed2 = pending[applied];
|
|
1906
|
+
const cause = applyError instanceof Error ? applyError.message : String(applyError);
|
|
1907
|
+
return {
|
|
1908
|
+
processed,
|
|
1909
|
+
error: new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, `failed to apply sync entry: ${cause}`, {
|
|
1910
|
+
indexName: failed2.indexName,
|
|
1911
|
+
partitionId: failed2.partitionId,
|
|
1912
|
+
seqNo: failed2.seqNo,
|
|
1913
|
+
cause
|
|
1914
|
+
})
|
|
1915
|
+
};
|
|
1796
1916
|
}
|
|
1797
|
-
return
|
|
1917
|
+
return { processed, error };
|
|
1798
1918
|
}
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
function validateArguments(indexName, partitionId, primaryNodeId) {
|
|
1803
|
-
if (typeof indexName !== "string" || indexName.length === 0) {
|
|
1804
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, "indexName must be a non-empty string", {
|
|
1805
|
-
indexName
|
|
1806
|
-
});
|
|
1919
|
+
function validateSyncEntryBatchTermination(entriesPayloads) {
|
|
1920
|
+
if (entriesPayloads.length === 0) {
|
|
1921
|
+
return null;
|
|
1807
1922
|
}
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
});
|
|
1923
|
+
for (let i = 0; i < entriesPayloads.length - 1; i += 1) {
|
|
1924
|
+
if (entriesPayloads[i].isLast) {
|
|
1925
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "sync_entries marked isLast before final batch");
|
|
1926
|
+
}
|
|
1813
1927
|
}
|
|
1814
|
-
if (
|
|
1815
|
-
return new NarsilError(
|
|
1816
|
-
|
|
1817
|
-
|
|
1928
|
+
if (!entriesPayloads[entriesPayloads.length - 1].isLast) {
|
|
1929
|
+
return new NarsilError(
|
|
1930
|
+
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
1931
|
+
"live sync stream ended before final sync_entries batch"
|
|
1932
|
+
);
|
|
1818
1933
|
}
|
|
1819
1934
|
return null;
|
|
1820
1935
|
}
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
try {
|
|
1824
|
-
targets = await resolveNodeTargets3(primaryNodeId);
|
|
1825
|
-
} catch (err) {
|
|
1826
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1827
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_NO_TARGETS, `resolveNodeTargets failed: ${cause}`, {
|
|
1828
|
-
indexName,
|
|
1829
|
-
primaryNodeId,
|
|
1830
|
-
cause
|
|
1831
|
-
});
|
|
1832
|
-
}
|
|
1833
|
-
if (targets.length === 0) {
|
|
1936
|
+
function validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log) {
|
|
1937
|
+
if (entry.indexName !== indexName || entry.partitionId !== partitionId) {
|
|
1834
1938
|
return new NarsilError(
|
|
1835
|
-
ErrorCodes.
|
|
1836
|
-
|
|
1837
|
-
{
|
|
1939
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
1940
|
+
"replication entry scope does not match bootstrap target",
|
|
1941
|
+
{
|
|
1942
|
+
expectedIndexName: indexName,
|
|
1943
|
+
expectedPartitionId: partitionId,
|
|
1944
|
+
entryIndexName: entry.indexName,
|
|
1945
|
+
entryPartitionId: entry.partitionId
|
|
1946
|
+
}
|
|
1838
1947
|
);
|
|
1839
1948
|
}
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1949
|
+
if (entry.seqNo !== expectedSeqNo) {
|
|
1950
|
+
return new NarsilError(
|
|
1951
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
1952
|
+
`Out-of-order sync entry ${entry.seqNo}; expected ${expectedSeqNo}`,
|
|
1953
|
+
{ indexName, partitionId, seqNo: entry.seqNo, expectedSeqNo }
|
|
1954
|
+
);
|
|
1845
1955
|
}
|
|
1846
|
-
const
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
);
|
|
1856
|
-
deps.onError(wrapped);
|
|
1956
|
+
const validation = validateReplicationEntry(entry, localPrimaryTerm, log);
|
|
1957
|
+
if (!validation.valid) {
|
|
1958
|
+
return new NarsilError(
|
|
1959
|
+
validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
1960
|
+
`Invalid sync entry ${entry.seqNo}`,
|
|
1961
|
+
{ indexName, partitionId, seqNo: entry.seqNo }
|
|
1962
|
+
);
|
|
1963
|
+
}
|
|
1964
|
+
return null;
|
|
1857
1965
|
}
|
|
1858
|
-
function
|
|
1859
|
-
|
|
1966
|
+
function createLiveSyncFrameState(indexName, partitionId, spoolDirectory) {
|
|
1967
|
+
return {
|
|
1968
|
+
snapshotState: createSnapshotStreamState({ indexName, partitionId, spoolDirectory }),
|
|
1969
|
+
syncEntries: [],
|
|
1970
|
+
sawSnapshotFrame: false,
|
|
1971
|
+
error: null
|
|
1972
|
+
};
|
|
1973
|
+
}
|
|
1974
|
+
function handleLiveSyncFrame(state, frame) {
|
|
1975
|
+
if (state.error !== null || state.snapshotState.failure !== null) {
|
|
1860
1976
|
return;
|
|
1861
1977
|
}
|
|
1862
|
-
|
|
1863
|
-
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, `bootstrap sync aborted for index '${indexName}'`, {
|
|
1864
|
-
indexName,
|
|
1865
|
-
primaryNodeId
|
|
1866
|
-
})
|
|
1867
|
-
);
|
|
1868
|
-
}
|
|
1869
|
-
async function executeEngineRestore(engine, indexName, primaryNodeId, source, remainingMs) {
|
|
1978
|
+
let decoded;
|
|
1870
1979
|
try {
|
|
1871
|
-
|
|
1872
|
-
return null;
|
|
1980
|
+
decoded = decode(frame);
|
|
1873
1981
|
} catch (err) {
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
}
|
|
1877
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1878
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `engine.restore failed: ${cause}`, {
|
|
1879
|
-
indexName,
|
|
1880
|
-
primaryNodeId,
|
|
1881
|
-
cause
|
|
1982
|
+
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_DECODE_FAILED, "failed to decode live sync frame", {
|
|
1983
|
+
cause: err instanceof Error ? err.message : String(err)
|
|
1882
1984
|
});
|
|
1985
|
+
return;
|
|
1883
1986
|
}
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
try {
|
|
1888
|
-
restoredSchema = engine.getStats(indexName).schema;
|
|
1889
|
-
} catch (err) {
|
|
1890
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1891
|
-
return new NarsilError(
|
|
1892
|
-
ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
|
|
1893
|
-
`failed to read restored schema for '${indexName}': ${cause}`,
|
|
1894
|
-
{ indexName, primaryNodeId, reason: "schema lookup failed", cause }
|
|
1895
|
-
);
|
|
1987
|
+
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
1988
|
+
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "live sync frame is not an object");
|
|
1989
|
+
return;
|
|
1896
1990
|
}
|
|
1897
|
-
const
|
|
1898
|
-
|
|
1899
|
-
|
|
1991
|
+
const record = decoded;
|
|
1992
|
+
const entriesPayload = decodeSyncEntriesRecord(record);
|
|
1993
|
+
if (entriesPayload instanceof NarsilError) {
|
|
1994
|
+
state.error = entriesPayload;
|
|
1995
|
+
return;
|
|
1900
1996
|
}
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
{
|
|
1905
|
-
indexName,
|
|
1906
|
-
primaryNodeId,
|
|
1907
|
-
reason: "schema mismatch",
|
|
1908
|
-
differences
|
|
1909
|
-
}
|
|
1910
|
-
);
|
|
1911
|
-
}
|
|
1912
|
-
async function dropRestoredIndexQuietly(engine, indexName, deps = {}) {
|
|
1913
|
-
try {
|
|
1914
|
-
const existing = engine.listIndexes().find((idx) => idx.name === indexName);
|
|
1915
|
-
if (existing === void 0) {
|
|
1916
|
-
return;
|
|
1917
|
-
}
|
|
1918
|
-
await engine.dropIndex(indexName);
|
|
1919
|
-
} catch (err) {
|
|
1920
|
-
if (localIndexIsGone(err)) {
|
|
1921
|
-
return;
|
|
1922
|
-
}
|
|
1923
|
-
if (deps.onError === void 0) {
|
|
1997
|
+
if (entriesPayload !== null) {
|
|
1998
|
+
if (state.sawSnapshotFrame && state.snapshotState.endPayload === null) {
|
|
1999
|
+
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received sync_entries before snapshot_end");
|
|
1924
2000
|
return;
|
|
1925
2001
|
}
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
new NarsilError(
|
|
1929
|
-
ErrorCodes.SNAPSHOT_SYNC_RESTORE_CLEANUP_FAILED,
|
|
1930
|
-
`failed to drop restored index '${indexName}' after bootstrap cleanup: ${cause}`,
|
|
1931
|
-
{ indexName, cause }
|
|
1932
|
-
)
|
|
1933
|
-
);
|
|
1934
|
-
}
|
|
1935
|
-
}
|
|
1936
|
-
async function loadCoordinatorIndexConfig(coordinator, indexName, primaryNodeId, abortPromise) {
|
|
1937
|
-
let schema;
|
|
1938
|
-
try {
|
|
1939
|
-
const winner = await Promise.race([coordinator.getSchema(indexName), abortPromise]);
|
|
1940
|
-
if (winner === ABORT_SENTINEL) {
|
|
1941
|
-
return "aborted";
|
|
1942
|
-
}
|
|
1943
|
-
schema = winner;
|
|
1944
|
-
} catch (err) {
|
|
1945
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1946
|
-
return {
|
|
1947
|
-
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE, `coordinator.getSchema failed: ${cause}`, {
|
|
1948
|
-
indexName,
|
|
1949
|
-
primaryNodeId,
|
|
1950
|
-
cause
|
|
1951
|
-
})
|
|
1952
|
-
};
|
|
1953
|
-
}
|
|
1954
|
-
if (schema === null) {
|
|
1955
|
-
return {
|
|
1956
|
-
error: new NarsilError(
|
|
1957
|
-
ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE,
|
|
1958
|
-
`coordinator has no schema for index '${indexName}'`,
|
|
1959
|
-
{ indexName, primaryNodeId }
|
|
1960
|
-
)
|
|
1961
|
-
};
|
|
2002
|
+
state.syncEntries.push(entriesPayload);
|
|
2003
|
+
return;
|
|
1962
2004
|
}
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
return
|
|
1969
|
-
error: new NarsilError(err.code, `coordinator schema for index '${indexName}' is invalid: ${err.message}`, {
|
|
1970
|
-
...err.details,
|
|
1971
|
-
indexName,
|
|
1972
|
-
primaryNodeId
|
|
1973
|
-
})
|
|
1974
|
-
};
|
|
2005
|
+
if (state.syncEntries.length > 0 && !state.sawSnapshotFrame) {
|
|
2006
|
+
state.error = new NarsilError(
|
|
2007
|
+
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
2008
|
+
"received snapshot frame after incremental sync_entries"
|
|
2009
|
+
);
|
|
2010
|
+
return;
|
|
1975
2011
|
}
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
}
|
|
1981
|
-
return { config: winner };
|
|
1982
|
-
} catch (err) {
|
|
1983
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
1984
|
-
return {
|
|
1985
|
-
error: new NarsilError(
|
|
1986
|
-
err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SCHEMA_UNAVAILABLE,
|
|
1987
|
-
`failed to read the coordinator index metadata for '${indexName}': ${cause}`,
|
|
1988
|
-
{ indexName, primaryNodeId, cause }
|
|
1989
|
-
)
|
|
1990
|
-
};
|
|
2012
|
+
state.sawSnapshotFrame = true;
|
|
2013
|
+
handleIncomingSnapshotRecord(state.snapshotState, record);
|
|
2014
|
+
if (state.snapshotState.failure !== null) {
|
|
2015
|
+
state.error = streamFailureToError(state.snapshotState.failure);
|
|
1991
2016
|
}
|
|
1992
2017
|
}
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
const existing = engine.listIndexes().find((idx) => idx.name === indexName);
|
|
1996
|
-
if (existing === void 0) {
|
|
1997
|
-
return null;
|
|
1998
|
-
}
|
|
1999
|
-
await engine.dropIndex(indexName);
|
|
2018
|
+
function decodeSyncEntriesRecord(record) {
|
|
2019
|
+
if (!("entries" in record) && !("isLast" in record)) {
|
|
2000
2020
|
return null;
|
|
2001
|
-
}
|
|
2002
|
-
|
|
2003
|
-
return null;
|
|
2004
|
-
}
|
|
2005
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
2021
|
+
}
|
|
2022
|
+
if (!Array.isArray(record.entries) || typeof record.isLast !== "boolean") {
|
|
2006
2023
|
return new NarsilError(
|
|
2007
|
-
ErrorCodes.
|
|
2008
|
-
|
|
2009
|
-
{ indexName, primaryNodeId, cause }
|
|
2024
|
+
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
2025
|
+
"Invalid SyncEntriesPayload: expected entries array and isLast boolean"
|
|
2010
2026
|
);
|
|
2011
2027
|
}
|
|
2028
|
+
const entries = [];
|
|
2029
|
+
for (const candidate of record.entries) {
|
|
2030
|
+
try {
|
|
2031
|
+
entries.push(validateEntryPayload({ entry: candidate }).entry);
|
|
2032
|
+
} catch (err) {
|
|
2033
|
+
return new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, err instanceof Error ? err.message : String(err));
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
return { entries, isLast: record.isLast };
|
|
2037
|
+
}
|
|
2038
|
+
function streamFailureToError(failure) {
|
|
2039
|
+
return new NarsilError(failure.code, failure.message, failure.details);
|
|
2012
2040
|
}
|
|
2013
2041
|
|
|
2014
|
-
// src/distribution/cluster-node/bootstrap-sync/
|
|
2015
|
-
function
|
|
2042
|
+
// src/distribution/cluster-node/bootstrap-sync/apply.ts
|
|
2043
|
+
async function applyLiveIncrementalSync(indexName, partitionId, target, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, coordinatorConfig, partitionCount, deps) {
|
|
2044
|
+
const ensureResult = await ensureLocalIndexForIncremental(indexName, target, coordinatorConfig, partitionCount, deps);
|
|
2045
|
+
if (ensureResult instanceof NarsilError) {
|
|
2046
|
+
return { ok: false, error: ensureResult };
|
|
2047
|
+
}
|
|
2048
|
+
const applyResult = await applySyncEntries(
|
|
2049
|
+
indexName,
|
|
2050
|
+
partitionId,
|
|
2051
|
+
entriesPayloads,
|
|
2052
|
+
expectedFirstSeqNo,
|
|
2053
|
+
localPrimaryTerm,
|
|
2054
|
+
deps
|
|
2055
|
+
);
|
|
2056
|
+
if (!applyResult.ok) {
|
|
2057
|
+
return applyResult;
|
|
2058
|
+
}
|
|
2016
2059
|
return {
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2060
|
+
ok: true,
|
|
2061
|
+
tier: "incremental",
|
|
2062
|
+
entriesApplied: applyResult.entriesApplied,
|
|
2063
|
+
newSeqNo: applyResult.newSeqNo,
|
|
2064
|
+
snapshotHeader: null
|
|
2020
2065
|
};
|
|
2021
2066
|
}
|
|
2022
|
-
function
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
return deps.getReplicationLog !== void 0 && deps.resetReplicationLog !== void 0 && deps.applyReplicationEntries !== void 0 && deps.restoreReplicationPartition !== void 0;
|
|
2027
|
-
}
|
|
2028
|
-
function clearBootstrapSyncIndex(state, indexName, partitionId) {
|
|
2029
|
-
const key = entryKey(indexName, partitionId);
|
|
2030
|
-
state.completed.delete(key);
|
|
2031
|
-
const inFlight = state.inFlight.get(key);
|
|
2032
|
-
if (inFlight === void 0) {
|
|
2033
|
-
state.generations.delete(key);
|
|
2034
|
-
return;
|
|
2067
|
+
async function applyLiveSnapshotSync(state, entry, indexName, partitionId, target, frameState, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2068
|
+
const finalized = finalizeSnapshotStream(frameState.snapshotState);
|
|
2069
|
+
if (!finalized.ok) {
|
|
2070
|
+
return { ok: false, error: streamFailureToError({ ...finalized, details: { ...finalized.details, target } }) };
|
|
2035
2071
|
}
|
|
2036
|
-
const
|
|
2037
|
-
|
|
2038
|
-
inFlight.aborted = true;
|
|
2039
|
-
inFlight.abortResolve();
|
|
2040
|
-
state.inFlight.delete(key);
|
|
2041
|
-
}
|
|
2042
|
-
function hasCompletedBootstrapSync(state, indexName, partitionId) {
|
|
2043
|
-
return state.completed.has(entryKey(indexName, partitionId));
|
|
2044
|
-
}
|
|
2045
|
-
function createEntry(state, indexName, partitionId, key) {
|
|
2046
|
-
const startGeneration = state.generations.get(key) ?? 0;
|
|
2047
|
-
let abortResolve = () => {
|
|
2048
|
-
};
|
|
2049
|
-
const abortPromise = new Promise((resolve) => {
|
|
2050
|
-
abortResolve = () => resolve(ABORT_SENTINEL);
|
|
2051
|
-
});
|
|
2052
|
-
return {
|
|
2072
|
+
const restoreError = await enqueuePartitionWrite(
|
|
2073
|
+
deps.partitionWriteQueues,
|
|
2053
2074
|
indexName,
|
|
2054
2075
|
partitionId,
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
abortResolve,
|
|
2059
|
-
abortPromise
|
|
2060
|
-
};
|
|
2061
|
-
}
|
|
2062
|
-
function anotherBootstrapOwnsKey(state, key, self) {
|
|
2063
|
-
const current = state.inFlight.get(key);
|
|
2064
|
-
if (current === void 0) {
|
|
2065
|
-
return false;
|
|
2066
|
-
}
|
|
2067
|
-
return current !== self;
|
|
2068
|
-
}
|
|
2069
|
-
|
|
2070
|
-
// src/distribution/cluster-node/write-routing/partition-queue.ts
|
|
2071
|
-
function createPartitionWriteQueues() {
|
|
2072
|
-
return { chains: /* @__PURE__ */ new Map() };
|
|
2073
|
-
}
|
|
2074
|
-
function enqueuePartitionWrite(queues, indexName, partitionId, task) {
|
|
2075
|
-
const key = `${indexName}:${partitionId}`;
|
|
2076
|
-
const previous = queues.chains.get(key) ?? Promise.resolve();
|
|
2077
|
-
const run = previous.then(task, task);
|
|
2078
|
-
const chain = run.then(
|
|
2079
|
-
() => void 0,
|
|
2080
|
-
() => void 0
|
|
2081
|
-
);
|
|
2082
|
-
queues.chains.set(key, chain);
|
|
2083
|
-
chain.then(() => {
|
|
2084
|
-
if (queues.chains.get(key) === chain) {
|
|
2085
|
-
queues.chains.delete(key);
|
|
2086
|
-
}
|
|
2087
|
-
});
|
|
2088
|
-
return run;
|
|
2089
|
-
}
|
|
2090
|
-
|
|
2091
|
-
// src/distribution/cluster-node/bootstrap-sync/entries.ts
|
|
2092
|
-
async function applySyncEntries(indexName, partitionId, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, deps) {
|
|
2093
|
-
const terminalError = validateSyncEntryBatchTermination(entriesPayloads);
|
|
2094
|
-
if (terminalError !== null) {
|
|
2095
|
-
return { ok: false, error: terminalError };
|
|
2096
|
-
}
|
|
2097
|
-
const log = deps.getReplicationLog(indexName, partitionId);
|
|
2098
|
-
let expectedSeqNo = expectedFirstSeqNo;
|
|
2099
|
-
let entriesApplied = 0;
|
|
2100
|
-
for (const payload of entriesPayloads) {
|
|
2101
|
-
const run = [];
|
|
2102
|
-
let entryError = null;
|
|
2103
|
-
for (const entry of payload.entries) {
|
|
2104
|
-
entryError = validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log);
|
|
2105
|
-
if (entryError !== null) break;
|
|
2106
|
-
run.push(entry);
|
|
2107
|
-
expectedSeqNo = entry.seqNo + 1;
|
|
2108
|
-
}
|
|
2109
|
-
const outcome = await enqueuePartitionWrite(
|
|
2110
|
-
deps.partitionWriteQueues,
|
|
2076
|
+
() => restartFromSnapshot(
|
|
2077
|
+
state,
|
|
2078
|
+
entry,
|
|
2111
2079
|
indexName,
|
|
2112
2080
|
partitionId,
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2081
|
+
target,
|
|
2082
|
+
finalized,
|
|
2083
|
+
coordinatorConfig,
|
|
2084
|
+
partitionCount,
|
|
2085
|
+
deadline,
|
|
2086
|
+
deps
|
|
2087
|
+
)
|
|
2088
|
+
);
|
|
2089
|
+
if (restoreError !== null) {
|
|
2090
|
+
return { ok: false, error: restoreError };
|
|
2091
|
+
}
|
|
2092
|
+
const applyResult = await applySyncEntries(
|
|
2093
|
+
indexName,
|
|
2094
|
+
partitionId,
|
|
2095
|
+
frameState.syncEntries,
|
|
2096
|
+
finalized.header.lastSeqNo + 1,
|
|
2097
|
+
finalized.header.primaryTerm,
|
|
2098
|
+
deps
|
|
2099
|
+
);
|
|
2100
|
+
if (!applyResult.ok) {
|
|
2101
|
+
return applyResult;
|
|
2120
2102
|
}
|
|
2121
2103
|
return {
|
|
2122
2104
|
ok: true,
|
|
2123
|
-
|
|
2124
|
-
|
|
2105
|
+
tier: "snapshot",
|
|
2106
|
+
entriesApplied: applyResult.entriesApplied,
|
|
2107
|
+
newSeqNo: applyResult.newSeqNo,
|
|
2108
|
+
snapshotHeader: finalized.header
|
|
2125
2109
|
};
|
|
2126
2110
|
}
|
|
2127
|
-
async function
|
|
2128
|
-
let
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
processed += 1;
|
|
2145
|
-
continue;
|
|
2146
|
-
}
|
|
2147
|
-
if (entry.seqNo !== tailSeqNo + 1) {
|
|
2148
|
-
error = new NarsilError(
|
|
2149
|
-
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
2150
|
-
`Out-of-order sync entry ${entry.seqNo}; the local log ends at ${tailSeqNo}`,
|
|
2151
|
-
where
|
|
2152
|
-
);
|
|
2153
|
-
break;
|
|
2154
|
-
}
|
|
2155
|
-
pending.push(entry);
|
|
2156
|
-
tailSeqNo = entry.seqNo;
|
|
2111
|
+
async function restartFromSnapshot(state, entry, indexName, partitionId, target, finalized, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2112
|
+
let restoreError;
|
|
2113
|
+
try {
|
|
2114
|
+
restoreError = await restoreLiveSnapshotPartition(
|
|
2115
|
+
state,
|
|
2116
|
+
entry,
|
|
2117
|
+
indexName,
|
|
2118
|
+
partitionId,
|
|
2119
|
+
target,
|
|
2120
|
+
finalized.snapshot.read(),
|
|
2121
|
+
coordinatorConfig,
|
|
2122
|
+
partitionCount,
|
|
2123
|
+
deadline,
|
|
2124
|
+
deps
|
|
2125
|
+
);
|
|
2126
|
+
} finally {
|
|
2127
|
+
await finalized.snapshot.discard();
|
|
2157
2128
|
}
|
|
2158
|
-
if (
|
|
2159
|
-
let applied;
|
|
2160
|
-
let applyError = null;
|
|
2129
|
+
if (restoreError !== null) return restoreError;
|
|
2161
2130
|
try {
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2131
|
+
await deps.restartLocalLog?.(indexName, partitionId, {
|
|
2132
|
+
seqNo: finalized.header.lastSeqNo,
|
|
2133
|
+
primaryTerm: finalized.header.primaryTerm
|
|
2134
|
+
});
|
|
2165
2135
|
} catch (err) {
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
const cause = applyError instanceof Error ? applyError.message : String(applyError);
|
|
2174
|
-
return {
|
|
2175
|
-
processed,
|
|
2176
|
-
error: new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, `failed to apply sync entry: ${cause}`, {
|
|
2177
|
-
indexName: failed2.indexName,
|
|
2178
|
-
partitionId: failed2.partitionId,
|
|
2179
|
-
seqNo: failed2.seqNo,
|
|
2180
|
-
cause
|
|
2181
|
-
})
|
|
2182
|
-
};
|
|
2183
|
-
}
|
|
2184
|
-
return { processed, error };
|
|
2185
|
-
}
|
|
2186
|
-
function validateSyncEntryBatchTermination(entriesPayloads) {
|
|
2187
|
-
if (entriesPayloads.length === 0) {
|
|
2188
|
-
return null;
|
|
2189
|
-
}
|
|
2190
|
-
for (let i = 0; i < entriesPayloads.length - 1; i += 1) {
|
|
2191
|
-
if (entriesPayloads[i].isLast) {
|
|
2192
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "sync_entries marked isLast before final batch");
|
|
2193
|
-
}
|
|
2194
|
-
}
|
|
2195
|
-
if (!entriesPayloads[entriesPayloads.length - 1].isLast) {
|
|
2196
|
-
return new NarsilError(
|
|
2197
|
-
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
2198
|
-
"live sync stream ended before final sync_entries batch"
|
|
2199
|
-
);
|
|
2136
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
2137
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `local log restart failed: ${cause}`, {
|
|
2138
|
+
indexName,
|
|
2139
|
+
partitionId,
|
|
2140
|
+
primaryNodeId: target,
|
|
2141
|
+
cause
|
|
2142
|
+
});
|
|
2200
2143
|
}
|
|
2144
|
+
deps.resetReplicationLog(indexName, partitionId, finalized.header.lastSeqNo + 1, finalized.header.primaryTerm);
|
|
2201
2145
|
return null;
|
|
2202
2146
|
}
|
|
2203
|
-
function
|
|
2204
|
-
|
|
2147
|
+
async function restoreLiveSnapshotPartition(state, entry, indexName, partitionId, primaryNodeId, source, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2148
|
+
const remainingMs = deadline - Date.now();
|
|
2149
|
+
if (remainingMs <= 0) {
|
|
2205
2150
|
return new NarsilError(
|
|
2206
|
-
ErrorCodes.
|
|
2207
|
-
"
|
|
2151
|
+
ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
2152
|
+
"bootstrap sync exceeded deadline before partition restore",
|
|
2208
2153
|
{
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
entryPartitionId: entry.partitionId
|
|
2154
|
+
indexName,
|
|
2155
|
+
partitionId,
|
|
2156
|
+
primaryNodeId
|
|
2213
2157
|
}
|
|
2214
2158
|
);
|
|
2215
2159
|
}
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
if (!validation.valid) {
|
|
2225
|
-
return new NarsilError(
|
|
2226
|
-
validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
2227
|
-
`Invalid sync entry ${entry.seqNo}`,
|
|
2228
|
-
{ indexName, partitionId, seqNo: entry.seqNo }
|
|
2229
|
-
);
|
|
2230
|
-
}
|
|
2231
|
-
return null;
|
|
2232
|
-
}
|
|
2233
|
-
function createLiveSyncFrameState(indexName, partitionId, spoolDirectory) {
|
|
2234
|
-
return {
|
|
2235
|
-
snapshotState: createSnapshotStreamState({ indexName, partitionId, spoolDirectory }),
|
|
2236
|
-
syncEntries: [],
|
|
2237
|
-
sawSnapshotFrame: false,
|
|
2238
|
-
error: null
|
|
2239
|
-
};
|
|
2240
|
-
}
|
|
2241
|
-
function handleLiveSyncFrame(state, frame) {
|
|
2242
|
-
if (state.error !== null || state.snapshotState.failure !== null) {
|
|
2243
|
-
return;
|
|
2244
|
-
}
|
|
2245
|
-
let decoded;
|
|
2246
|
-
try {
|
|
2247
|
-
decoded = decode(frame);
|
|
2248
|
-
} catch (err) {
|
|
2249
|
-
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_DECODE_FAILED, "failed to decode live sync frame", {
|
|
2250
|
-
cause: err instanceof Error ? err.message : String(err)
|
|
2251
|
-
});
|
|
2252
|
-
return;
|
|
2253
|
-
}
|
|
2254
|
-
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
2255
|
-
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "live sync frame is not an object");
|
|
2256
|
-
return;
|
|
2257
|
-
}
|
|
2258
|
-
const record = decoded;
|
|
2259
|
-
const entriesPayload = decodeSyncEntriesRecord(record);
|
|
2260
|
-
if (entriesPayload instanceof NarsilError) {
|
|
2261
|
-
state.error = entriesPayload;
|
|
2262
|
-
return;
|
|
2263
|
-
}
|
|
2264
|
-
if (entriesPayload !== null) {
|
|
2265
|
-
if (state.sawSnapshotFrame && state.snapshotState.endPayload === null) {
|
|
2266
|
-
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received sync_entries before snapshot_end");
|
|
2267
|
-
return;
|
|
2268
|
-
}
|
|
2269
|
-
state.syncEntries.push(entriesPayload);
|
|
2270
|
-
return;
|
|
2271
|
-
}
|
|
2272
|
-
if (state.syncEntries.length > 0 && !state.sawSnapshotFrame) {
|
|
2273
|
-
state.error = new NarsilError(
|
|
2274
|
-
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
2275
|
-
"received snapshot frame after incremental sync_entries"
|
|
2276
|
-
);
|
|
2277
|
-
return;
|
|
2278
|
-
}
|
|
2279
|
-
state.sawSnapshotFrame = true;
|
|
2280
|
-
handleIncomingSnapshotRecord(state.snapshotState, record);
|
|
2281
|
-
if (state.snapshotState.failure !== null) {
|
|
2282
|
-
state.error = streamFailureToError(state.snapshotState.failure);
|
|
2283
|
-
}
|
|
2284
|
-
}
|
|
2285
|
-
function decodeSyncEntriesRecord(record) {
|
|
2286
|
-
if (!("entries" in record) && !("isLast" in record)) {
|
|
2287
|
-
return null;
|
|
2288
|
-
}
|
|
2289
|
-
if (!Array.isArray(record.entries) || typeof record.isLast !== "boolean") {
|
|
2290
|
-
return new NarsilError(
|
|
2291
|
-
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
2292
|
-
"Invalid SyncEntriesPayload: expected entries array and isLast boolean"
|
|
2293
|
-
);
|
|
2294
|
-
}
|
|
2295
|
-
const entries = [];
|
|
2296
|
-
for (const candidate of record.entries) {
|
|
2297
|
-
try {
|
|
2298
|
-
entries.push(validateEntryPayload({ entry: candidate }).entry);
|
|
2299
|
-
} catch (err) {
|
|
2300
|
-
return new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, err instanceof Error ? err.message : String(err));
|
|
2301
|
-
}
|
|
2302
|
-
}
|
|
2303
|
-
return { entries, isLast: record.isLast };
|
|
2304
|
-
}
|
|
2305
|
-
function streamFailureToError(failure) {
|
|
2306
|
-
return new NarsilError(failure.code, failure.message, failure.details);
|
|
2307
|
-
}
|
|
2308
|
-
|
|
2309
|
-
// src/distribution/cluster-node/bootstrap-sync/apply.ts
|
|
2310
|
-
async function applyLiveIncrementalSync(indexName, partitionId, target, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, coordinatorConfig, partitionCount, deps) {
|
|
2311
|
-
const ensureResult = await ensureLocalIndexForIncremental(indexName, target, coordinatorConfig, partitionCount, deps);
|
|
2312
|
-
if (ensureResult instanceof NarsilError) {
|
|
2313
|
-
return { ok: false, error: ensureResult };
|
|
2314
|
-
}
|
|
2315
|
-
const applyResult = await applySyncEntries(
|
|
2316
|
-
indexName,
|
|
2317
|
-
partitionId,
|
|
2318
|
-
entriesPayloads,
|
|
2319
|
-
expectedFirstSeqNo,
|
|
2320
|
-
localPrimaryTerm,
|
|
2321
|
-
deps
|
|
2322
|
-
);
|
|
2323
|
-
if (!applyResult.ok) {
|
|
2324
|
-
return applyResult;
|
|
2325
|
-
}
|
|
2326
|
-
return {
|
|
2327
|
-
ok: true,
|
|
2328
|
-
tier: "incremental",
|
|
2329
|
-
entriesApplied: applyResult.entriesApplied,
|
|
2330
|
-
newSeqNo: applyResult.newSeqNo,
|
|
2331
|
-
snapshotHeader: null
|
|
2332
|
-
};
|
|
2333
|
-
}
|
|
2334
|
-
async function applyLiveSnapshotSync(state, entry, indexName, partitionId, target, frameState, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2335
|
-
const finalized = finalizeSnapshotStream(frameState.snapshotState);
|
|
2336
|
-
if (!finalized.ok) {
|
|
2337
|
-
return { ok: false, error: streamFailureToError({ ...finalized, details: { ...finalized.details, target } }) };
|
|
2338
|
-
}
|
|
2339
|
-
const restoreError = await enqueuePartitionWrite(
|
|
2340
|
-
deps.partitionWriteQueues,
|
|
2341
|
-
indexName,
|
|
2342
|
-
partitionId,
|
|
2343
|
-
() => restartFromSnapshot(
|
|
2344
|
-
state,
|
|
2345
|
-
entry,
|
|
2346
|
-
indexName,
|
|
2347
|
-
partitionId,
|
|
2348
|
-
target,
|
|
2349
|
-
finalized,
|
|
2350
|
-
coordinatorConfig,
|
|
2351
|
-
partitionCount,
|
|
2352
|
-
deadline,
|
|
2353
|
-
deps
|
|
2354
|
-
)
|
|
2355
|
-
);
|
|
2356
|
-
if (restoreError !== null) {
|
|
2357
|
-
return { ok: false, error: restoreError };
|
|
2358
|
-
}
|
|
2359
|
-
const applyResult = await applySyncEntries(
|
|
2360
|
-
indexName,
|
|
2361
|
-
partitionId,
|
|
2362
|
-
frameState.syncEntries,
|
|
2363
|
-
finalized.header.lastSeqNo + 1,
|
|
2364
|
-
finalized.header.primaryTerm,
|
|
2365
|
-
deps
|
|
2366
|
-
);
|
|
2367
|
-
if (!applyResult.ok) {
|
|
2368
|
-
return applyResult;
|
|
2369
|
-
}
|
|
2370
|
-
return {
|
|
2371
|
-
ok: true,
|
|
2372
|
-
tier: "snapshot",
|
|
2373
|
-
entriesApplied: applyResult.entriesApplied,
|
|
2374
|
-
newSeqNo: applyResult.newSeqNo,
|
|
2375
|
-
snapshotHeader: finalized.header
|
|
2376
|
-
};
|
|
2377
|
-
}
|
|
2378
|
-
async function restartFromSnapshot(state, entry, indexName, partitionId, target, finalized, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2379
|
-
let restoreError;
|
|
2380
|
-
try {
|
|
2381
|
-
restoreError = await restoreLiveSnapshotPartition(
|
|
2382
|
-
state,
|
|
2383
|
-
entry,
|
|
2384
|
-
indexName,
|
|
2385
|
-
partitionId,
|
|
2386
|
-
target,
|
|
2387
|
-
finalized.snapshot.read(),
|
|
2388
|
-
coordinatorConfig,
|
|
2389
|
-
partitionCount,
|
|
2390
|
-
deadline,
|
|
2391
|
-
deps
|
|
2392
|
-
);
|
|
2393
|
-
} finally {
|
|
2394
|
-
await finalized.snapshot.discard();
|
|
2395
|
-
}
|
|
2396
|
-
if (restoreError !== null) return restoreError;
|
|
2397
|
-
try {
|
|
2398
|
-
await deps.restartLocalLog?.(indexName, partitionId, {
|
|
2399
|
-
seqNo: finalized.header.lastSeqNo,
|
|
2400
|
-
primaryTerm: finalized.header.primaryTerm
|
|
2401
|
-
});
|
|
2402
|
-
} catch (err) {
|
|
2403
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
2404
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `local log restart failed: ${cause}`, {
|
|
2405
|
-
indexName,
|
|
2406
|
-
partitionId,
|
|
2407
|
-
primaryNodeId: target,
|
|
2408
|
-
cause
|
|
2409
|
-
});
|
|
2410
|
-
}
|
|
2411
|
-
deps.resetReplicationLog(indexName, partitionId, finalized.header.lastSeqNo + 1, finalized.header.primaryTerm);
|
|
2412
|
-
return null;
|
|
2413
|
-
}
|
|
2414
|
-
async function restoreLiveSnapshotPartition(state, entry, indexName, partitionId, primaryNodeId, source, coordinatorConfig, partitionCount, deadline, deps) {
|
|
2415
|
-
const remainingMs = deadline - Date.now();
|
|
2416
|
-
if (remainingMs <= 0) {
|
|
2417
|
-
return new NarsilError(
|
|
2418
|
-
ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
2419
|
-
"bootstrap sync exceeded deadline before partition restore",
|
|
2420
|
-
{
|
|
2421
|
-
indexName,
|
|
2422
|
-
partitionId,
|
|
2423
|
-
primaryNodeId
|
|
2424
|
-
}
|
|
2425
|
-
);
|
|
2426
|
-
}
|
|
2427
|
-
const key = entryKey(indexName, partitionId);
|
|
2428
|
-
const generationBeforeRestore = state.generations.get(key) ?? 0;
|
|
2429
|
-
if (generationBeforeRestore !== entry.generation || entry.aborted) {
|
|
2430
|
-
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, "bootstrap sync aborted before partition restore", {
|
|
2431
|
-
indexName,
|
|
2432
|
-
partitionId,
|
|
2433
|
-
primaryNodeId
|
|
2434
|
-
});
|
|
2160
|
+
const key = entryKey(indexName, partitionId);
|
|
2161
|
+
const generationBeforeRestore = state.generations.get(key) ?? 0;
|
|
2162
|
+
if (generationBeforeRestore !== entry.generation || entry.aborted) {
|
|
2163
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, "bootstrap sync aborted before partition restore", {
|
|
2164
|
+
indexName,
|
|
2165
|
+
partitionId,
|
|
2166
|
+
primaryNodeId
|
|
2167
|
+
});
|
|
2435
2168
|
}
|
|
2436
2169
|
try {
|
|
2437
2170
|
await withDeadline(
|
|
@@ -2682,6 +2415,9 @@ async function syncFrameStateFromTarget(state, entry, indexName, partitionId, ta
|
|
|
2682
2415
|
});
|
|
2683
2416
|
await withDeadline(streamWork, remainingMs, indexName, "live-sync");
|
|
2684
2417
|
} catch (err) {
|
|
2418
|
+
if (err instanceof NarsilError && err.code === ErrorCodes.SNAPSHOT_SYNC_TIMEOUT) {
|
|
2419
|
+
return { ok: false, error: err };
|
|
2420
|
+
}
|
|
2685
2421
|
const cause = err instanceof Error ? err.message : String(err);
|
|
2686
2422
|
return {
|
|
2687
2423
|
ok: false,
|
|
@@ -2876,67 +2612,6 @@ async function executeLiveBootstrapSync(state, entry, indexName, partitionId, pr
|
|
|
2876
2612
|
return true;
|
|
2877
2613
|
}
|
|
2878
2614
|
|
|
2879
|
-
// src/distribution/cluster-node/bootstrap-sync/snapshot-restore.ts
|
|
2880
|
-
async function fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps) {
|
|
2881
|
-
const dropError = await dropExistingIndex(deps.engine, indexName, primaryNodeId);
|
|
2882
|
-
if (dropError !== null) {
|
|
2883
|
-
surfaceError(deps, indexName, primaryNodeId, dropError);
|
|
2884
|
-
return null;
|
|
2885
|
-
}
|
|
2886
|
-
if (entry.aborted) {
|
|
2887
|
-
return "aborted";
|
|
2888
|
-
}
|
|
2889
|
-
const configResult = await loadCoordinatorIndexConfig(deps.coordinator, indexName, primaryNodeId, entry.abortPromise);
|
|
2890
|
-
if (configResult === "aborted") {
|
|
2891
|
-
return "aborted";
|
|
2892
|
-
}
|
|
2893
|
-
if ("error" in configResult) {
|
|
2894
|
-
surfaceError(deps, indexName, primaryNodeId, configResult.error);
|
|
2895
|
-
return null;
|
|
2896
|
-
}
|
|
2897
|
-
return configResult.config.schema;
|
|
2898
|
-
}
|
|
2899
|
-
async function applyRestore(state, entry, indexName, primaryNodeId, source, coordinatorSchema, deadline, deps) {
|
|
2900
|
-
const remainingMs = deadline - Date.now();
|
|
2901
|
-
if (remainingMs <= 0) {
|
|
2902
|
-
surfaceError(
|
|
2903
|
-
deps,
|
|
2904
|
-
indexName,
|
|
2905
|
-
primaryNodeId,
|
|
2906
|
-
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before restore", {
|
|
2907
|
-
indexName
|
|
2908
|
-
})
|
|
2909
|
-
);
|
|
2910
|
-
return false;
|
|
2911
|
-
}
|
|
2912
|
-
const key = entryKey(indexName, entry.partitionId);
|
|
2913
|
-
const generationBeforeRestore = state.generations.get(key) ?? 0;
|
|
2914
|
-
if (generationBeforeRestore !== entry.generation || entry.aborted) {
|
|
2915
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
2916
|
-
return false;
|
|
2917
|
-
}
|
|
2918
|
-
const restoreError = await executeEngineRestore(deps.engine, indexName, primaryNodeId, source, remainingMs);
|
|
2919
|
-
if (restoreError !== null) {
|
|
2920
|
-
surfaceError(deps, indexName, primaryNodeId, restoreError);
|
|
2921
|
-
return false;
|
|
2922
|
-
}
|
|
2923
|
-
const generationAfterRestore = state.generations.get(key) ?? 0;
|
|
2924
|
-
if (generationAfterRestore !== entry.generation || entry.aborted) {
|
|
2925
|
-
if (!anotherBootstrapOwnsKey(state, key, entry)) {
|
|
2926
|
-
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
2927
|
-
}
|
|
2928
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
2929
|
-
return false;
|
|
2930
|
-
}
|
|
2931
|
-
const schemaError = validateRestoredSchema(deps.engine, indexName, primaryNodeId, coordinatorSchema);
|
|
2932
|
-
if (schemaError !== null) {
|
|
2933
|
-
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
2934
|
-
surfaceError(deps, indexName, primaryNodeId, schemaError);
|
|
2935
|
-
return false;
|
|
2936
|
-
}
|
|
2937
|
-
return true;
|
|
2938
|
-
}
|
|
2939
|
-
|
|
2940
2615
|
// src/distribution/cluster-node/bootstrap-sync/index.ts
|
|
2941
2616
|
async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
|
|
2942
2617
|
const validationError = validateArguments(indexName, partitionId, primaryNodeId);
|
|
@@ -2994,113 +2669,17 @@ async function executeBootstrapSync(state, entry, indexName, partitionId, primar
|
|
|
2994
2669
|
}
|
|
2995
2670
|
return entry.aborted;
|
|
2996
2671
|
};
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
entry,
|
|
3001
|
-
indexName,
|
|
3002
|
-
partitionId,
|
|
3003
|
-
primaryNodeId,
|
|
3004
|
-
deps,
|
|
3005
|
-
deadlineMs,
|
|
3006
|
-
deadline,
|
|
3007
|
-
abortCheck
|
|
3008
|
-
);
|
|
3009
|
-
}
|
|
3010
|
-
const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
|
|
3011
|
-
if (schemaResult === "aborted") {
|
|
3012
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3013
|
-
return false;
|
|
3014
|
-
}
|
|
3015
|
-
if (schemaResult === null) {
|
|
3016
|
-
return false;
|
|
3017
|
-
}
|
|
3018
|
-
if (abortCheck()) {
|
|
3019
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3020
|
-
return false;
|
|
3021
|
-
}
|
|
3022
|
-
if (Date.now() >= deadline) {
|
|
3023
|
-
surfaceError(
|
|
3024
|
-
deps,
|
|
3025
|
-
indexName,
|
|
3026
|
-
primaryNodeId,
|
|
3027
|
-
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
|
|
3028
|
-
indexName,
|
|
3029
|
-
deadlineMs
|
|
3030
|
-
})
|
|
3031
|
-
);
|
|
3032
|
-
return false;
|
|
3033
|
-
}
|
|
3034
|
-
const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
|
|
3035
|
-
if (targetsResult instanceof NarsilError) {
|
|
3036
|
-
surfaceError(deps, indexName, primaryNodeId, targetsResult);
|
|
3037
|
-
return false;
|
|
3038
|
-
}
|
|
3039
|
-
const indexUuid = (await deps.coordinator.getAllocation(indexName))?.indexUuid;
|
|
3040
|
-
const fetchDeps = {
|
|
3041
|
-
transport: deps.transport,
|
|
3042
|
-
sourceNodeId: deps.sourceNodeId,
|
|
3043
|
-
spoolDirectory: deps.snapshotSpoolDirectory,
|
|
3044
|
-
...indexUuid === void 0 ? {} : { indexUuid }
|
|
3045
|
-
};
|
|
3046
|
-
const fetchResult = await fetchSnapshotFromAnyTarget(
|
|
2672
|
+
return executeLiveBootstrapSync(
|
|
2673
|
+
state,
|
|
2674
|
+
entry,
|
|
3047
2675
|
indexName,
|
|
2676
|
+
partitionId,
|
|
3048
2677
|
primaryNodeId,
|
|
3049
|
-
|
|
2678
|
+
deps,
|
|
2679
|
+
deadlineMs,
|
|
3050
2680
|
deadline,
|
|
3051
|
-
|
|
3052
|
-
abortCheck,
|
|
3053
|
-
partitionId
|
|
2681
|
+
abortCheck
|
|
3054
2682
|
);
|
|
3055
|
-
if (!fetchResult.ok) {
|
|
3056
|
-
if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
|
|
3057
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3058
|
-
return false;
|
|
3059
|
-
}
|
|
3060
|
-
surfaceError(
|
|
3061
|
-
deps,
|
|
3062
|
-
indexName,
|
|
3063
|
-
primaryNodeId,
|
|
3064
|
-
new NarsilError(fetchResult.code, fetchResult.message, {
|
|
3065
|
-
indexName,
|
|
3066
|
-
primaryNodeId,
|
|
3067
|
-
...fetchResult.details
|
|
3068
|
-
})
|
|
3069
|
-
);
|
|
3070
|
-
return false;
|
|
3071
|
-
}
|
|
3072
|
-
let restoreSucceeded;
|
|
3073
|
-
try {
|
|
3074
|
-
if (abortCheck()) {
|
|
3075
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3076
|
-
return false;
|
|
3077
|
-
}
|
|
3078
|
-
restoreSucceeded = await applyRestore(
|
|
3079
|
-
state,
|
|
3080
|
-
entry,
|
|
3081
|
-
indexName,
|
|
3082
|
-
primaryNodeId,
|
|
3083
|
-
fetchResult.snapshot.read(),
|
|
3084
|
-
schemaResult,
|
|
3085
|
-
deadline,
|
|
3086
|
-
deps
|
|
3087
|
-
);
|
|
3088
|
-
} finally {
|
|
3089
|
-
await fetchResult.snapshot.discard();
|
|
3090
|
-
}
|
|
3091
|
-
if (!restoreSucceeded) {
|
|
3092
|
-
return false;
|
|
3093
|
-
}
|
|
3094
|
-
if (abortCheck()) {
|
|
3095
|
-
if (!anotherBootstrapOwnsKey(state, key, entry)) {
|
|
3096
|
-
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
3097
|
-
}
|
|
3098
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
3099
|
-
return false;
|
|
3100
|
-
}
|
|
3101
|
-
deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
|
|
3102
|
-
state.completed.add(key);
|
|
3103
|
-
return true;
|
|
3104
2683
|
}
|
|
3105
2684
|
var REFUSE = { outcome: "refuse" };
|
|
3106
2685
|
var ACCEPT = { outcome: "accept" };
|
|
@@ -7051,6 +6630,22 @@ function noteSupersedingPrimary(log, indexName, partitionId, primaryNodeId, prim
|
|
|
7051
6630
|
}
|
|
7052
6631
|
}
|
|
7053
6632
|
|
|
6633
|
+
// src/distribution/cluster-node/write-routing/deadline.ts
|
|
6634
|
+
function replicationTimeoutOf(deps) {
|
|
6635
|
+
return deps.transport.timeouts?.replicationTimeout ?? DEFAULT_TRANSPORT_CONFIG.replicationTimeout;
|
|
6636
|
+
}
|
|
6637
|
+
async function withinDeadline(work, timeoutMs, expired) {
|
|
6638
|
+
let timer;
|
|
6639
|
+
const timedOut = new Promise((_, reject) => {
|
|
6640
|
+
timer = setTimeout(() => reject(expired()), Math.max(0, timeoutMs));
|
|
6641
|
+
});
|
|
6642
|
+
try {
|
|
6643
|
+
return await Promise.race([work, timedOut]);
|
|
6644
|
+
} finally {
|
|
6645
|
+
clearTimeout(timer);
|
|
6646
|
+
}
|
|
6647
|
+
}
|
|
6648
|
+
|
|
7054
6649
|
// src/distribution/cluster-node/write-routing/replication.ts
|
|
7055
6650
|
async function assertPrimaryWriteAuthority(entry, deps) {
|
|
7056
6651
|
const table = await deps.coordinator.getAllocation(entry.indexName);
|
|
@@ -7099,6 +6694,18 @@ async function removeFailedReplicasFromInsync(entry, failedReplicas, deps) {
|
|
|
7099
6694
|
if (failedReplicas.length === 0) {
|
|
7100
6695
|
return;
|
|
7101
6696
|
}
|
|
6697
|
+
const timeoutMs = replicationTimeoutOf(deps);
|
|
6698
|
+
await withinDeadline(
|
|
6699
|
+
removeEachFailedReplica(entry, failedReplicas, deps),
|
|
6700
|
+
timeoutMs,
|
|
6701
|
+
() => new NarsilError(
|
|
6702
|
+
ErrorCodes.REPLICATION_INSYNC_REMOVAL_FAILED,
|
|
6703
|
+
`Removing failed replicas from the in-sync set of index '${entry.indexName}' partition ${entry.partitionId} did not finish within ${timeoutMs}ms`,
|
|
6704
|
+
{ indexName: entry.indexName, partitionId: entry.partitionId, failedReplicas, timeoutMs }
|
|
6705
|
+
)
|
|
6706
|
+
);
|
|
6707
|
+
}
|
|
6708
|
+
async function removeEachFailedReplica(entry, failedReplicas, deps) {
|
|
7102
6709
|
const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
7103
6710
|
if (controllerNodeId === null) {
|
|
7104
6711
|
await deps.refreshAllocation?.(entry.indexName);
|
|
@@ -10573,6 +10180,112 @@ async function sendToNode(nodeId, message, deps) {
|
|
|
10573
10180
|
});
|
|
10574
10181
|
}
|
|
10575
10182
|
|
|
10183
|
+
// src/distribution/cluster-node/write-routing/rollback.ts
|
|
10184
|
+
function outcomeUnknown(scope, originalError, reason, because) {
|
|
10185
|
+
return new NarsilError(
|
|
10186
|
+
ErrorCodes.REPLICATION_OUTCOME_UNKNOWN,
|
|
10187
|
+
`A write to partition ${scope.partitionId} of index '${scope.indexName}' failed after the primary applied it, and ${because}, so the write may or may not have taken effect. Read the document before you retry the write.`,
|
|
10188
|
+
{
|
|
10189
|
+
indexName: scope.indexName,
|
|
10190
|
+
partitionId: scope.partitionId,
|
|
10191
|
+
primaryTerm: scope.assignment.primaryTerm,
|
|
10192
|
+
reason,
|
|
10193
|
+
originalError: describeError(originalError)
|
|
10194
|
+
}
|
|
10195
|
+
);
|
|
10196
|
+
}
|
|
10197
|
+
function isOutcomeUnknown(error) {
|
|
10198
|
+
return error instanceof NarsilError && error.code === ErrorCodes.REPLICATION_OUTCOME_UNKNOWN;
|
|
10199
|
+
}
|
|
10200
|
+
function abandonedWithTerm(error) {
|
|
10201
|
+
return isOutcomeUnknown(error) && error instanceof NarsilError && error.details.reason === "term_ended";
|
|
10202
|
+
}
|
|
10203
|
+
function lostTerm(error) {
|
|
10204
|
+
return error instanceof NarsilError && error.code === ErrorCodes.PARTITION_NOT_PRIMARY;
|
|
10205
|
+
}
|
|
10206
|
+
async function leadingAssignment(scope) {
|
|
10207
|
+
const { deps, indexName, partitionId } = scope;
|
|
10208
|
+
const table = deps.refreshAllocation !== void 0 ? await deps.refreshAllocation(indexName) : await deps.coordinator.getAllocation(indexName);
|
|
10209
|
+
const assignment = table?.assignments.get(partitionId);
|
|
10210
|
+
if (assignment !== void 0 && assignment.primary === deps.nodeId && assignment.primaryTerm === scope.assignment.primaryTerm && deps.copyFailed?.(indexName, partitionId) !== true) {
|
|
10211
|
+
return assignment;
|
|
10212
|
+
}
|
|
10213
|
+
throw new NarsilError(
|
|
10214
|
+
ErrorCodes.PARTITION_NOT_PRIMARY,
|
|
10215
|
+
`Node '${deps.nodeId}' no longer leads partition ${partitionId} of index '${indexName}' under term ${scope.assignment.primaryTerm}`,
|
|
10216
|
+
{
|
|
10217
|
+
indexName,
|
|
10218
|
+
partitionId,
|
|
10219
|
+
localNodeId: deps.nodeId,
|
|
10220
|
+
expectedPrimaryTerm: scope.assignment.primaryTerm,
|
|
10221
|
+
currentPrimaryNodeId: assignment?.primary ?? null,
|
|
10222
|
+
currentPrimaryTerm: assignment?.primaryTerm ?? null
|
|
10223
|
+
}
|
|
10224
|
+
);
|
|
10225
|
+
}
|
|
10226
|
+
async function sendRollback(scope, entries) {
|
|
10227
|
+
const assignment = await leadingAssignment(scope);
|
|
10228
|
+
for (const chunk of chunkReplicationEntries(entries.map((entry) => ({ entry })))) {
|
|
10229
|
+
await replicateEntryBatch(
|
|
10230
|
+
chunk.map((item) => item.entry),
|
|
10231
|
+
assignment,
|
|
10232
|
+
scope.deps,
|
|
10233
|
+
scope.indexUuid
|
|
10234
|
+
);
|
|
10235
|
+
}
|
|
10236
|
+
}
|
|
10237
|
+
async function tryRollback(scope, entries, timeoutMs, delayMs) {
|
|
10238
|
+
try {
|
|
10239
|
+
await withinDeadline(
|
|
10240
|
+
sendRollback(scope, entries),
|
|
10241
|
+
timeoutMs,
|
|
10242
|
+
() => new Error(`A try to complete a rollback did not finish within ${timeoutMs}ms`)
|
|
10243
|
+
);
|
|
10244
|
+
return true;
|
|
10245
|
+
} catch (error) {
|
|
10246
|
+
if (lostTerm(error)) throw error;
|
|
10247
|
+
}
|
|
10248
|
+
await pause(delayMs);
|
|
10249
|
+
return false;
|
|
10250
|
+
}
|
|
10251
|
+
async function completeRollback(scope, fromSeqNo, timeoutMs) {
|
|
10252
|
+
const log = scope.deps.getReplicationLog(scope.indexName, scope.partitionId);
|
|
10253
|
+
let delayMs = ROLLBACK_RETRY_INITIAL_DELAY_MS;
|
|
10254
|
+
for (; ; ) {
|
|
10255
|
+
if (scope.deps.stopped?.() === true) return "stopped";
|
|
10256
|
+
const entries = log.getEntriesFrom(fromSeqNo);
|
|
10257
|
+
const last = entries.at(-1);
|
|
10258
|
+
if (last === void 0) return "superseded";
|
|
10259
|
+
try {
|
|
10260
|
+
if (await untilSuperseded(log, last, tryRollback(scope, entries, timeoutMs, delayMs))) return "complete";
|
|
10261
|
+
} catch (error) {
|
|
10262
|
+
if (lostTerm(error)) return "superseded";
|
|
10263
|
+
}
|
|
10264
|
+
delayMs = Math.min(delayMs * 2, ROLLBACK_RETRY_DELAY_CAP_MS);
|
|
10265
|
+
}
|
|
10266
|
+
}
|
|
10267
|
+
async function settleRollback(scope, fromSeqNo, originalError, failedAt) {
|
|
10268
|
+
const timeoutMs = replicationTimeoutOf(scope.deps);
|
|
10269
|
+
const rollback = completeRollback(scope, fromSeqNo, timeoutMs);
|
|
10270
|
+
scope.holdQueue?.(rollback);
|
|
10271
|
+
let timer;
|
|
10272
|
+
const stillOpen = new Promise((resolve) => {
|
|
10273
|
+
timer = setTimeout(() => resolve("open"), Math.max(0, failedAt + timeoutMs - Date.now()));
|
|
10274
|
+
});
|
|
10275
|
+
const end = await Promise.race([rollback, stillOpen]).finally(() => clearTimeout(timer));
|
|
10276
|
+
if (end === "complete") return originalError;
|
|
10277
|
+
if (end === "superseded") {
|
|
10278
|
+
return outcomeUnknown(
|
|
10279
|
+
scope,
|
|
10280
|
+
originalError,
|
|
10281
|
+
"term_ended",
|
|
10282
|
+
"the primary lost its term before every in-sync copy dropped the write"
|
|
10283
|
+
);
|
|
10284
|
+
}
|
|
10285
|
+
const because = end === "stopped" ? "the primary shut down before every in-sync copy dropped the write" : `the primary could not confirm within ${timeoutMs}ms that every in-sync copy dropped it`;
|
|
10286
|
+
return outcomeUnknown(scope, originalError, "rollback_open", because);
|
|
10287
|
+
}
|
|
10288
|
+
|
|
10576
10289
|
// src/distribution/cluster-node/write-routing/sent-fields.ts
|
|
10577
10290
|
function isRecord3(value) {
|
|
10578
10291
|
return typeof value === "object" && value !== null && !Array.isArray(value) && !ArrayBuffer.isView(value);
|
|
@@ -10627,8 +10340,9 @@ function storedCopyMatchesSent(sent, stored, derivedFields) {
|
|
|
10627
10340
|
}
|
|
10628
10341
|
|
|
10629
10342
|
// src/distribution/cluster-node/write-routing/primary-change.ts
|
|
10630
|
-
function refusedByPrimaryChange(error) {
|
|
10631
|
-
|
|
10343
|
+
function refusedByPrimaryChange(error, retriesEndedTerm = false) {
|
|
10344
|
+
if (error instanceof NarsilError && error.code === ErrorCodes.PARTITION_NOT_PRIMARY) return true;
|
|
10345
|
+
return retriesEndedTerm && abandonedWithTerm(error);
|
|
10632
10346
|
}
|
|
10633
10347
|
function refusedForAnotherIndex(error) {
|
|
10634
10348
|
return refusedByPrimaryChange(error) && error instanceof NarsilError && typeof error.details.heldIndexUuid === "string";
|
|
@@ -10685,7 +10399,7 @@ async function awaitNewPrimary(deps, indexName, refusing, deadline) {
|
|
|
10685
10399
|
}
|
|
10686
10400
|
return false;
|
|
10687
10401
|
}
|
|
10688
|
-
async function retryOnPrimaryChange(deps, indexName, docId, attempt, settleRetried, settledValue) {
|
|
10402
|
+
async function retryOnPrimaryChange(deps, indexName, docId, attempt, settleRetried, settledValue, retriesEndedTerm = false) {
|
|
10689
10403
|
const deadline = Date.now() + PRIMARY_CHANGE_RETRY_WINDOW_MS;
|
|
10690
10404
|
let retryDelayMs = FORWARD_RETRY_INITIAL_DELAY_MS;
|
|
10691
10405
|
let retried = false;
|
|
@@ -10695,7 +10409,9 @@ async function retryOnPrimaryChange(deps, indexName, docId, attempt, settleRetri
|
|
|
10695
10409
|
return await attempt();
|
|
10696
10410
|
} catch (error) {
|
|
10697
10411
|
if (retried && (await settleRetried([{ docId, error }])).has(docId)) return settledValue();
|
|
10698
|
-
if (!(refusedByPrimaryChange(error) || forwardUndelivered(error)) || refusing === null)
|
|
10412
|
+
if (!(refusedByPrimaryChange(error, retriesEndedTerm) || forwardUndelivered(error)) || refusing === null) {
|
|
10413
|
+
throw error;
|
|
10414
|
+
}
|
|
10699
10415
|
let retryable;
|
|
10700
10416
|
if (retriedAfterDelay(error)) {
|
|
10701
10417
|
retryable = await pauseAndRereadTable(deps, indexName, retryDelayMs, deadline);
|
|
@@ -10708,7 +10424,7 @@ async function retryOnPrimaryChange(deps, indexName, docId, attempt, settleRetri
|
|
|
10708
10424
|
}
|
|
10709
10425
|
}
|
|
10710
10426
|
}
|
|
10711
|
-
async function retryBatchOnPrimaryChange(deps, indexName, items, docIdOf, attempt, settleRetried, deadline) {
|
|
10427
|
+
async function retryBatchOnPrimaryChange(deps, indexName, items, docIdOf, attempt, settleRetried, deadline, retriesEndedTerm) {
|
|
10712
10428
|
const succeeded2 = [];
|
|
10713
10429
|
const failed2 = [];
|
|
10714
10430
|
let pending = [...items];
|
|
@@ -10728,7 +10444,7 @@ async function retryBatchOnPrimaryChange(deps, indexName, items, docIdOf, attemp
|
|
|
10728
10444
|
}
|
|
10729
10445
|
const item = pendingById.get(failure.docId);
|
|
10730
10446
|
const refusing = primaryFor(table, failure.docId);
|
|
10731
|
-
if (item === void 0 || refusing === null || !(refusedByPrimaryChange(failure.error) || forwardUndelivered(failure.error))) {
|
|
10447
|
+
if (item === void 0 || refusing === null || !(refusedByPrimaryChange(failure.error, retriesEndedTerm) || forwardUndelivered(failure.error))) {
|
|
10732
10448
|
failed2.push(failure);
|
|
10733
10449
|
continue;
|
|
10734
10450
|
}
|
|
@@ -10783,12 +10499,12 @@ async function settleEachBounded(inputs, limit, run) {
|
|
|
10783
10499
|
await Promise.all(Array.from({ length: Math.min(limit, inputs.length) }, work));
|
|
10784
10500
|
return outcomes;
|
|
10785
10501
|
}
|
|
10786
|
-
async function retryEachPartition(deps, indexName, table, items, docIdOf, attempt, settleRetried) {
|
|
10502
|
+
async function retryEachPartition(deps, indexName, table, items, docIdOf, attempt, settleRetried, retriesEndedTerm = false) {
|
|
10787
10503
|
const deadline = Date.now() + PRIMARY_CHANGE_RETRY_WINDOW_MS;
|
|
10788
10504
|
const outcomes = await settleEachBounded(
|
|
10789
10505
|
groupByPartition(table, items, docIdOf),
|
|
10790
10506
|
PARTITION_WRITE_CONCURRENCY,
|
|
10791
|
-
(group) => retryBatchOnPrimaryChange(deps, indexName, group, docIdOf, attempt, settleRetried, deadline)
|
|
10507
|
+
(group) => retryBatchOnPrimaryChange(deps, indexName, group, docIdOf, attempt, settleRetried, deadline, retriesEndedTerm)
|
|
10792
10508
|
);
|
|
10793
10509
|
const merged = { succeeded: [], failed: [] };
|
|
10794
10510
|
for (const outcome of outcomes) {
|
|
@@ -10858,7 +10574,7 @@ function createGroupCommit(scope, appended, failed2) {
|
|
|
10858
10574
|
}
|
|
10859
10575
|
};
|
|
10860
10576
|
}
|
|
10861
|
-
async function replicateAppendedWrites(scope, appended, failed2,
|
|
10577
|
+
async function replicateAppendedWrites(scope, appended, failed2, compensate, deps) {
|
|
10862
10578
|
const succeeded2 = [];
|
|
10863
10579
|
const chunks = chunkReplicationEntries(appended);
|
|
10864
10580
|
const abandoned = [];
|
|
@@ -10880,14 +10596,23 @@ async function replicateAppendedWrites(scope, appended, failed2, rollback, deps)
|
|
|
10880
10596
|
}
|
|
10881
10597
|
abandoned.push(...chunk);
|
|
10882
10598
|
}
|
|
10599
|
+
const first = abandoned[0];
|
|
10600
|
+
if (first === void 0) return succeeded2;
|
|
10601
|
+
const failedAt = Date.now();
|
|
10602
|
+
const compensated = [];
|
|
10883
10603
|
for (let index = abandoned.length - 1; index >= 0; index--) {
|
|
10884
10604
|
const item = abandoned[index];
|
|
10885
10605
|
try {
|
|
10886
|
-
await
|
|
10606
|
+
await compensate(item, abortError);
|
|
10607
|
+
compensated.push(item);
|
|
10887
10608
|
} catch (err) {
|
|
10888
10609
|
failed2.push({ docId: item.docId, error: asWriteError(err, ErrorCodes.DOC_VALIDATION_FAILED, item.docId) });
|
|
10889
10610
|
}
|
|
10890
10611
|
}
|
|
10612
|
+
const answer = await settleRollback(scope, first.entry.seqNo, abortError, failedAt);
|
|
10613
|
+
for (const item of compensated) {
|
|
10614
|
+
failed2.push({ docId: item.docId, error: asWriteError(answer, ErrorCodes.DOC_VALIDATION_FAILED, item.docId) });
|
|
10615
|
+
}
|
|
10891
10616
|
return succeeded2;
|
|
10892
10617
|
}
|
|
10893
10618
|
|
|
@@ -10971,7 +10696,7 @@ function createRollbackFailure(operation, scope, documentId, originalError, roll
|
|
|
10971
10696
|
}
|
|
10972
10697
|
);
|
|
10973
10698
|
}
|
|
10974
|
-
async function
|
|
10699
|
+
async function compensateInsert(scope, documentId, originalError) {
|
|
10975
10700
|
try {
|
|
10976
10701
|
await numberedWrite(
|
|
10977
10702
|
scope,
|
|
@@ -10983,9 +10708,8 @@ async function rollbackPrimaryInsert(scope, documentId, originalError) {
|
|
|
10983
10708
|
throw createRollbackFailure("insert", scope, documentId, originalError, rollbackError);
|
|
10984
10709
|
}
|
|
10985
10710
|
}
|
|
10986
|
-
throwWriteFailure(originalError);
|
|
10987
10711
|
}
|
|
10988
|
-
async function
|
|
10712
|
+
async function compensateUpdate(scope, documentId, previousDocument, originalError) {
|
|
10989
10713
|
if (previousDocument === void 0) {
|
|
10990
10714
|
throw createRollbackFailure(
|
|
10991
10715
|
"update",
|
|
@@ -11004,9 +10728,8 @@ async function rollbackPrimaryUpdate(scope, documentId, previousDocument, origin
|
|
|
11004
10728
|
} catch (rollbackError) {
|
|
11005
10729
|
throw createRollbackFailure("update", scope, documentId, originalError, rollbackError);
|
|
11006
10730
|
}
|
|
11007
|
-
throwWriteFailure(originalError);
|
|
11008
10731
|
}
|
|
11009
|
-
async function
|
|
10732
|
+
async function compensateRemove(scope, documentId, previousDocument, originalError) {
|
|
11010
10733
|
if (previousDocument === void 0) {
|
|
11011
10734
|
throw createRollbackFailure(
|
|
11012
10735
|
"remove",
|
|
@@ -11025,7 +10748,6 @@ async function rollbackPrimaryRemove(scope, documentId, previousDocument, origin
|
|
|
11025
10748
|
} catch (rollbackError) {
|
|
11026
10749
|
throw createRollbackFailure("remove", scope, documentId, originalError, rollbackError);
|
|
11027
10750
|
}
|
|
11028
|
-
throwWriteFailure(originalError);
|
|
11029
10751
|
}
|
|
11030
10752
|
|
|
11031
10753
|
// src/distribution/cluster-node/reshard/state.ts
|
|
@@ -11157,6 +10879,9 @@ async function leadershipUnconfirmed(scope) {
|
|
|
11157
10879
|
if (assignment === void 0) return true;
|
|
11158
10880
|
return assignment.primary !== deps.nodeId || assignment.primaryTerm > scope.assignment.primaryTerm;
|
|
11159
10881
|
}
|
|
10882
|
+
function logEndOf(scope) {
|
|
10883
|
+
return scope.deps.getReplicationLog(scope.indexName, scope.partitionId).localLogEnd;
|
|
10884
|
+
}
|
|
11160
10885
|
function copyFailedIn(scope) {
|
|
11161
10886
|
return scope.deps.copyFailed?.(scope.indexName, scope.partitionId) === true;
|
|
11162
10887
|
}
|
|
@@ -11203,23 +10928,35 @@ async function currentScope(scope) {
|
|
|
11203
10928
|
}
|
|
11204
10929
|
async function inLeadingPartitionQueue(scope, task, failed2) {
|
|
11205
10930
|
let current = scope;
|
|
10931
|
+
let logEndBefore;
|
|
11206
10932
|
let result;
|
|
11207
10933
|
try {
|
|
11208
10934
|
result = await enqueuePartitionWrite(
|
|
11209
10935
|
scope.deps.partitionWriteQueues,
|
|
11210
10936
|
scope.indexName,
|
|
11211
10937
|
scope.partitionId,
|
|
11212
|
-
async () => {
|
|
11213
|
-
current = await currentScope(scope);
|
|
10938
|
+
async (hold) => {
|
|
10939
|
+
current = { ...await currentScope(scope), holdQueue: hold };
|
|
10940
|
+
logEndBefore = logEndOf(current);
|
|
11214
10941
|
return task(current);
|
|
11215
10942
|
}
|
|
11216
10943
|
);
|
|
11217
10944
|
} catch (error) {
|
|
10945
|
+
if (isOutcomeUnknown(error)) throw error;
|
|
11218
10946
|
if (copyFailedIn(current)) throw notPrimary(current);
|
|
11219
10947
|
if (refusedFromLocalCopy(error) && await leadershipUnconfirmed(current)) throw notPrimary(current);
|
|
11220
10948
|
throw error;
|
|
11221
10949
|
}
|
|
11222
|
-
if (await registrationLeaseLapsed(current.deps))
|
|
10950
|
+
if (await registrationLeaseLapsed(current.deps)) {
|
|
10951
|
+
const refusal = notPrimary(current);
|
|
10952
|
+
if (logEndBefore === void 0 || logEndOf(current) === logEndBefore) throw refusal;
|
|
10953
|
+
throw outcomeUnknown(
|
|
10954
|
+
current,
|
|
10955
|
+
refusal,
|
|
10956
|
+
"lease_lapsed",
|
|
10957
|
+
"its registration lease lapsed before it could acknowledge the write"
|
|
10958
|
+
);
|
|
10959
|
+
}
|
|
11223
10960
|
if (failed2 !== void 0 && copyFailedIn(current)) refuseFromFailedCopy(current, failed2);
|
|
11224
10961
|
if (failed2 !== void 0) await confirmBatchRefusals(current, failed2);
|
|
11225
10962
|
return result;
|
|
@@ -11509,7 +11246,9 @@ async function applyPrimaryInsert(indexName, document, docId, partitionId, assig
|
|
|
11509
11246
|
try {
|
|
11510
11247
|
await replicateEntry(inserted.entry, scope.assignment, deps, scope.indexUuid);
|
|
11511
11248
|
} catch (error) {
|
|
11512
|
-
|
|
11249
|
+
const failedAt = Date.now();
|
|
11250
|
+
await compensateInsert(scope, inserted.value, error);
|
|
11251
|
+
throwWriteFailure(await settleRollback(scope, inserted.entry.seqNo, error, failedAt));
|
|
11513
11252
|
}
|
|
11514
11253
|
return inserted.value;
|
|
11515
11254
|
} finally {
|
|
@@ -11530,7 +11269,9 @@ async function applyPrimaryRemove(indexName, docId, partitionId, assignment, dep
|
|
|
11530
11269
|
try {
|
|
11531
11270
|
await replicateEntry(removed.entry, scope.assignment, deps, scope.indexUuid);
|
|
11532
11271
|
} catch (error) {
|
|
11533
|
-
|
|
11272
|
+
const failedAt = Date.now();
|
|
11273
|
+
await compensateRemove(scope, docId, previousDocument, error);
|
|
11274
|
+
throwWriteFailure(await settleRollback(scope, removed.entry.seqNo, error, failedAt));
|
|
11534
11275
|
}
|
|
11535
11276
|
});
|
|
11536
11277
|
}
|
|
@@ -11549,7 +11290,9 @@ async function applyPrimaryUpdate(indexName, docId, document, partitionId, assig
|
|
|
11549
11290
|
try {
|
|
11550
11291
|
await replicateEntry(updated.entry, scope.assignment, deps, scope.indexUuid);
|
|
11551
11292
|
} catch (error) {
|
|
11552
|
-
|
|
11293
|
+
const failedAt = Date.now();
|
|
11294
|
+
await compensateUpdate(scope, docId, previousDocument, error);
|
|
11295
|
+
throwWriteFailure(await settleRollback(scope, updated.entry.seqNo, error, failedAt));
|
|
11553
11296
|
}
|
|
11554
11297
|
} finally {
|
|
11555
11298
|
projected.release();
|
|
@@ -11590,7 +11333,7 @@ async function applyPrimaryInsertBatch(indexName, items, partitionId, assignment
|
|
|
11590
11333
|
scope,
|
|
11591
11334
|
appended,
|
|
11592
11335
|
failed2,
|
|
11593
|
-
(write, error) =>
|
|
11336
|
+
(write, error) => compensateInsert(scope, write.docId, error),
|
|
11594
11337
|
deps
|
|
11595
11338
|
);
|
|
11596
11339
|
},
|
|
@@ -11636,7 +11379,7 @@ async function applyPrimaryUpdateBatch(indexName, updates, partitionId, assignme
|
|
|
11636
11379
|
scope,
|
|
11637
11380
|
appended,
|
|
11638
11381
|
failed2,
|
|
11639
|
-
(write, error) =>
|
|
11382
|
+
(write, error) => compensateUpdate(scope, write.docId, write.previousDocument, error),
|
|
11640
11383
|
deps
|
|
11641
11384
|
);
|
|
11642
11385
|
},
|
|
@@ -11672,7 +11415,7 @@ async function applyPrimaryRemoveBatch(indexName, docIds, partitionId, assignmen
|
|
|
11672
11415
|
scope,
|
|
11673
11416
|
appended,
|
|
11674
11417
|
failed2,
|
|
11675
|
-
(write, error) =>
|
|
11418
|
+
(write, error) => compensateRemove(scope, write.docId, write.previousDocument, error),
|
|
11676
11419
|
deps
|
|
11677
11420
|
);
|
|
11678
11421
|
},
|
|
@@ -11939,7 +11682,8 @@ async function routeAdmittedInsertBatch(indexName, documents, deps, options) {
|
|
|
11939
11682
|
withIds,
|
|
11940
11683
|
(item) => item.docId,
|
|
11941
11684
|
async (pending) => routeInsertItems(await requireRoutableTable(deps, indexName), indexName, pending, deps, options),
|
|
11942
|
-
settleEqualInserts(deps, indexName, sentById)
|
|
11685
|
+
settleEqualInserts(deps, indexName, sentById),
|
|
11686
|
+
true
|
|
11943
11687
|
);
|
|
11944
11688
|
}
|
|
11945
11689
|
async function requireRoutableTable(deps, indexName) {
|
|
@@ -12434,7 +12178,8 @@ async function routeAdmittedInsert(indexName, document, docId, deps, options) {
|
|
|
12434
12178
|
resolvedDocId,
|
|
12435
12179
|
() => insertOnce(indexName, document, resolvedDocId, deps, options),
|
|
12436
12180
|
settleEqualInserts(deps, indexName, /* @__PURE__ */ new Map([[resolvedDocId, document]])),
|
|
12437
|
-
() => resolvedDocId
|
|
12181
|
+
() => resolvedDocId,
|
|
12182
|
+
true
|
|
12438
12183
|
);
|
|
12439
12184
|
}
|
|
12440
12185
|
async function insertOnce(indexName, document, resolvedDocId, deps, options) {
|
|
@@ -12508,11 +12253,14 @@ async function routeAdmittedUpdate(indexName, docId, document, deps) {
|
|
|
12508
12253
|
|
|
12509
12254
|
// src/distribution/cluster-node/node-messaging.ts
|
|
12510
12255
|
async function resolveNodeTargets2(config, targetNodeId) {
|
|
12256
|
+
const targets = await resolveRegisteredTargets(config, targetNodeId);
|
|
12257
|
+
return targets.length > 0 ? targets : [targetNodeId];
|
|
12258
|
+
}
|
|
12259
|
+
async function resolveRegisteredTargets(config, targetNodeId) {
|
|
12511
12260
|
const nodes = await config.coordinator.listNodes();
|
|
12512
12261
|
const registration = nodes.find((node) => node.nodeId === targetNodeId);
|
|
12513
|
-
if (registration === void 0
|
|
12514
|
-
|
|
12515
|
-
}
|
|
12262
|
+
if (registration === void 0) return [];
|
|
12263
|
+
if (registration.address.length === 0 || registration.address === targetNodeId) return [targetNodeId];
|
|
12516
12264
|
return [registration.address, targetNodeId];
|
|
12517
12265
|
}
|
|
12518
12266
|
async function sendToNode2(config, targetNodeId, message, failure) {
|
|
@@ -13033,7 +12781,7 @@ async function purgeGeneration(deps, job) {
|
|
|
13033
12781
|
}
|
|
13034
12782
|
|
|
13035
12783
|
// src/distribution/cluster-node/reshard/leadership.ts
|
|
13036
|
-
async function
|
|
12784
|
+
async function leadingAssignment2(deps, indexName, partitionId) {
|
|
13037
12785
|
let assignment = (await deps.coordinator.getAllocation(indexName))?.assignments.get(partitionId);
|
|
13038
12786
|
if (assignment?.primary !== deps.nodeId && deps.refreshAllocation !== void 0) {
|
|
13039
12787
|
assignment = (await deps.refreshAllocation(indexName))?.assignments.get(partitionId);
|
|
@@ -13229,7 +12977,7 @@ async function runCopy(deps, job, announce) {
|
|
|
13229
12977
|
}
|
|
13230
12978
|
async function startReshardCopy(deps, payload) {
|
|
13231
12979
|
const state = reshardState(deps);
|
|
13232
|
-
const assignment = await
|
|
12980
|
+
const assignment = await leadingAssignment2(deps.writeDeps, payload.sourceGeneration, payload.partitionId);
|
|
13233
12981
|
const key = jobKey(payload.jobId, payload.partitionId);
|
|
13234
12982
|
const existing = state.jobs.get(key);
|
|
13235
12983
|
if (existing !== void 0 && !existing.stopped && existing.primaryTerm === assignment.primaryTerm) {
|
|
@@ -13278,7 +13026,7 @@ async function applyReshardCutover(deps, payload) {
|
|
|
13278
13026
|
);
|
|
13279
13027
|
}
|
|
13280
13028
|
const source = record.sourceGeneration;
|
|
13281
|
-
const assignment = await
|
|
13029
|
+
const assignment = await leadingAssignment2(deps.writeDeps, source, payload.partitionId);
|
|
13282
13030
|
setReshardBlock(state, source, payload.jobId, payload.partitionId);
|
|
13283
13031
|
await enqueuePartitionWrite(deps.writeDeps.partitionWriteQueues, source, payload.partitionId, async () => void 0);
|
|
13284
13032
|
return forwardableEnd(deps.writeDeps.getReplicationLog(source, payload.partitionId), assignment.primaryTerm);
|
|
@@ -13301,7 +13049,7 @@ function failed(documentId, error) {
|
|
|
13301
13049
|
async function applyInPartition(deps, engine, indexName, partitionId, queued, results) {
|
|
13302
13050
|
try {
|
|
13303
13051
|
await enqueuePartitionWrite(deps.partitionWriteQueues, indexName, partitionId, async () => {
|
|
13304
|
-
const assignment = await
|
|
13052
|
+
const assignment = await leadingAssignment2(deps, indexName, partitionId);
|
|
13305
13053
|
const log = deps.getReplicationLog(indexName, partitionId);
|
|
13306
13054
|
noteLedTerm(log, assignment.primaryTerm);
|
|
13307
13055
|
const applied = [];
|
|
@@ -13531,1067 +13279,810 @@ async function handleReshardPurge(message, respond, deps) {
|
|
|
13531
13279
|
const result = { removed };
|
|
13532
13280
|
await respond(reply(deps, message, result));
|
|
13533
13281
|
}
|
|
13534
|
-
|
|
13535
|
-
|
|
13536
|
-
|
|
13537
|
-
|
|
13282
|
+
async function handleIndexSettings(message, respond, deps) {
|
|
13283
|
+
const payload = validateIndexSettingsPayload(decode(message.payload));
|
|
13284
|
+
const applied = payload !== null && await applyStoredSynonyms(deps.engine, deps.coordinator, payload.indexName, payload.indexUuid);
|
|
13285
|
+
const indexName = payload?.indexName ?? "";
|
|
13286
|
+
await respond(createIndexSettingsResultMessage({ indexName, applied }, deps.nodeId, message.requestId));
|
|
13538
13287
|
}
|
|
13539
13288
|
|
|
13540
|
-
// src/distribution/
|
|
13541
|
-
var
|
|
13542
|
-
|
|
13543
|
-
|
|
13544
|
-
|
|
13545
|
-
|
|
13546
|
-
} catch (err) {
|
|
13547
|
-
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13548
|
-
return {
|
|
13549
|
-
outcome: "denied",
|
|
13550
|
-
code: ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
13551
|
-
reason: `allocation lookup failed: ${errMessage}`
|
|
13552
|
-
};
|
|
13553
|
-
}
|
|
13554
|
-
if (allocation === null) {
|
|
13555
|
-
return {
|
|
13556
|
-
outcome: "denied",
|
|
13557
|
-
code: ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
13558
|
-
reason: `index '${indexName}' has no allocation`
|
|
13559
|
-
};
|
|
13560
|
-
}
|
|
13561
|
-
let sawAssignment = false;
|
|
13562
|
-
for (const assignment of allocation.assignments.values()) {
|
|
13563
|
-
const isAssigned = assignment.primary === sourceId || assignment.replicas.includes(sourceId);
|
|
13564
|
-
if (!isAssigned) {
|
|
13565
|
-
continue;
|
|
13566
|
-
}
|
|
13567
|
-
sawAssignment = true;
|
|
13568
|
-
if (!BOOTSTRAPPABLE_STATES.has(assignment.state)) {
|
|
13569
|
-
continue;
|
|
13570
|
-
}
|
|
13571
|
-
return { outcome: "authorized" };
|
|
13572
|
-
}
|
|
13573
|
-
if (sawAssignment) {
|
|
13574
|
-
return {
|
|
13575
|
-
outcome: "denied",
|
|
13576
|
-
code: ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
13577
|
-
reason: `node '${sourceId}' is listed for index '${indexName}' but not in a bootstrappable state`
|
|
13578
|
-
};
|
|
13579
|
-
}
|
|
13580
|
-
return {
|
|
13581
|
-
outcome: "denied",
|
|
13582
|
-
code: ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
13583
|
-
reason: `node '${sourceId}' is not an assigned replica for index '${indexName}'`
|
|
13584
|
-
};
|
|
13585
|
-
}
|
|
13289
|
+
// src/distribution/query/types.ts
|
|
13290
|
+
var DEFAULT_QUERY_CONFIG = {
|
|
13291
|
+
allowPartialResults: true,
|
|
13292
|
+
partitionTimeout: 5e3,
|
|
13293
|
+
defaultFacetSize: 10
|
|
13294
|
+
};
|
|
13586
13295
|
|
|
13587
|
-
// src/distribution/cluster-node/
|
|
13588
|
-
function
|
|
13589
|
-
return { build, release: () => build.discard() };
|
|
13590
|
-
}
|
|
13591
|
-
function sharedLease(entry, build) {
|
|
13592
|
-
let released = false;
|
|
13296
|
+
// src/distribution/cluster-node/query-conversion.ts
|
|
13297
|
+
function wireParamsToLocal(wire, facetShardSize) {
|
|
13593
13298
|
return {
|
|
13594
|
-
|
|
13595
|
-
|
|
13596
|
-
|
|
13597
|
-
|
|
13598
|
-
|
|
13599
|
-
|
|
13600
|
-
|
|
13299
|
+
term: wire.term ?? void 0,
|
|
13300
|
+
fields: wire.fields ?? void 0,
|
|
13301
|
+
filters: wire.filters ?? void 0,
|
|
13302
|
+
boost: wire.boost ?? void 0,
|
|
13303
|
+
scoring: wire.scoring,
|
|
13304
|
+
tolerance: wire.tolerance ?? void 0,
|
|
13305
|
+
minScore: wire.threshold ?? void 0,
|
|
13306
|
+
includeScores: wire.includeScores ?? void 0,
|
|
13307
|
+
termMatch: wire.termMatch ?? void 0,
|
|
13308
|
+
prefixLength: wire.prefixLength ?? void 0,
|
|
13309
|
+
prefix: wire.prefix ?? void 0,
|
|
13310
|
+
partialWords: wire.partialWords ?? void 0,
|
|
13311
|
+
exact: wire.exact ?? void 0,
|
|
13312
|
+
pinned: wire.pinned !== null ? wire.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : void 0,
|
|
13313
|
+
mode: wire.mode ?? void 0,
|
|
13314
|
+
limit: wire.limit,
|
|
13315
|
+
offset: wire.offset,
|
|
13316
|
+
searchAfter: wire.searchAfter ?? void 0,
|
|
13317
|
+
sort: convertWireSortToLocal(wire.sort),
|
|
13318
|
+
group: convertWireGroupToLocal(wire.group),
|
|
13319
|
+
facets: convertWireFacetConfigToLocal(wire.facets, facetShardSize ?? wire.facetSize),
|
|
13320
|
+
vector: convertWireVectorToLocal(wire.vector),
|
|
13321
|
+
hybrid: convertWireHybridToLocal(wire.hybrid)
|
|
13601
13322
|
};
|
|
13602
13323
|
}
|
|
13603
|
-
|
|
13604
|
-
|
|
13605
|
-
|
|
13324
|
+
function convertWireGroupToLocal(group) {
|
|
13325
|
+
if (group === null) {
|
|
13326
|
+
return void 0;
|
|
13327
|
+
}
|
|
13328
|
+
const local = { fields: [...group.fields], maxPerGroup: group.maxPerGroup };
|
|
13329
|
+
if (group.limit !== null) local.limit = group.limit;
|
|
13330
|
+
return local;
|
|
13606
13331
|
}
|
|
13607
|
-
function
|
|
13608
|
-
if (
|
|
13609
|
-
|
|
13610
|
-
maxConcurrent
|
|
13611
|
-
});
|
|
13332
|
+
function convertWireVectorToLocal(vector) {
|
|
13333
|
+
if (vector === null) {
|
|
13334
|
+
return void 0;
|
|
13612
13335
|
}
|
|
13613
|
-
|
|
13614
|
-
|
|
13615
|
-
|
|
13616
|
-
|
|
13336
|
+
const local = { field: vector.field };
|
|
13337
|
+
if (vector.value !== null) local.value = vector.value;
|
|
13338
|
+
if (vector.text !== null) local.text = vector.text;
|
|
13339
|
+
if (vector.similarity !== null) local.similarity = vector.similarity;
|
|
13340
|
+
if (vector.metric !== null) local.metric = vector.metric;
|
|
13341
|
+
if (vector.efSearch !== null) local.efSearch = vector.efSearch;
|
|
13342
|
+
if (vector.oversample !== null) local.oversample = vector.oversample;
|
|
13343
|
+
return local;
|
|
13344
|
+
}
|
|
13345
|
+
function convertWireHybridToLocal(hybrid) {
|
|
13346
|
+
if (hybrid === null) {
|
|
13347
|
+
return void 0;
|
|
13617
13348
|
}
|
|
13618
|
-
|
|
13619
|
-
|
|
13620
|
-
|
|
13621
|
-
|
|
13622
|
-
|
|
13623
|
-
return
|
|
13624
|
-
cache: /* @__PURE__ */ new Map(),
|
|
13625
|
-
active: 0,
|
|
13626
|
-
perSource: /* @__PURE__ */ new Map(),
|
|
13627
|
-
streamActive: /* @__PURE__ */ new Map(),
|
|
13628
|
-
maxConcurrent,
|
|
13629
|
-
maxPerSource,
|
|
13630
|
-
maxStreamsPerIndex
|
|
13631
|
-
};
|
|
13632
|
-
}
|
|
13633
|
-
function sourceSlotKey(sourceId, indexName) {
|
|
13634
|
-
return `${sourceId}\0${indexName}`;
|
|
13635
|
-
}
|
|
13636
|
-
function acquireSourceSlot(state, sourceId, indexName) {
|
|
13637
|
-
const key = sourceSlotKey(sourceId, indexName);
|
|
13638
|
-
const current = state.perSource.get(key) ?? 0;
|
|
13639
|
-
if (current >= state.maxPerSource) {
|
|
13640
|
-
throw new NarsilError(
|
|
13641
|
-
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
13642
|
-
`snapshot producer per-source cap reached for '${sourceId}' index '${indexName}'`,
|
|
13643
|
-
{ sourceId, indexName, cap: state.maxPerSource }
|
|
13644
|
-
);
|
|
13645
|
-
}
|
|
13646
|
-
state.perSource.set(key, current + 1);
|
|
13647
|
-
return { key, released: false };
|
|
13349
|
+
const local = {};
|
|
13350
|
+
if (hybrid.strategy !== null) local.strategy = hybrid.strategy;
|
|
13351
|
+
if (hybrid.k !== null) local.k = hybrid.k;
|
|
13352
|
+
if (hybrid.alpha !== null) local.alpha = hybrid.alpha;
|
|
13353
|
+
if (hybrid.depth !== null) local.depth = hybrid.depth;
|
|
13354
|
+
return local;
|
|
13648
13355
|
}
|
|
13649
|
-
function
|
|
13650
|
-
if (
|
|
13651
|
-
return;
|
|
13652
|
-
}
|
|
13653
|
-
handle.released = true;
|
|
13654
|
-
const current = state.perSource.get(handle.key);
|
|
13655
|
-
if (current === void 0) {
|
|
13656
|
-
return;
|
|
13657
|
-
}
|
|
13658
|
-
if (current <= 1) {
|
|
13659
|
-
state.perSource.delete(handle.key);
|
|
13660
|
-
return;
|
|
13356
|
+
function convertWireSortToLocal(wireSort) {
|
|
13357
|
+
if (wireSort === null || wireSort.length === 0) {
|
|
13358
|
+
return void 0;
|
|
13661
13359
|
}
|
|
13662
|
-
|
|
13360
|
+
return normalizeSort(wireSort);
|
|
13663
13361
|
}
|
|
13664
|
-
function
|
|
13665
|
-
|
|
13666
|
-
if (current >= state.maxStreamsPerIndex) {
|
|
13667
|
-
throw new NarsilError(
|
|
13668
|
-
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
13669
|
-
`snapshot stream fan-out cap reached for index '${indexName}'`,
|
|
13670
|
-
{ indexName, cap: state.maxStreamsPerIndex }
|
|
13671
|
-
);
|
|
13672
|
-
}
|
|
13673
|
-
state.streamActive.set(indexName, current + 1);
|
|
13674
|
-
return { indexName, released: false };
|
|
13362
|
+
function countsEveryValue(facet) {
|
|
13363
|
+
return facet.ranges !== null || facet.sort === "asc";
|
|
13675
13364
|
}
|
|
13676
|
-
function
|
|
13677
|
-
if (
|
|
13678
|
-
return;
|
|
13679
|
-
}
|
|
13680
|
-
handle.released = true;
|
|
13681
|
-
const current = state.streamActive.get(handle.indexName);
|
|
13682
|
-
if (current === void 0) {
|
|
13683
|
-
return;
|
|
13365
|
+
function convertWireFacetConfigToLocal(facets, limit) {
|
|
13366
|
+
if (facets === null || facets.length === 0) {
|
|
13367
|
+
return void 0;
|
|
13684
13368
|
}
|
|
13685
|
-
|
|
13686
|
-
|
|
13687
|
-
|
|
13369
|
+
const result = {};
|
|
13370
|
+
for (const facet of facets) {
|
|
13371
|
+
const options = {};
|
|
13372
|
+
if (countsEveryValue(facet)) options.limit = MAX_FACET_BUCKETS;
|
|
13373
|
+
else if (limit !== null) options.limit = limit;
|
|
13374
|
+
if (facet.sort !== null) options.sort = facet.sort;
|
|
13375
|
+
if (facet.ranges !== null) options.ranges = facet.ranges.map((range) => ({ from: range.from, to: range.to }));
|
|
13376
|
+
result[facet.field] = options;
|
|
13688
13377
|
}
|
|
13689
|
-
|
|
13378
|
+
return result;
|
|
13690
13379
|
}
|
|
13691
|
-
|
|
13692
|
-
const
|
|
13693
|
-
|
|
13694
|
-
|
|
13695
|
-
}
|
|
13696
|
-
if (state.active >= state.maxConcurrent) {
|
|
13697
|
-
throw new NarsilError(
|
|
13698
|
-
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
13699
|
-
"snapshot producer capacity exhausted; retry later",
|
|
13700
|
-
{ active: state.active, max: state.maxConcurrent }
|
|
13701
|
-
);
|
|
13380
|
+
function ascendingFacetFields(facets) {
|
|
13381
|
+
const ascending = /* @__PURE__ */ new Set();
|
|
13382
|
+
for (const facet of facets ?? []) {
|
|
13383
|
+
if (facet.sort === "asc") ascending.add(facet.field);
|
|
13702
13384
|
}
|
|
13703
|
-
|
|
13704
|
-
|
|
13705
|
-
|
|
13706
|
-
|
|
13707
|
-
|
|
13708
|
-
|
|
13709
|
-
|
|
13710
|
-
|
|
13711
|
-
|
|
13712
|
-
|
|
13713
|
-
|
|
13714
|
-
|
|
13385
|
+
return ascending;
|
|
13386
|
+
}
|
|
13387
|
+
function localParamsToWire(params) {
|
|
13388
|
+
return {
|
|
13389
|
+
term: params.term ?? null,
|
|
13390
|
+
filters: params.filters ?? null,
|
|
13391
|
+
sort: convertLocalSortToWire(params.sort),
|
|
13392
|
+
group: convertLocalGroupToWire(params.group),
|
|
13393
|
+
facets: convertLocalFacetsToWire(params.facets),
|
|
13394
|
+
facetSize: wireFacetSize(params.facets),
|
|
13395
|
+
limit: clampRowCount(params.limit, DEFAULT_PAGE_SIZE),
|
|
13396
|
+
offset: clampRowCount(params.offset, 0),
|
|
13397
|
+
searchAfter: params.searchAfter ?? null,
|
|
13398
|
+
fields: params.fields ?? null,
|
|
13399
|
+
boost: params.boost ?? null,
|
|
13400
|
+
tolerance: params.tolerance ?? null,
|
|
13401
|
+
threshold: params.minScore ?? null,
|
|
13402
|
+
includeScores: params.includeScores ?? null,
|
|
13403
|
+
scoring: params.scoring ?? "local",
|
|
13404
|
+
termMatch: params.termMatch ?? null,
|
|
13405
|
+
prefixLength: params.prefixLength ?? null,
|
|
13406
|
+
prefix: params.prefix ?? null,
|
|
13407
|
+
partialWords: params.partialWords ?? null,
|
|
13408
|
+
exact: params.exact ?? null,
|
|
13409
|
+
pinned: params.pinned !== void 0 ? params.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : null,
|
|
13410
|
+
mode: params.mode ?? null,
|
|
13411
|
+
vector: convertLocalVectorToWire(params.vector),
|
|
13412
|
+
hybrid: convertLocalHybridToWire(params.hybrid)
|
|
13715
13413
|
};
|
|
13716
|
-
const rawBuild = (async () => builder())();
|
|
13717
|
-
const entry = { promise: rawBuild, holders: 0 };
|
|
13718
|
-
entry.promise = rawBuild.then(
|
|
13719
|
-
(result) => {
|
|
13720
|
-
clearEntry(entry);
|
|
13721
|
-
return result;
|
|
13722
|
-
},
|
|
13723
|
-
(err) => {
|
|
13724
|
-
clearEntry(entry);
|
|
13725
|
-
throw err;
|
|
13726
|
-
}
|
|
13727
|
-
);
|
|
13728
|
-
state.cache.set(indexName, entry);
|
|
13729
|
-
return joinSnapshotBuild(entry);
|
|
13730
13414
|
}
|
|
13731
|
-
|
|
13732
|
-
|
|
13733
|
-
|
|
13734
|
-
|
|
13735
|
-
|
|
13736
|
-
|
|
13737
|
-
return respond(response);
|
|
13738
|
-
});
|
|
13739
|
-
sink.closed = false;
|
|
13740
|
-
return sink;
|
|
13415
|
+
function countIsExactFor(params) {
|
|
13416
|
+
const vector = params.vector;
|
|
13417
|
+
if (vector === void 0 || vector.value === void 0 && vector.text === void 0) return true;
|
|
13418
|
+
const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
|
|
13419
|
+
if (params.mode === "hybrid" || hasTerm) return false;
|
|
13420
|
+
return vector.similarity === void 0;
|
|
13741
13421
|
}
|
|
13742
|
-
|
|
13743
|
-
|
|
13744
|
-
|
|
13745
|
-
|
|
13746
|
-
|
|
13747
|
-
|
|
13748
|
-
|
|
13749
|
-
|
|
13750
|
-
|
|
13751
|
-
|
|
13752
|
-
|
|
13753
|
-
|
|
13754
|
-
|
|
13755
|
-
|
|
13756
|
-
|
|
13757
|
-
|
|
13758
|
-
|
|
13759
|
-
|
|
13760
|
-
for (let from = 0; from < source.byteLength; from += SNAPSHOT_CHUNK_SIZE) {
|
|
13761
|
-
const chunk = source.subarray(from, Math.min(from + SNAPSHOT_CHUNK_SIZE, source.byteLength));
|
|
13762
|
-
const chunkPayload = {
|
|
13763
|
-
partitionId: metadata.partitionId,
|
|
13764
|
-
indexName,
|
|
13765
|
-
offset,
|
|
13766
|
-
data: chunk
|
|
13767
|
-
};
|
|
13768
|
-
const chunkBytes = encode(chunkPayload);
|
|
13769
|
-
assertMessageWithinLimit(chunkBytes, "SNAPSHOT_CHUNK");
|
|
13770
|
-
await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_CHUNK, nodeId, requestId, chunkBytes);
|
|
13771
|
-
offset += chunk.byteLength;
|
|
13772
|
-
if (offset < totalBytes && now2() - lastYieldAt >= SNAPSHOT_CHUNK_YIELD_INTERVAL_MS) {
|
|
13773
|
-
await yieldToEventLoop2();
|
|
13774
|
-
lastYieldAt = now2();
|
|
13775
|
-
}
|
|
13776
|
-
}
|
|
13777
|
-
}
|
|
13778
|
-
await yieldToEventLoop2();
|
|
13779
|
-
const endPayload = {
|
|
13780
|
-
partitionId: metadata.partitionId,
|
|
13781
|
-
indexName,
|
|
13782
|
-
totalBytes,
|
|
13783
|
-
checksum
|
|
13422
|
+
function distributedCountIsExact(params, coverage) {
|
|
13423
|
+
if (coverage.queriedPartitions < coverage.totalPartitions) return false;
|
|
13424
|
+
if (coverage.timedOutPartitions > 0 || coverage.failedPartitions > 0) return false;
|
|
13425
|
+
return countIsExactFor(params);
|
|
13426
|
+
}
|
|
13427
|
+
function distributedResultToLocal(result, countExact, documents = /* @__PURE__ */ new Map(), requestedFacets) {
|
|
13428
|
+
return {
|
|
13429
|
+
hits: result.scored.map((entry) => ({
|
|
13430
|
+
id: entry.docId,
|
|
13431
|
+
score: entry.score ?? void 0,
|
|
13432
|
+
document: documents.get(entry.docId) ?? {}
|
|
13433
|
+
})),
|
|
13434
|
+
count: result.totalHits,
|
|
13435
|
+
countExact,
|
|
13436
|
+
elapsed: 0,
|
|
13437
|
+
cursor: result.cursor ?? void 0,
|
|
13438
|
+
facets: result.facets !== null ? convertWireFacetsToLocal(result, result.facets, requestedFacets) : void 0,
|
|
13439
|
+
coverage: result.coverage
|
|
13784
13440
|
};
|
|
13785
|
-
const endBytes = encode(endPayload);
|
|
13786
|
-
assertMessageWithinLimit(endBytes, "SNAPSHOT_END");
|
|
13787
|
-
await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_END, nodeId, requestId, endBytes);
|
|
13788
|
-
if (options.closeOnEnd !== false) {
|
|
13789
|
-
sink.closed = true;
|
|
13790
|
-
}
|
|
13791
13441
|
}
|
|
13792
|
-
function
|
|
13793
|
-
|
|
13794
|
-
|
|
13795
|
-
}
|
|
13796
|
-
return Date.now();
|
|
13442
|
+
function convertLocalSortToWire(sort) {
|
|
13443
|
+
const fields = normalizeSort(sort);
|
|
13444
|
+
return fields.length === 0 ? null : fields;
|
|
13797
13445
|
}
|
|
13798
|
-
function
|
|
13799
|
-
|
|
13800
|
-
|
|
13801
|
-
|
|
13802
|
-
|
|
13803
|
-
|
|
13804
|
-
|
|
13805
|
-
|
|
13446
|
+
function convertLocalGroupToWire(group) {
|
|
13447
|
+
if (group === void 0) {
|
|
13448
|
+
return null;
|
|
13449
|
+
}
|
|
13450
|
+
return {
|
|
13451
|
+
fields: [...group.fields],
|
|
13452
|
+
maxPerGroup: group.reduce !== void 0 ? MAX_LIMIT : hitsKeptPerGroup(group.maxPerGroup),
|
|
13453
|
+
limit: group.limit ?? null
|
|
13454
|
+
};
|
|
13806
13455
|
}
|
|
13807
|
-
function
|
|
13808
|
-
return
|
|
13456
|
+
function wireFacetOrder(sort) {
|
|
13457
|
+
return sort === "asc" || sort === "desc" ? sort : null;
|
|
13809
13458
|
}
|
|
13810
|
-
function
|
|
13811
|
-
if (
|
|
13812
|
-
|
|
13813
|
-
ErrorCodes.CONFIG_INVALID,
|
|
13814
|
-
`${label} payload (${bytes.byteLength} bytes) exceeds transport message limit (${MAX_MESSAGE_SIZE_BYTES})`
|
|
13815
|
-
);
|
|
13459
|
+
function convertLocalFacetsToWire(facets) {
|
|
13460
|
+
if (facets === void 0) {
|
|
13461
|
+
return null;
|
|
13816
13462
|
}
|
|
13463
|
+
return Object.entries(facets).map(([field, options]) => ({
|
|
13464
|
+
field,
|
|
13465
|
+
sort: wireFacetOrder(options?.sort),
|
|
13466
|
+
ranges: options?.ranges?.map((range) => ({ from: range.from, to: range.to })) ?? null
|
|
13467
|
+
}));
|
|
13817
13468
|
}
|
|
13818
|
-
|
|
13819
|
-
|
|
13820
|
-
|
|
13821
|
-
sourceId,
|
|
13822
|
-
requestId,
|
|
13823
|
-
payload: encodeErrorReply(code, message, details)
|
|
13824
|
-
};
|
|
13825
|
-
await sink(response);
|
|
13826
|
-
sink.closed = true;
|
|
13827
|
-
}
|
|
13828
|
-
|
|
13829
|
-
// src/distribution/replication/snapshot-constants.ts
|
|
13830
|
-
var SNAPSHOT_HEADER_SENTINEL_SEQNO = Number.MAX_SAFE_INTEGER;
|
|
13831
|
-
var SNAPSHOT_HEADER_SENTINEL_PARTITION_ID = 4294967295;
|
|
13832
|
-
|
|
13833
|
-
// src/distribution/cluster-node/snapshot-sync-handler/metadata.ts
|
|
13834
|
-
async function defaultSnapshotHeaderMetadataProvider(coordinator, indexName) {
|
|
13835
|
-
try {
|
|
13836
|
-
const allocation = await coordinator.getAllocation(indexName);
|
|
13837
|
-
if (allocation !== null) {
|
|
13838
|
-
for (const assignment of allocation.assignments.values()) {
|
|
13839
|
-
if (Number.isInteger(assignment.primaryTerm) && assignment.primaryTerm >= 0) {
|
|
13840
|
-
return {
|
|
13841
|
-
partitionId: SNAPSHOT_HEADER_SENTINEL_PARTITION_ID,
|
|
13842
|
-
primaryTerm: assignment.primaryTerm,
|
|
13843
|
-
lastSeqNo: SNAPSHOT_HEADER_SENTINEL_SEQNO
|
|
13844
|
-
};
|
|
13845
|
-
}
|
|
13846
|
-
}
|
|
13847
|
-
}
|
|
13848
|
-
} catch (_) {
|
|
13469
|
+
function convertLocalVectorToWire(vector) {
|
|
13470
|
+
if (vector === void 0) {
|
|
13471
|
+
return null;
|
|
13849
13472
|
}
|
|
13850
13473
|
return {
|
|
13851
|
-
|
|
13852
|
-
|
|
13853
|
-
|
|
13474
|
+
field: vector.field,
|
|
13475
|
+
value: vector.value === void 0 ? null : Array.from(vector.value),
|
|
13476
|
+
text: vector.text ?? null,
|
|
13477
|
+
similarity: vector.similarity ?? null,
|
|
13478
|
+
metric: vector.metric ?? null,
|
|
13479
|
+
efSearch: vector.efSearch ?? null,
|
|
13480
|
+
oversample: vector.oversample ?? null,
|
|
13481
|
+
rescoreWindow: null
|
|
13854
13482
|
};
|
|
13855
13483
|
}
|
|
13856
|
-
|
|
13857
|
-
|
|
13858
|
-
|
|
13859
|
-
return fallback;
|
|
13860
|
-
}
|
|
13861
|
-
const localIndex = deps.engine.listIndexes().find((index) => index.name === indexName);
|
|
13862
|
-
const release = localIndex !== void 0 ? await deps.engine.acquireIndexForReplication(indexName) : null;
|
|
13863
|
-
try {
|
|
13864
|
-
const allocation = await deps.coordinator.getAllocation(indexName);
|
|
13865
|
-
const assignment = allocation?.assignments.get(partitionId);
|
|
13866
|
-
const log = deps.getReplicationLog(indexName, partitionId);
|
|
13867
|
-
return {
|
|
13868
|
-
partitionId,
|
|
13869
|
-
primaryTerm: assignment?.primaryTerm ?? fallback.primaryTerm,
|
|
13870
|
-
lastSeqNo: log.newestSeqNo ?? 0
|
|
13871
|
-
};
|
|
13872
|
-
} finally {
|
|
13873
|
-
release?.();
|
|
13484
|
+
function convertLocalHybridToWire(hybrid) {
|
|
13485
|
+
if (hybrid === void 0) {
|
|
13486
|
+
return null;
|
|
13874
13487
|
}
|
|
13488
|
+
return {
|
|
13489
|
+
strategy: hybrid.strategy ?? null,
|
|
13490
|
+
k: hybrid.k ?? null,
|
|
13491
|
+
alpha: hybrid.alpha ?? null,
|
|
13492
|
+
depth: hybrid.depth ?? null
|
|
13493
|
+
};
|
|
13875
13494
|
}
|
|
13876
|
-
|
|
13877
|
-
const
|
|
13878
|
-
|
|
13879
|
-
|
|
13495
|
+
function facetLimitOf(options) {
|
|
13496
|
+
const limit = options?.limit;
|
|
13497
|
+
const ranges = options?.ranges;
|
|
13498
|
+
if ((limit === void 0 || !Number.isFinite(limit)) && Array.isArray(ranges)) {
|
|
13499
|
+
return Math.min(Math.max(ranges.length, 1), MAX_FACET_SIZE);
|
|
13880
13500
|
}
|
|
13881
|
-
|
|
13882
|
-
|
|
13883
|
-
|
|
13884
|
-
|
|
13885
|
-
|
|
13886
|
-
|
|
13887
|
-
|
|
13888
|
-
|
|
13501
|
+
if (limit === void 0 || !Number.isFinite(limit)) return DEFAULT_QUERY_CONFIG.defaultFacetSize;
|
|
13502
|
+
return Math.min(Math.max(Math.floor(limit), 1), MAX_FACET_SIZE);
|
|
13503
|
+
}
|
|
13504
|
+
function wireFacetSize(facets) {
|
|
13505
|
+
if (facets === void 0) return null;
|
|
13506
|
+
let size = 0;
|
|
13507
|
+
for (const options of Object.values(facets)) size = Math.max(size, facetLimitOf(options));
|
|
13508
|
+
return size === 0 ? null : size;
|
|
13509
|
+
}
|
|
13510
|
+
function convertWireFacetsToLocal(result, wireFacets, requested) {
|
|
13511
|
+
const converted = {};
|
|
13512
|
+
for (const [field, buckets] of Object.entries(wireFacets)) {
|
|
13513
|
+
const mergedBound = result.facetErrorBounds?.[field] ?? 0;
|
|
13514
|
+
const kept = requested === void 0 ? buckets.length : Math.min(buckets.length, facetLimitOf(requested[field]));
|
|
13515
|
+
const values = {};
|
|
13516
|
+
for (let index = 0; index < kept; index++) values[buckets[index].value] = buckets[index].count;
|
|
13517
|
+
let largestCut = 0;
|
|
13518
|
+
for (let index = kept; index < buckets.length; index++) largestCut = Math.max(largestCut, buckets[index].count);
|
|
13519
|
+
const errorBound = kept < buckets.length ? (result.facetUndercounts?.[field] ?? mergedBound) + largestCut : mergedBound;
|
|
13520
|
+
converted[field] = { values, count: kept, errorBound: Math.max(errorBound, mergedBound) };
|
|
13889
13521
|
}
|
|
13522
|
+
return converted;
|
|
13890
13523
|
}
|
|
13891
13524
|
|
|
13892
|
-
// src/distribution/cluster-node/
|
|
13893
|
-
async function
|
|
13894
|
-
const
|
|
13895
|
-
|
|
13896
|
-
|
|
13897
|
-
|
|
13898
|
-
|
|
13899
|
-
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
|
|
13900
|
-
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13901
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
13902
|
-
return;
|
|
13525
|
+
// src/distribution/cluster-node/message-handler/queries.ts
|
|
13526
|
+
async function handleSearch(message, respond, deps) {
|
|
13527
|
+
const decoded = decode(message.payload);
|
|
13528
|
+
const payload = validateSearchPayload(decoded);
|
|
13529
|
+
const queryParams = wireParamsToLocal(payload.params, payload.facetShardSize);
|
|
13530
|
+
if (queryParams.group?.limit !== void 0) {
|
|
13531
|
+
queryParams.group = { ...queryParams.group, limit: oversampledShardSize(queryParams.group.limit) };
|
|
13903
13532
|
}
|
|
13904
|
-
|
|
13905
|
-
|
|
13906
|
-
|
|
13907
|
-
const buildForRequest = async () => buildSnapshotForRequest(deps, indexName, options);
|
|
13908
|
-
lease = options.disableBuildCache === true ? soleSnapshotLease(await buildForRequest()) : await acquireSnapshotBuild(deps.state, indexName, buildForRequest);
|
|
13909
|
-
} catch (err) {
|
|
13910
|
-
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
|
|
13911
|
-
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13912
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
13913
|
-
return;
|
|
13914
|
-
}
|
|
13915
|
-
try {
|
|
13916
|
-
await streamLeasedSnapshot(message, sink, request, deps, options, lease.build);
|
|
13917
|
-
} finally {
|
|
13918
|
-
await lease.release();
|
|
13919
|
-
}
|
|
13920
|
-
} finally {
|
|
13921
|
-
releaseSourceSlot(deps.state, sourceHandle);
|
|
13533
|
+
const rescoreWindow = payload.params.vector?.rescoreWindow ?? void 0;
|
|
13534
|
+
if (rescoreWindow !== void 0 && queryParams.sort === void 0 && queryParams.group === void 0) {
|
|
13535
|
+
queryParams.document = false;
|
|
13922
13536
|
}
|
|
13537
|
+
const queryResult = await deps.engine.queryPartitions(
|
|
13538
|
+
payload.indexName,
|
|
13539
|
+
queryParams,
|
|
13540
|
+
payload.partitionIds,
|
|
13541
|
+
payload.globalStats ?? void 0,
|
|
13542
|
+
rescoreWindow
|
|
13543
|
+
);
|
|
13544
|
+
const normalizedSort = queryParams.sort !== void 0 ? normalizeSort(queryParams.sort) : null;
|
|
13545
|
+
const sortFields = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
13546
|
+
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
13547
|
+
const scored = queryResult.hits.map((hit) => ({
|
|
13548
|
+
docId: hit.id,
|
|
13549
|
+
score: hit.score ?? null,
|
|
13550
|
+
sortValues: sortFields !== null ? readSortValues(hit.document, sortFields, sortModes).map(toWireSortValue) : null
|
|
13551
|
+
}));
|
|
13552
|
+
const results = payload.partitionIds.map((partitionId) => ({
|
|
13553
|
+
partitionId,
|
|
13554
|
+
scored: partitionId === payload.partitionIds[0] ? scored : [],
|
|
13555
|
+
totalHits: partitionId === payload.partitionIds[0] ? queryResult.count : 0
|
|
13556
|
+
}));
|
|
13557
|
+
const resultPayload = {
|
|
13558
|
+
results,
|
|
13559
|
+
facets: convertLocalFacetsToWire2(queryResult.facets),
|
|
13560
|
+
facetErrorBounds: convertLocalFacetBoundsToWire(queryResult.facets),
|
|
13561
|
+
groups: await convertLocalGroupsToWire(queryResult.groups, sortFields, sortModes, deps, payload.indexName)
|
|
13562
|
+
};
|
|
13563
|
+
await respond({
|
|
13564
|
+
type: QueryMessageTypes.SEARCH_RESULT,
|
|
13565
|
+
sourceId: deps.nodeId,
|
|
13566
|
+
requestId: message.requestId,
|
|
13567
|
+
payload: encodeDocumentReply(resultPayload)
|
|
13568
|
+
});
|
|
13923
13569
|
}
|
|
13924
|
-
async function
|
|
13925
|
-
|
|
13926
|
-
|
|
13927
|
-
try {
|
|
13928
|
-
streamHandle = acquireStreamSlot(deps.state, indexName);
|
|
13929
|
-
} catch (err) {
|
|
13930
|
-
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
|
|
13931
|
-
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13932
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
13933
|
-
return;
|
|
13570
|
+
async function convertLocalGroupsToWire(groups, sortFields, sortModes, deps, indexName) {
|
|
13571
|
+
if (groups === void 0) {
|
|
13572
|
+
return null;
|
|
13934
13573
|
}
|
|
13935
|
-
|
|
13936
|
-
|
|
13937
|
-
|
|
13938
|
-
|
|
13939
|
-
|
|
13940
|
-
|
|
13941
|
-
|
|
13942
|
-
|
|
13943
|
-
|
|
13574
|
+
const unreadIds = sortFields === null ? [] : groups.flatMap((group) => group.hits.filter((hit) => hit.document === void 0).map((hit) => hit.id));
|
|
13575
|
+
const readDocuments = unreadIds.length > 0 ? await deps.engine.getMultiple(indexName, unreadIds) : /* @__PURE__ */ new Map();
|
|
13576
|
+
const wireGroups = [];
|
|
13577
|
+
for (const group of groups) {
|
|
13578
|
+
const scored = group.hits.map((hit) => {
|
|
13579
|
+
if (sortFields === null) return { docId: hit.id, score: hit.score ?? null, sortValues: null };
|
|
13580
|
+
const document = hit.document ?? readDocuments.get(hit.id);
|
|
13581
|
+
const sortValues = readSortValues(document, sortFields, sortModes).map(toWireSortValue);
|
|
13582
|
+
return { docId: hit.id, score: hit.score ?? null, sortValues };
|
|
13944
13583
|
});
|
|
13945
|
-
|
|
13946
|
-
if (options.closeOnEnd === false) {
|
|
13947
|
-
sink.closed = true;
|
|
13948
|
-
}
|
|
13949
|
-
} finally {
|
|
13950
|
-
releaseStreamSlot(deps.state, streamHandle);
|
|
13584
|
+
wireGroups.push({ values: group.values, scored });
|
|
13951
13585
|
}
|
|
13586
|
+
return wireGroups;
|
|
13952
13587
|
}
|
|
13953
|
-
async function
|
|
13954
|
-
|
|
13955
|
-
|
|
13956
|
-
|
|
13957
|
-
|
|
13958
|
-
|
|
13959
|
-
|
|
13960
|
-
if (
|
|
13961
|
-
|
|
13588
|
+
async function handleFetch(message, respond, deps) {
|
|
13589
|
+
const decoded = decode(message.payload);
|
|
13590
|
+
const payload = validateFetchPayload(decoded);
|
|
13591
|
+
const projection = resolveProjection(payload.fields === null ? void 0 : { include: payload.fields });
|
|
13592
|
+
const documents = [];
|
|
13593
|
+
for (const docRef of payload.documentIds) {
|
|
13594
|
+
const doc = await deps.engine.get(payload.indexName, docRef.docId);
|
|
13595
|
+
if (doc !== void 0) {
|
|
13596
|
+
documents.push({
|
|
13597
|
+
docId: docRef.docId,
|
|
13598
|
+
document: applyProjection(doc, projection),
|
|
13599
|
+
highlights: null
|
|
13600
|
+
});
|
|
13962
13601
|
}
|
|
13963
|
-
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13964
|
-
throw new NarsilError(ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED, `engine.snapshot failed: ${errMessage}`, {
|
|
13965
|
-
indexName,
|
|
13966
|
-
cause: errMessage
|
|
13967
|
-
});
|
|
13968
13602
|
}
|
|
13603
|
+
const resultPayload = { documents };
|
|
13604
|
+
await respond({
|
|
13605
|
+
type: QueryMessageTypes.FETCH_RESULT,
|
|
13606
|
+
sourceId: deps.nodeId,
|
|
13607
|
+
requestId: message.requestId,
|
|
13608
|
+
payload: encodeDocumentReply(resultPayload)
|
|
13609
|
+
});
|
|
13969
13610
|
}
|
|
13970
|
-
function
|
|
13971
|
-
|
|
13972
|
-
|
|
13973
|
-
|
|
13974
|
-
|
|
13611
|
+
async function handleStats(message, respond, deps) {
|
|
13612
|
+
const decoded = decode(message.payload);
|
|
13613
|
+
const payload = validateStatsPayload(decoded);
|
|
13614
|
+
const resultPayload = await deps.engine.collectQueryStats(
|
|
13615
|
+
payload.indexName,
|
|
13616
|
+
payload.terms,
|
|
13617
|
+
payload.partitionIds,
|
|
13618
|
+
{ expansion: payload.expansion ?? void 0, tokens: payload.tokens ?? void 0 }
|
|
13619
|
+
);
|
|
13620
|
+
await respond({
|
|
13621
|
+
type: QueryMessageTypes.STATS_RESULT,
|
|
13622
|
+
sourceId: deps.nodeId,
|
|
13623
|
+
requestId: message.requestId,
|
|
13624
|
+
payload: encode(resultPayload)
|
|
13625
|
+
});
|
|
13975
13626
|
}
|
|
13976
|
-
function
|
|
13977
|
-
|
|
13978
|
-
|
|
13979
|
-
|
|
13980
|
-
|
|
13981
|
-
}
|
|
13982
|
-
if (code >= 127 && code <= 159) {
|
|
13983
|
-
return true;
|
|
13984
|
-
}
|
|
13985
|
-
if (code === 8232 || code === 8233 || code === 65279) {
|
|
13986
|
-
return true;
|
|
13987
|
-
}
|
|
13988
|
-
}
|
|
13989
|
-
return false;
|
|
13627
|
+
function toWireSortValue(value) {
|
|
13628
|
+
if (typeof value === "string") return value;
|
|
13629
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
13630
|
+
if (typeof value === "boolean") return value;
|
|
13631
|
+
return null;
|
|
13990
13632
|
}
|
|
13991
|
-
|
|
13992
|
-
|
|
13993
|
-
maxArrayLength: 16,
|
|
13994
|
-
maxStrLength: MAX_INDEX_NAME_LENGTH,
|
|
13995
|
-
maxBinLength: 0,
|
|
13996
|
-
maxExtLength: 0,
|
|
13997
|
-
keyDecoder: null
|
|
13998
|
-
};
|
|
13999
|
-
async function decodeRequest(message, sink, deps) {
|
|
14000
|
-
try {
|
|
14001
|
-
const decoded = decode(message.payload, REQUEST_DECODE_OPTIONS);
|
|
14002
|
-
return validateSnapshotSyncRequestPayload(decoded);
|
|
14003
|
-
} catch (err) {
|
|
14004
|
-
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID;
|
|
14005
|
-
const errMessage = err instanceof Error ? err.message : String(err);
|
|
14006
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
13633
|
+
function convertLocalFacetBoundsToWire(facets) {
|
|
13634
|
+
if (facets === void 0) {
|
|
14007
13635
|
return null;
|
|
14008
13636
|
}
|
|
14009
|
-
}
|
|
14010
|
-
|
|
14011
|
-
|
|
14012
|
-
return "request sourceId is missing";
|
|
13637
|
+
const result = {};
|
|
13638
|
+
for (const [field, facet] of Object.entries(facets)) {
|
|
13639
|
+
result[field] = facet.errorBound;
|
|
14013
13640
|
}
|
|
14014
|
-
|
|
14015
|
-
|
|
13641
|
+
return result;
|
|
13642
|
+
}
|
|
13643
|
+
function convertLocalFacetsToWire2(facets) {
|
|
13644
|
+
if (facets === void 0) {
|
|
13645
|
+
return null;
|
|
14016
13646
|
}
|
|
14017
|
-
|
|
14018
|
-
|
|
13647
|
+
const result = {};
|
|
13648
|
+
for (const [field, facet] of Object.entries(facets)) {
|
|
13649
|
+
result[field] = Object.entries(facet.values).map(([value, count2]) => ({ value, count: count2 }));
|
|
14019
13650
|
}
|
|
14020
|
-
return
|
|
13651
|
+
return result;
|
|
14021
13652
|
}
|
|
14022
|
-
function
|
|
14023
|
-
|
|
14024
|
-
|
|
14025
|
-
|
|
14026
|
-
|
|
14027
|
-
|
|
14028
|
-
|
|
14029
|
-
|
|
14030
|
-
const
|
|
14031
|
-
|
|
14032
|
-
throw new NarsilError(
|
|
14033
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14034
|
-
'Invalid SnapshotSyncRequestPayload: "indexName" must be a non-empty string'
|
|
14035
|
-
);
|
|
14036
|
-
}
|
|
14037
|
-
if (indexName.length > MAX_INDEX_NAME_LENGTH) {
|
|
14038
|
-
throw new NarsilError(
|
|
14039
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14040
|
-
`Invalid SnapshotSyncRequestPayload: "indexName" must be at most ${MAX_INDEX_NAME_LENGTH} characters`,
|
|
14041
|
-
{ length: indexName.length, limit: MAX_INDEX_NAME_LENGTH }
|
|
14042
|
-
);
|
|
14043
|
-
}
|
|
14044
|
-
if (!INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
14045
|
-
throw new NarsilError(
|
|
14046
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14047
|
-
'Invalid SnapshotSyncRequestPayload: "indexName" contains invalid characters'
|
|
14048
|
-
);
|
|
14049
|
-
}
|
|
14050
|
-
const indexUuid = record.indexUuid;
|
|
14051
|
-
if (indexUuid !== void 0 && indexUuid !== null && (typeof indexUuid !== "string" || indexUuid.length === 0)) {
|
|
14052
|
-
throw new NarsilError(
|
|
14053
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14054
|
-
'Invalid SnapshotSyncRequestPayload: "indexUuid" must be a non-empty string when provided'
|
|
14055
|
-
);
|
|
14056
|
-
}
|
|
14057
|
-
const identity = typeof indexUuid === "string" ? { indexUuid } : {};
|
|
14058
|
-
const partitionId = record.partitionId;
|
|
14059
|
-
if (partitionId !== void 0 && partitionId !== null) {
|
|
14060
|
-
if (typeof partitionId !== "number" || !Number.isInteger(partitionId) || partitionId < 0) {
|
|
14061
|
-
throw new NarsilError(
|
|
14062
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
14063
|
-
'Invalid SnapshotSyncRequestPayload: "partitionId" must be a non-negative integer when provided'
|
|
14064
|
-
);
|
|
14065
|
-
}
|
|
14066
|
-
return { indexName, partitionId, ...identity };
|
|
14067
|
-
}
|
|
14068
|
-
return { indexName, partitionId: null, ...identity };
|
|
14069
|
-
}
|
|
14070
|
-
|
|
14071
|
-
// src/distribution/cluster-node/snapshot-sync-handler/index.ts
|
|
14072
|
-
function createSnapshotSyncHandlerState(maxConcurrent = DEFAULT_MAX_CONCURRENT_SNAPSHOTS, maxPerSource = DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, maxStreamsPerIndex = DEFAULT_MAX_STREAMS_PER_INDEX) {
|
|
14073
|
-
return createSnapshotCacheState(maxConcurrent, maxPerSource, maxStreamsPerIndex);
|
|
13653
|
+
async function handleCount(message, respond, deps) {
|
|
13654
|
+
const payload = validateCountPayload(decode(message.payload));
|
|
13655
|
+
const requested = new Set(payload.partitionIds);
|
|
13656
|
+
const partitions = (await deps.engine.partitionStatsForRead(payload.indexName)).filter((partition) => requested.has(partition.partitionId)).map((partition) => ({
|
|
13657
|
+
partitionId: partition.partitionId,
|
|
13658
|
+
documentCount: partition.documentCount,
|
|
13659
|
+
estimatedMemoryBytes: partition.estimatedMemoryBytes
|
|
13660
|
+
}));
|
|
13661
|
+
const language = deps.engine.getStats(payload.indexName).language;
|
|
13662
|
+
await respond(createCountResultMessage({ partitions, language }, deps.nodeId, message.requestId));
|
|
14074
13663
|
}
|
|
14075
|
-
async function
|
|
14076
|
-
const
|
|
14077
|
-
|
|
14078
|
-
|
|
14079
|
-
|
|
14080
|
-
|
|
14081
|
-
|
|
14082
|
-
|
|
14083
|
-
|
|
14084
|
-
|
|
14085
|
-
|
|
14086
|
-
|
|
14087
|
-
|
|
14088
|
-
|
|
14089
|
-
|
|
14090
|
-
|
|
14091
|
-
|
|
14092
|
-
|
|
14093
|
-
|
|
13664
|
+
async function handleList(message, respond, deps) {
|
|
13665
|
+
const payload = validateListPayload(decode(message.payload));
|
|
13666
|
+
const listParams = {
|
|
13667
|
+
cursor: payload.cursor ?? void 0,
|
|
13668
|
+
limit: payload.limit,
|
|
13669
|
+
filters: payload.filters ?? void 0,
|
|
13670
|
+
sort: payload.sort ?? void 0,
|
|
13671
|
+
document: payload.fields === null ? void 0 : { include: payload.fields }
|
|
13672
|
+
};
|
|
13673
|
+
const result = await deps.engine.listPartitions(payload.indexName, listParams, payload.partitionIds);
|
|
13674
|
+
const normalizedSort = payload.sort === null ? null : normalizeSort(payload.sort);
|
|
13675
|
+
const sortFields = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
13676
|
+
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
13677
|
+
const entries = result.documents.map((listed) => ({
|
|
13678
|
+
docId: listed.id,
|
|
13679
|
+
document: listed.document,
|
|
13680
|
+
sortValues: sortFields === null ? null : readSortValues(listed.document, sortFields, sortModes).map(toWireSortValue2)
|
|
13681
|
+
}));
|
|
13682
|
+
await respond(
|
|
13683
|
+
createListResultMessage(
|
|
13684
|
+
{ entries, total: result.total, hasMore: result.cursor !== null },
|
|
13685
|
+
deps.nodeId,
|
|
13686
|
+
message.requestId
|
|
13687
|
+
)
|
|
13688
|
+
);
|
|
14094
13689
|
}
|
|
14095
|
-
async function
|
|
14096
|
-
const
|
|
14097
|
-
|
|
14098
|
-
|
|
14099
|
-
|
|
14100
|
-
|
|
13690
|
+
async function handleSuggest(message, respond, deps) {
|
|
13691
|
+
const payload = validateSuggestPayload(decode(message.payload));
|
|
13692
|
+
const result = await deps.engine.suggestPartitions(
|
|
13693
|
+
payload.indexName,
|
|
13694
|
+
{ prefix: payload.prefix, limit: payload.limit },
|
|
13695
|
+
payload.partitionIds
|
|
13696
|
+
);
|
|
13697
|
+
await respond(
|
|
13698
|
+
createSuggestResultMessage(
|
|
13699
|
+
{ terms: result.terms, analysisStale: result.analysisStale === true },
|
|
13700
|
+
deps.nodeId,
|
|
13701
|
+
message.requestId
|
|
13702
|
+
)
|
|
13703
|
+
);
|
|
14101
13704
|
}
|
|
14102
|
-
async function
|
|
14103
|
-
const
|
|
14104
|
-
|
|
14105
|
-
|
|
14106
|
-
|
|
14107
|
-
|
|
14108
|
-
|
|
14109
|
-
if (authResult.outcome === "denied") {
|
|
14110
|
-
await respondError(sink, deps.nodeId, message.requestId, authResult.code, authResult.reason);
|
|
14111
|
-
return;
|
|
14112
|
-
}
|
|
14113
|
-
const existingIndex = deps.engine.listIndexes().find((idx) => idx.name === request.indexName);
|
|
14114
|
-
if (existingIndex === void 0 || holdsAnotherIndex(deps.indexUuidOf?.(request.indexName), request.indexUuid)) {
|
|
14115
|
-
await respondError(
|
|
14116
|
-
sink,
|
|
13705
|
+
async function handlePreflight(message, respond, deps) {
|
|
13706
|
+
const payload = validatePreflightPayload(decode(message.payload));
|
|
13707
|
+
const params = wireParamsToLocal(payload.params);
|
|
13708
|
+
const result = await deps.engine.preflightPartitions(payload.indexName, params, payload.partitionIds);
|
|
13709
|
+
await respond(
|
|
13710
|
+
createPreflightResultMessage(
|
|
13711
|
+
{ count: result.count, analysisStale: result.analysisStale === true },
|
|
14117
13712
|
deps.nodeId,
|
|
14118
|
-
message.requestId
|
|
14119
|
-
|
|
14120
|
-
|
|
14121
|
-
);
|
|
14122
|
-
return;
|
|
14123
|
-
}
|
|
14124
|
-
await acquireAndStream(message, sink, request, deps, options);
|
|
13713
|
+
message.requestId
|
|
13714
|
+
)
|
|
13715
|
+
);
|
|
14125
13716
|
}
|
|
14126
|
-
|
|
14127
|
-
|
|
14128
|
-
|
|
14129
|
-
|
|
14130
|
-
|
|
13717
|
+
function toWireSortValue2(value) {
|
|
13718
|
+
if (typeof value === "string") return value;
|
|
13719
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
13720
|
+
if (typeof value === "boolean") return value;
|
|
13721
|
+
return null;
|
|
14131
13722
|
}
|
|
14132
13723
|
|
|
14133
|
-
// src/distribution/
|
|
14134
|
-
|
|
14135
|
-
|
|
14136
|
-
|
|
14137
|
-
defaultFacetSize: 10
|
|
14138
|
-
};
|
|
14139
|
-
|
|
14140
|
-
// src/distribution/cluster-node/query-conversion.ts
|
|
14141
|
-
function wireParamsToLocal(wire, facetShardSize) {
|
|
14142
|
-
return {
|
|
14143
|
-
term: wire.term ?? void 0,
|
|
14144
|
-
fields: wire.fields ?? void 0,
|
|
14145
|
-
filters: wire.filters ?? void 0,
|
|
14146
|
-
boost: wire.boost ?? void 0,
|
|
14147
|
-
scoring: wire.scoring,
|
|
14148
|
-
tolerance: wire.tolerance ?? void 0,
|
|
14149
|
-
minScore: wire.threshold ?? void 0,
|
|
14150
|
-
includeScores: wire.includeScores ?? void 0,
|
|
14151
|
-
termMatch: wire.termMatch ?? void 0,
|
|
14152
|
-
prefixLength: wire.prefixLength ?? void 0,
|
|
14153
|
-
prefix: wire.prefix ?? void 0,
|
|
14154
|
-
partialWords: wire.partialWords ?? void 0,
|
|
14155
|
-
exact: wire.exact ?? void 0,
|
|
14156
|
-
pinned: wire.pinned !== null ? wire.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : void 0,
|
|
14157
|
-
mode: wire.mode ?? void 0,
|
|
14158
|
-
limit: wire.limit,
|
|
14159
|
-
offset: wire.offset,
|
|
14160
|
-
searchAfter: wire.searchAfter ?? void 0,
|
|
14161
|
-
sort: convertWireSortToLocal(wire.sort),
|
|
14162
|
-
group: convertWireGroupToLocal(wire.group),
|
|
14163
|
-
facets: convertWireFacetConfigToLocal(wire.facets, facetShardSize ?? wire.facetSize),
|
|
14164
|
-
vector: convertWireVectorToLocal(wire.vector),
|
|
14165
|
-
hybrid: convertWireHybridToLocal(wire.hybrid)
|
|
14166
|
-
};
|
|
14167
|
-
}
|
|
14168
|
-
function convertWireGroupToLocal(group) {
|
|
14169
|
-
if (group === null) {
|
|
14170
|
-
return void 0;
|
|
14171
|
-
}
|
|
14172
|
-
const local = { fields: [...group.fields], maxPerGroup: group.maxPerGroup };
|
|
14173
|
-
if (group.limit !== null) local.limit = group.limit;
|
|
14174
|
-
return local;
|
|
14175
|
-
}
|
|
14176
|
-
function convertWireVectorToLocal(vector) {
|
|
14177
|
-
if (vector === null) {
|
|
14178
|
-
return void 0;
|
|
13724
|
+
// src/distribution/replication/sync-primary.ts
|
|
13725
|
+
function decideSyncTier(log, replicaLastSeqNo, replicaLastPrimaryTerm) {
|
|
13726
|
+
if (replicaLastSeqNo === 0) {
|
|
13727
|
+
return log.localLogEnd === 0 || log.oldestSeqNo === 1 ? "incremental" : "snapshot";
|
|
14179
13728
|
}
|
|
14180
|
-
const
|
|
14181
|
-
|
|
14182
|
-
if (vector.text !== null) local.text = vector.text;
|
|
14183
|
-
if (vector.similarity !== null) local.similarity = vector.similarity;
|
|
14184
|
-
if (vector.metric !== null) local.metric = vector.metric;
|
|
14185
|
-
if (vector.efSearch !== null) local.efSearch = vector.efSearch;
|
|
14186
|
-
if (vector.oversample !== null) local.oversample = vector.oversample;
|
|
14187
|
-
return local;
|
|
13729
|
+
const primaryTerm = log.termAt(replicaLastSeqNo);
|
|
13730
|
+
return primaryTerm !== void 0 && primaryTerm === replicaLastPrimaryTerm ? "incremental" : "snapshot";
|
|
14188
13731
|
}
|
|
14189
|
-
function
|
|
14190
|
-
if (
|
|
14191
|
-
|
|
13732
|
+
function validateSyncRequest(decoded) {
|
|
13733
|
+
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
13734
|
+
throw new Error("Invalid SyncRequestPayload: expected an object");
|
|
14192
13735
|
}
|
|
14193
|
-
const
|
|
14194
|
-
if (
|
|
14195
|
-
|
|
14196
|
-
if (hybrid.alpha !== null) local.alpha = hybrid.alpha;
|
|
14197
|
-
if (hybrid.depth !== null) local.depth = hybrid.depth;
|
|
14198
|
-
return local;
|
|
14199
|
-
}
|
|
14200
|
-
function convertWireSortToLocal(wireSort) {
|
|
14201
|
-
if (wireSort === null || wireSort.length === 0) {
|
|
14202
|
-
return void 0;
|
|
13736
|
+
const record = decoded;
|
|
13737
|
+
if (typeof record.indexName !== "string") {
|
|
13738
|
+
throw new Error('Invalid SyncRequestPayload: "indexName" must be a string');
|
|
14203
13739
|
}
|
|
14204
|
-
|
|
14205
|
-
|
|
14206
|
-
function countsEveryValue(facet) {
|
|
14207
|
-
return facet.ranges !== null || facet.sort === "asc";
|
|
14208
|
-
}
|
|
14209
|
-
function convertWireFacetConfigToLocal(facets, limit) {
|
|
14210
|
-
if (facets === null || facets.length === 0) {
|
|
14211
|
-
return void 0;
|
|
13740
|
+
if (typeof record.partitionId !== "number" || !Number.isInteger(record.partitionId) || record.partitionId < 0) {
|
|
13741
|
+
throw new Error('Invalid SyncRequestPayload: "partitionId" must be a non-negative integer');
|
|
14212
13742
|
}
|
|
14213
|
-
|
|
14214
|
-
|
|
14215
|
-
const options = {};
|
|
14216
|
-
if (countsEveryValue(facet)) options.limit = MAX_FACET_BUCKETS;
|
|
14217
|
-
else if (limit !== null) options.limit = limit;
|
|
14218
|
-
if (facet.sort !== null) options.sort = facet.sort;
|
|
14219
|
-
if (facet.ranges !== null) options.ranges = facet.ranges.map((range) => ({ from: range.from, to: range.to }));
|
|
14220
|
-
result[facet.field] = options;
|
|
13743
|
+
if (typeof record.lastSeqNo !== "number" || !Number.isInteger(record.lastSeqNo) || record.lastSeqNo < 0) {
|
|
13744
|
+
throw new Error('Invalid SyncRequestPayload: "lastSeqNo" must be a non-negative integer');
|
|
14221
13745
|
}
|
|
14222
|
-
|
|
14223
|
-
|
|
14224
|
-
function ascendingFacetFields(facets) {
|
|
14225
|
-
const ascending = /* @__PURE__ */ new Set();
|
|
14226
|
-
for (const facet of facets ?? []) {
|
|
14227
|
-
if (facet.sort === "asc") ascending.add(facet.field);
|
|
13746
|
+
if (typeof record.lastPrimaryTerm !== "number" || !Number.isInteger(record.lastPrimaryTerm) || record.lastPrimaryTerm < 0) {
|
|
13747
|
+
throw new Error('Invalid SyncRequestPayload: "lastPrimaryTerm" must be a non-negative integer');
|
|
14228
13748
|
}
|
|
14229
|
-
|
|
14230
|
-
|
|
14231
|
-
function localParamsToWire(params) {
|
|
14232
|
-
return {
|
|
14233
|
-
term: params.term ?? null,
|
|
14234
|
-
filters: params.filters ?? null,
|
|
14235
|
-
sort: convertLocalSortToWire(params.sort),
|
|
14236
|
-
group: convertLocalGroupToWire(params.group),
|
|
14237
|
-
facets: convertLocalFacetsToWire(params.facets),
|
|
14238
|
-
facetSize: wireFacetSize(params.facets),
|
|
14239
|
-
limit: clampRowCount(params.limit, DEFAULT_PAGE_SIZE),
|
|
14240
|
-
offset: clampRowCount(params.offset, 0),
|
|
14241
|
-
searchAfter: params.searchAfter ?? null,
|
|
14242
|
-
fields: params.fields ?? null,
|
|
14243
|
-
boost: params.boost ?? null,
|
|
14244
|
-
tolerance: params.tolerance ?? null,
|
|
14245
|
-
threshold: params.minScore ?? null,
|
|
14246
|
-
includeScores: params.includeScores ?? null,
|
|
14247
|
-
scoring: params.scoring ?? "local",
|
|
14248
|
-
termMatch: params.termMatch ?? null,
|
|
14249
|
-
prefixLength: params.prefixLength ?? null,
|
|
14250
|
-
prefix: params.prefix ?? null,
|
|
14251
|
-
partialWords: params.partialWords ?? null,
|
|
14252
|
-
exact: params.exact ?? null,
|
|
14253
|
-
pinned: params.pinned !== void 0 ? params.pinned.map((entry) => ({ docId: entry.docId, position: entry.position })) : null,
|
|
14254
|
-
mode: params.mode ?? null,
|
|
14255
|
-
vector: convertLocalVectorToWire(params.vector),
|
|
14256
|
-
hybrid: convertLocalHybridToWire(params.hybrid)
|
|
14257
|
-
};
|
|
14258
|
-
}
|
|
14259
|
-
function countIsExactFor(params) {
|
|
14260
|
-
const vector = params.vector;
|
|
14261
|
-
if (vector === void 0 || vector.value === void 0 && vector.text === void 0) return true;
|
|
14262
|
-
const hasTerm = params.term !== void 0 && params.term.trim().length > 0;
|
|
14263
|
-
if (params.mode === "hybrid" || hasTerm) return false;
|
|
14264
|
-
return vector.similarity === void 0;
|
|
14265
|
-
}
|
|
14266
|
-
function distributedCountIsExact(params, coverage) {
|
|
14267
|
-
if (coverage.queriedPartitions < coverage.totalPartitions) return false;
|
|
14268
|
-
if (coverage.timedOutPartitions > 0 || coverage.failedPartitions > 0) return false;
|
|
14269
|
-
return countIsExactFor(params);
|
|
14270
|
-
}
|
|
14271
|
-
function distributedResultToLocal(result, countExact, documents = /* @__PURE__ */ new Map(), requestedFacets) {
|
|
14272
|
-
return {
|
|
14273
|
-
hits: result.scored.map((entry) => ({
|
|
14274
|
-
id: entry.docId,
|
|
14275
|
-
score: entry.score ?? void 0,
|
|
14276
|
-
document: documents.get(entry.docId) ?? {}
|
|
14277
|
-
})),
|
|
14278
|
-
count: result.totalHits,
|
|
14279
|
-
countExact,
|
|
14280
|
-
elapsed: 0,
|
|
14281
|
-
cursor: result.cursor ?? void 0,
|
|
14282
|
-
facets: result.facets !== null ? convertWireFacetsToLocal(result, result.facets, requestedFacets) : void 0,
|
|
14283
|
-
coverage: result.coverage
|
|
14284
|
-
};
|
|
13749
|
+
optionalIndexUuid(record.indexUuid, "SyncRequestPayload");
|
|
13750
|
+
return record;
|
|
14285
13751
|
}
|
|
14286
|
-
|
|
14287
|
-
|
|
14288
|
-
|
|
13752
|
+
|
|
13753
|
+
// src/distribution/cluster-node/request-identity.ts
|
|
13754
|
+
function holdsAnotherIndex(localUuid, requestedUuid) {
|
|
13755
|
+
return typeof requestedUuid === "string" && typeof localUuid === "string" && localUuid !== requestedUuid;
|
|
14289
13756
|
}
|
|
14290
|
-
|
|
14291
|
-
|
|
14292
|
-
|
|
13757
|
+
|
|
13758
|
+
// src/distribution/cluster-node/snapshot-auth.ts
|
|
13759
|
+
var BOOTSTRAPPABLE_STATES = /* @__PURE__ */ new Set(["ACTIVE", "INITIALISING", "MIGRATING"]);
|
|
13760
|
+
async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
|
|
13761
|
+
let allocation;
|
|
13762
|
+
try {
|
|
13763
|
+
allocation = await coordinator.getAllocation(indexName);
|
|
13764
|
+
} catch (err) {
|
|
13765
|
+
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13766
|
+
return {
|
|
13767
|
+
outcome: "denied",
|
|
13768
|
+
code: ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
13769
|
+
reason: `allocation lookup failed: ${errMessage}`
|
|
13770
|
+
};
|
|
13771
|
+
}
|
|
13772
|
+
if (allocation === null) {
|
|
13773
|
+
return {
|
|
13774
|
+
outcome: "denied",
|
|
13775
|
+
code: ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
13776
|
+
reason: `index '${indexName}' has no allocation`
|
|
13777
|
+
};
|
|
13778
|
+
}
|
|
13779
|
+
let sawAssignment = false;
|
|
13780
|
+
for (const assignment of allocation.assignments.values()) {
|
|
13781
|
+
const isAssigned = assignment.primary === sourceId || assignment.replicas.includes(sourceId);
|
|
13782
|
+
if (!isAssigned) {
|
|
13783
|
+
continue;
|
|
13784
|
+
}
|
|
13785
|
+
sawAssignment = true;
|
|
13786
|
+
if (!BOOTSTRAPPABLE_STATES.has(assignment.state)) {
|
|
13787
|
+
continue;
|
|
13788
|
+
}
|
|
13789
|
+
return { outcome: "authorized" };
|
|
13790
|
+
}
|
|
13791
|
+
if (sawAssignment) {
|
|
13792
|
+
return {
|
|
13793
|
+
outcome: "denied",
|
|
13794
|
+
code: ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
13795
|
+
reason: `node '${sourceId}' is listed for index '${indexName}' but not in a bootstrappable state`
|
|
13796
|
+
};
|
|
14293
13797
|
}
|
|
14294
13798
|
return {
|
|
14295
|
-
|
|
14296
|
-
|
|
14297
|
-
|
|
13799
|
+
outcome: "denied",
|
|
13800
|
+
code: ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
13801
|
+
reason: `node '${sourceId}' is not an assigned replica for index '${indexName}'`
|
|
14298
13802
|
};
|
|
14299
13803
|
}
|
|
14300
|
-
function
|
|
14301
|
-
|
|
14302
|
-
|
|
14303
|
-
|
|
14304
|
-
|
|
14305
|
-
return
|
|
14306
|
-
}
|
|
14307
|
-
|
|
14308
|
-
|
|
14309
|
-
sort: wireFacetOrder(options?.sort),
|
|
14310
|
-
ranges: options?.ranges?.map((range) => ({ from: range.from, to: range.to })) ?? null
|
|
14311
|
-
}));
|
|
13804
|
+
function createSingleResponseSink(respond) {
|
|
13805
|
+
const sink = ((response) => {
|
|
13806
|
+
if (sink.closed) {
|
|
13807
|
+
return Promise.resolve();
|
|
13808
|
+
}
|
|
13809
|
+
return respond(response);
|
|
13810
|
+
});
|
|
13811
|
+
sink.closed = false;
|
|
13812
|
+
return sink;
|
|
14312
13813
|
}
|
|
14313
|
-
function
|
|
14314
|
-
|
|
14315
|
-
|
|
14316
|
-
|
|
14317
|
-
|
|
14318
|
-
|
|
14319
|
-
|
|
14320
|
-
|
|
14321
|
-
|
|
14322
|
-
metric: vector.metric ?? null,
|
|
14323
|
-
efSearch: vector.efSearch ?? null,
|
|
14324
|
-
oversample: vector.oversample ?? null,
|
|
14325
|
-
rescoreWindow: null
|
|
13814
|
+
async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build, metadata) {
|
|
13815
|
+
const totalBytes = build.totalBytes;
|
|
13816
|
+
const checksum = build.checksum;
|
|
13817
|
+
const header = {
|
|
13818
|
+
lastSeqNo: metadata.lastSeqNo,
|
|
13819
|
+
primaryTerm: metadata.primaryTerm,
|
|
13820
|
+
partitionId: metadata.partitionId,
|
|
13821
|
+
indexName,
|
|
13822
|
+
checksum
|
|
14326
13823
|
};
|
|
14327
|
-
}
|
|
14328
|
-
|
|
14329
|
-
|
|
14330
|
-
|
|
13824
|
+
const startPayload = { header, totalBytes };
|
|
13825
|
+
const startBytes = encode(startPayload);
|
|
13826
|
+
assertMessageWithinLimit(startBytes, "SNAPSHOT_START");
|
|
13827
|
+
await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_START, nodeId, requestId, startBytes);
|
|
13828
|
+
await yieldToEventLoop2();
|
|
13829
|
+
let offset = 0;
|
|
13830
|
+
let lastYieldAt = now2();
|
|
13831
|
+
for await (const source of build.read()) {
|
|
13832
|
+
for (let from = 0; from < source.byteLength; from += SNAPSHOT_CHUNK_SIZE) {
|
|
13833
|
+
const chunk = source.subarray(from, Math.min(from + SNAPSHOT_CHUNK_SIZE, source.byteLength));
|
|
13834
|
+
const chunkPayload = {
|
|
13835
|
+
partitionId: metadata.partitionId,
|
|
13836
|
+
indexName,
|
|
13837
|
+
offset,
|
|
13838
|
+
data: chunk
|
|
13839
|
+
};
|
|
13840
|
+
const chunkBytes = encode(chunkPayload);
|
|
13841
|
+
assertMessageWithinLimit(chunkBytes, "SNAPSHOT_CHUNK");
|
|
13842
|
+
await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_CHUNK, nodeId, requestId, chunkBytes);
|
|
13843
|
+
offset += chunk.byteLength;
|
|
13844
|
+
if (offset < totalBytes && now2() - lastYieldAt >= SNAPSHOT_CHUNK_YIELD_INTERVAL_MS) {
|
|
13845
|
+
await yieldToEventLoop2();
|
|
13846
|
+
lastYieldAt = now2();
|
|
13847
|
+
}
|
|
13848
|
+
}
|
|
14331
13849
|
}
|
|
14332
|
-
|
|
14333
|
-
|
|
14334
|
-
|
|
14335
|
-
|
|
14336
|
-
|
|
13850
|
+
await yieldToEventLoop2();
|
|
13851
|
+
const endPayload = {
|
|
13852
|
+
partitionId: metadata.partitionId,
|
|
13853
|
+
indexName,
|
|
13854
|
+
totalBytes,
|
|
13855
|
+
checksum
|
|
14337
13856
|
};
|
|
13857
|
+
const endBytes = encode(endPayload);
|
|
13858
|
+
assertMessageWithinLimit(endBytes, "SNAPSHOT_END");
|
|
13859
|
+
await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_END, nodeId, requestId, endBytes);
|
|
14338
13860
|
}
|
|
14339
|
-
function
|
|
14340
|
-
|
|
14341
|
-
|
|
14342
|
-
if ((limit === void 0 || !Number.isFinite(limit)) && Array.isArray(ranges)) {
|
|
14343
|
-
return Math.min(Math.max(ranges.length, 1), MAX_FACET_SIZE);
|
|
13861
|
+
function now2() {
|
|
13862
|
+
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
13863
|
+
return performance.now();
|
|
14344
13864
|
}
|
|
14345
|
-
|
|
14346
|
-
return Math.min(Math.max(Math.floor(limit), 1), MAX_FACET_SIZE);
|
|
13865
|
+
return Date.now();
|
|
14347
13866
|
}
|
|
14348
|
-
function
|
|
14349
|
-
|
|
14350
|
-
|
|
14351
|
-
|
|
14352
|
-
|
|
13867
|
+
function yieldToEventLoop2() {
|
|
13868
|
+
return new Promise((resolve) => {
|
|
13869
|
+
if (typeof setImmediate === "function") {
|
|
13870
|
+
setImmediate(resolve);
|
|
13871
|
+
return;
|
|
13872
|
+
}
|
|
13873
|
+
setTimeout(resolve, 0);
|
|
13874
|
+
});
|
|
14353
13875
|
}
|
|
14354
|
-
function
|
|
14355
|
-
|
|
14356
|
-
for (const [field, buckets] of Object.entries(wireFacets)) {
|
|
14357
|
-
const mergedBound = result.facetErrorBounds?.[field] ?? 0;
|
|
14358
|
-
const kept = requested === void 0 ? buckets.length : Math.min(buckets.length, facetLimitOf(requested[field]));
|
|
14359
|
-
const values = {};
|
|
14360
|
-
for (let index = 0; index < kept; index++) values[buckets[index].value] = buckets[index].count;
|
|
14361
|
-
let largestCut = 0;
|
|
14362
|
-
for (let index = kept; index < buckets.length; index++) largestCut = Math.max(largestCut, buckets[index].count);
|
|
14363
|
-
const errorBound = kept < buckets.length ? (result.facetUndercounts?.[field] ?? mergedBound) + largestCut : mergedBound;
|
|
14364
|
-
converted[field] = { values, count: kept, errorBound: Math.max(errorBound, mergedBound) };
|
|
14365
|
-
}
|
|
14366
|
-
return converted;
|
|
13876
|
+
function respondMessage(sink, type, sourceId, requestId, payload) {
|
|
13877
|
+
return sink({ type, sourceId, requestId, payload });
|
|
14367
13878
|
}
|
|
14368
|
-
|
|
14369
|
-
|
|
14370
|
-
|
|
14371
|
-
|
|
14372
|
-
|
|
14373
|
-
|
|
14374
|
-
if (queryParams.group?.limit !== void 0) {
|
|
14375
|
-
queryParams.group = { ...queryParams.group, limit: oversampledShardSize(queryParams.group.limit) };
|
|
14376
|
-
}
|
|
14377
|
-
const rescoreWindow = payload.params.vector?.rescoreWindow ?? void 0;
|
|
14378
|
-
if (rescoreWindow !== void 0 && queryParams.sort === void 0 && queryParams.group === void 0) {
|
|
14379
|
-
queryParams.document = false;
|
|
13879
|
+
function assertMessageWithinLimit(bytes, label) {
|
|
13880
|
+
if (bytes.byteLength > MAX_MESSAGE_SIZE_BYTES) {
|
|
13881
|
+
throw new NarsilError(
|
|
13882
|
+
ErrorCodes.CONFIG_INVALID,
|
|
13883
|
+
`${label} payload (${bytes.byteLength} bytes) exceeds transport message limit (${MAX_MESSAGE_SIZE_BYTES})`
|
|
13884
|
+
);
|
|
14380
13885
|
}
|
|
14381
|
-
|
|
14382
|
-
|
|
14383
|
-
|
|
14384
|
-
|
|
14385
|
-
|
|
14386
|
-
|
|
14387
|
-
|
|
14388
|
-
const normalizedSort = queryParams.sort !== void 0 ? normalizeSort(queryParams.sort) : null;
|
|
14389
|
-
const sortFields = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
14390
|
-
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
14391
|
-
const scored = queryResult.hits.map((hit) => ({
|
|
14392
|
-
docId: hit.id,
|
|
14393
|
-
score: hit.score ?? null,
|
|
14394
|
-
sortValues: sortFields !== null ? readSortValues(hit.document, sortFields, sortModes).map(toWireSortValue) : null
|
|
14395
|
-
}));
|
|
14396
|
-
const results = payload.partitionIds.map((partitionId) => ({
|
|
14397
|
-
partitionId,
|
|
14398
|
-
scored: partitionId === payload.partitionIds[0] ? scored : [],
|
|
14399
|
-
totalHits: partitionId === payload.partitionIds[0] ? queryResult.count : 0
|
|
14400
|
-
}));
|
|
14401
|
-
const resultPayload = {
|
|
14402
|
-
results,
|
|
14403
|
-
facets: convertLocalFacetsToWire2(queryResult.facets),
|
|
14404
|
-
facetErrorBounds: convertLocalFacetBoundsToWire(queryResult.facets),
|
|
14405
|
-
groups: await convertLocalGroupsToWire(queryResult.groups, sortFields, sortModes, deps, payload.indexName)
|
|
13886
|
+
}
|
|
13887
|
+
async function respondError(sink, sourceId, request, code, message, details) {
|
|
13888
|
+
const response = {
|
|
13889
|
+
type: `${request.type}.error`,
|
|
13890
|
+
sourceId,
|
|
13891
|
+
requestId: request.requestId,
|
|
13892
|
+
payload: encodeErrorReply(code, message, details)
|
|
14406
13893
|
};
|
|
14407
|
-
await
|
|
14408
|
-
|
|
14409
|
-
sourceId: deps.nodeId,
|
|
14410
|
-
requestId: message.requestId,
|
|
14411
|
-
payload: encodeDocumentReply(resultPayload)
|
|
14412
|
-
});
|
|
13894
|
+
await sink(response);
|
|
13895
|
+
sink.closed = true;
|
|
14413
13896
|
}
|
|
14414
|
-
|
|
14415
|
-
|
|
14416
|
-
|
|
14417
|
-
|
|
14418
|
-
|
|
14419
|
-
|
|
14420
|
-
const wireGroups = [];
|
|
14421
|
-
for (const group of groups) {
|
|
14422
|
-
const scored = group.hits.map((hit) => {
|
|
14423
|
-
if (sortFields === null) return { docId: hit.id, score: hit.score ?? null, sortValues: null };
|
|
14424
|
-
const document = hit.document ?? readDocuments.get(hit.id);
|
|
14425
|
-
const sortValues = readSortValues(document, sortFields, sortModes).map(toWireSortValue);
|
|
14426
|
-
return { docId: hit.id, score: hit.score ?? null, sortValues };
|
|
13897
|
+
|
|
13898
|
+
// src/distribution/cluster-node/snapshot-cache.ts
|
|
13899
|
+
function createSnapshotCacheState(maxPerSource = DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, maxStreamsPerIndex = DEFAULT_MAX_STREAMS_PER_INDEX) {
|
|
13900
|
+
if (!Number.isInteger(maxPerSource) || maxPerSource <= 0) {
|
|
13901
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "maxPerSource must be a positive integer", {
|
|
13902
|
+
maxPerSource
|
|
14427
13903
|
});
|
|
14428
|
-
wireGroups.push({ values: group.values, scored });
|
|
14429
|
-
}
|
|
14430
|
-
return wireGroups;
|
|
14431
|
-
}
|
|
14432
|
-
async function handleFetch(message, respond, deps) {
|
|
14433
|
-
const decoded = decode(message.payload);
|
|
14434
|
-
const payload = validateFetchPayload(decoded);
|
|
14435
|
-
const projection = resolveProjection(payload.fields === null ? void 0 : { include: payload.fields });
|
|
14436
|
-
const documents = [];
|
|
14437
|
-
for (const docRef of payload.documentIds) {
|
|
14438
|
-
const doc = await deps.engine.get(payload.indexName, docRef.docId);
|
|
14439
|
-
if (doc !== void 0) {
|
|
14440
|
-
documents.push({
|
|
14441
|
-
docId: docRef.docId,
|
|
14442
|
-
document: applyProjection(doc, projection),
|
|
14443
|
-
highlights: null
|
|
14444
|
-
});
|
|
14445
|
-
}
|
|
14446
13904
|
}
|
|
14447
|
-
|
|
14448
|
-
|
|
14449
|
-
|
|
14450
|
-
|
|
14451
|
-
|
|
14452
|
-
|
|
14453
|
-
|
|
14454
|
-
|
|
14455
|
-
|
|
14456
|
-
|
|
14457
|
-
|
|
14458
|
-
const resultPayload = await deps.engine.collectQueryStats(
|
|
14459
|
-
payload.indexName,
|
|
14460
|
-
payload.terms,
|
|
14461
|
-
payload.partitionIds,
|
|
14462
|
-
{ expansion: payload.expansion ?? void 0, tokens: payload.tokens ?? void 0 }
|
|
14463
|
-
);
|
|
14464
|
-
await respond({
|
|
14465
|
-
type: QueryMessageTypes.STATS_RESULT,
|
|
14466
|
-
sourceId: deps.nodeId,
|
|
14467
|
-
requestId: message.requestId,
|
|
14468
|
-
payload: encode(resultPayload)
|
|
14469
|
-
});
|
|
13905
|
+
if (!Number.isInteger(maxStreamsPerIndex) || maxStreamsPerIndex <= 0) {
|
|
13906
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "maxStreamsPerIndex must be a positive integer", {
|
|
13907
|
+
maxStreamsPerIndex
|
|
13908
|
+
});
|
|
13909
|
+
}
|
|
13910
|
+
return {
|
|
13911
|
+
perSource: /* @__PURE__ */ new Map(),
|
|
13912
|
+
streamActive: /* @__PURE__ */ new Map(),
|
|
13913
|
+
maxPerSource,
|
|
13914
|
+
maxStreamsPerIndex
|
|
13915
|
+
};
|
|
14470
13916
|
}
|
|
14471
|
-
function
|
|
14472
|
-
|
|
14473
|
-
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
14474
|
-
if (typeof value === "boolean") return value;
|
|
14475
|
-
return null;
|
|
13917
|
+
function sourceSlotKey(sourceId, indexName) {
|
|
13918
|
+
return `${sourceId}\0${indexName}`;
|
|
14476
13919
|
}
|
|
14477
|
-
function
|
|
14478
|
-
|
|
14479
|
-
|
|
14480
|
-
|
|
14481
|
-
|
|
14482
|
-
|
|
14483
|
-
|
|
13920
|
+
function acquireSourceSlot(state, sourceId, indexName) {
|
|
13921
|
+
const key = sourceSlotKey(sourceId, indexName);
|
|
13922
|
+
const current = state.perSource.get(key) ?? 0;
|
|
13923
|
+
if (current >= state.maxPerSource) {
|
|
13924
|
+
throw new NarsilError(
|
|
13925
|
+
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
13926
|
+
`snapshot producer per-source cap reached for '${sourceId}' index '${indexName}'`,
|
|
13927
|
+
{ sourceId, indexName, cap: state.maxPerSource }
|
|
13928
|
+
);
|
|
14484
13929
|
}
|
|
14485
|
-
|
|
13930
|
+
state.perSource.set(key, current + 1);
|
|
13931
|
+
return { key, released: false };
|
|
14486
13932
|
}
|
|
14487
|
-
function
|
|
14488
|
-
if (
|
|
14489
|
-
return
|
|
13933
|
+
function releaseSourceSlot(state, handle) {
|
|
13934
|
+
if (handle.released) {
|
|
13935
|
+
return;
|
|
14490
13936
|
}
|
|
14491
|
-
|
|
14492
|
-
|
|
14493
|
-
|
|
13937
|
+
handle.released = true;
|
|
13938
|
+
const current = state.perSource.get(handle.key);
|
|
13939
|
+
if (current === void 0) {
|
|
13940
|
+
return;
|
|
14494
13941
|
}
|
|
14495
|
-
|
|
14496
|
-
|
|
14497
|
-
|
|
14498
|
-
|
|
14499
|
-
|
|
14500
|
-
const partitions = (await deps.engine.partitionStatsForRead(payload.indexName)).filter((partition) => requested.has(partition.partitionId)).map((partition) => ({
|
|
14501
|
-
partitionId: partition.partitionId,
|
|
14502
|
-
documentCount: partition.documentCount,
|
|
14503
|
-
estimatedMemoryBytes: partition.estimatedMemoryBytes
|
|
14504
|
-
}));
|
|
14505
|
-
const language = deps.engine.getStats(payload.indexName).language;
|
|
14506
|
-
await respond(createCountResultMessage({ partitions, language }, deps.nodeId, message.requestId));
|
|
13942
|
+
if (current <= 1) {
|
|
13943
|
+
state.perSource.delete(handle.key);
|
|
13944
|
+
return;
|
|
13945
|
+
}
|
|
13946
|
+
state.perSource.set(handle.key, current - 1);
|
|
14507
13947
|
}
|
|
14508
|
-
|
|
14509
|
-
const
|
|
14510
|
-
|
|
14511
|
-
|
|
14512
|
-
|
|
14513
|
-
|
|
14514
|
-
|
|
14515
|
-
|
|
14516
|
-
}
|
|
14517
|
-
|
|
14518
|
-
|
|
14519
|
-
const sortFields = normalizedSort === null ? null : normalizedSort.map((entry) => entry.field);
|
|
14520
|
-
const sortModes = normalizedSort === null ? [] : sortModesOf(normalizedSort);
|
|
14521
|
-
const entries = result.documents.map((listed) => ({
|
|
14522
|
-
docId: listed.id,
|
|
14523
|
-
document: listed.document,
|
|
14524
|
-
sortValues: sortFields === null ? null : readSortValues(listed.document, sortFields, sortModes).map(toWireSortValue2)
|
|
14525
|
-
}));
|
|
14526
|
-
await respond(
|
|
14527
|
-
createListResultMessage(
|
|
14528
|
-
{ entries, total: result.total, hasMore: result.cursor !== null },
|
|
14529
|
-
deps.nodeId,
|
|
14530
|
-
message.requestId
|
|
14531
|
-
)
|
|
14532
|
-
);
|
|
13948
|
+
function acquireStreamSlot(state, indexName) {
|
|
13949
|
+
const current = state.streamActive.get(indexName) ?? 0;
|
|
13950
|
+
if (current >= state.maxStreamsPerIndex) {
|
|
13951
|
+
throw new NarsilError(
|
|
13952
|
+
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
13953
|
+
`snapshot stream fan-out cap reached for index '${indexName}'`,
|
|
13954
|
+
{ indexName, cap: state.maxStreamsPerIndex }
|
|
13955
|
+
);
|
|
13956
|
+
}
|
|
13957
|
+
state.streamActive.set(indexName, current + 1);
|
|
13958
|
+
return { indexName, released: false };
|
|
14533
13959
|
}
|
|
14534
|
-
|
|
14535
|
-
|
|
14536
|
-
|
|
14537
|
-
|
|
14538
|
-
|
|
14539
|
-
|
|
14540
|
-
)
|
|
14541
|
-
|
|
14542
|
-
|
|
14543
|
-
|
|
14544
|
-
|
|
14545
|
-
|
|
14546
|
-
|
|
14547
|
-
);
|
|
13960
|
+
function releaseStreamSlot(state, handle) {
|
|
13961
|
+
if (handle.released) {
|
|
13962
|
+
return;
|
|
13963
|
+
}
|
|
13964
|
+
handle.released = true;
|
|
13965
|
+
const current = state.streamActive.get(handle.indexName);
|
|
13966
|
+
if (current === void 0) {
|
|
13967
|
+
return;
|
|
13968
|
+
}
|
|
13969
|
+
if (current <= 1) {
|
|
13970
|
+
state.streamActive.delete(handle.indexName);
|
|
13971
|
+
return;
|
|
13972
|
+
}
|
|
13973
|
+
state.streamActive.set(handle.indexName, current - 1);
|
|
14548
13974
|
}
|
|
14549
|
-
|
|
14550
|
-
|
|
14551
|
-
|
|
14552
|
-
|
|
14553
|
-
|
|
14554
|
-
|
|
14555
|
-
|
|
14556
|
-
|
|
14557
|
-
|
|
14558
|
-
)
|
|
14559
|
-
|
|
13975
|
+
|
|
13976
|
+
// src/distribution/cluster-node/snapshot-sync-handler/stream.ts
|
|
13977
|
+
async function acquireAndStream(message, sink, indexName, deps, options) {
|
|
13978
|
+
let sourceHandle;
|
|
13979
|
+
try {
|
|
13980
|
+
sourceHandle = acquireSourceSlot(deps.state, message.sourceId, indexName);
|
|
13981
|
+
} catch (err) {
|
|
13982
|
+
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
|
|
13983
|
+
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13984
|
+
await respondError(sink, deps.nodeId, message, code, errMessage, transportableDetails(err));
|
|
13985
|
+
return;
|
|
13986
|
+
}
|
|
13987
|
+
try {
|
|
13988
|
+
let build;
|
|
13989
|
+
try {
|
|
13990
|
+
build = await options.buildSnapshot(indexName);
|
|
13991
|
+
} catch (err) {
|
|
13992
|
+
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
|
|
13993
|
+
const errMessage = err instanceof Error ? err.message : String(err);
|
|
13994
|
+
await respondError(sink, deps.nodeId, message, code, errMessage, transportableDetails(err));
|
|
13995
|
+
return;
|
|
13996
|
+
}
|
|
13997
|
+
try {
|
|
13998
|
+
await streamBuiltSnapshot(message, sink, indexName, deps, options, build);
|
|
13999
|
+
} finally {
|
|
14000
|
+
await build.discard();
|
|
14001
|
+
}
|
|
14002
|
+
} finally {
|
|
14003
|
+
releaseSourceSlot(deps.state, sourceHandle);
|
|
14004
|
+
}
|
|
14560
14005
|
}
|
|
14561
|
-
function
|
|
14562
|
-
|
|
14563
|
-
|
|
14564
|
-
|
|
14565
|
-
|
|
14006
|
+
async function streamBuiltSnapshot(message, sink, indexName, deps, options, build) {
|
|
14007
|
+
let streamHandle;
|
|
14008
|
+
try {
|
|
14009
|
+
streamHandle = acquireStreamSlot(deps.state, indexName);
|
|
14010
|
+
} catch (err) {
|
|
14011
|
+
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
|
|
14012
|
+
const errMessage = err instanceof Error ? err.message : String(err);
|
|
14013
|
+
await respondError(sink, deps.nodeId, message, code, errMessage, transportableDetails(err));
|
|
14014
|
+
return;
|
|
14015
|
+
}
|
|
14016
|
+
try {
|
|
14017
|
+
const metadata = {
|
|
14018
|
+
...options.metadata,
|
|
14019
|
+
lastSeqNo: build.lastSeqNo ?? options.metadata.lastSeqNo,
|
|
14020
|
+
primaryTerm: build.primaryTerm ?? options.metadata.primaryTerm
|
|
14021
|
+
};
|
|
14022
|
+
await streamSnapshotToReplica(sink, deps.nodeId, message.requestId, indexName, build, metadata);
|
|
14023
|
+
await options.afterSnapshot(sink, build);
|
|
14024
|
+
sink.closed = true;
|
|
14025
|
+
} finally {
|
|
14026
|
+
releaseStreamSlot(deps.state, streamHandle);
|
|
14027
|
+
}
|
|
14566
14028
|
}
|
|
14567
14029
|
|
|
14568
|
-
// src/distribution/
|
|
14569
|
-
function
|
|
14570
|
-
|
|
14571
|
-
|
|
14030
|
+
// src/distribution/cluster-node/snapshot-sync-handler/validation.ts
|
|
14031
|
+
function containsControlCharacter(value) {
|
|
14032
|
+
for (let i = 0; i < value.length; i++) {
|
|
14033
|
+
const code = value.charCodeAt(i);
|
|
14034
|
+
if (code < 32) {
|
|
14035
|
+
return true;
|
|
14036
|
+
}
|
|
14037
|
+
if (code >= 127 && code <= 159) {
|
|
14038
|
+
return true;
|
|
14039
|
+
}
|
|
14040
|
+
if (code === 8232 || code === 8233 || code === 65279) {
|
|
14041
|
+
return true;
|
|
14042
|
+
}
|
|
14572
14043
|
}
|
|
14573
|
-
|
|
14574
|
-
return primaryTerm !== void 0 && primaryTerm === replicaLastPrimaryTerm ? "incremental" : "snapshot";
|
|
14044
|
+
return false;
|
|
14575
14045
|
}
|
|
14576
|
-
function
|
|
14577
|
-
if (
|
|
14578
|
-
|
|
14046
|
+
function validateSourceId(sourceId) {
|
|
14047
|
+
if (typeof sourceId !== "string" || sourceId.length === 0) {
|
|
14048
|
+
return "request sourceId is missing";
|
|
14579
14049
|
}
|
|
14580
|
-
|
|
14581
|
-
|
|
14582
|
-
throw new Error('Invalid SyncRequestPayload: "indexName" must be a string');
|
|
14050
|
+
if (sourceId.length > MAX_SOURCE_ID_LENGTH) {
|
|
14051
|
+
return `request sourceId exceeds ${MAX_SOURCE_ID_LENGTH} characters`;
|
|
14583
14052
|
}
|
|
14584
|
-
if (
|
|
14585
|
-
|
|
14053
|
+
if (containsControlCharacter(sourceId)) {
|
|
14054
|
+
return "request sourceId contains control characters";
|
|
14586
14055
|
}
|
|
14587
|
-
|
|
14588
|
-
|
|
14056
|
+
return null;
|
|
14057
|
+
}
|
|
14058
|
+
|
|
14059
|
+
// src/distribution/cluster-node/snapshot-sync-handler/index.ts
|
|
14060
|
+
function createSnapshotSyncHandlerState() {
|
|
14061
|
+
return createSnapshotCacheState();
|
|
14062
|
+
}
|
|
14063
|
+
async function streamValidatedSnapshotRequest(message, sink, indexName, deps, options) {
|
|
14064
|
+
const sourceIdError = validateSourceId(message.sourceId);
|
|
14065
|
+
if (sourceIdError !== null) {
|
|
14066
|
+
await respondError(sink, deps.nodeId, message, ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, sourceIdError);
|
|
14067
|
+
return;
|
|
14589
14068
|
}
|
|
14590
|
-
|
|
14591
|
-
|
|
14069
|
+
const authResult = await authorizeSnapshotRequest(deps.coordinator, indexName, message.sourceId);
|
|
14070
|
+
if (authResult.outcome === "denied") {
|
|
14071
|
+
await respondError(sink, deps.nodeId, message, authResult.code, authResult.reason);
|
|
14072
|
+
return;
|
|
14592
14073
|
}
|
|
14593
|
-
|
|
14594
|
-
|
|
14074
|
+
const existingIndex = deps.engine.listIndexes().find((idx) => idx.name === indexName);
|
|
14075
|
+
if (existingIndex === void 0) {
|
|
14076
|
+
await respondError(
|
|
14077
|
+
sink,
|
|
14078
|
+
deps.nodeId,
|
|
14079
|
+
message,
|
|
14080
|
+
ErrorCodes.SNAPSHOT_SYNC_INDEX_NOT_FOUND,
|
|
14081
|
+
`Index '${indexName}' is not hosted on this node`
|
|
14082
|
+
);
|
|
14083
|
+
return;
|
|
14084
|
+
}
|
|
14085
|
+
await acquireAndStream(message, sink, indexName, deps, options);
|
|
14595
14086
|
}
|
|
14596
14087
|
|
|
14597
14088
|
// src/distribution/cluster-node/message-handler/sync.ts
|
|
@@ -14686,14 +14177,12 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
14686
14177
|
await streamValidatedSnapshotRequest(
|
|
14687
14178
|
message,
|
|
14688
14179
|
sink,
|
|
14689
|
-
|
|
14180
|
+
request.indexName,
|
|
14690
14181
|
{
|
|
14691
14182
|
nodeId: deps.nodeId,
|
|
14692
14183
|
engine: deps.engine,
|
|
14693
14184
|
coordinator: deps.coordinator,
|
|
14694
|
-
state: deps.snapshotSyncState
|
|
14695
|
-
resolveHeaderMetadata: deps.resolveHeaderMetadata,
|
|
14696
|
-
spoolDirectory: deps.snapshotSpoolDirectory
|
|
14185
|
+
state: deps.snapshotSyncState
|
|
14697
14186
|
},
|
|
14698
14187
|
{
|
|
14699
14188
|
metadata: {
|
|
@@ -14701,8 +14190,6 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
14701
14190
|
lastSeqNo: snapshotSeqNo,
|
|
14702
14191
|
primaryTerm: snapshotPrimaryTerm
|
|
14703
14192
|
},
|
|
14704
|
-
closeOnEnd: false,
|
|
14705
|
-
disableBuildCache: true,
|
|
14706
14193
|
buildSnapshot: async () => {
|
|
14707
14194
|
const archive = await deps.engine.archiveReplicationPartition(request.indexName, request.partitionId, () => {
|
|
14708
14195
|
snapshotPrimaryTerm = log.termAt(log.localLogEnd) ?? 0;
|
|
@@ -14967,18 +14454,6 @@ function validateForwardPayload(decoded) {
|
|
|
14967
14454
|
// src/distribution/cluster-node/message-handler/index.ts
|
|
14968
14455
|
function createDataNodeHandler(deps) {
|
|
14969
14456
|
return async (message, respond) => {
|
|
14970
|
-
if (message.type === ReplicationMessageTypes.SNAPSHOT_SYNC_REQUEST) {
|
|
14971
|
-
await handleSnapshotSyncRequest(message, respond, {
|
|
14972
|
-
nodeId: deps.nodeId,
|
|
14973
|
-
engine: deps.engine,
|
|
14974
|
-
coordinator: deps.coordinator,
|
|
14975
|
-
state: deps.snapshotSyncState,
|
|
14976
|
-
resolveHeaderMetadata: deps.resolveHeaderMetadata,
|
|
14977
|
-
spoolDirectory: deps.snapshotSpoolDirectory,
|
|
14978
|
-
indexUuidOf: (indexName) => deps.engine.indexUuidOf(indexName)
|
|
14979
|
-
});
|
|
14980
|
-
return;
|
|
14981
|
-
}
|
|
14982
14457
|
try {
|
|
14983
14458
|
switch (message.type) {
|
|
14984
14459
|
case ReplicationMessageTypes.SYNC_REQUEST:
|
|
@@ -16421,7 +15896,7 @@ async function queryCluster(deps, indexName, params) {
|
|
|
16421
15896
|
return deps.engine.query(indexName, params);
|
|
16422
15897
|
}
|
|
16423
15898
|
if (params.rerank !== void 0) {
|
|
16424
|
-
const { rerankClusterQuery } = await import('./rerank-
|
|
15899
|
+
const { rerankClusterQuery } = await import('./rerank-WVFFZV76.mjs');
|
|
16425
15900
|
return rerankClusterQuery(
|
|
16426
15901
|
deps,
|
|
16427
15902
|
indexName,
|
|
@@ -17186,7 +16661,8 @@ function createNodeWiring(input) {
|
|
|
17186
16661
|
getReplicationLog: getReplicationLog2,
|
|
17187
16662
|
seedReplicationLog: seedReplicationLog2,
|
|
17188
16663
|
existingReplicationLog: (indexName, partitionId) => replicationLogs.get(replicationLogKey(indexName, partitionId)),
|
|
17189
|
-
resolveNodeTargets: resolveNodeTargets3
|
|
16664
|
+
resolveNodeTargets: resolveNodeTargets3,
|
|
16665
|
+
resolveRegisteredNodeTargets: (targetNodeId) => resolveRegisteredTargets(routingConfig2, targetNodeId)
|
|
17190
16666
|
};
|
|
17191
16667
|
}
|
|
17192
16668
|
|
|
@@ -17547,6 +17023,7 @@ function createNotReadyHandler(nodeId) {
|
|
|
17547
17023
|
function createMultiplexedControllerTransport(baseTransport, nodeId) {
|
|
17548
17024
|
let controllerHandler = null;
|
|
17549
17025
|
const transport = {
|
|
17026
|
+
timeouts: baseTransport.timeouts,
|
|
17550
17027
|
send(target, message) {
|
|
17551
17028
|
return baseTransport.send(target, message);
|
|
17552
17029
|
},
|
|
@@ -17798,6 +17275,7 @@ async function createClusterNode(callerConfig) {
|
|
|
17798
17275
|
});
|
|
17799
17276
|
writeDeps.copyFailed = copyFailures.isFailed;
|
|
17800
17277
|
writeDeps.copyUnlisted = copyFailures.isUnlisted;
|
|
17278
|
+
writeDeps.stopped = () => isShutdown;
|
|
17801
17279
|
engine.isolateCopyFailures(copyFailures.record);
|
|
17802
17280
|
const bootstrapPartition2 = createPartitionBootstrap({
|
|
17803
17281
|
nodeId,
|
|
@@ -17805,7 +17283,7 @@ async function createClusterNode(callerConfig) {
|
|
|
17805
17283
|
coordinator: config.coordinator,
|
|
17806
17284
|
transport: config.transport,
|
|
17807
17285
|
state: bootstrapSyncState,
|
|
17808
|
-
resolveNodeTargets:
|
|
17286
|
+
resolveNodeTargets: wiring.resolveRegisteredNodeTargets,
|
|
17809
17287
|
getReplicationLog: getReplicationLog2,
|
|
17810
17288
|
seedReplicationLog: seedReplicationLog2,
|
|
17811
17289
|
partitionWriteQueues: writeDeps.partitionWriteQueues,
|
|
@@ -17940,7 +17418,6 @@ async function createClusterNode(callerConfig) {
|
|
|
17940
17418
|
writeDeps,
|
|
17941
17419
|
snapshotSyncState: snapshotSyncHandlerState,
|
|
17942
17420
|
snapshotSpoolDirectory: snapshotSpool.path,
|
|
17943
|
-
resolveHeaderMetadata: resolveSnapshotHeaderMetadata,
|
|
17944
17421
|
isBootstrapSynced: (indexName, partitionId) => hasCompletedBootstrapSync(bootstrapSyncState, indexName, partitionId),
|
|
17945
17422
|
onError: forwardOnError
|
|
17946
17423
|
});
|
|
@@ -17991,13 +17468,6 @@ async function createClusterNode(callerConfig) {
|
|
|
17991
17468
|
});
|
|
17992
17469
|
bindRerankWindowSource(node, () => rerankWindowCapOf(engine));
|
|
17993
17470
|
return node;
|
|
17994
|
-
function resolveSnapshotHeaderMetadata(indexName, partitionId) {
|
|
17995
|
-
return nodeSnapshotHeaderMetadata(
|
|
17996
|
-
{ coordinator: config.coordinator, engine, getReplicationLog: getReplicationLog2 },
|
|
17997
|
-
indexName,
|
|
17998
|
-
partitionId
|
|
17999
|
-
);
|
|
18000
|
-
}
|
|
18001
17471
|
}
|
|
18002
17472
|
|
|
18003
17473
|
// src/distribution/cluster-node/server-engine.ts
|