@delali/narsil 0.3.1-next.185 → 0.3.1-next.189
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/adapters/filesystem.mjs +2 -2
- package/dist/adapters/gcs.mjs +2 -2
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/s3.mjs +2 -2
- package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
- package/dist/browser/{chunk-4ZICWS4V.mjs → chunk-QAOVAYAG.mjs} +1 -1
- package/dist/browser/{chunk-CPSHS46N.mjs → chunk-WFQLSNCO.mjs} +27 -22
- package/dist/browser/{chunk-2Q45HI5H.mjs → chunk-YINZFPLT.mjs} +7 -0
- package/dist/browser/{module-XVLHJ7T7.mjs → module-VOTN3Q6X.mjs} +15 -0
- package/dist/browser/{optional-writes-ZTBHYQMN.mjs → optional-writes-J5BHKEQL.mjs} +21 -8
- package/dist/browser/{storage-leaf-4XMCITJ6.mjs → storage-leaf-62LA56U2.mjs} +14558 -13739
- package/dist/{capabilities-DAHXphRn.d.ts → capabilities-CLnQHIqV.d.ts} +51 -9
- package/dist/{chunk-KLVMK3R5.mjs → chunk-2CKDCC65.mjs} +1 -1
- package/dist/{chunk-7WXGUTWK.mjs → chunk-3JRN23HA.mjs} +17 -6
- package/dist/{chunk-5BUEBIXR.mjs → chunk-3OZ2UQJV.mjs} +18 -5
- package/dist/{chunk-VYSWVQCE.mjs → chunk-5HPILB7R.mjs} +2 -2
- package/dist/{chunk-FBQYC5OX.mjs → chunk-6G3VRWH5.mjs} +16 -13
- package/dist/{chunk-QJGDB3WR.mjs → chunk-6GHX35ZM.mjs} +2 -2
- package/dist/{chunk-H4QYQ5DE.mjs → chunk-7NCVV3ST.mjs} +2 -2
- package/dist/{chunk-X5NHWOXT.mjs → chunk-7P5YWDRQ.mjs} +2 -2
- package/dist/{chunk-F4OOBPKA.mjs → chunk-AOYQTX6W.mjs} +1 -1
- package/dist/{chunk-D6FYUTFN.mjs → chunk-ASZLNVWW.mjs} +1 -1
- package/dist/{chunk-DU23J5JG.mjs → chunk-EFS5GVBK.mjs} +6 -6
- package/dist/{chunk-FETZYCH7.mjs → chunk-F6F4G322.mjs} +26 -1
- package/dist/{chunk-IE3Q62QD.mjs → chunk-GR5GMFJJ.mjs} +1 -1
- package/dist/{chunk-Y7QUFIGV.mjs → chunk-GXED6R3Q.mjs} +2 -2
- package/dist/{chunk-LR6JJVML.mjs → chunk-GYLRKKUT.mjs} +764 -699
- package/dist/{react-native/chunks/chunk-ZUFOYZ5P.mjs → chunk-IDWDYS2N.mjs} +1 -1
- package/dist/{chunk-YM2HI5N5.mjs → chunk-IL5DRCNU.mjs} +2 -2
- package/dist/{chunk-BVZWWMOU.mjs → chunk-IWKYAO45.mjs} +2 -2
- package/dist/chunk-JEMK23OY.mjs +602 -0
- package/dist/{chunk-2YWWHMVC.mjs → chunk-JOWXHENJ.mjs} +2 -2
- package/dist/{chunk-SL53DHVE.mjs → chunk-JXDY3ZA3.mjs} +66 -40
- package/dist/{chunk-WLFOLAQI.mjs → chunk-KBU3ZFJN.mjs} +2 -2
- package/dist/{chunk-N3M4NPML.mjs → chunk-M2Z5WVWW.mjs} +1 -1
- package/dist/{chunk-2KCCQK4R.mjs → chunk-NFAWR2O4.mjs} +2 -2
- package/dist/{chunk-EWGIF6FQ.mjs → chunk-NJJN4FMM.mjs} +1 -1
- package/dist/{chunk-MHBG33R5.mjs → chunk-NM2ITITI.mjs} +76 -46
- package/dist/{chunk-WSIU6YXQ.mjs → chunk-NULDX3ID.mjs} +1732 -743
- package/dist/{chunk-II6XF6M4.mjs → chunk-P2IPUYT2.mjs} +6 -6
- package/dist/{chunk-C7YQHCSO.mjs → chunk-P4F4UWLC.mjs} +1 -1
- package/dist/{chunk-A3OAFUZI.mjs → chunk-PLLKT5NU.mjs} +2 -2
- package/dist/{chunk-FCQMZDRI.mjs → chunk-PPD4J2JI.mjs} +3 -3
- package/dist/{chunk-EDEE7ETQ.mjs → chunk-PZYBWMCL.mjs} +3 -3
- package/dist/{chunk-I2WJCRIH.mjs → chunk-R2PJKSNL.mjs} +1 -1
- package/dist/{chunk-M5I64AQW.mjs → chunk-REBDKXAH.mjs} +33 -33
- package/dist/{chunk-6SZQETSC.mjs → chunk-RORNUNXW.mjs} +5 -2
- package/dist/{chunk-CYCVPTTR.mjs → chunk-SCZI7COG.mjs} +2 -2
- package/dist/{chunk-O337MDRJ.mjs → chunk-U5IMECEX.mjs} +38 -13
- package/dist/{chunk-CODGHJVW.mjs → chunk-X2NE4VJI.mjs} +40 -12
- package/dist/{chunk-2GONI5HL.mjs → chunk-YFMM7QOT.mjs} +5 -4
- package/dist/{chunk-JDG6CCXM.mjs → chunk-YQ4RZIIN.mjs} +2 -2
- package/dist/{chunk-G3XRHAJA.mjs → chunk-ZGMCJXY5.mjs} +3 -3
- package/dist/{chunk-Q3ZZH5Z7.mjs → chunk-ZKOQR7EA.mjs} +7 -0
- package/dist/{chunk-CKEOTSF5.mjs → chunk-ZS2YUB2L.mjs} +6 -3
- package/dist/{client-DpqtT2XG.d.ts → client-uB0kp2yT.d.ts} +88 -5
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +3 -1265
- package/dist/{config-D9WR3MhO.d.ts → config-BvHQ-C7U.d.ts} +3 -3
- package/dist/{config-C687-reW.d.ts → config-PqgNY1H7.d.ts} +1 -1
- package/dist/{core-leaf-NEUM7PU3.mjs → core-leaf-TAR3GJVE.mjs} +3 -3
- package/dist/{direct-executor-37IH3ISS.mjs → direct-executor-QLBSVTEE.mjs} +8 -8
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +35 -15
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/coordinator/in-memory.mjs +16 -6
- package/dist/distribution/transport/grpc.mjs +2 -2
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +70 -9
- package/dist/distribution.mjs +1694 -679
- package/dist/{durability-config-NWU2G2SJ.mjs → durability-config-VWS2K24E.mjs} +1 -1
- package/dist/{react-native/chunks/embed-4LBSQD7U.mjs → embed-4PHHKGBK.mjs} +1 -1
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-De6JzTW8.d.ts → engine-qjAxJSKk.d.ts} +206 -7
- package/dist/{index-CBUl4z3E.d.ts → index-BaQIAnk0.d.ts} +137 -111
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +4 -4
- package/dist/index.mjs +8 -8
- package/dist/index.react-native.mjs +9 -9
- package/dist/{long-query-thread-DIPJBMV7.mjs → long-query-thread-IKD4Z37M.mjs} +21 -21
- package/dist/{module-Z3CL4GJ4.mjs → module-NIVZ4SF2.mjs} +22 -10
- package/dist/{msgpack-leaf-7KBYOVTF.mjs → msgpack-leaf-H5ZYW7S7.mjs} +4 -4
- package/dist/{optional-reads-ENKMVQ4Q.mjs → optional-reads-5HFMF2QZ.mjs} +2 -2
- package/dist/{optional-writes-YIVJMQ4D.mjs → optional-writes-DVTVX2XY.mjs} +31 -18
- package/dist/{paged-partition-archive-IZ6U7DVN.mjs → paged-partition-archive-ES4Z5SM4.mjs} +20 -20
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +8 -8
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +18 -18
- package/dist/{phrase-module-ZY3TLTQA.mjs → phrase-module-EFESWOCR.mjs} +2 -2
- package/dist/{query-actions-7EOKT3WD.mjs → query-actions-IC5UEWZG.mjs} +8 -8
- package/dist/query-suggestions/merge-worker.mjs +11 -11
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +1 -1
- package/dist/query-suggestions.mjs +9 -9
- package/dist/react-native/chunks/{chunk-T5HF3FW3.mjs → chunk-3553RYCS.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-TQ26BKBH.mjs → chunk-43R4GCM7.mjs} +7 -0
- package/dist/react-native/chunks/{chunk-BN6FUB7N.mjs → chunk-AN4HKLB4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-PVMEHBQO.mjs → chunk-CW2ZFXSE.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-ZQPSXN6C.mjs → chunk-FJHZWUCB.mjs} +5 -5
- package/dist/react-native/chunks/{chunk-NLEFMPY3.mjs → chunk-G434H5WG.mjs} +15 -4
- package/dist/react-native/chunks/{chunk-IEY4DARA.mjs → chunk-GXY4R3GA.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-SK3DPAI4.mjs → chunk-HTBY4GON.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-7TPO4FLE.mjs → chunk-JZOHEHBD.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-ZDDJVR7X.mjs → chunk-MRH36RN3.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-YOTP2FPE.mjs → chunk-UO44MA5M.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-WFQ6YQZA.mjs → chunk-UYKIV2KT.mjs} +1 -1
- package/dist/{chunk-4JBTALUC.mjs → react-native/chunks/chunk-VSN76LWE.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-CBU4M3EU.mjs → chunk-XLYQ2ZMC.mjs} +222 -47
- package/dist/react-native/chunks/{chunk-OZ3XEM6D.mjs → chunk-ZUJCUPVS.mjs} +1 -1
- package/dist/react-native/chunks/{core-leaf-TPBRRWVN.mjs → core-leaf-5PVST5PR.mjs} +4 -4
- package/dist/react-native/chunks/{durability-config-RVTWSXPH.mjs → durability-config-FCSD4MXY.mjs} +1 -1
- package/dist/{embed-2QMYCD2Q.mjs → react-native/chunks/embed-HA2I7RCF.mjs} +1 -1
- package/dist/react-native/chunks/{gzip.react-native-DLO4WW27.mjs → gzip.react-native-CVM4NQG2.mjs} +2 -2
- package/dist/react-native/chunks/{module-COLNJNML.mjs → module-TN7HXWJ3.mjs} +21 -6
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-74HYI4DO.mjs → optional-reads-AFPO2FEB.mjs} +3 -3
- package/dist/react-native/chunks/{optional-writes-QETIM6BI.mjs → optional-writes-2UPNQNAN.mjs} +31 -18
- package/dist/react-native/chunks/{phrase-module-UC4YGEBB.mjs → phrase-module-IGYBK7XH.mjs} +3 -3
- package/dist/react-native/chunks/{resolve-vector-text-FU7D4QP5.mjs → resolve-vector-text-GE2CRJKI.mjs} +1 -1
- package/dist/react-native/chunks/{restore-queue-3E2MJIJS.mjs → restore-queue-DEAWTVHW.mjs} +12 -12
- package/dist/react-native/chunks/{segment-codec-leaf-IE4DXUZV.mjs → segment-codec-leaf-3LZ6HZFD.mjs} +10 -10
- package/dist/react-native/chunks/{segment-merge-leaf-62LQ5WZB.mjs → segment-merge-leaf-5OIWAPCI.mjs} +11 -11
- package/dist/react-native/chunks/{storage-leaf-LNOLKWZB.mjs → storage-leaf-LC4RZU3T.mjs} +15614 -14805
- package/dist/react-native/chunks/{store-leaf-Y7CBOPFZ.mjs → store-leaf-LQ6LWM2Q.mjs} +11 -11
- package/dist/react-native/chunks/{suggestions-leaf-PSUGL6MI.mjs → suggestions-leaf-GPNU6I72.mjs} +11 -11
- package/dist/react-native/chunks/{worker-copies-MW5O5U3D.mjs → worker-copies-32TCHOYP.mjs} +10 -10
- package/dist/react-native/partial-words.mjs +9 -9
- package/dist/react-native/pattern.mjs +5 -5
- package/dist/react-native/query-suggestions.mjs +9 -9
- package/dist/react-native/sync.mjs +433 -51
- package/dist/react-native/synonyms.mjs +9 -9
- package/dist/react.d.ts +8 -8
- package/dist/react.mjs +7 -0
- package/dist/{rerank-KTV5L7MZ.mjs → rerank-ELMGHQ3T.mjs} +8 -8
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{resolve-vector-text-SD6BEMKZ.mjs → resolve-vector-text-OD4J3WHF.mjs} +2 -2
- package/dist/{restore-queue-ES7TDJ5V.mjs → restore-queue-6VCVZT5C.mjs} +11 -11
- package/dist/{segment-codec-leaf-DO75VKEV.mjs → segment-codec-leaf-XNMCUIJF.mjs} +9 -9
- package/dist/{segment-merge-leaf-JGND6LG5.mjs → segment-merge-leaf-RA4S7CC6.mjs} +9 -9
- package/dist/server.d.ts +13 -9
- package/dist/server.mjs +875 -348
- package/dist/{storage-leaf-ZI3H2WJY.mjs → storage-leaf-36SAEVV3.mjs} +5243 -5846
- package/dist/{store-leaf-YIH36Y5J.mjs → store-leaf-QAGHHLEK.mjs} +10 -10
- package/dist/{suggestions-leaf-FP5HSTFP.mjs → suggestions-leaf-JG2IP2VQ.mjs} +12 -12
- package/dist/sync/filesystem.d.ts +2 -2
- package/dist/sync/filesystem.mjs +16 -16
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +2 -2
- package/dist/sync/indexeddb.mjs +3 -3
- package/dist/sync.browser.mjs +35 -1108
- package/dist/sync.d.ts +3 -3
- package/dist/sync.mjs +430 -51
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +8 -8
- package/dist/{tasks-X2IJQc8J.d.ts → tasks-C65V7lIr.d.ts} +3 -3
- package/dist/thread-HAE2XXFR.mjs +31 -0
- package/dist/{types-D0YasjoX.d.ts → types-CpCg-Gq-.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-WYPRCNB7.mjs → worker-copies-IXC43Z4Q.mjs} +11 -11
- package/dist/workers/entry.mjs +14 -14
- package/openapi.json +144 -0
- package/package.json +7 -7
- package/dist/chunk-URVH6BLO.mjs +0 -217
- package/dist/react-native/chunks/msgpack-leaf-GTKLF6GJ.mjs +0 -7
- package/dist/thread-NQRNW3IH.mjs +0 -31
- /package/dist/{chunk-CV3Q2EKO.mjs → chunk-S5QMWECL.mjs} +0 -0
package/dist/distribution.mjs
CHANGED
|
@@ -1,55 +1,55 @@
|
|
|
1
|
-
import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-
|
|
2
|
-
import { emptySerializablePartition } from './chunk-
|
|
3
|
-
import { applyStoredIndexBm25 } from './chunk-
|
|
4
|
-
import { keyBytesOf, querySuggestionsOf, createQuerySuggestionsRuntime, publicBuildOf, buildRecordToWire, buildRecordFromWire, SUGGESTIONS_SCHEMA } from './chunk-
|
|
5
|
-
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, archiveStream, buildArchiveFromMemory, indexMetadataOf, applyIndexEntry, applyDeleteEntry, resolveAnalysisTarget, requireConfigurationWithinSizeLimit, memoryCheckpointStore, createByteReader, writeArchiveFile, namedSettingsOf, createDetachedIndex, loadPartitionSegments, holdsSettings, configOfNamedSettings, closePartitionWriter, requireRestoreMemory, archiveFootprint, storedBytesOfSegments, validateReplicationEntry, readSegmentManifest, replaySyncedWal, nestedSchemaOf } from './chunk-
|
|
6
|
-
import { readVectorFile } from './chunk-
|
|
7
|
-
import { addBackgroundAdmission, workWaitingForMemory, resolveHybridFusion, fuseRankings } from './chunk-
|
|
1
|
+
import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-PPD4J2JI.mjs';
|
|
2
|
+
import { emptySerializablePartition } from './chunk-PLLKT5NU.mjs';
|
|
3
|
+
import { applyStoredIndexBm25 } from './chunk-IWKYAO45.mjs';
|
|
4
|
+
import { keyBytesOf, querySuggestionsOf, createQuerySuggestionsRuntime, publicBuildOf, buildRecordToWire, buildRecordFromWire, SUGGESTIONS_SCHEMA } from './chunk-EFS5GVBK.mjs';
|
|
5
|
+
import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, archiveStream, buildArchiveFromMemory, indexMetadataOf, applyIndexEntry, applyDeleteEntry, documentExportStream, followSourceChanges, requireChangesSetting, resolveAnalysisTarget, requireConfigurationWithinSizeLimit, memoryCheckpointStore, createByteReader, writeArchiveFile, namedSettingsOf, createDetachedIndex, loadPartitionSegments, changesStatusOf, holdsSettings, configOfNamedSettings, closePartitionWriter, requireRestoreMemory, archiveFootprint, storedBytesOfSegments, validateReplicationEntry, readSegmentManifest, replaySyncedWal, nestedSchemaOf } from './chunk-NULDX3ID.mjs';
|
|
6
|
+
import { readVectorFile } from './chunk-P2IPUYT2.mjs';
|
|
7
|
+
import { addBackgroundAdmission, workWaitingForMemory, resolveHybridFusion, fuseRankings } from './chunk-GXED6R3Q.mjs';
|
|
8
8
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
9
|
-
import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName,
|
|
10
|
-
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName,
|
|
11
|
-
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, refuseEncodeWithoutRoom, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, MAX_SOURCE_ID_LENGTH, ADMISSION_TIMEOUT_MS, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, OUT_OF_STEP_REMOVAL_RETRY_INITIAL_DELAY_MS, OUT_OF_STEP_REMOVAL_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-
|
|
12
|
-
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-
|
|
13
|
-
import { encodeIndexConfiguration, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireSyncSetting, requireNoQuerySuggestions,
|
|
14
|
-
import { restoredConfigFields, restoredEmbedding } from './chunk-
|
|
15
|
-
import {
|
|
9
|
+
import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, changesServingSourceOf, bindChangesServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, CHANGES_BACKUP_TAKER, BACKUP_MANIFEST_READS_AT_ONCE, storedFilesOf, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, isBackupTaker, readRestorePlacement, SYNC_BACKUP_TAKER, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-3OZ2UQJV.mjs';
|
|
10
|
+
import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, indexConfigKey, metadataWithSync, suggestionStateOfSettings, indexConfigFromSettings, metadataWithBm25, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, isRecord2, decodeIndexSettings, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, storedSynonymSetOf, RESHARD_FORWARD_BATCH_DOCUMENTS, RESHARD_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, SYNC_EPOCH_CAS_ATTEMPTS, syncSettingsOfSnapshot, epochFeaturesOn, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS, metadataWithSuggestionState, UNFED_REPLICA_CHECK_INTERVAL_MS, UNFED_REPLICA_RESYNC_MS } from './chunk-X2NE4VJI.mjs';
|
|
11
|
+
import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, refuseEncodeWithoutRoom, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, MAX_SOURCE_ID_LENGTH, ADMISSION_TIMEOUT_MS, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, ADMISSION_RETRIES_AFTER_IN_SYNC_CHANGE, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, OUT_OF_STEP_REMOVAL_RETRY_INITIAL_DELAY_MS, OUT_OF_STEP_REMOVAL_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-YFMM7QOT.mjs';
|
|
12
|
+
export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-YFMM7QOT.mjs';
|
|
13
|
+
import { encodeIndexConfiguration, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, decodeIndexConfiguration, requireSyncSetting, requireNoQuerySuggestions, validChangeFilter, createDocumentTest, INDEX_CONFIGURATION_ENVELOPE_VERSION, changesNotEnabled } from './chunk-JEMK23OY.mjs';
|
|
14
|
+
import { restoredConfigFields, restoredEmbedding } from './chunk-M2Z5WVWW.mjs';
|
|
15
|
+
import { readSnapshotArchive, archiveLayout, openManifestPayload, sourceIndexNameOf, listedKeysOf, decodeManifestPayload, estimatedArchiveBytes } from './chunk-3JRN23HA.mjs';
|
|
16
16
|
import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
|
|
17
17
|
import './chunk-T4HMSZCX.mjs';
|
|
18
|
-
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload,
|
|
19
|
-
import { directoryCheckpointStore, readListedFile, encodeSegmentManifest } from './chunk-
|
|
18
|
+
import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, DEFAULT_LOG_RETENTION_BYTES, buildEntry, optionalIndexUuid, entryByteEstimate, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, createInsyncRemoveMessage, SNAPSHOT_CHUNK_SIZE, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, checkpointLastSeqNo, checkpointPrimaryTerm, INSYNC_CAS_ATTEMPTS, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-GYLRKKUT.mjs';
|
|
19
|
+
import { directoryCheckpointStore, readListedFile, encodeSegmentManifest } from './chunk-REBDKXAH.mjs';
|
|
20
20
|
import { TransportErrorCodes, ClusterMessageTypes, ReplicationMessageTypes, SuggestionMessageTypes, SyncMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-5FLRXF73.mjs';
|
|
21
21
|
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
|
|
22
|
-
import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-
|
|
22
|
+
import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-NFAWR2O4.mjs';
|
|
23
23
|
import { getPath, getFs } from './chunk-HLPRHOSO.mjs';
|
|
24
24
|
import './chunk-MSIN4NM3.mjs';
|
|
25
|
-
import './chunk-
|
|
25
|
+
import './chunk-S5QMWECL.mjs';
|
|
26
26
|
import './chunk-LNGE7SLY.mjs';
|
|
27
|
-
import './chunk-
|
|
28
|
-
import './chunk-
|
|
29
|
-
import { deserializePartition } from './chunk-
|
|
30
|
-
import { decodeVectorFilePayload, bytesToVectors, isDead, MIN_EF_SEARCH, MAX_EF_SEARCH, MIN_OVERSAMPLE } from './chunk-
|
|
31
|
-
import './chunk-
|
|
32
|
-
import { RestoredHistory } from './chunk-
|
|
27
|
+
import './chunk-R2PJKSNL.mjs';
|
|
28
|
+
import './chunk-IDWDYS2N.mjs';
|
|
29
|
+
import { deserializePartition } from './chunk-7NCVV3ST.mjs';
|
|
30
|
+
import { decodeVectorFilePayload, bytesToVectors, isDead, MIN_EF_SEARCH, MAX_EF_SEARCH, MIN_OVERSAMPLE } from './chunk-KBU3ZFJN.mjs';
|
|
31
|
+
import './chunk-JOWXHENJ.mjs';
|
|
32
|
+
import { RestoredHistory } from './chunk-ZGMCJXY5.mjs';
|
|
33
33
|
import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
|
|
34
|
-
import { historyFilesOf, removeHistoryFolder, QUERY_HISTORY_FOLDER, RESTORE_STAGING_INFIX, openChecksummedEnvelope, historyFolderOf, listHistoryFolder, historyFileOf, COUNTS_STREAM_CHUNK_BYTES, BUILD_WRITE_BATCH, restoreStagingPrefix, crc32Init, crc32Final, crc32Update, MAX_COUNTS_STREAM_KEY_BYTES, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, storedStateOf } from './chunk-
|
|
34
|
+
import { historyFilesOf, removeHistoryFolder, QUERY_HISTORY_FOLDER, RESTORE_STAGING_INFIX, openChecksummedEnvelope, historyFolderOf, listHistoryFolder, historyFileOf, COUNTS_STREAM_CHUNK_BYTES, BUILD_WRITE_BATCH, restoreStagingPrefix, crc32Init, crc32Final, crc32Update, MAX_COUNTS_STREAM_KEY_BYTES, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, storedStateOf } from './chunk-PZYBWMCL.mjs';
|
|
35
35
|
import { boundQuerySuggestionsOf, bindQuerySuggestions } from './chunk-S2POFTLN.mjs';
|
|
36
|
-
import { oversampledShardSize, runtimeHost, bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, importOptionalReads, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, emitEngineEvent, synonymModule, createEngineIndex, generateId, getLanguage, resolveIndexAnalysis, memoryFractionsOf, createMemoryMonitor, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, bm25SetsValue, prepareConfiguredSynonyms, storedArrayPositionGap, updateDocumentVectors, arrayPositionsAreStale, fnv1a, readRecentMemoryFigures, normalizeSort, sortModesOf, resolveProjection, requireLoadedOptionalReads, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, validateBM25Shape, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, hasLanguage, FIELD_FILTER_OPERATORS, yieldToEventLoop, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, customTokensOf, visitTokens, clampRowCount, now, sortSignatureOf, listBindingOf, requireMatchingCursor, decodePageCursor, encodePageCursor, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, clampLimit, requireWithinResultWindow, providedDocId, holdsQuotedText, queryBindingOf, engineCoreOf, sortSignatureEntry, fnv1aUint32s, loadRerankRuntime, mergePartitionStats, mergeEntries, regionsOverlap, compareScores } from './chunk-
|
|
37
|
-
import { encodeDocument, encodeDocumentReply } from './chunk-
|
|
38
|
-
import { collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, onRefusalAfterCollection, reservedHeapBytes, reserveHeapBytes } from './chunk-
|
|
36
|
+
import { oversampledShardSize, runtimeHost, bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, importOptionalReads, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, emitEngineEvent, synonymModule, createEngineIndex, generateId, documentChurnOf, isCompositePartition, getLanguage, resolveIndexAnalysis, memoryFractionsOf, createMemoryMonitor, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, bm25SetsValue, prepareConfiguredSynonyms, storedArrayPositionGap, updateDocumentVectors, arrayPositionsAreStale, fnv1a, readRecentMemoryFigures, normalizeSort, sortModesOf, resolveProjection, noteDocumentChurn, MAP_TABLE_MIN_CAPACITY, MAP_TABLE_SHRINK_DIVISOR, requireLoadedOptionalReads, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, validateBM25Shape, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, hasLanguage, FIELD_FILTER_OPERATORS, yieldToEventLoop, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, customTokensOf, visitTokens, getVectorFieldPaths, resolvePartitionInsertOptions, clampRowCount, now, sortSignatureOf, listBindingOf, requireMatchingCursor, decodePageCursor, encodePageCursor, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, clampLimit, requireWithinResultWindow, providedDocId, MAP_TABLE_HEADER_BYTES, MAP_TABLE_SLOT_BYTES, holdsQuotedText, queryBindingOf, engineCoreOf, sortSignatureEntry, fnv1aUint32s, loadRerankRuntime, mergePartitionStats, mergeEntries, regionsOverlap, compareScores } from './chunk-JXDY3ZA3.mjs';
|
|
37
|
+
import { encodeDocument, encodeDocumentReply } from './chunk-2CKDCC65.mjs';
|
|
38
|
+
import { collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, onRefusalAfterCollection, reservedHeapBytes, reserveHeapBytes } from './chunk-AOYQTX6W.mjs';
|
|
39
39
|
import { decode } from './chunk-GXCDWLQI.mjs';
|
|
40
40
|
import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
41
41
|
import './chunk-7T4LAYT4.mjs';
|
|
42
|
-
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE, buildStoredValues } from './chunk-
|
|
43
|
-
import { loadMessagePack, validateSchema, requireCreatableSchema, schemaFieldsOf, getNestedValue, MAX_NESTING_DEPTH, MAX_STORED_VECTOR_DIMENSION, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-
|
|
42
|
+
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE, buildStoredValues } from './chunk-IL5DRCNU.mjs';
|
|
43
|
+
import { loadMessagePack, validateSchema, requireCreatableSchema, schemaFieldsOf, getNestedValue, MAX_NESTING_DEPTH, MAX_STORED_VECTOR_DIMENSION, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-7P5YWDRQ.mjs';
|
|
44
44
|
import './chunk-U2BKYEC7.mjs';
|
|
45
|
-
import './chunk-
|
|
45
|
+
import { shardEntryRangeOf } from './chunk-RORNUNXW.mjs';
|
|
46
46
|
import { collectAndReturnPages } from './chunk-UPDNNOTG.mjs';
|
|
47
47
|
import { BATCH_CHUNK_SIZE, MAX_SORT_FIELDS, MAX_DOC_ID_LENGTH, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, DEFAULT_PAGE_SIZE, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, HYBRID_RESCORE_WINDOW, DEFAULT_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT } from './chunk-3ECIRFS5.mjs';
|
|
48
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback } from './chunk-
|
|
48
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback } from './chunk-ASZLNVWW.mjs';
|
|
49
49
|
import './chunk-ZXX5GZEX.mjs';
|
|
50
|
-
import { joinChunks, readWhole } from './chunk-
|
|
50
|
+
import { joinChunks, readWhole } from './chunk-GR5GMFJJ.mjs';
|
|
51
51
|
import { MAX_DOCUMENT_ENCODING_DEPTH } from './chunk-4PY4JYMY.mjs';
|
|
52
|
-
import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-
|
|
52
|
+
import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-ZKOQR7EA.mjs';
|
|
53
53
|
import { holdsUnpairedSurrogate, compareCodePoints, readSortField, isSortMode, SORT_MODES, exceedsCodePoints, toComparableSortValue } from './chunk-TGPKS2UM.mjs';
|
|
54
54
|
import { encode } from '@msgpack/msgpack';
|
|
55
55
|
import { randomUUID, createHash } from 'crypto';
|
|
@@ -225,12 +225,13 @@ async function clearRebalanceProgress(coordinator, indexName, partitionId) {
|
|
|
225
225
|
}
|
|
226
226
|
|
|
227
227
|
// src/distribution/cluster/version.ts
|
|
228
|
-
var SPEC_VERSION = "1.
|
|
228
|
+
var SPEC_VERSION = "1.5";
|
|
229
229
|
var PHRASE_SEARCH_SPEC_VERSION = "1.1";
|
|
230
230
|
var BM25_TUNING_SPEC_VERSION = "1.1";
|
|
231
231
|
var BACKUP_SPEC_VERSION = "1.2";
|
|
232
232
|
var DEVICE_SYNC_SPEC_VERSION = "1.3";
|
|
233
233
|
var QUERY_SUGGESTIONS_SPEC_VERSION = "1.4";
|
|
234
|
+
var CHANGE_CAPTURE_SPEC_VERSION = "1.5";
|
|
234
235
|
var VERSION_PATTERN = /^(\d+)\.(\d+)$/;
|
|
235
236
|
function versionPartsOf(version) {
|
|
236
237
|
const match = VERSION_PATTERN.exec(version);
|
|
@@ -3203,6 +3204,50 @@ function chunkByBudget(items, budget) {
|
|
|
3203
3204
|
return chunks;
|
|
3204
3205
|
}
|
|
3205
3206
|
|
|
3207
|
+
// src/distribution/cluster/in-sync-version.ts
|
|
3208
|
+
function inSyncVersionOf(assignment) {
|
|
3209
|
+
return assignment.inSyncVersion ?? 0;
|
|
3210
|
+
}
|
|
3211
|
+
function holdSameMembers(left, right) {
|
|
3212
|
+
const leftMembers = new Set(left);
|
|
3213
|
+
const rightMembers = new Set(right);
|
|
3214
|
+
if (leftMembers.size !== rightMembers.size) return false;
|
|
3215
|
+
for (const nodeId of rightMembers) {
|
|
3216
|
+
if (!leftMembers.has(nodeId)) return false;
|
|
3217
|
+
}
|
|
3218
|
+
return true;
|
|
3219
|
+
}
|
|
3220
|
+
function dropsAReplica(previous, next) {
|
|
3221
|
+
const nextReplicas = new Set(next.replicas);
|
|
3222
|
+
return previous.replicas.some((nodeId) => !nextReplicas.has(nodeId));
|
|
3223
|
+
}
|
|
3224
|
+
function stampedInSyncVersion(previous, next) {
|
|
3225
|
+
if (previous === void 0) return inSyncVersionOf(next);
|
|
3226
|
+
const stored = inSyncVersionOf(previous);
|
|
3227
|
+
const unchanged = holdSameMembers(previous.inSyncSet, next.inSyncSet) && !dropsAReplica(previous, next);
|
|
3228
|
+
const raised = unchanged ? stored : stored + 1;
|
|
3229
|
+
return Math.max(raised, inSyncVersionOf(next));
|
|
3230
|
+
}
|
|
3231
|
+
function withInSyncVersions(previous, next) {
|
|
3232
|
+
let stamped = null;
|
|
3233
|
+
for (const [partitionId, assignment] of next.assignments) {
|
|
3234
|
+
const version = stampedInSyncVersion(previous?.assignments.get(partitionId), assignment);
|
|
3235
|
+
if (version === inSyncVersionOf(assignment)) continue;
|
|
3236
|
+
stamped ??= new Map(next.assignments);
|
|
3237
|
+
stamped.set(partitionId, { ...assignment, inSyncVersion: version });
|
|
3238
|
+
}
|
|
3239
|
+
return stamped === null ? next : { ...next, assignments: stamped };
|
|
3240
|
+
}
|
|
3241
|
+
function putAllocationOver(coordinator, indexName, previous, next, writerNodeId, replicaRegistration) {
|
|
3242
|
+
return coordinator.putAllocation(
|
|
3243
|
+
indexName,
|
|
3244
|
+
withInSyncVersions(previous, next),
|
|
3245
|
+
previous === null ? null : previous.version,
|
|
3246
|
+
writerNodeId,
|
|
3247
|
+
replicaRegistration
|
|
3248
|
+
);
|
|
3249
|
+
}
|
|
3250
|
+
|
|
3206
3251
|
// src/distribution/cluster/table-identity.ts
|
|
3207
3252
|
function belongsToAnotherIndex(table, indexUuid) {
|
|
3208
3253
|
return table.indexUuid !== void 0 && indexUuid !== void 0 && table.indexUuid !== indexUuid;
|
|
@@ -3217,16 +3262,10 @@ async function rewriteForCurrentIndex(coordinator, table) {
|
|
|
3217
3262
|
return stampedWith(table, indexUuid);
|
|
3218
3263
|
}
|
|
3219
3264
|
|
|
3220
|
-
// src/distribution/replication/constants.ts
|
|
3221
|
-
var REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES = 40;
|
|
3222
|
-
var DEFAULT_LOG_RETENTION_BYTES = 268435456;
|
|
3223
|
-
var INSYNC_CAS_ATTEMPTS = 5;
|
|
3224
|
-
var SNAPSHOT_CHUNK_SIZE = 65536;
|
|
3225
|
-
|
|
3226
3265
|
// src/distribution/replication/insync.ts
|
|
3227
3266
|
var REFUSE = { outcome: "refuse" };
|
|
3228
3267
|
var ACCEPT = { outcome: "accept" };
|
|
3229
|
-
async function updateInsyncSet(request, coordinator, controllerNodeId, decide) {
|
|
3268
|
+
async function updateInsyncSet(request, coordinator, controllerNodeId, decide, replicaRegistration) {
|
|
3230
3269
|
const refused = {
|
|
3231
3270
|
indexName: request.indexName,
|
|
3232
3271
|
partitionId: request.partitionId,
|
|
@@ -3261,7 +3300,14 @@ async function updateInsyncSet(request, coordinator, controllerNodeId, decide) {
|
|
|
3261
3300
|
if (rewritten === null) {
|
|
3262
3301
|
return refused;
|
|
3263
3302
|
}
|
|
3264
|
-
const written = await
|
|
3303
|
+
const written = await putAllocationOver(
|
|
3304
|
+
coordinator,
|
|
3305
|
+
request.indexName,
|
|
3306
|
+
table,
|
|
3307
|
+
rewritten,
|
|
3308
|
+
controllerNodeId,
|
|
3309
|
+
replicaRegistration
|
|
3310
|
+
);
|
|
3265
3311
|
if (written) {
|
|
3266
3312
|
return { ...refused, accepted: true };
|
|
3267
3313
|
}
|
|
@@ -3286,7 +3332,7 @@ function handleInsyncRemoval(payload, coordinator, controllerNodeId) {
|
|
|
3286
3332
|
return REFUSE;
|
|
3287
3333
|
}
|
|
3288
3334
|
if (!assignment.inSyncSet.includes(payload.replicaNodeId)) {
|
|
3289
|
-
return
|
|
3335
|
+
return { outcome: "write", assignment: { ...assignment, inSyncVersion: inSyncVersionOf(assignment) + 1 } };
|
|
3290
3336
|
}
|
|
3291
3337
|
return {
|
|
3292
3338
|
outcome: "write",
|
|
@@ -3305,16 +3351,21 @@ async function requestInsyncAdmission(payload, controllerNodeId, transport, sour
|
|
|
3305
3351
|
}
|
|
3306
3352
|
async function handleInsyncAdmission(payload, coordinator, controllerNodeId) {
|
|
3307
3353
|
const registered = await coordinator.listNodes();
|
|
3308
|
-
|
|
3354
|
+
const registration = registered.find((node) => node.nodeId === payload.replicaNodeId);
|
|
3355
|
+
if (registration === void 0 || registration.startedAt !== payload.replicaStartedAt) {
|
|
3309
3356
|
return { indexName: payload.indexName, partitionId: payload.partitionId, accepted: false };
|
|
3310
3357
|
}
|
|
3311
|
-
|
|
3358
|
+
const replicaRegistration = { nodeId: payload.replicaNodeId, startedAt: payload.replicaStartedAt };
|
|
3359
|
+
const decide = (assignment) => {
|
|
3312
3360
|
if (assignment.state !== "ACTIVE") {
|
|
3313
3361
|
return REFUSE;
|
|
3314
3362
|
}
|
|
3315
3363
|
if (!assignment.replicas.includes(payload.replicaNodeId)) {
|
|
3316
3364
|
return REFUSE;
|
|
3317
3365
|
}
|
|
3366
|
+
if (payload.inSyncVersion !== inSyncVersionOf(assignment)) {
|
|
3367
|
+
return REFUSE;
|
|
3368
|
+
}
|
|
3318
3369
|
if (payload.appliedSeqNo < assignment.commitPoint || payload.appliedSeqNo < payload.commitPoint) {
|
|
3319
3370
|
return REFUSE;
|
|
3320
3371
|
}
|
|
@@ -3331,7 +3382,8 @@ async function handleInsyncAdmission(payload, coordinator, controllerNodeId) {
|
|
|
3331
3382
|
commitPoint
|
|
3332
3383
|
}
|
|
3333
3384
|
};
|
|
3334
|
-
}
|
|
3385
|
+
};
|
|
3386
|
+
return updateInsyncSet(payload, coordinator, controllerNodeId, decide, replicaRegistration);
|
|
3335
3387
|
}
|
|
3336
3388
|
|
|
3337
3389
|
// src/distribution/replication/primary.ts
|
|
@@ -3344,7 +3396,7 @@ async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeI
|
|
|
3344
3396
|
}
|
|
3345
3397
|
async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3, indexUuid, previousPrimaryTerm) {
|
|
3346
3398
|
if (entries.length === 0) {
|
|
3347
|
-
return { acknowledged: [], failed: [] };
|
|
3399
|
+
return { acknowledged: [], failed: [], refusedAsInvalid: [] };
|
|
3348
3400
|
}
|
|
3349
3401
|
if (entries.length === 1) {
|
|
3350
3402
|
return replicateToReplicas(
|
|
@@ -3364,13 +3416,14 @@ async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sour
|
|
|
3364
3416
|
async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport, resolveNodeTargets3) {
|
|
3365
3417
|
const uniqueReplicas = [...new Set(inSyncReplicas)];
|
|
3366
3418
|
if (uniqueReplicas.length === 0) {
|
|
3367
|
-
return { acknowledged: [], failed: [] };
|
|
3419
|
+
return { acknowledged: [], failed: [], refusedAsInvalid: [] };
|
|
3368
3420
|
}
|
|
3369
3421
|
const sendResults = await Promise.allSettled(
|
|
3370
3422
|
uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
|
|
3371
3423
|
);
|
|
3372
3424
|
const acknowledged = [];
|
|
3373
3425
|
const failed2 = [];
|
|
3426
|
+
const refusedAsInvalid = [];
|
|
3374
3427
|
for (let i = 0; i < sendResults.length; i++) {
|
|
3375
3428
|
const result = sendResults[i];
|
|
3376
3429
|
const replicaNodeId = uniqueReplicas[i];
|
|
@@ -3386,9 +3439,10 @@ async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport,
|
|
|
3386
3439
|
acknowledged.push(replicaNodeId);
|
|
3387
3440
|
} else {
|
|
3388
3441
|
failed2.push(replicaNodeId);
|
|
3442
|
+
if (refusesAsInvalid(response)) refusedAsInvalid.push(replicaNodeId);
|
|
3389
3443
|
}
|
|
3390
3444
|
}
|
|
3391
|
-
return { acknowledged, failed: failed2 };
|
|
3445
|
+
return { acknowledged, failed: failed2, refusedAsInvalid };
|
|
3392
3446
|
}
|
|
3393
3447
|
async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
|
|
3394
3448
|
const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
|
|
@@ -3406,6 +3460,15 @@ async function sendToReplica(transport, replicaNodeId, message, resolveNodeTarge
|
|
|
3406
3460
|
}
|
|
3407
3461
|
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
3408
3462
|
}
|
|
3463
|
+
function refusesAsInvalid(response) {
|
|
3464
|
+
if (!response.type.endsWith(".error")) return false;
|
|
3465
|
+
try {
|
|
3466
|
+
const payload = decode(response.payload);
|
|
3467
|
+
return isRecord(payload) && payload.code === ErrorCodes.REPLICATION_ENTRY_INVALID;
|
|
3468
|
+
} catch (_) {
|
|
3469
|
+
return false;
|
|
3470
|
+
}
|
|
3471
|
+
}
|
|
3409
3472
|
function isMatchingAck(response, entry) {
|
|
3410
3473
|
if (response.type !== ReplicationMessageTypes.ACK) {
|
|
3411
3474
|
return false;
|
|
@@ -3425,6 +3488,7 @@ function partitionKey(indexName, partitionId) {
|
|
|
3425
3488
|
function createCatchUpState() {
|
|
3426
3489
|
return {
|
|
3427
3490
|
cursors: /* @__PURE__ */ new Map(),
|
|
3491
|
+
syncSessions: /* @__PURE__ */ new Map(),
|
|
3428
3492
|
pendingAdmissions: /* @__PURE__ */ new Map(),
|
|
3429
3493
|
admittedReplicas: /* @__PURE__ */ new Map(),
|
|
3430
3494
|
admissionsToWithdraw: /* @__PURE__ */ new Map(),
|
|
@@ -3451,17 +3515,26 @@ function noteAcknowledgedPosition(state, key, replicaNodeId, appliedSeqNo) {
|
|
|
3451
3515
|
acknowledged.set(replicaNodeId, appliedSeqNo);
|
|
3452
3516
|
state.acknowledgedSeqNos.set(key, acknowledged);
|
|
3453
3517
|
}
|
|
3454
|
-
function recordReplicaPosition(state, indexName, partitionId, replicaNodeId, appliedSeqNo, logEnd) {
|
|
3518
|
+
function recordReplicaPosition(state, indexName, partitionId, replicaNodeId, appliedSeqNo, logEnd, confirmsPosition) {
|
|
3455
3519
|
const key = partitionKey(indexName, partitionId);
|
|
3456
3520
|
const reported = Math.min(appliedSeqNo, logEnd);
|
|
3457
3521
|
const partition = state.cursors.get(key) ?? /* @__PURE__ */ new Map();
|
|
3458
3522
|
const existing = partition.get(replicaNodeId);
|
|
3459
3523
|
if (existing === void 0) {
|
|
3460
|
-
partition.set(replicaNodeId, {
|
|
3461
|
-
|
|
3462
|
-
|
|
3524
|
+
partition.set(replicaNodeId, {
|
|
3525
|
+
appliedSeqNo: reported,
|
|
3526
|
+
confirmed: confirmsPosition,
|
|
3527
|
+
busy: false,
|
|
3528
|
+
syncEpoch: 0,
|
|
3529
|
+
feed: null
|
|
3530
|
+
});
|
|
3463
3531
|
} else {
|
|
3464
|
-
existing.appliedSeqNo
|
|
3532
|
+
if (!confirmsPosition || reported < existing.appliedSeqNo) {
|
|
3533
|
+
openFreshSyncSession(existing, reported);
|
|
3534
|
+
} else {
|
|
3535
|
+
existing.appliedSeqNo = reported;
|
|
3536
|
+
}
|
|
3537
|
+
existing.confirmed = confirmsPosition;
|
|
3465
3538
|
}
|
|
3466
3539
|
state.cursors.set(key, partition);
|
|
3467
3540
|
noteAcknowledgedPosition(state, key, replicaNodeId, reported);
|
|
@@ -3486,7 +3559,10 @@ function forgetReplica(state, indexName, partitionId, replicaNodeId) {
|
|
|
3486
3559
|
state.cursors.delete(key);
|
|
3487
3560
|
}
|
|
3488
3561
|
}
|
|
3489
|
-
|
|
3562
|
+
const pending = pendingAdmissionOf(state, indexName, partitionId, replicaNodeId);
|
|
3563
|
+
if (pending !== void 0 && pending.answer === null) {
|
|
3564
|
+
clearPendingAdmission(state, indexName, partitionId, replicaNodeId, pending);
|
|
3565
|
+
}
|
|
3490
3566
|
}
|
|
3491
3567
|
function forgetKeysOfIndex(entries, indexName) {
|
|
3492
3568
|
for (const key of [...entries.keys()]) {
|
|
@@ -3499,6 +3575,7 @@ function forgetCatchUpIndex(state, indexName) {
|
|
|
3499
3575
|
for (const cursor of replicas.values()) cursor.feed?.cancel();
|
|
3500
3576
|
state.cursors.delete(key);
|
|
3501
3577
|
}
|
|
3578
|
+
forgetKeysOfIndex(state.syncSessions, indexName);
|
|
3502
3579
|
forgetKeysOfIndex(state.pendingAdmissions, indexName);
|
|
3503
3580
|
forgetKeysOfIndex(state.admittedReplicas, indexName);
|
|
3504
3581
|
forgetKeysOfIndex(state.admissionsToWithdraw, indexName);
|
|
@@ -3521,28 +3598,18 @@ function feedsReplica(state, indexName, partitionId, replicaNodeId) {
|
|
|
3521
3598
|
}
|
|
3522
3599
|
function getPendingAdmissions(state, indexName, partitionId) {
|
|
3523
3600
|
const pending = state.pendingAdmissions.get(partitionKey(indexName, partitionId));
|
|
3524
|
-
return pending === void 0 ? [] : [...pending];
|
|
3601
|
+
return pending === void 0 ? [] : [...pending.keys()];
|
|
3525
3602
|
}
|
|
3526
|
-
function
|
|
3527
|
-
|
|
3528
|
-
const owed = state.admissionsToWithdraw.get(key) ?? /* @__PURE__ */ new Map();
|
|
3529
|
-
owed.set(replicaNodeId, { primaryTerm, withdrawing: false });
|
|
3530
|
-
state.admissionsToWithdraw.set(key, owed);
|
|
3531
|
-
}
|
|
3532
|
-
function forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId) {
|
|
3533
|
-
const key = partitionKey(indexName, partitionId);
|
|
3534
|
-
const owed = state.admissionsToWithdraw.get(key);
|
|
3535
|
-
if (owed === void 0) return;
|
|
3536
|
-
owed.delete(replicaNodeId);
|
|
3537
|
-
if (owed.size === 0) state.admissionsToWithdraw.delete(key);
|
|
3603
|
+
function pendingAdmissionOf(state, indexName, partitionId, replicaNodeId) {
|
|
3604
|
+
return state.pendingAdmissions.get(partitionKey(indexName, partitionId))?.get(replicaNodeId);
|
|
3538
3605
|
}
|
|
3539
3606
|
function markPendingAdmission(state, indexName, partitionId, replicaNodeId) {
|
|
3540
3607
|
const key = partitionKey(indexName, partitionId);
|
|
3541
|
-
const pending = state.pendingAdmissions.get(key) ?? /* @__PURE__ */ new
|
|
3608
|
+
const pending = state.pendingAdmissions.get(key) ?? /* @__PURE__ */ new Map();
|
|
3542
3609
|
if (pending.has(replicaNodeId)) {
|
|
3543
3610
|
return false;
|
|
3544
3611
|
}
|
|
3545
|
-
pending.
|
|
3612
|
+
pending.set(replicaNodeId, { failedWrite: false, unanswered: false, answer: null });
|
|
3546
3613
|
state.pendingAdmissions.set(key, pending);
|
|
3547
3614
|
forgetRemovedReplica(state, indexName, partitionId, replicaNodeId);
|
|
3548
3615
|
advanceAdmissionGeneration(state, indexName, partitionId, replicaNodeId);
|
|
@@ -3570,21 +3637,10 @@ function forgetRemovedReplica(state, indexName, partitionId, replicaNodeId) {
|
|
|
3570
3637
|
removed.delete(replicaNodeId);
|
|
3571
3638
|
if (removed.size === 0) state.removedReplicas.delete(key);
|
|
3572
3639
|
}
|
|
3573
|
-
function
|
|
3574
|
-
const key = partitionKey(indexName, partitionId);
|
|
3575
|
-
const removed = state.removedReplicas.get(key) ?? /* @__PURE__ */ new Map();
|
|
3576
|
-
removed.set(replicaNodeId, primaryTerm);
|
|
3577
|
-
state.removedReplicas.set(key, removed);
|
|
3578
|
-
}
|
|
3579
|
-
function withoutRemovedReplicas(state, indexName, partitionId, primaryTerm, targets) {
|
|
3580
|
-
const removed = state.removedReplicas.get(partitionKey(indexName, partitionId));
|
|
3581
|
-
if (removed === void 0) return targets;
|
|
3582
|
-
return targets.filter((replicaNodeId) => removed.get(replicaNodeId) !== primaryTerm);
|
|
3583
|
-
}
|
|
3584
|
-
function clearPendingAdmission(state, indexName, partitionId, replicaNodeId) {
|
|
3640
|
+
function clearPendingAdmission(state, indexName, partitionId, replicaNodeId, claim) {
|
|
3585
3641
|
const key = partitionKey(indexName, partitionId);
|
|
3586
3642
|
const pending = state.pendingAdmissions.get(key);
|
|
3587
|
-
if (pending === void 0) {
|
|
3643
|
+
if (pending === void 0 || claim !== void 0 && pending.get(replicaNodeId) !== claim) {
|
|
3588
3644
|
return;
|
|
3589
3645
|
}
|
|
3590
3646
|
pending.delete(replicaNodeId);
|
|
@@ -3618,6 +3674,47 @@ function keepAcknowledgementsOf(state, ledPartitionKeys) {
|
|
|
3618
3674
|
}
|
|
3619
3675
|
|
|
3620
3676
|
// src/distribution/cluster-node/catch-up/admitted.ts
|
|
3677
|
+
function admissionToWithdrawOf(state, indexName, partitionId, replicaNodeId) {
|
|
3678
|
+
return state.admissionsToWithdraw.get(partitionKey(indexName, partitionId))?.get(replicaNodeId);
|
|
3679
|
+
}
|
|
3680
|
+
function withUnansweredAdmissions(state, indexName, partitionId, localNodeId, targets) {
|
|
3681
|
+
const pending = state.pendingAdmissions.get(partitionKey(indexName, partitionId));
|
|
3682
|
+
if (pending === void 0) return targets;
|
|
3683
|
+
const unanswered = [...pending].filter(([replicaNodeId, admission]) => admission.unanswered && replicaNodeId !== localNodeId).map(([replicaNodeId]) => replicaNodeId);
|
|
3684
|
+
return [.../* @__PURE__ */ new Set([...targets, ...unanswered])];
|
|
3685
|
+
}
|
|
3686
|
+
function settleFencedAdmission(state, indexName, partitionId, replicaNodeId, fenced) {
|
|
3687
|
+
if (fenced.pending?.unanswered === true) {
|
|
3688
|
+
clearPendingAdmission(state, indexName, partitionId, replicaNodeId, fenced.pending);
|
|
3689
|
+
}
|
|
3690
|
+
if (fenced.owed !== void 0 && admissionToWithdrawOf(state, indexName, partitionId, replicaNodeId) === fenced.owed) {
|
|
3691
|
+
forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId);
|
|
3692
|
+
}
|
|
3693
|
+
}
|
|
3694
|
+
function recordAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId, primaryTerm) {
|
|
3695
|
+
const key = partitionKey(indexName, partitionId);
|
|
3696
|
+
const owed = state.admissionsToWithdraw.get(key) ?? /* @__PURE__ */ new Map();
|
|
3697
|
+
owed.set(replicaNodeId, { primaryTerm, withdrawing: false });
|
|
3698
|
+
state.admissionsToWithdraw.set(key, owed);
|
|
3699
|
+
}
|
|
3700
|
+
function forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId) {
|
|
3701
|
+
const key = partitionKey(indexName, partitionId);
|
|
3702
|
+
const owed = state.admissionsToWithdraw.get(key);
|
|
3703
|
+
if (owed === void 0) return;
|
|
3704
|
+
owed.delete(replicaNodeId);
|
|
3705
|
+
if (owed.size === 0) state.admissionsToWithdraw.delete(key);
|
|
3706
|
+
}
|
|
3707
|
+
function recordRemovedReplica(state, indexName, partitionId, replicaNodeId, primaryTerm) {
|
|
3708
|
+
const key = partitionKey(indexName, partitionId);
|
|
3709
|
+
const removed = state.removedReplicas.get(key) ?? /* @__PURE__ */ new Map();
|
|
3710
|
+
removed.set(replicaNodeId, primaryTerm);
|
|
3711
|
+
state.removedReplicas.set(key, removed);
|
|
3712
|
+
}
|
|
3713
|
+
function withoutRemovedReplicas(state, indexName, partitionId, primaryTerm, targets) {
|
|
3714
|
+
const removed = state.removedReplicas.get(partitionKey(indexName, partitionId));
|
|
3715
|
+
if (removed === void 0) return targets;
|
|
3716
|
+
return targets.filter((replicaNodeId) => removed.get(replicaNodeId) !== primaryTerm);
|
|
3717
|
+
}
|
|
3621
3718
|
function recordAdmittedReplica(state, indexName, partitionId, replicaNodeId, primaryTerm, generation) {
|
|
3622
3719
|
const key = partitionKey(indexName, partitionId);
|
|
3623
3720
|
const admitted = state.admittedReplicas.get(key) ?? /* @__PURE__ */ new Map();
|
|
@@ -3652,6 +3749,371 @@ function admittedBeyondTable(state, indexName, partitionId, assignment, generati
|
|
|
3652
3749
|
return beyond;
|
|
3653
3750
|
}
|
|
3654
3751
|
|
|
3752
|
+
// src/distribution/cluster-node/catch-up/feed.ts
|
|
3753
|
+
function inSyncSignature(assignment) {
|
|
3754
|
+
return [...assignment.inSyncSet].sort().join("\n");
|
|
3755
|
+
}
|
|
3756
|
+
function createCatchUpFeed(assignment, replicaStartedAt) {
|
|
3757
|
+
let release = () => {
|
|
3758
|
+
};
|
|
3759
|
+
const whenCancelled = new Promise((resolve) => {
|
|
3760
|
+
release = resolve;
|
|
3761
|
+
});
|
|
3762
|
+
const feed = {
|
|
3763
|
+
primaryTerm: assignment.primaryTerm,
|
|
3764
|
+
inSyncSet: inSyncSignature(assignment),
|
|
3765
|
+
replicaStartedAt,
|
|
3766
|
+
whenCancelled,
|
|
3767
|
+
cancelled: false,
|
|
3768
|
+
cancel: () => {
|
|
3769
|
+
feed.cancelled = true;
|
|
3770
|
+
release();
|
|
3771
|
+
}
|
|
3772
|
+
};
|
|
3773
|
+
return feed;
|
|
3774
|
+
}
|
|
3775
|
+
function feedIsStale(feed, assignment, registered, replicaNodeId) {
|
|
3776
|
+
const replicaLeft = registered !== null && registered.get(replicaNodeId) !== feed.replicaStartedAt;
|
|
3777
|
+
return replicaLeft || assignment.primaryTerm !== feed.primaryTerm || inSyncSignature(assignment) !== feed.inSyncSet;
|
|
3778
|
+
}
|
|
3779
|
+
function untilCancelled(feed, work, fallback) {
|
|
3780
|
+
if (feed.cancelled) {
|
|
3781
|
+
work.catch(() => {
|
|
3782
|
+
});
|
|
3783
|
+
return Promise.resolve(fallback);
|
|
3784
|
+
}
|
|
3785
|
+
return Promise.race([work, feed.whenCancelled.then(() => fallback)]);
|
|
3786
|
+
}
|
|
3787
|
+
|
|
3788
|
+
// src/distribution/cluster-node/catch-up/batches.ts
|
|
3789
|
+
function batchBytes(entries) {
|
|
3790
|
+
let total = 0;
|
|
3791
|
+
for (const entry of entries) {
|
|
3792
|
+
total += entryByteEstimate(entry);
|
|
3793
|
+
}
|
|
3794
|
+
return total;
|
|
3795
|
+
}
|
|
3796
|
+
function nextWireBatch(state, pending) {
|
|
3797
|
+
const chunks = chunkByBudget(pending, {
|
|
3798
|
+
...WIRE_BATCH_BUDGET,
|
|
3799
|
+
payloadBytesOf: (entry) => entry.document?.byteLength ?? 0,
|
|
3800
|
+
breaksRun: (entry, previous) => entry.seqNo !== previous.seqNo + 1
|
|
3801
|
+
});
|
|
3802
|
+
const batch = chunks[0] ?? [];
|
|
3803
|
+
if (batch.length === 0) {
|
|
3804
|
+
return [];
|
|
3805
|
+
}
|
|
3806
|
+
const available = MAX_CATCH_UP_IN_FLIGHT_BYTES - state.inFlightBytes;
|
|
3807
|
+
return batchBytes(batch) > available ? [] : batch;
|
|
3808
|
+
}
|
|
3809
|
+
async function sendBatch(state, deps, batch, replicaNodeId, cursor, feed) {
|
|
3810
|
+
const reservedBytes = batchBytes(batch);
|
|
3811
|
+
const sendingEpoch = cursor.syncEpoch;
|
|
3812
|
+
state.inFlightBytes += reservedBytes;
|
|
3813
|
+
const delivery = (async () => {
|
|
3814
|
+
const first = batch[0];
|
|
3815
|
+
const indexName = first?.indexName;
|
|
3816
|
+
const table = indexName === void 0 ? null : await deps.coordinator.getAllocation(indexName);
|
|
3817
|
+
if (feed.cancelled) {
|
|
3818
|
+
return false;
|
|
3819
|
+
}
|
|
3820
|
+
const result = await replicateBatchToReplicas(
|
|
3821
|
+
batch,
|
|
3822
|
+
[replicaNodeId],
|
|
3823
|
+
deps.transport,
|
|
3824
|
+
deps.nodeId,
|
|
3825
|
+
deps.resolveNodeTargets,
|
|
3826
|
+
table?.indexUuid,
|
|
3827
|
+
first === void 0 ? void 0 : previousTermOf(deps.getReplicationLog(first.indexName, first.partitionId), first.seqNo)
|
|
3828
|
+
);
|
|
3829
|
+
return result.acknowledged.includes(replicaNodeId);
|
|
3830
|
+
})().finally(() => {
|
|
3831
|
+
state.inFlightBytes -= reservedBytes;
|
|
3832
|
+
});
|
|
3833
|
+
const acknowledged = await untilCancelled(feed, delivery, false);
|
|
3834
|
+
if (!acknowledged || feed.cancelled || cursor.syncEpoch !== sendingEpoch) {
|
|
3835
|
+
return false;
|
|
3836
|
+
}
|
|
3837
|
+
cursor.appliedSeqNo = Math.max(cursor.appliedSeqNo, batch[batch.length - 1].seqNo);
|
|
3838
|
+
cursor.confirmed = true;
|
|
3839
|
+
return true;
|
|
3840
|
+
}
|
|
3841
|
+
async function sendHeldEntryAgain(state, deps, entry, replicaNodeId) {
|
|
3842
|
+
const reservedBytes = entryByteEstimate(entry);
|
|
3843
|
+
if (reservedBytes > MAX_CATCH_UP_IN_FLIGHT_BYTES - state.inFlightBytes) return "unconfirmed";
|
|
3844
|
+
state.inFlightBytes += reservedBytes;
|
|
3845
|
+
try {
|
|
3846
|
+
const table = await deps.coordinator.getAllocation(entry.indexName);
|
|
3847
|
+
const result = await replicateBatchToReplicas(
|
|
3848
|
+
[entry],
|
|
3849
|
+
[replicaNodeId],
|
|
3850
|
+
deps.transport,
|
|
3851
|
+
deps.nodeId,
|
|
3852
|
+
deps.resolveNodeTargets,
|
|
3853
|
+
table?.indexUuid,
|
|
3854
|
+
previousTermOf(deps.getReplicationLog(entry.indexName, entry.partitionId), entry.seqNo)
|
|
3855
|
+
);
|
|
3856
|
+
if (result.acknowledged.includes(replicaNodeId)) return "acknowledged";
|
|
3857
|
+
return result.refusedAsInvalid.includes(replicaNodeId) ? "refused-as-invalid" : "unconfirmed";
|
|
3858
|
+
} finally {
|
|
3859
|
+
state.inFlightBytes -= reservedBytes;
|
|
3860
|
+
}
|
|
3861
|
+
}
|
|
3862
|
+
function validatePartitionStoresPayload(decoded) {
|
|
3863
|
+
if (!isRecord(decoded)) {
|
|
3864
|
+
return null;
|
|
3865
|
+
}
|
|
3866
|
+
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
3867
|
+
return null;
|
|
3868
|
+
}
|
|
3869
|
+
return { indexName: decoded.indexName };
|
|
3870
|
+
}
|
|
3871
|
+
function validatePartitionStoresResultPayload(decoded) {
|
|
3872
|
+
if (!isRecord(decoded)) {
|
|
3873
|
+
return null;
|
|
3874
|
+
}
|
|
3875
|
+
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
3876
|
+
return null;
|
|
3877
|
+
}
|
|
3878
|
+
if (decoded.indexUuid !== null && typeof decoded.indexUuid !== "string") {
|
|
3879
|
+
return null;
|
|
3880
|
+
}
|
|
3881
|
+
if (!Array.isArray(decoded.partitionIds)) {
|
|
3882
|
+
return null;
|
|
3883
|
+
}
|
|
3884
|
+
const partitionIds = [];
|
|
3885
|
+
for (const partitionId of decoded.partitionIds) {
|
|
3886
|
+
if (typeof partitionId !== "number" || !Number.isInteger(partitionId) || partitionId < 0) {
|
|
3887
|
+
return null;
|
|
3888
|
+
}
|
|
3889
|
+
partitionIds.push(partitionId);
|
|
3890
|
+
}
|
|
3891
|
+
return { indexName: decoded.indexName, indexUuid: decoded.indexUuid, partitionIds };
|
|
3892
|
+
}
|
|
3893
|
+
function createPartitionStoresMessage(payload, sourceNodeId, requestId) {
|
|
3894
|
+
return {
|
|
3895
|
+
type: ClusterMessageTypes.PARTITION_STORES,
|
|
3896
|
+
sourceId: sourceNodeId,
|
|
3897
|
+
requestId,
|
|
3898
|
+
payload: encode(payload)
|
|
3899
|
+
};
|
|
3900
|
+
}
|
|
3901
|
+
function createPartitionStoresResultMessage(result, sourceNodeId, requestId) {
|
|
3902
|
+
return {
|
|
3903
|
+
type: ClusterMessageTypes.PARTITION_STORES,
|
|
3904
|
+
sourceId: sourceNodeId,
|
|
3905
|
+
requestId,
|
|
3906
|
+
payload: encode(result)
|
|
3907
|
+
};
|
|
3908
|
+
}
|
|
3909
|
+
|
|
3910
|
+
// src/distribution/cluster-node/catch-up/listed-copy.ts
|
|
3911
|
+
function answerOf(payload) {
|
|
3912
|
+
try {
|
|
3913
|
+
return validatePartitionStoresResultPayload(decode(payload));
|
|
3914
|
+
} catch (_) {
|
|
3915
|
+
return null;
|
|
3916
|
+
}
|
|
3917
|
+
}
|
|
3918
|
+
async function replicaListsPartition(deps, indexName, partitionId, replicaNodeId) {
|
|
3919
|
+
const table = await deps.coordinator.getAllocation(indexName);
|
|
3920
|
+
const message = createPartitionStoresMessage({ indexName }, deps.nodeId, generateId());
|
|
3921
|
+
for (const target of await resolveNodeTargets(replicaNodeId, deps)) {
|
|
3922
|
+
let payload;
|
|
3923
|
+
try {
|
|
3924
|
+
payload = (await deps.transport.send(target, message)).payload;
|
|
3925
|
+
} catch (_) {
|
|
3926
|
+
continue;
|
|
3927
|
+
}
|
|
3928
|
+
const answer2 = answerOf(payload);
|
|
3929
|
+
if (answer2 === null || answer2.indexName !== indexName || answer2.indexUuid === null) return false;
|
|
3930
|
+
if (table?.indexUuid !== void 0 && answer2.indexUuid !== table.indexUuid) return false;
|
|
3931
|
+
return answer2.partitionIds.includes(partitionId);
|
|
3932
|
+
}
|
|
3933
|
+
return false;
|
|
3934
|
+
}
|
|
3935
|
+
|
|
3936
|
+
// src/distribution/cluster-node/catch-up/admission.ts
|
|
3937
|
+
async function sendToController(payload, controllerNodeId, deps) {
|
|
3938
|
+
const targets = await resolveNodeTargets(controllerNodeId, deps);
|
|
3939
|
+
let sentWithoutAnswer = false;
|
|
3940
|
+
for (const target of targets) {
|
|
3941
|
+
try {
|
|
3942
|
+
const result = await requestInsyncAdmission(payload, target, deps.transport, deps.nodeId);
|
|
3943
|
+
if (result.accepted) return "accepted";
|
|
3944
|
+
return sentWithoutAnswer ? "unanswered" : "refused";
|
|
3945
|
+
} catch (_) {
|
|
3946
|
+
sentWithoutAnswer = true;
|
|
3947
|
+
}
|
|
3948
|
+
}
|
|
3949
|
+
return "unanswered";
|
|
3950
|
+
}
|
|
3951
|
+
function withTimeout(work, timeoutMs, fallback) {
|
|
3952
|
+
return new Promise((resolve) => {
|
|
3953
|
+
const timer = setTimeout(() => resolve(fallback), timeoutMs);
|
|
3954
|
+
timer.unref?.();
|
|
3955
|
+
work.then((value) => {
|
|
3956
|
+
clearTimeout(timer);
|
|
3957
|
+
resolve(value);
|
|
3958
|
+
}).catch(() => {
|
|
3959
|
+
clearTimeout(timer);
|
|
3960
|
+
resolve(fallback);
|
|
3961
|
+
});
|
|
3962
|
+
});
|
|
3963
|
+
}
|
|
3964
|
+
async function withdrawAdmission(removal, controllerNodeId, deps) {
|
|
3965
|
+
const targets = await resolveNodeTargets(controllerNodeId, deps);
|
|
3966
|
+
for (const target of targets) {
|
|
3967
|
+
try {
|
|
3968
|
+
const result = await requestInsyncRemoval(
|
|
3969
|
+
removal.indexName,
|
|
3970
|
+
removal.partitionId,
|
|
3971
|
+
removal.replicaNodeId,
|
|
3972
|
+
removal.primaryTerm,
|
|
3973
|
+
target,
|
|
3974
|
+
deps.transport,
|
|
3975
|
+
deps.nodeId
|
|
3976
|
+
);
|
|
3977
|
+
if (result.accepted) {
|
|
3978
|
+
return true;
|
|
3979
|
+
}
|
|
3980
|
+
} catch (_) {
|
|
3981
|
+
}
|
|
3982
|
+
}
|
|
3983
|
+
return false;
|
|
3984
|
+
}
|
|
3985
|
+
async function registeredStartedAt(deps, replicaNodeId) {
|
|
3986
|
+
try {
|
|
3987
|
+
const registered = await deps.coordinator.listNodes();
|
|
3988
|
+
return registered.find((node) => node.nodeId === replicaNodeId)?.startedAt ?? null;
|
|
3989
|
+
} catch (_) {
|
|
3990
|
+
return null;
|
|
3991
|
+
}
|
|
3992
|
+
}
|
|
3993
|
+
function claimHolds(claim, payload) {
|
|
3994
|
+
const current = pendingAdmissionOf(claim.state, payload.indexName, payload.partitionId, payload.replicaNodeId);
|
|
3995
|
+
return current === claim.pending && !claim.pending.failedWrite && claim.cursor.syncEpoch === claim.proposedEpoch;
|
|
3996
|
+
}
|
|
3997
|
+
async function changedInSyncVersion(claim, payload) {
|
|
3998
|
+
const { deps } = claim;
|
|
3999
|
+
let assignment;
|
|
4000
|
+
try {
|
|
4001
|
+
const table = await (deps.readLiveAllocation?.(payload.indexName) ?? deps.coordinator.getAllocation(payload.indexName));
|
|
4002
|
+
assignment = table?.assignments.get(payload.partitionId);
|
|
4003
|
+
} catch (_) {
|
|
4004
|
+
return null;
|
|
4005
|
+
}
|
|
4006
|
+
if (assignment === void 0) return null;
|
|
4007
|
+
const inSyncVersion = inSyncVersionOf(assignment);
|
|
4008
|
+
const qualifies = inSyncVersion !== payload.inSyncVersion && assignment.primary === deps.nodeId && assignment.primaryTerm === payload.primaryTerm && assignment.state === "ACTIVE" && assignment.replicas.includes(payload.replicaNodeId) && !assignment.inSyncSet.includes(payload.replicaNodeId) && payload.appliedSeqNo >= assignment.commitPoint;
|
|
4009
|
+
return qualifies ? inSyncVersion : null;
|
|
4010
|
+
}
|
|
4011
|
+
async function provePositionAfterRead(claim, request) {
|
|
4012
|
+
const { state, cursor, deps } = claim;
|
|
4013
|
+
if (request.appliedSeqNo === 0) return true;
|
|
4014
|
+
const entry = deps.getReplicationLog(request.indexName, request.partitionId).getEntry(request.appliedSeqNo);
|
|
4015
|
+
const provingEpoch = cursor.syncEpoch;
|
|
4016
|
+
if (entry === void 0) {
|
|
4017
|
+
const listed = await replicaListsPartition(
|
|
4018
|
+
deps,
|
|
4019
|
+
request.indexName,
|
|
4020
|
+
request.partitionId,
|
|
4021
|
+
request.replicaNodeId
|
|
4022
|
+
).catch(() => false);
|
|
4023
|
+
return listed && cursor.syncEpoch === provingEpoch;
|
|
4024
|
+
}
|
|
4025
|
+
const answer2 = await sendHeldEntryAgain(state, deps, entry, request.replicaNodeId).catch(
|
|
4026
|
+
() => "unconfirmed"
|
|
4027
|
+
);
|
|
4028
|
+
if (answer2 === "refused-as-invalid" && cursor.syncEpoch === provingEpoch) {
|
|
4029
|
+
forgetReplica(state, request.indexName, request.partitionId, request.replicaNodeId);
|
|
4030
|
+
}
|
|
4031
|
+
return answer2 === "acknowledged" && cursor.syncEpoch === provingEpoch;
|
|
4032
|
+
}
|
|
4033
|
+
async function askController(claim, payload, controllerNodeId) {
|
|
4034
|
+
let request = payload;
|
|
4035
|
+
let retries = 0;
|
|
4036
|
+
while (claimHolds(claim, request)) {
|
|
4037
|
+
const replicaStartedAt = await registeredStartedAt(claim.deps, request.replicaNodeId);
|
|
4038
|
+
if (replicaStartedAt === null) return "refused";
|
|
4039
|
+
if (!await provePositionAfterRead(claim, request) || !claimHolds(claim, request)) return "refused";
|
|
4040
|
+
const answer2 = await withTimeout(
|
|
4041
|
+
sendToController({ ...request, replicaStartedAt }, controllerNodeId, claim.deps),
|
|
4042
|
+
ADMISSION_TIMEOUT_MS,
|
|
4043
|
+
"unanswered"
|
|
4044
|
+
);
|
|
4045
|
+
if (answer2 !== "refused" || retries >= ADMISSION_RETRIES_AFTER_IN_SYNC_CHANGE) return answer2;
|
|
4046
|
+
const inSyncVersion = await changedInSyncVersion(claim, request);
|
|
4047
|
+
if (inSyncVersion === null) return answer2;
|
|
4048
|
+
request = { ...request, inSyncVersion };
|
|
4049
|
+
retries += 1;
|
|
4050
|
+
}
|
|
4051
|
+
return "refused";
|
|
4052
|
+
}
|
|
4053
|
+
async function proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
|
|
4054
|
+
if (claimAdmission(state, deps, indexName, partitionId, replicaNodeId, cursor)) {
|
|
4055
|
+
await requestClaimedAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
|
|
4056
|
+
}
|
|
4057
|
+
}
|
|
4058
|
+
function claimAdmission(state, deps, indexName, partitionId, replicaNodeId, cursor) {
|
|
4059
|
+
const log = deps.getReplicationLog(indexName, partitionId);
|
|
4060
|
+
if (!cursor.confirmed || cursor.appliedSeqNo < log.commitPoint) {
|
|
4061
|
+
return false;
|
|
4062
|
+
}
|
|
4063
|
+
return markPendingAdmission(state, indexName, partitionId, replicaNodeId);
|
|
4064
|
+
}
|
|
4065
|
+
async function requestClaimedAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
|
|
4066
|
+
const pending = pendingAdmissionOf(state, indexName, partitionId, replicaNodeId);
|
|
4067
|
+
if (pending === void 0) return;
|
|
4068
|
+
let settle = () => {
|
|
4069
|
+
};
|
|
4070
|
+
pending.answer = new Promise((resolve) => {
|
|
4071
|
+
settle = resolve;
|
|
4072
|
+
});
|
|
4073
|
+
const claim = { state, deps, cursor, proposedEpoch: cursor.syncEpoch, pending };
|
|
4074
|
+
try {
|
|
4075
|
+
const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
4076
|
+
if (controllerNodeId === null) {
|
|
4077
|
+
return;
|
|
4078
|
+
}
|
|
4079
|
+
const payload = {
|
|
4080
|
+
indexName,
|
|
4081
|
+
partitionId,
|
|
4082
|
+
replicaNodeId,
|
|
4083
|
+
primaryTerm: assignment.primaryTerm,
|
|
4084
|
+
appliedSeqNo: cursor.appliedSeqNo,
|
|
4085
|
+
commitPoint: deps.getReplicationLog(indexName, partitionId).commitPoint,
|
|
4086
|
+
inSyncVersion: inSyncVersionOf(assignment)
|
|
4087
|
+
};
|
|
4088
|
+
const answer2 = await askController(claim, payload, controllerNodeId);
|
|
4089
|
+
pending.unanswered = answer2 === "unanswered";
|
|
4090
|
+
settle(answer2);
|
|
4091
|
+
if (answer2 === "accepted" && pending.failedWrite) {
|
|
4092
|
+
recordAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId, assignment.primaryTerm);
|
|
4093
|
+
return;
|
|
4094
|
+
}
|
|
4095
|
+
const outlived = answer2 === "accepted" && cursor.syncEpoch !== claim.proposedEpoch;
|
|
4096
|
+
const withdrawn = outlived && await withTimeout(withdrawAdmission(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
|
|
4097
|
+
if (answer2 === "unanswered" || outlived && !withdrawn) {
|
|
4098
|
+
recordAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId, assignment.primaryTerm);
|
|
4099
|
+
} else if (answer2 === "accepted" && !outlived) {
|
|
4100
|
+
forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId);
|
|
4101
|
+
forgetRemovedReplica(state, indexName, partitionId, replicaNodeId);
|
|
4102
|
+
recordAdmittedReplica(
|
|
4103
|
+
state,
|
|
4104
|
+
indexName,
|
|
4105
|
+
partitionId,
|
|
4106
|
+
replicaNodeId,
|
|
4107
|
+
assignment.primaryTerm,
|
|
4108
|
+
advanceAdmissionGeneration(state, indexName, partitionId, replicaNodeId)
|
|
4109
|
+
);
|
|
4110
|
+
}
|
|
4111
|
+
} finally {
|
|
4112
|
+
settle("refused");
|
|
4113
|
+
if (!pending.unanswered) clearPendingAdmission(state, indexName, partitionId, replicaNodeId, pending);
|
|
4114
|
+
}
|
|
4115
|
+
}
|
|
4116
|
+
|
|
3655
4117
|
// src/distribution/cluster-node/write-routing/replication.ts
|
|
3656
4118
|
async function assertPrimaryWriteAuthority(entry, deps) {
|
|
3657
4119
|
const table = await deps.coordinator.getAllocation(entry.indexName);
|
|
@@ -3765,7 +4227,13 @@ function replicaTargetsOf(entry, assignment, deps) {
|
|
|
3765
4227
|
entry.indexName,
|
|
3766
4228
|
entry.partitionId,
|
|
3767
4229
|
entry.primaryTerm,
|
|
3768
|
-
|
|
4230
|
+
withUnansweredAdmissions(
|
|
4231
|
+
deps.catchUp,
|
|
4232
|
+
entry.indexName,
|
|
4233
|
+
entry.partitionId,
|
|
4234
|
+
deps.nodeId,
|
|
4235
|
+
getInSyncReplicaTargets(assignment, deps.nodeId, [...pending, ...admitted])
|
|
4236
|
+
)
|
|
3769
4237
|
);
|
|
3770
4238
|
return { admitted, targets };
|
|
3771
4239
|
}
|
|
@@ -3815,21 +4283,70 @@ async function replicateEntry(entry, assignment, deps, indexUuid) {
|
|
|
3815
4283
|
)
|
|
3816
4284
|
);
|
|
3817
4285
|
recordAcknowledgements(deps.catchUp, entry.indexName, entry.partitionId, result.acknowledged, entry.seqNo);
|
|
3818
|
-
|
|
3819
|
-
await removeFailedReplicasFromInsync(entry, inSyncFailures, deps);
|
|
4286
|
+
await removeInSyncFailures(entry, result.failed, assignment, admitted, deps);
|
|
3820
4287
|
await assertPrimaryWriteAuthority(entry, deps);
|
|
3821
4288
|
deps.getReplicationLog(entry.indexName, entry.partitionId).advanceCommitPoint(entry.seqNo);
|
|
3822
4289
|
}
|
|
3823
|
-
function
|
|
4290
|
+
async function inSyncFailuresOf(entry, failedReplicas, assignment, admitted, deps) {
|
|
3824
4291
|
const inSyncFailures = [];
|
|
4292
|
+
const fenced = [];
|
|
4293
|
+
const answers = [];
|
|
4294
|
+
const admittedNow = admittedBeyondTable(deps.catchUp, entry.indexName, entry.partitionId, assignment);
|
|
3825
4295
|
for (const replicaNodeId of failedReplicas) {
|
|
3826
|
-
if (assignment.inSyncSet.includes(replicaNodeId) || admitted.includes(replicaNodeId)) {
|
|
4296
|
+
if (assignment.inSyncSet.includes(replicaNodeId) || admitted.includes(replicaNodeId) || admittedNow.includes(replicaNodeId)) {
|
|
4297
|
+
inSyncFailures.push(replicaNodeId);
|
|
4298
|
+
continue;
|
|
4299
|
+
}
|
|
4300
|
+
const pending = pendingAdmissionOf(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId);
|
|
4301
|
+
if (pending === void 0) continue;
|
|
4302
|
+
if (pending.unanswered) {
|
|
3827
4303
|
inSyncFailures.push(replicaNodeId);
|
|
4304
|
+
const owed = admissionToWithdrawOf(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId);
|
|
4305
|
+
fenced.push({ replicaNodeId, pending, owed });
|
|
3828
4306
|
continue;
|
|
3829
4307
|
}
|
|
3830
|
-
|
|
4308
|
+
pending.failedWrite = true;
|
|
4309
|
+
if (pending.answer === null) {
|
|
4310
|
+
clearPendingAdmission(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId, pending);
|
|
4311
|
+
} else {
|
|
4312
|
+
answers.push({ replicaNodeId, answer: pending.answer });
|
|
4313
|
+
}
|
|
3831
4314
|
}
|
|
3832
|
-
|
|
4315
|
+
const timeoutMs = replicationTimeoutOf(deps);
|
|
4316
|
+
const settled = await Promise.all(
|
|
4317
|
+
answers.map(async ({ replicaNodeId, answer: answer2 }) => ({
|
|
4318
|
+
replicaNodeId,
|
|
4319
|
+
answer: await withTimeout(answer2, timeoutMs, "unanswered")
|
|
4320
|
+
}))
|
|
4321
|
+
);
|
|
4322
|
+
const unanswered = [];
|
|
4323
|
+
for (const { replicaNodeId, answer: answer2 } of settled) {
|
|
4324
|
+
if (answer2 === "accepted") inSyncFailures.push(replicaNodeId);
|
|
4325
|
+
if (answer2 === "unanswered") unanswered.push(replicaNodeId);
|
|
4326
|
+
}
|
|
4327
|
+
return { inSyncFailures, unanswered, fenced };
|
|
4328
|
+
}
|
|
4329
|
+
async function removeInSyncFailures(entry, failedReplicas, assignment, admitted, deps) {
|
|
4330
|
+
const { inSyncFailures, unanswered, fenced } = await inSyncFailuresOf(
|
|
4331
|
+
entry,
|
|
4332
|
+
failedReplicas,
|
|
4333
|
+
assignment,
|
|
4334
|
+
admitted,
|
|
4335
|
+
deps
|
|
4336
|
+
);
|
|
4337
|
+
await removeFailedReplicasFromInsync(entry, inSyncFailures, deps);
|
|
4338
|
+
for (const { replicaNodeId, pending, owed } of fenced) {
|
|
4339
|
+
settleFencedAdmission(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId, { pending, owed });
|
|
4340
|
+
}
|
|
4341
|
+
if (unanswered.length === 0) return;
|
|
4342
|
+
await deps.refreshAllocation?.(entry.indexName);
|
|
4343
|
+
await assertPrimaryWriteAuthority(entry, deps);
|
|
4344
|
+
const timeoutMs = replicationTimeoutOf(deps);
|
|
4345
|
+
throw new NarsilError(
|
|
4346
|
+
ErrorCodes.REPLICATION_INSYNC_REMOVAL_FAILED,
|
|
4347
|
+
`The controller did not answer the admission of ${unanswered.map((nodeId) => `'${nodeId}'`).join(", ")} to index '${entry.indexName}' partition ${entry.partitionId} within ${timeoutMs}ms after a failed write, so the primary cannot tell whether to remove it`,
|
|
4348
|
+
{ indexName: entry.indexName, partitionId: entry.partitionId, unansweredReplicas: unanswered, timeoutMs }
|
|
4349
|
+
);
|
|
3833
4350
|
}
|
|
3834
4351
|
async function replicateEntryBatch(entries, assignment, deps, indexUuid) {
|
|
3835
4352
|
if (entries.length === 0) {
|
|
@@ -3852,8 +4369,7 @@ async function replicateEntryBatch(entries, assignment, deps, indexUuid) {
|
|
|
3852
4369
|
)
|
|
3853
4370
|
);
|
|
3854
4371
|
recordAcknowledgements(deps.catchUp, lastEntry.indexName, lastEntry.partitionId, result.acknowledged, lastEntry.seqNo);
|
|
3855
|
-
|
|
3856
|
-
await removeFailedReplicasFromInsync(lastEntry, inSyncFailures, deps);
|
|
4372
|
+
await removeInSyncFailures(lastEntry, result.failed, assignment, admitted, deps);
|
|
3857
4373
|
await assertPrimaryWriteAuthority(lastEntry, deps);
|
|
3858
4374
|
deps.getReplicationLog(lastEntry.indexName, lastEntry.partitionId).advanceCommitPoint(lastEntry.seqNo);
|
|
3859
4375
|
}
|
|
@@ -5304,15 +5820,28 @@ function processRoomIn(figures, fractions) {
|
|
|
5304
5820
|
}
|
|
5305
5821
|
function createWriteProjection(options) {
|
|
5306
5822
|
const bounds = /* @__PURE__ */ new Map();
|
|
5823
|
+
const pendingDocuments = /* @__PURE__ */ new Map();
|
|
5307
5824
|
const { refusal: refusal2 } = options;
|
|
5825
|
+
function settlePending(indexName, count2) {
|
|
5826
|
+
const remaining = (pendingDocuments.get(indexName) ?? 0) - count2;
|
|
5827
|
+
if (remaining > 0) pendingDocuments.set(indexName, remaining);
|
|
5828
|
+
else pendingDocuments.delete(indexName);
|
|
5829
|
+
}
|
|
5308
5830
|
function fittingCount(indexName, figures, groupBounds, start, mostDocuments, countExactly) {
|
|
5309
5831
|
const end = Math.min(groupBounds.length, start + mostDocuments);
|
|
5310
5832
|
if (figures === null) return end - start;
|
|
5311
5833
|
const reserved = reservedHeapBytes();
|
|
5312
5834
|
const heapRoom = heapRoomIn(figures, options.fractions) - reserved;
|
|
5313
5835
|
const processRoom = processRoomIn(figures, options.fractions) - reserved;
|
|
5314
|
-
const
|
|
5315
|
-
const
|
|
5836
|
+
const pending = pendingDocuments.get(indexName) ?? 0;
|
|
5837
|
+
const mostTableGrowth = options.documentTableGrowthBytesOf(indexName, end - start, pending);
|
|
5838
|
+
const mostVectorGrowth = options.vectorGrowthBytesOf(indexName, end - start);
|
|
5839
|
+
const fits = (next, count3) => {
|
|
5840
|
+
const most = next + mostTableGrowth;
|
|
5841
|
+
if (most <= heapRoom && most + mostVectorGrowth <= processRoom) return true;
|
|
5842
|
+
const withTables = next + options.documentTableGrowthBytesOf(indexName, count3 + 1, pending);
|
|
5843
|
+
return withTables <= heapRoom && withTables + options.vectorGrowthBytesOf(indexName, count3 + 1) <= processRoom;
|
|
5844
|
+
};
|
|
5316
5845
|
let taken = 0;
|
|
5317
5846
|
let count2 = 0;
|
|
5318
5847
|
while (start + count2 < end) {
|
|
@@ -5327,10 +5856,22 @@ function createWriteProjection(options) {
|
|
|
5327
5856
|
return count2;
|
|
5328
5857
|
}
|
|
5329
5858
|
function reserve(indexName, groupBounds, start, count2) {
|
|
5330
|
-
|
|
5859
|
+
if (count2 === 0) return UNPROJECTED;
|
|
5860
|
+
const pending = pendingDocuments.get(indexName) ?? 0;
|
|
5861
|
+
let bytes = options.vectorGrowthBytesOf(indexName, count2) + options.documentTableGrowthBytesOf(indexName, count2, pending);
|
|
5331
5862
|
for (let index = start; index < start + count2; index++) bytes += groupBounds[index];
|
|
5332
5863
|
const reservation = reserveHeapBytes(bytes);
|
|
5333
|
-
|
|
5864
|
+
pendingDocuments.set(indexName, pending + count2);
|
|
5865
|
+
let released = false;
|
|
5866
|
+
return {
|
|
5867
|
+
count: count2,
|
|
5868
|
+
release: () => {
|
|
5869
|
+
if (released) return;
|
|
5870
|
+
released = true;
|
|
5871
|
+
reservation.release();
|
|
5872
|
+
settlePending(indexName, count2);
|
|
5873
|
+
}
|
|
5874
|
+
};
|
|
5334
5875
|
}
|
|
5335
5876
|
function blockBytesOf(figures) {
|
|
5336
5877
|
return Math.floor(
|
|
@@ -6955,7 +7496,7 @@ async function runBackupCreation(context, request) {
|
|
|
6955
7496
|
const indexName = jobIndexNameOf(context);
|
|
6956
7497
|
const generation = await servingNameOf(coordinator, indexName);
|
|
6957
7498
|
const named = (await readIndexMetadataSnapshot(coordinator, indexName))?.settings;
|
|
6958
|
-
const syncEpoch = named
|
|
7499
|
+
const syncEpoch = epochFeaturesOn(named) ? named?.syncEpoch : void 0;
|
|
6959
7500
|
const metadata = await requireIndexMetadata(context, indexName, generation);
|
|
6960
7501
|
const clusterUuid = await clusterUuidOf(coordinator);
|
|
6961
7502
|
await reportJobProgress(context, 0, metadata.partitionCount);
|
|
@@ -7255,9 +7796,9 @@ async function runBackupRestore(context, request) {
|
|
|
7255
7796
|
}
|
|
7256
7797
|
|
|
7257
7798
|
// src/distribution/backups/sync-retention.ts
|
|
7258
|
-
async function
|
|
7799
|
+
async function surplusTakenBackups(store, indexName, takenBy) {
|
|
7259
7800
|
const taken = (await readEveryReadableManifest(store)).filter(
|
|
7260
|
-
(manifest) => manifest.indexName === indexName && manifest.takenBy ===
|
|
7801
|
+
(manifest) => manifest.indexName === indexName && manifest.takenBy === takenBy
|
|
7261
7802
|
);
|
|
7262
7803
|
taken.sort((a, b) => b.completedAt - a.completedAt || b.startedAt - a.startedAt);
|
|
7263
7804
|
return taken.slice(SYNC_BACKUPS_KEPT).map((manifest) => manifest.name);
|
|
@@ -7337,8 +7878,10 @@ async function startJob(deps, store, job, run, afterJob) {
|
|
|
7337
7878
|
}
|
|
7338
7879
|
function takenByOf(request) {
|
|
7339
7880
|
if (request.takenBy === void 0) return void 0;
|
|
7340
|
-
if (request.takenBy
|
|
7341
|
-
|
|
7881
|
+
if (!isBackupTaker(request.takenBy)) {
|
|
7882
|
+
throw malformed3("backup", `"takenBy" must be "${SYNC_BACKUP_TAKER}" or "${CHANGES_BACKUP_TAKER}"`);
|
|
7883
|
+
}
|
|
7884
|
+
return request.takenBy;
|
|
7342
7885
|
}
|
|
7343
7886
|
function queryHistoryOf(request) {
|
|
7344
7887
|
if (request.queryHistory === void 0) return false;
|
|
@@ -7348,8 +7891,8 @@ function queryHistoryOf(request) {
|
|
|
7348
7891
|
function quietRefusal(error) {
|
|
7349
7892
|
return error instanceof NarsilError && (error.code === ErrorCodes.BACKUP_IN_PROGRESS || error.code === ErrorCodes.BACKUP_NOT_FOUND);
|
|
7350
7893
|
}
|
|
7351
|
-
async function
|
|
7352
|
-
for (const backupName of await
|
|
7894
|
+
async function deleteSurplusTakenBackups(deps, store, indexName, takenBy) {
|
|
7895
|
+
for (const backupName of await surplusTakenBackups(store, indexName, takenBy)) {
|
|
7353
7896
|
if (!deps.isActive()) return;
|
|
7354
7897
|
try {
|
|
7355
7898
|
await admitDeletion(deps.config.coordinator, store, backupName);
|
|
@@ -7385,7 +7928,7 @@ async function acceptRequest(deps, kind, message) {
|
|
|
7385
7928
|
const takenBy = takenByOf(request);
|
|
7386
7929
|
const queryHistory = queryHistoryOf(request);
|
|
7387
7930
|
await admitBackup(coordinator, store, backupName, indexName);
|
|
7388
|
-
const afterJob = takenBy === void 0 ? void 0 : () =>
|
|
7931
|
+
const afterJob = takenBy === void 0 ? void 0 : () => deleteSurplusTakenBackups(deps, store, indexName, takenBy);
|
|
7389
7932
|
return jobIdOf(
|
|
7390
7933
|
startJob(
|
|
7391
7934
|
deps,
|
|
@@ -7915,8 +8458,8 @@ async function runJob(deps, request, options) {
|
|
|
7915
8458
|
function requestBackup(deps, indexName, backupName, options) {
|
|
7916
8459
|
return backUp(deps, indexName, backupName, options?.queryHistory === true ? { queryHistory: true } : {}, options);
|
|
7917
8460
|
}
|
|
7918
|
-
async function
|
|
7919
|
-
await backUp(deps, indexName, backupName, { takenBy
|
|
8461
|
+
async function requestTakenBackup(deps, indexName, backupName, takenBy) {
|
|
8462
|
+
await backUp(deps, indexName, backupName, { takenBy });
|
|
7920
8463
|
}
|
|
7921
8464
|
async function backUp(deps, indexName, backupName, extra, options) {
|
|
7922
8465
|
const store = requireBackupStore(deps.store);
|
|
@@ -8228,7 +8771,7 @@ function createAnalysisView(deps) {
|
|
|
8228
8771
|
answerAt(indexName, requested, read) {
|
|
8229
8772
|
const generation = engine.analysisGenerationOf(indexName);
|
|
8230
8773
|
if (generation === void 0) return read();
|
|
8231
|
-
const readAtRequested = (
|
|
8774
|
+
const readAtRequested = (answerOf3) => answerOf3().then((answer2) => {
|
|
8232
8775
|
const after = engine.analysisGenerationOf(indexName) ?? requested;
|
|
8233
8776
|
if (after !== requested) throw generationMissing(indexName, requested, after);
|
|
8234
8777
|
return answer2;
|
|
@@ -8681,6 +9224,11 @@ function applyStartPayload(state, record) {
|
|
|
8681
9224
|
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "totalBytes must be a non-negative integer");
|
|
8682
9225
|
return;
|
|
8683
9226
|
}
|
|
9227
|
+
const syncSession = record.syncSession;
|
|
9228
|
+
if (syncSession !== void 0 && syncSession !== null && (typeof syncSession !== "string" || syncSession === "")) {
|
|
9229
|
+
fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "syncSession must be a non-empty string or absent");
|
|
9230
|
+
return;
|
|
9231
|
+
}
|
|
8684
9232
|
try {
|
|
8685
9233
|
state.spool = openSnapshotSpoolWriter(state.expectation.spoolDirectory);
|
|
8686
9234
|
} catch (err) {
|
|
@@ -8689,6 +9237,7 @@ function applyStartPayload(state, record) {
|
|
|
8689
9237
|
}
|
|
8690
9238
|
state.header = header;
|
|
8691
9239
|
state.totalBytes = start.totalBytes;
|
|
9240
|
+
state.syncSession = typeof syncSession === "string" ? syncSession : null;
|
|
8692
9241
|
}
|
|
8693
9242
|
function applyChunkPayload(state, payload) {
|
|
8694
9243
|
if (state.totalBytes === null || state.header === null) {
|
|
@@ -8837,7 +9386,8 @@ function createSnapshotStreamState(expectation) {
|
|
|
8837
9386
|
expectedNextOffset: 0,
|
|
8838
9387
|
endPayload: null,
|
|
8839
9388
|
failure: null,
|
|
8840
|
-
firstFrameSeen: false
|
|
9389
|
+
firstFrameSeen: false,
|
|
9390
|
+
syncSession: null
|
|
8841
9391
|
};
|
|
8842
9392
|
}
|
|
8843
9393
|
function handleIncomingSnapshotRecord(state, record) {
|
|
@@ -8953,7 +9503,8 @@ var TRANSIENT_CODES = /* @__PURE__ */ new Set([
|
|
|
8953
9503
|
var TRANSIENT_PRIMARY_CODES = /* @__PURE__ */ new Set([
|
|
8954
9504
|
ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED,
|
|
8955
9505
|
ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
|
|
8956
|
-
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE
|
|
9506
|
+
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
9507
|
+
ErrorCodes.REPLICATION_SYNC_FAILED
|
|
8957
9508
|
]);
|
|
8958
9509
|
function isTransientFailure(code, details) {
|
|
8959
9510
|
if (TRANSIENT_CODES.has(code)) {
|
|
@@ -9133,6 +9684,7 @@ function createLiveSyncFrameState(indexName, partitionId, spoolDirectory) {
|
|
|
9133
9684
|
return {
|
|
9134
9685
|
snapshotState: createSnapshotStreamState({ indexName, partitionId, spoolDirectory }),
|
|
9135
9686
|
syncEntries: [],
|
|
9687
|
+
syncSession: null,
|
|
9136
9688
|
sawSnapshotFrame: false,
|
|
9137
9689
|
error: null
|
|
9138
9690
|
};
|
|
@@ -9165,7 +9717,8 @@ function handleLiveSyncFrame(state, frame) {
|
|
|
9165
9717
|
state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received sync_entries before snapshot_end");
|
|
9166
9718
|
return;
|
|
9167
9719
|
}
|
|
9168
|
-
state.
|
|
9720
|
+
if (!noteSyncSession(state, entriesPayload.syncSession)) return;
|
|
9721
|
+
state.syncEntries.push({ entries: entriesPayload.entries, isLast: entriesPayload.isLast });
|
|
9169
9722
|
return;
|
|
9170
9723
|
}
|
|
9171
9724
|
if (state.syncEntries.length > 0 && !state.sawSnapshotFrame) {
|
|
@@ -9179,7 +9732,22 @@ function handleLiveSyncFrame(state, frame) {
|
|
|
9179
9732
|
handleIncomingSnapshotRecord(state.snapshotState, record);
|
|
9180
9733
|
if (state.snapshotState.failure !== null) {
|
|
9181
9734
|
state.error = streamFailureToError(state.snapshotState.failure);
|
|
9735
|
+
return;
|
|
9736
|
+
}
|
|
9737
|
+
noteSyncSession(state, state.snapshotState.syncSession);
|
|
9738
|
+
}
|
|
9739
|
+
function noteSyncSession(state, syncSession) {
|
|
9740
|
+
if (syncSession === null) return true;
|
|
9741
|
+
if (state.syncSession === null) {
|
|
9742
|
+
state.syncSession = syncSession;
|
|
9743
|
+
return true;
|
|
9182
9744
|
}
|
|
9745
|
+
if (state.syncSession === syncSession) return true;
|
|
9746
|
+
state.error = new NarsilError(
|
|
9747
|
+
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
9748
|
+
"one sync answer carried two different sync sessions"
|
|
9749
|
+
);
|
|
9750
|
+
return false;
|
|
9183
9751
|
}
|
|
9184
9752
|
function decodeSyncEntriesRecord(record) {
|
|
9185
9753
|
if (!("entries" in record) && !("isLast" in record)) {
|
|
@@ -9191,6 +9759,13 @@ function decodeSyncEntriesRecord(record) {
|
|
|
9191
9759
|
"Invalid SyncEntriesPayload: expected entries array and isLast boolean"
|
|
9192
9760
|
);
|
|
9193
9761
|
}
|
|
9762
|
+
const syncSession = record.syncSession ?? null;
|
|
9763
|
+
if (syncSession !== null && (typeof syncSession !== "string" || syncSession === "")) {
|
|
9764
|
+
return new NarsilError(
|
|
9765
|
+
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
9766
|
+
"Invalid SyncEntriesPayload: syncSession must be a non-empty string or absent"
|
|
9767
|
+
);
|
|
9768
|
+
}
|
|
9194
9769
|
const entries = [];
|
|
9195
9770
|
for (const candidate of record.entries) {
|
|
9196
9771
|
try {
|
|
@@ -9199,7 +9774,7 @@ function decodeSyncEntriesRecord(record) {
|
|
|
9199
9774
|
return new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, err instanceof Error ? err.message : String(err));
|
|
9200
9775
|
}
|
|
9201
9776
|
}
|
|
9202
|
-
return { entries, isLast: record.isLast };
|
|
9777
|
+
return { entries, isLast: record.isLast, syncSession };
|
|
9203
9778
|
}
|
|
9204
9779
|
function streamFailureToError(failure) {
|
|
9205
9780
|
return new NarsilError(failure.code, failure.message, failure.details);
|
|
@@ -9434,7 +10009,7 @@ async function withRoutedIndexUuid(payload, deps) {
|
|
|
9434
10009
|
const indexUuid = (await deps.coordinator.getAllocation(payload.indexName))?.indexUuid;
|
|
9435
10010
|
return indexUuid === void 0 ? payload : { ...payload, indexUuid };
|
|
9436
10011
|
}
|
|
9437
|
-
async function reportSyncedPosition(indexName, partitionId, target, deadline, deps) {
|
|
10012
|
+
async function reportSyncedPosition(indexName, partitionId, target, syncSession, deadline, deps) {
|
|
9438
10013
|
const logState = readLocalLogState(indexName, partitionId, deps);
|
|
9439
10014
|
const details = { indexName, partitionId, target };
|
|
9440
10015
|
const budgetMs = Math.min(SYNCED_POSITION_REPORT_TIMEOUT_MS, deadline - Date.now());
|
|
@@ -9449,7 +10024,8 @@ async function reportSyncedPosition(indexName, partitionId, target, deadline, de
|
|
|
9449
10024
|
indexName,
|
|
9450
10025
|
partitionId,
|
|
9451
10026
|
lastSeqNo: logState.lastSeqNo,
|
|
9452
|
-
lastPrimaryTerm: logState.lastPrimaryTerm
|
|
10027
|
+
lastPrimaryTerm: logState.lastPrimaryTerm,
|
|
10028
|
+
...syncSession === null ? {} : { syncSession }
|
|
9453
10029
|
};
|
|
9454
10030
|
const request = {
|
|
9455
10031
|
type: ReplicationMessageTypes.SYNC_REQUEST,
|
|
@@ -9481,6 +10057,9 @@ async function reportSyncedPosition(indexName, partitionId, target, deadline, de
|
|
|
9481
10057
|
} finally {
|
|
9482
10058
|
abandonSnapshotStream(frameState.snapshotState);
|
|
9483
10059
|
}
|
|
10060
|
+
if (frameState.error !== null) {
|
|
10061
|
+
return frameState.error;
|
|
10062
|
+
}
|
|
9484
10063
|
if (frameState.sawSnapshotFrame) {
|
|
9485
10064
|
return new NarsilError(
|
|
9486
10065
|
ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED,
|
|
@@ -9488,7 +10067,7 @@ async function reportSyncedPosition(indexName, partitionId, target, deadline, de
|
|
|
9488
10067
|
details
|
|
9489
10068
|
);
|
|
9490
10069
|
}
|
|
9491
|
-
return
|
|
10070
|
+
return null;
|
|
9492
10071
|
}
|
|
9493
10072
|
async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNodeId, targets, coordinatorConfig, partitionCount, deadline, deps, abortCheck) {
|
|
9494
10073
|
let lastError = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "no live sync attempts were made", {
|
|
@@ -9523,7 +10102,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
|
|
|
9523
10102
|
)
|
|
9524
10103
|
};
|
|
9525
10104
|
}
|
|
9526
|
-
const attempt = await syncFromTarget(
|
|
10105
|
+
const { attempt, syncSession } = await syncFromTarget(
|
|
9527
10106
|
state,
|
|
9528
10107
|
entry,
|
|
9529
10108
|
indexName,
|
|
@@ -9535,8 +10114,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
|
|
|
9535
10114
|
deps,
|
|
9536
10115
|
abortCheck
|
|
9537
10116
|
);
|
|
9538
|
-
const
|
|
9539
|
-
const failure = !attempt.ok ? attempt.error : movedThePosition ? await reportSyncedPosition(indexName, partitionId, target, deadline, deps) : null;
|
|
10117
|
+
const failure = attempt.ok ? await reportSyncedPosition(indexName, partitionId, target, syncSession, deadline, deps) : attempt.error;
|
|
9540
10118
|
if (failure === null) {
|
|
9541
10119
|
return attempt;
|
|
9542
10120
|
}
|
|
@@ -9554,7 +10132,7 @@ async function syncFromTarget(state, entry, indexName, partitionId, target, coor
|
|
|
9554
10132
|
const spoolDirectory = deps.snapshotSpoolDirectory ?? defaultSpoolDirectory();
|
|
9555
10133
|
const frameState = createLiveSyncFrameState(indexName, partitionId, spoolDirectory);
|
|
9556
10134
|
try {
|
|
9557
|
-
|
|
10135
|
+
const attempt = await syncFrameStateFromTarget(
|
|
9558
10136
|
state,
|
|
9559
10137
|
entry,
|
|
9560
10138
|
indexName,
|
|
@@ -9567,6 +10145,7 @@ async function syncFromTarget(state, entry, indexName, partitionId, target, coor
|
|
|
9567
10145
|
abortCheck,
|
|
9568
10146
|
frameState
|
|
9569
10147
|
);
|
|
10148
|
+
return { attempt, syncSession: frameState.syncSession };
|
|
9570
10149
|
} finally {
|
|
9571
10150
|
abandonSnapshotStream(frameState.snapshotState);
|
|
9572
10151
|
}
|
|
@@ -9869,157 +10448,39 @@ async function executeBootstrapSync(state, entry, indexName, partitionId, primar
|
|
|
9869
10448
|
);
|
|
9870
10449
|
}
|
|
9871
10450
|
|
|
9872
|
-
// src/distribution/cluster-node/catch-up/
|
|
9873
|
-
|
|
9874
|
-
const
|
|
9875
|
-
|
|
9876
|
-
for (const
|
|
9877
|
-
|
|
9878
|
-
const result = await requestInsyncAdmission(payload, target, deps.transport, deps.nodeId);
|
|
9879
|
-
if (result.accepted) return "accepted";
|
|
9880
|
-
return sentWithoutAnswer ? "unanswered" : "refused";
|
|
9881
|
-
} catch (_) {
|
|
9882
|
-
sentWithoutAnswer = true;
|
|
9883
|
-
}
|
|
9884
|
-
}
|
|
9885
|
-
return "unanswered";
|
|
9886
|
-
}
|
|
9887
|
-
function withTimeout(work, timeoutMs, fallback) {
|
|
9888
|
-
return new Promise((resolve) => {
|
|
9889
|
-
const timer = setTimeout(() => resolve(fallback), timeoutMs);
|
|
9890
|
-
timer.unref?.();
|
|
9891
|
-
work.then((value) => {
|
|
9892
|
-
clearTimeout(timer);
|
|
9893
|
-
resolve(value);
|
|
9894
|
-
}).catch(() => {
|
|
9895
|
-
clearTimeout(timer);
|
|
9896
|
-
resolve(fallback);
|
|
9897
|
-
});
|
|
9898
|
-
});
|
|
9899
|
-
}
|
|
9900
|
-
async function withdrawAdmission(removal, controllerNodeId, deps) {
|
|
9901
|
-
const targets = await resolveNodeTargets(controllerNodeId, deps);
|
|
9902
|
-
for (const target of targets) {
|
|
9903
|
-
try {
|
|
9904
|
-
const result = await requestInsyncRemoval(
|
|
9905
|
-
removal.indexName,
|
|
9906
|
-
removal.partitionId,
|
|
9907
|
-
removal.replicaNodeId,
|
|
9908
|
-
removal.primaryTerm,
|
|
9909
|
-
target,
|
|
9910
|
-
deps.transport,
|
|
9911
|
-
deps.nodeId
|
|
9912
|
-
);
|
|
9913
|
-
if (result.accepted) {
|
|
9914
|
-
return true;
|
|
9915
|
-
}
|
|
9916
|
-
} catch (_) {
|
|
9917
|
-
}
|
|
9918
|
-
}
|
|
9919
|
-
return false;
|
|
9920
|
-
}
|
|
9921
|
-
async function proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
|
|
9922
|
-
if (claimAdmission(state, deps, indexName, partitionId, replicaNodeId, cursor)) {
|
|
9923
|
-
await requestClaimedAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
|
|
9924
|
-
}
|
|
9925
|
-
}
|
|
9926
|
-
function claimAdmission(state, deps, indexName, partitionId, replicaNodeId, cursor) {
|
|
9927
|
-
const log = deps.getReplicationLog(indexName, partitionId);
|
|
9928
|
-
if (cursor.appliedSeqNo < log.commitPoint) {
|
|
9929
|
-
return false;
|
|
9930
|
-
}
|
|
9931
|
-
return markPendingAdmission(state, indexName, partitionId, replicaNodeId);
|
|
9932
|
-
}
|
|
9933
|
-
async function requestClaimedAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
|
|
9934
|
-
const log = deps.getReplicationLog(indexName, partitionId);
|
|
9935
|
-
const proposedEpoch = cursor.syncEpoch;
|
|
9936
|
-
try {
|
|
9937
|
-
const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
9938
|
-
if (controllerNodeId === null) {
|
|
9939
|
-
return;
|
|
9940
|
-
}
|
|
9941
|
-
const payload = {
|
|
9942
|
-
indexName,
|
|
9943
|
-
partitionId,
|
|
9944
|
-
replicaNodeId,
|
|
9945
|
-
primaryTerm: assignment.primaryTerm,
|
|
9946
|
-
appliedSeqNo: cursor.appliedSeqNo,
|
|
9947
|
-
commitPoint: log.commitPoint
|
|
9948
|
-
};
|
|
9949
|
-
const answer2 = await withTimeout(
|
|
9950
|
-
sendToController(payload, controllerNodeId, deps),
|
|
9951
|
-
ADMISSION_TIMEOUT_MS,
|
|
9952
|
-
"unanswered"
|
|
9953
|
-
);
|
|
9954
|
-
const outlived = answer2 === "accepted" && cursor.syncEpoch !== proposedEpoch;
|
|
9955
|
-
const withdrawn = outlived && await withTimeout(withdrawAdmission(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
|
|
9956
|
-
if (answer2 === "unanswered" || outlived && !withdrawn) {
|
|
9957
|
-
recordAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId, assignment.primaryTerm);
|
|
9958
|
-
} else if (answer2 === "accepted" && !outlived) {
|
|
9959
|
-
forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId);
|
|
9960
|
-
forgetRemovedReplica(state, indexName, partitionId, replicaNodeId);
|
|
9961
|
-
recordAdmittedReplica(
|
|
9962
|
-
state,
|
|
9963
|
-
indexName,
|
|
9964
|
-
partitionId,
|
|
9965
|
-
replicaNodeId,
|
|
9966
|
-
assignment.primaryTerm,
|
|
9967
|
-
advanceAdmissionGeneration(state, indexName, partitionId, replicaNodeId)
|
|
9968
|
-
);
|
|
9969
|
-
}
|
|
9970
|
-
} finally {
|
|
9971
|
-
clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
|
|
10451
|
+
// src/distribution/cluster-node/catch-up/sessions.ts
|
|
10452
|
+
function openSyncSession(state, indexName, partitionId, replicaNodeId, primaryTerm, assignedNodeIds) {
|
|
10453
|
+
const key = partitionKey(indexName, partitionId);
|
|
10454
|
+
const sessions = state.syncSessions.get(key) ?? /* @__PURE__ */ new Map();
|
|
10455
|
+
for (const [nodeId, session] of sessions) {
|
|
10456
|
+
if (session.primaryTerm !== primaryTerm || !assignedNodeIds.includes(nodeId)) sessions.delete(nodeId);
|
|
9972
10457
|
}
|
|
10458
|
+
const id = generateId();
|
|
10459
|
+
sessions.set(replicaNodeId, { id, primaryTerm });
|
|
10460
|
+
state.syncSessions.set(key, sessions);
|
|
10461
|
+
return id;
|
|
9973
10462
|
}
|
|
9974
|
-
|
|
9975
|
-
|
|
9976
|
-
|
|
9977
|
-
return [...assignment.inSyncSet].sort().join("\n");
|
|
9978
|
-
}
|
|
9979
|
-
function createCatchUpFeed(assignment, replicaStartedAt) {
|
|
9980
|
-
let release = () => {
|
|
9981
|
-
};
|
|
9982
|
-
const whenCancelled = new Promise((resolve) => {
|
|
9983
|
-
release = resolve;
|
|
9984
|
-
});
|
|
9985
|
-
const feed = {
|
|
9986
|
-
primaryTerm: assignment.primaryTerm,
|
|
9987
|
-
inSyncSet: inSyncSignature(assignment),
|
|
9988
|
-
replicaStartedAt,
|
|
9989
|
-
whenCancelled,
|
|
9990
|
-
cancelled: false,
|
|
9991
|
-
cancel: () => {
|
|
9992
|
-
feed.cancelled = true;
|
|
9993
|
-
release();
|
|
9994
|
-
}
|
|
9995
|
-
};
|
|
9996
|
-
return feed;
|
|
9997
|
-
}
|
|
9998
|
-
function feedIsStale(feed, assignment, registered, replicaNodeId) {
|
|
9999
|
-
const replicaLeft = registered !== null && registered.get(replicaNodeId) !== feed.replicaStartedAt;
|
|
10000
|
-
return replicaLeft || assignment.primaryTerm !== feed.primaryTerm || inSyncSignature(assignment) !== feed.inSyncSet;
|
|
10463
|
+
function isCurrentSyncSession(state, indexName, partitionId, replicaNodeId, syncSession, primaryTerm) {
|
|
10464
|
+
const session = state.syncSessions.get(partitionKey(indexName, partitionId))?.get(replicaNodeId);
|
|
10465
|
+
return session !== void 0 && session.id === syncSession && session.primaryTerm === primaryTerm;
|
|
10001
10466
|
}
|
|
10002
|
-
function
|
|
10003
|
-
|
|
10004
|
-
work.catch(() => {
|
|
10005
|
-
});
|
|
10006
|
-
return Promise.resolve(fallback);
|
|
10007
|
-
}
|
|
10008
|
-
return Promise.race([work, feed.whenCancelled.then(() => fallback)]);
|
|
10467
|
+
function forgetSyncSessions(state, indexName, partitionId) {
|
|
10468
|
+
state.syncSessions.delete(partitionKey(indexName, partitionId));
|
|
10009
10469
|
}
|
|
10010
10470
|
|
|
10011
10471
|
// src/distribution/cluster-node/catch-up/withdrawals.ts
|
|
10012
10472
|
async function withdrawAppliedAdmission(state, deps, indexName, partitionId, replicaNodeId, owed) {
|
|
10013
10473
|
owed.withdrawing = true;
|
|
10474
|
+
const pending = pendingAdmissionOf(state, indexName, partitionId, replicaNodeId);
|
|
10014
10475
|
try {
|
|
10015
10476
|
const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
|
|
10016
10477
|
if (controllerNodeId === null) return;
|
|
10017
10478
|
const removal = { indexName, partitionId, replicaNodeId, primaryTerm: owed.primaryTerm };
|
|
10018
10479
|
const withdrawn = await withTimeout(withdrawAdmission(removal, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
|
|
10019
|
-
if (!withdrawn
|
|
10020
|
-
|
|
10021
|
-
}
|
|
10022
|
-
|
|
10480
|
+
if (!withdrawn) return;
|
|
10481
|
+
const stillOwed = admissionToWithdrawOf(state, indexName, partitionId, replicaNodeId) === owed;
|
|
10482
|
+
settleFencedAdmission(state, indexName, partitionId, replicaNodeId, { pending, owed });
|
|
10483
|
+
if (!stillOwed) return;
|
|
10023
10484
|
forgetAdmittedReplica(state, indexName, partitionId, replicaNodeId);
|
|
10024
10485
|
if (!getPendingAdmissions(state, indexName, partitionId).includes(replicaNodeId)) {
|
|
10025
10486
|
recordRemovedReplica(state, indexName, partitionId, replicaNodeId, owed.primaryTerm);
|
|
@@ -10031,14 +10492,14 @@ async function withdrawAppliedAdmission(state, deps, indexName, partitionId, rep
|
|
|
10031
10492
|
function withdrawAppliedAdmissions(state, deps, indexName, partitionId, assignment) {
|
|
10032
10493
|
const owedByReplica = state.admissionsToWithdraw.get(partitionKey(indexName, partitionId));
|
|
10033
10494
|
if (owedByReplica === void 0) return [];
|
|
10034
|
-
const pending = getPendingAdmissions(state, indexName, partitionId);
|
|
10035
10495
|
const started = [];
|
|
10036
10496
|
for (const [replicaNodeId, owed] of [...owedByReplica]) {
|
|
10037
|
-
|
|
10038
|
-
|
|
10497
|
+
const pending = pendingAdmissionOf(state, indexName, partitionId, replicaNodeId);
|
|
10498
|
+
if (owed.primaryTerm !== assignment.primaryTerm) {
|
|
10499
|
+
settleFencedAdmission(state, indexName, partitionId, replicaNodeId, { pending, owed });
|
|
10039
10500
|
continue;
|
|
10040
10501
|
}
|
|
10041
|
-
if (owed.withdrawing || pending
|
|
10502
|
+
if (owed.withdrawing || pending !== void 0 && !pending.unanswered) continue;
|
|
10042
10503
|
started.push(withdrawAppliedAdmission(state, deps, indexName, partitionId, replicaNodeId, owed).catch(() => {
|
|
10043
10504
|
}));
|
|
10044
10505
|
}
|
|
@@ -10046,16 +10507,6 @@ function withdrawAppliedAdmissions(state, deps, indexName, partitionId, assignme
|
|
|
10046
10507
|
}
|
|
10047
10508
|
|
|
10048
10509
|
// src/distribution/cluster-node/catch-up/pump.ts
|
|
10049
|
-
function entryBytes(entry) {
|
|
10050
|
-
return REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
10051
|
-
}
|
|
10052
|
-
function batchBytes(entries) {
|
|
10053
|
-
let total = 0;
|
|
10054
|
-
for (const entry of entries) {
|
|
10055
|
-
total += entryBytes(entry);
|
|
10056
|
-
}
|
|
10057
|
-
return total;
|
|
10058
|
-
}
|
|
10059
10510
|
function replicaHasFallenOutOfRetention(log, cursor) {
|
|
10060
10511
|
const oldest = log.oldestSeqNo;
|
|
10061
10512
|
if (oldest === void 0) {
|
|
@@ -10063,19 +10514,6 @@ function replicaHasFallenOutOfRetention(log, cursor) {
|
|
|
10063
10514
|
}
|
|
10064
10515
|
return cursor.appliedSeqNo + 1 < oldest;
|
|
10065
10516
|
}
|
|
10066
|
-
function nextWireBatch(state, pending) {
|
|
10067
|
-
const chunks = chunkByBudget(pending, {
|
|
10068
|
-
...WIRE_BATCH_BUDGET,
|
|
10069
|
-
payloadBytesOf: (entry) => entry.document?.byteLength ?? 0,
|
|
10070
|
-
breaksRun: (entry, previous) => entry.seqNo !== previous.seqNo + 1
|
|
10071
|
-
});
|
|
10072
|
-
const batch = chunks[0] ?? [];
|
|
10073
|
-
if (batch.length === 0) {
|
|
10074
|
-
return [];
|
|
10075
|
-
}
|
|
10076
|
-
const available = MAX_CATCH_UP_IN_FLIGHT_BYTES - state.inFlightBytes;
|
|
10077
|
-
return batchBytes(batch) > available ? [] : batch;
|
|
10078
|
-
}
|
|
10079
10517
|
async function pumpReplica(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor, replicaStartedAt) {
|
|
10080
10518
|
if (cursor.busy) {
|
|
10081
10519
|
return;
|
|
@@ -10102,6 +10540,7 @@ async function feedReplica(state, deps, indexName, partitionId, assignment, repl
|
|
|
10102
10540
|
return;
|
|
10103
10541
|
}
|
|
10104
10542
|
if (cursor.appliedSeqNo >= log.localLogEnd) {
|
|
10543
|
+
if (!cursor.confirmed) return;
|
|
10105
10544
|
const proposal = proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
|
|
10106
10545
|
await untilCancelled(feed, proposal, void 0);
|
|
10107
10546
|
return;
|
|
@@ -10152,37 +10591,6 @@ async function closeGapInPartitionQueue(state, deps, indexName, partitionId, ass
|
|
|
10152
10591
|
);
|
|
10153
10592
|
}
|
|
10154
10593
|
}
|
|
10155
|
-
async function sendBatch(state, deps, batch, replicaNodeId, cursor, feed) {
|
|
10156
|
-
const reservedBytes = batchBytes(batch);
|
|
10157
|
-
const sendingEpoch = cursor.syncEpoch;
|
|
10158
|
-
state.inFlightBytes += reservedBytes;
|
|
10159
|
-
const delivery = (async () => {
|
|
10160
|
-
const first = batch[0];
|
|
10161
|
-
const indexName = first?.indexName;
|
|
10162
|
-
const table = indexName === void 0 ? null : await deps.coordinator.getAllocation(indexName);
|
|
10163
|
-
if (feed.cancelled) {
|
|
10164
|
-
return false;
|
|
10165
|
-
}
|
|
10166
|
-
const result = await replicateBatchToReplicas(
|
|
10167
|
-
batch,
|
|
10168
|
-
[replicaNodeId],
|
|
10169
|
-
deps.transport,
|
|
10170
|
-
deps.nodeId,
|
|
10171
|
-
deps.resolveNodeTargets,
|
|
10172
|
-
table?.indexUuid,
|
|
10173
|
-
first === void 0 ? void 0 : previousTermOf(deps.getReplicationLog(first.indexName, first.partitionId), first.seqNo)
|
|
10174
|
-
);
|
|
10175
|
-
return result.acknowledged.includes(replicaNodeId);
|
|
10176
|
-
})().finally(() => {
|
|
10177
|
-
state.inFlightBytes -= reservedBytes;
|
|
10178
|
-
});
|
|
10179
|
-
const acknowledged = await untilCancelled(feed, delivery, false);
|
|
10180
|
-
if (!acknowledged || feed.cancelled || cursor.syncEpoch !== sendingEpoch) {
|
|
10181
|
-
return false;
|
|
10182
|
-
}
|
|
10183
|
-
cursor.appliedSeqNo = Math.max(cursor.appliedSeqNo, batch[batch.length - 1].seqNo);
|
|
10184
|
-
return true;
|
|
10185
|
-
}
|
|
10186
10594
|
function cancelFeeds(replicas) {
|
|
10187
10595
|
for (const cursor of replicas.values()) {
|
|
10188
10596
|
cursor.feed?.cancel();
|
|
@@ -10206,6 +10614,7 @@ async function pumpPartition(state, deps, key, replicas, started, registered) {
|
|
|
10206
10614
|
if (assignment === void 0 || assignment.primary !== deps.nodeId) {
|
|
10207
10615
|
cancelFeeds(replicas);
|
|
10208
10616
|
state.cursors.delete(key);
|
|
10617
|
+
forgetSyncSessions(state, indexName, partitionId);
|
|
10209
10618
|
state.pendingAdmissions.delete(key);
|
|
10210
10619
|
state.admissionsToWithdraw.delete(key);
|
|
10211
10620
|
return;
|
|
@@ -11285,6 +11694,7 @@ function cloneAssignments(assignments) {
|
|
|
11285
11694
|
primary: assignment.primary,
|
|
11286
11695
|
replicas: [...assignment.replicas],
|
|
11287
11696
|
inSyncSet: [...assignment.inSyncSet],
|
|
11697
|
+
...assignment.inSyncVersion !== void 0 ? { inSyncVersion: assignment.inSyncVersion } : {},
|
|
11288
11698
|
commitPoint: assignment.commitPoint,
|
|
11289
11699
|
state: assignment.state,
|
|
11290
11700
|
primaryTerm: assignment.primaryTerm,
|
|
@@ -11550,6 +11960,37 @@ function finishDepartureRun(state, run, onExpiry) {
|
|
|
11550
11960
|
state.departureTimer.unref?.();
|
|
11551
11961
|
}
|
|
11552
11962
|
|
|
11963
|
+
// src/distribution/cluster/controller/event-loop/previous-document-placement.ts
|
|
11964
|
+
async function recordsPreviousDocuments(coordinator, indexName) {
|
|
11965
|
+
try {
|
|
11966
|
+
const changes = (await readIndexMetadataSnapshot(coordinator, indexName))?.settings?.changes;
|
|
11967
|
+
return changes?.enabled === true && changes.previousDocument === true;
|
|
11968
|
+
} catch {
|
|
11969
|
+
return true;
|
|
11970
|
+
}
|
|
11971
|
+
}
|
|
11972
|
+
async function namesRecordingPreviousDocuments(coordinator, indexNames) {
|
|
11973
|
+
const recording = /* @__PURE__ */ new Set();
|
|
11974
|
+
for (const indexName of indexNames) {
|
|
11975
|
+
if (isHiddenGeneration(indexName) || !await recordsPreviousDocuments(coordinator, indexName)) continue;
|
|
11976
|
+
recording.add(indexName);
|
|
11977
|
+
const generation = (await coordinator.getAllocation(indexName))?.generation;
|
|
11978
|
+
if (generation !== void 0) recording.add(generation);
|
|
11979
|
+
const record = await readRebalanceRecord(coordinator, indexName);
|
|
11980
|
+
if (record !== null) {
|
|
11981
|
+
recording.add(record.sourceGeneration);
|
|
11982
|
+
recording.add(record.targetGeneration);
|
|
11983
|
+
}
|
|
11984
|
+
}
|
|
11985
|
+
return recording;
|
|
11986
|
+
}
|
|
11987
|
+
async function nodesForEachIndex(coordinator, nodes, indexNames) {
|
|
11988
|
+
const capable = nodes.filter((node) => reachesVersion(node.version, CHANGE_CAPTURE_SPEC_VERSION));
|
|
11989
|
+
if (capable.length === nodes.length) return () => nodes;
|
|
11990
|
+
const recording = await namesRecordingPreviousDocuments(coordinator, indexNames);
|
|
11991
|
+
return (indexName) => recording.has(indexName) ? capable : nodes;
|
|
11992
|
+
}
|
|
11993
|
+
|
|
11553
11994
|
// src/distribution/cluster/controller/event-loop/state.ts
|
|
11554
11995
|
function createEventLoopState(initialIndexNames, nodeLeftDelayMs = DEFAULT_CONTROLLER_CONFIG.nodeLeftDelayMs) {
|
|
11555
11996
|
return {
|
|
@@ -11708,53 +12149,6 @@ async function tearDownAllocation(indexName, coordinator, state, isActive, round
|
|
|
11708
12149
|
{ indexName, attempts: TEARDOWN_CAS_ATTEMPTS * TEARDOWN_RETRY_ROUNDS }
|
|
11709
12150
|
);
|
|
11710
12151
|
}
|
|
11711
|
-
function validatePartitionStoresPayload(decoded) {
|
|
11712
|
-
if (!isRecord(decoded)) {
|
|
11713
|
-
return null;
|
|
11714
|
-
}
|
|
11715
|
-
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
11716
|
-
return null;
|
|
11717
|
-
}
|
|
11718
|
-
return { indexName: decoded.indexName };
|
|
11719
|
-
}
|
|
11720
|
-
function validatePartitionStoresResultPayload(decoded) {
|
|
11721
|
-
if (!isRecord(decoded)) {
|
|
11722
|
-
return null;
|
|
11723
|
-
}
|
|
11724
|
-
if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
|
|
11725
|
-
return null;
|
|
11726
|
-
}
|
|
11727
|
-
if (decoded.indexUuid !== null && typeof decoded.indexUuid !== "string") {
|
|
11728
|
-
return null;
|
|
11729
|
-
}
|
|
11730
|
-
if (!Array.isArray(decoded.partitionIds)) {
|
|
11731
|
-
return null;
|
|
11732
|
-
}
|
|
11733
|
-
const partitionIds = [];
|
|
11734
|
-
for (const partitionId of decoded.partitionIds) {
|
|
11735
|
-
if (typeof partitionId !== "number" || !Number.isInteger(partitionId) || partitionId < 0) {
|
|
11736
|
-
return null;
|
|
11737
|
-
}
|
|
11738
|
-
partitionIds.push(partitionId);
|
|
11739
|
-
}
|
|
11740
|
-
return { indexName: decoded.indexName, indexUuid: decoded.indexUuid, partitionIds };
|
|
11741
|
-
}
|
|
11742
|
-
function createPartitionStoresMessage(payload, sourceNodeId, requestId) {
|
|
11743
|
-
return {
|
|
11744
|
-
type: ClusterMessageTypes.PARTITION_STORES,
|
|
11745
|
-
sourceId: sourceNodeId,
|
|
11746
|
-
requestId,
|
|
11747
|
-
payload: encode(payload)
|
|
11748
|
-
};
|
|
11749
|
-
}
|
|
11750
|
-
function createPartitionStoresResultMessage(result, sourceNodeId, requestId) {
|
|
11751
|
-
return {
|
|
11752
|
-
type: ClusterMessageTypes.PARTITION_STORES,
|
|
11753
|
-
sourceId: sourceNodeId,
|
|
11754
|
-
requestId,
|
|
11755
|
-
payload: encode(result)
|
|
11756
|
-
};
|
|
11757
|
-
}
|
|
11758
12152
|
|
|
11759
12153
|
// src/distribution/cluster/controller/event-loop/unassigned-recovery.ts
|
|
11760
12154
|
function liveHoldersOfUnassignedPartitions(table, liveNodeIds) {
|
|
@@ -11917,7 +12311,7 @@ async function writeRecoveryOutcome(deps, indexName, liveNodeIds, answers) {
|
|
|
11917
12311
|
if (next === null) {
|
|
11918
12312
|
return false;
|
|
11919
12313
|
}
|
|
11920
|
-
const written = await deps.coordinator
|
|
12314
|
+
const written = await putAllocationOver(deps.coordinator, indexName, table, next, deps.controllerNodeId);
|
|
11921
12315
|
if (written) {
|
|
11922
12316
|
return anyPromoted;
|
|
11923
12317
|
}
|
|
@@ -11996,7 +12390,7 @@ async function runAllocatorForIndex(coordinator, writerNodeId, indexName, nodes,
|
|
|
11996
12390
|
return;
|
|
11997
12391
|
}
|
|
11998
12392
|
const table = stampedWith(result.table, metadata.indexUuid);
|
|
11999
|
-
if (await coordinator
|
|
12393
|
+
if (await putAllocationOver(coordinator, indexName, storedTable, table, writerNodeId)) {
|
|
12000
12394
|
return;
|
|
12001
12395
|
}
|
|
12002
12396
|
if (attempt + 1 < ALLOCATION_CAS_ATTEMPTS) {
|
|
@@ -12035,6 +12429,7 @@ async function runAllocatorForAllIndexes(state, coordinator, isActive, onError)
|
|
|
12035
12429
|
if (!holdsDataNode(nodes) || !isActive()) {
|
|
12036
12430
|
return;
|
|
12037
12431
|
}
|
|
12432
|
+
const nodesFor = await nodesForEachIndex(coordinator, nodes, state.knownIndexes);
|
|
12038
12433
|
const departures = startDepartureRun(new Set(nodes.map((node) => node.nodeId)));
|
|
12039
12434
|
const retainedFor = (table) => retainedNodesFor(state, departures, table);
|
|
12040
12435
|
let anyIndexFailed = false;
|
|
@@ -12043,8 +12438,10 @@ async function runAllocatorForAllIndexes(state, coordinator, isActive, onError)
|
|
|
12043
12438
|
return;
|
|
12044
12439
|
}
|
|
12045
12440
|
try {
|
|
12046
|
-
|
|
12047
|
-
|
|
12441
|
+
const placeable = nodesFor(indexName);
|
|
12442
|
+
if (!holdsDataNode(placeable)) continue;
|
|
12443
|
+
await runAllocatorForIndex(coordinator, writerNodeId, indexName, placeable, isActive, retainedFor);
|
|
12444
|
+
await recoverIndexPartitions(state, coordinator, indexName, placeable, isActive, retainedFor);
|
|
12048
12445
|
} catch (error) {
|
|
12049
12446
|
anyIndexFailed = true;
|
|
12050
12447
|
if (onError !== void 0) {
|
|
@@ -12086,7 +12483,10 @@ async function allocateIndexNow(state, coordinator, indexName, isActive, onError
|
|
|
12086
12483
|
if (holdsDataNode(nodes) && isActive()) {
|
|
12087
12484
|
const departures = startDepartureRun(new Set(nodes.map((node) => node.nodeId)));
|
|
12088
12485
|
const retainedFor = (table) => retainedNodesFor(state, departures, table);
|
|
12089
|
-
await
|
|
12486
|
+
const placeable = (await nodesForEachIndex(coordinator, nodes, state.knownIndexes))(indexName);
|
|
12487
|
+
if (holdsDataNode(placeable)) {
|
|
12488
|
+
await runAllocatorForIndex(coordinator, writerNodeId, indexName, placeable, isActive, retainedFor);
|
|
12489
|
+
}
|
|
12090
12490
|
}
|
|
12091
12491
|
} finally {
|
|
12092
12492
|
if (isActive()) {
|
|
@@ -12116,8 +12516,12 @@ async function handleSchemaCreated(indexName, coordinator, writerNodeId, knownIn
|
|
|
12116
12516
|
return;
|
|
12117
12517
|
}
|
|
12118
12518
|
const currentTable = ownTable === null || ownTable.assignments.size === 0 ? null : ownTable;
|
|
12519
|
+
const placeable = (await nodesForEachIndex(coordinator, nodes, knownIndexes))(indexName);
|
|
12520
|
+
if (!holdsDataNode(placeable) || !isActive()) {
|
|
12521
|
+
return;
|
|
12522
|
+
}
|
|
12119
12523
|
const result = allocate(
|
|
12120
|
-
|
|
12524
|
+
placeable,
|
|
12121
12525
|
currentTable,
|
|
12122
12526
|
indexName,
|
|
12123
12527
|
metadata.partitionCount,
|
|
@@ -12127,16 +12531,16 @@ async function handleSchemaCreated(indexName, coordinator, writerNodeId, knownIn
|
|
|
12127
12531
|
if (!isActive()) {
|
|
12128
12532
|
return;
|
|
12129
12533
|
}
|
|
12130
|
-
|
|
12131
|
-
|
|
12534
|
+
if (await putAllocationOver(
|
|
12535
|
+
coordinator,
|
|
12132
12536
|
indexName,
|
|
12537
|
+
storedTable,
|
|
12133
12538
|
stampedWith(result.table, metadata.indexUuid),
|
|
12134
|
-
expectedVersion,
|
|
12135
12539
|
writerNodeId
|
|
12136
12540
|
)) {
|
|
12137
12541
|
return;
|
|
12138
12542
|
}
|
|
12139
|
-
await runAllocatorForIndex(coordinator, writerNodeId, indexName,
|
|
12543
|
+
await runAllocatorForIndex(coordinator, writerNodeId, indexName, placeable, isActive);
|
|
12140
12544
|
}
|
|
12141
12545
|
async function handleSchemaEvent(event, coordinator, state, isActive, onError) {
|
|
12142
12546
|
if (event.type === "schema_created") {
|
|
@@ -12207,7 +12611,7 @@ function metadataPublishingAnalysis(snapshot, pending) {
|
|
|
12207
12611
|
...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
|
|
12208
12612
|
...pending.stopWords !== void 0 ? { stopWords: pending.stopWords } : {},
|
|
12209
12613
|
...pending.stopWordList !== void 0 ? { stopWordList: pending.stopWordList } : {},
|
|
12210
|
-
...isRecord2(rest.sync) && rest.sync.enabled === true ? { syncActive: false, syncEpoch: newSyncEpoch() } : {}
|
|
12614
|
+
...isRecord2(rest.sync) && rest.sync.enabled === true || isRecord2(rest.changes) && rest.changes.enabled === true ? { syncActive: false, syncEpoch: newSyncEpoch() } : {}
|
|
12211
12615
|
});
|
|
12212
12616
|
}
|
|
12213
12617
|
function metadataWithAnalysisFailure(snapshot, failure) {
|
|
@@ -12465,7 +12869,7 @@ async function processBootstrapComplete(payload, coordinator, controllerNodeId)
|
|
|
12465
12869
|
if (rewritten === null) {
|
|
12466
12870
|
return false;
|
|
12467
12871
|
}
|
|
12468
|
-
const written = await coordinator
|
|
12872
|
+
const written = await putAllocationOver(coordinator, payload.indexName, table, rewritten, controllerNodeId);
|
|
12469
12873
|
if (written) {
|
|
12470
12874
|
return true;
|
|
12471
12875
|
}
|
|
@@ -12509,7 +12913,19 @@ function validateInsyncAddPayload(decoded) {
|
|
|
12509
12913
|
if (!isValidInteger(decoded.commitPoint) || decoded.commitPoint < 0) {
|
|
12510
12914
|
return null;
|
|
12511
12915
|
}
|
|
12512
|
-
|
|
12916
|
+
if (!isValidInteger(decoded.inSyncVersion) || decoded.inSyncVersion < 0) {
|
|
12917
|
+
return null;
|
|
12918
|
+
}
|
|
12919
|
+
if (typeof decoded.replicaStartedAt !== "string" || decoded.replicaStartedAt.length === 0) {
|
|
12920
|
+
return null;
|
|
12921
|
+
}
|
|
12922
|
+
return {
|
|
12923
|
+
...base,
|
|
12924
|
+
appliedSeqNo: decoded.appliedSeqNo,
|
|
12925
|
+
commitPoint: decoded.commitPoint,
|
|
12926
|
+
inSyncVersion: decoded.inSyncVersion,
|
|
12927
|
+
replicaStartedAt: decoded.replicaStartedAt
|
|
12928
|
+
};
|
|
12513
12929
|
}
|
|
12514
12930
|
function decodeMessage(message) {
|
|
12515
12931
|
try {
|
|
@@ -12644,7 +13060,7 @@ async function resign(payload, sourceId, coordinator, controllerNodeId) {
|
|
|
12644
13060
|
assignments.set(payload.partitionId, failedOver(assignment, sourceId, registered));
|
|
12645
13061
|
const rewritten = await rewriteForCurrentIndex(coordinator, { ...table, version: table.version + 1, assignments });
|
|
12646
13062
|
if (rewritten === null) return false;
|
|
12647
|
-
if (await coordinator
|
|
13063
|
+
if (await putAllocationOver(coordinator, payload.indexName, table, rewritten, controllerNodeId)) return true;
|
|
12648
13064
|
}
|
|
12649
13065
|
return false;
|
|
12650
13066
|
}
|
|
@@ -12751,9 +13167,9 @@ async function renewClusterSyncEpoch(coordinator, indexName) {
|
|
|
12751
13167
|
for (let attempt = 1; attempt <= SYNC_EPOCH_CAS_ATTEMPTS; attempt++) {
|
|
12752
13168
|
const current = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
12753
13169
|
if (current === null) return false;
|
|
12754
|
-
const
|
|
12755
|
-
if (
|
|
12756
|
-
const next = metadataWithSync(current, {
|
|
13170
|
+
const held = syncSettingsOfSnapshot(current);
|
|
13171
|
+
if (!epochFeaturesOn(held)) return true;
|
|
13172
|
+
const next = metadataWithSync(current, { ...held, syncActive: false, syncEpoch: newSyncEpoch() });
|
|
12757
13173
|
if (await coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) return true;
|
|
12758
13174
|
}
|
|
12759
13175
|
return false;
|
|
@@ -13682,9 +14098,337 @@ var DEVICE_SYNC_HEAD_POLL_MS = 1e3;
|
|
|
13682
14098
|
var DEVICE_SYNC_BOUNDS_REUSE_MS = 250;
|
|
13683
14099
|
var DEVICE_SYNC_ARCHIVE_IDLE_MS = 6e5;
|
|
13684
14100
|
var DEVICE_SYNC_SNAPSHOT_RETRY_AFTER_MS = 5e3;
|
|
13685
|
-
var
|
|
14101
|
+
var CHANGES_EXPORT_RETRY_AFTER_MS = 5e3;
|
|
13686
14102
|
var CACHED_BACKUP_MANIFEST_LIMIT = 4096;
|
|
13687
14103
|
|
|
14104
|
+
// src/distribution/cluster-node/device-sync/backup-catalog.ts
|
|
14105
|
+
var TAKEN_BACKUP_NAME = new RegExp(`^(${SYNC_BACKUP_TAKER}|${CHANGES_BACKUP_TAKER})-[0-9a-f]{32}-`);
|
|
14106
|
+
function takenBackupPrefix(takenBy, indexName) {
|
|
14107
|
+
return `${takenBy}-${sha256Hex(new TextEncoder().encode(indexName)).slice(0, 32)}-`;
|
|
14108
|
+
}
|
|
14109
|
+
function mayHoldIndex(name, indexName) {
|
|
14110
|
+
const taken = TAKEN_BACKUP_NAME.exec(name);
|
|
14111
|
+
if (taken === null) return true;
|
|
14112
|
+
const taker = taken[1] === CHANGES_BACKUP_TAKER ? CHANGES_BACKUP_TAKER : SYNC_BACKUP_TAKER;
|
|
14113
|
+
return name.startsWith(takenBackupPrefix(taker, indexName));
|
|
14114
|
+
}
|
|
14115
|
+
function createBackupCatalog(deps) {
|
|
14116
|
+
const manifests = /* @__PURE__ */ new Map();
|
|
14117
|
+
const backingUp = /* @__PURE__ */ new Map();
|
|
14118
|
+
let closed = false;
|
|
14119
|
+
async function manifestOf(store, name) {
|
|
14120
|
+
if (manifests.has(name)) return manifests.get(name) ?? null;
|
|
14121
|
+
let read;
|
|
14122
|
+
try {
|
|
14123
|
+
read = await readBackupManifest(store, name);
|
|
14124
|
+
} catch (error) {
|
|
14125
|
+
if (!(error instanceof NarsilError)) throw error;
|
|
14126
|
+
if (error.code !== ErrorCodes.BACKUP_DAMAGED && error.code !== ErrorCodes.BACKUP_NOT_FOUND) throw error;
|
|
14127
|
+
read = null;
|
|
14128
|
+
}
|
|
14129
|
+
if (manifests.size >= CACHED_BACKUP_MANIFEST_LIMIT) {
|
|
14130
|
+
const oldest = manifests.keys().next().value;
|
|
14131
|
+
if (oldest !== void 0) manifests.delete(oldest);
|
|
14132
|
+
}
|
|
14133
|
+
manifests.set(name, read);
|
|
14134
|
+
return read;
|
|
14135
|
+
}
|
|
14136
|
+
async function matching(store, indexName, partitionCount, epoch) {
|
|
14137
|
+
const names = (await listBackupNames(store)).filter((name) => mayHoldIndex(name, indexName));
|
|
14138
|
+
const found = [];
|
|
14139
|
+
for (const name of names) {
|
|
14140
|
+
const manifest = await manifestOf(store, name);
|
|
14141
|
+
if (manifest !== null && manifest.indexName === indexName && manifest.syncEpoch === epoch && manifest.partitionCount === partitionCount) {
|
|
14142
|
+
found.push(manifest);
|
|
14143
|
+
}
|
|
14144
|
+
}
|
|
14145
|
+
return found.sort((a, b) => b.completedAt - a.completedAt || b.startedAt - a.startedAt);
|
|
14146
|
+
}
|
|
14147
|
+
return {
|
|
14148
|
+
matching,
|
|
14149
|
+
async qualifying(store, indexName, partitionCount, epoch) {
|
|
14150
|
+
const candidates = await matching(store, indexName, partitionCount, epoch);
|
|
14151
|
+
if (candidates.length === 0) return null;
|
|
14152
|
+
const oldest = [];
|
|
14153
|
+
for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
|
|
14154
|
+
oldest.push((await deps.bounds(indexName, partitionId)).oldestSeqNo);
|
|
14155
|
+
}
|
|
14156
|
+
const qualifies = (backup) => {
|
|
14157
|
+
const covered = new Set(backup.partitions.map((partition) => partition.partitionId));
|
|
14158
|
+
if (covered.size !== partitionCount || oldest.some((_, partitionId) => !covered.has(partitionId))) return false;
|
|
14159
|
+
return backup.partitions.every((partition) => partition.lastSeqNo >= (oldest[partition.partitionId] ?? 0) - 1);
|
|
14160
|
+
};
|
|
14161
|
+
return candidates.find(qualifies) ?? null;
|
|
14162
|
+
},
|
|
14163
|
+
async startBackup(indexName, takenBy) {
|
|
14164
|
+
const key = `${takenBy}\0${indexName}`;
|
|
14165
|
+
if (backingUp.has(key) || closed) return;
|
|
14166
|
+
for (const job of (await readBackupJobs(deps.coordinator)).values()) {
|
|
14167
|
+
if (job.kind === "backup" && job.status === "running" && job.indexName === indexName) return;
|
|
14168
|
+
}
|
|
14169
|
+
if (backingUp.has(key) || closed) return;
|
|
14170
|
+
const backupName = `${takenBackupPrefix(takenBy, indexName)}${generateId()}`;
|
|
14171
|
+
const run = requestTakenBackup(deps.requests, indexName, backupName, takenBy).catch(deps.onError).finally(() => backingUp.delete(key));
|
|
14172
|
+
backingUp.set(key, run);
|
|
14173
|
+
},
|
|
14174
|
+
async close() {
|
|
14175
|
+
closed = true;
|
|
14176
|
+
await Promise.all([...backingUp.values()]);
|
|
14177
|
+
}
|
|
14178
|
+
};
|
|
14179
|
+
}
|
|
14180
|
+
|
|
14181
|
+
// src/distribution/cluster-node/device-sync/state.ts
|
|
14182
|
+
function syncStateOfSettings(settings) {
|
|
14183
|
+
const setting = settings?.sync;
|
|
14184
|
+
if (setting === void 0 || !setting.enabled || settings?.syncEpoch === void 0) return null;
|
|
14185
|
+
return {
|
|
14186
|
+
...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
|
|
14187
|
+
...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {},
|
|
14188
|
+
active: settings?.syncActive === true,
|
|
14189
|
+
epoch: settings.syncEpoch
|
|
14190
|
+
};
|
|
14191
|
+
}
|
|
14192
|
+
function changesStateOfSettings(settings) {
|
|
14193
|
+
const setting = settings?.changes;
|
|
14194
|
+
if (setting === void 0 || !setting.enabled || settings?.syncEpoch === void 0) return null;
|
|
14195
|
+
return {
|
|
14196
|
+
previousDocument: setting.previousDocument === true,
|
|
14197
|
+
...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
|
|
14198
|
+
...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {},
|
|
14199
|
+
epoch: settings.syncEpoch
|
|
14200
|
+
};
|
|
14201
|
+
}
|
|
14202
|
+
function changesStatusOfSettings(settings) {
|
|
14203
|
+
return changesStatusOf(changesStateOfSettings(settings));
|
|
14204
|
+
}
|
|
14205
|
+
function syncStatusOfSettings(settings) {
|
|
14206
|
+
const state = syncStateOfSettings(settings);
|
|
14207
|
+
if (state === null) return { enabled: false, active: false, epoch: null };
|
|
14208
|
+
return { enabled: true, ...state };
|
|
14209
|
+
}
|
|
14210
|
+
function analysisRevisionOf(language) {
|
|
14211
|
+
if (!hasLanguage(language)) return void 0;
|
|
14212
|
+
return getLanguage(language).revision;
|
|
14213
|
+
}
|
|
14214
|
+
function feedSettingsOf(settings, synonyms) {
|
|
14215
|
+
const language = settings?.language ?? getLanguage("english").name;
|
|
14216
|
+
const revision = analysisRevisionOf(language);
|
|
14217
|
+
return {
|
|
14218
|
+
language,
|
|
14219
|
+
...settings?.tokenizer !== void 0 ? { tokenizer: settings.tokenizer } : {},
|
|
14220
|
+
...settings?.stopWords !== void 0 ? { stopWords: settings.stopWords } : {},
|
|
14221
|
+
...settings?.stopWordList !== void 0 ? { stopWordList: settings.stopWordList } : {},
|
|
14222
|
+
...revision !== void 0 ? { analysisRevision: revision } : {},
|
|
14223
|
+
...settings?.bm25 !== void 0 ? { bm25: settings.bm25 } : {},
|
|
14224
|
+
...synonyms.length > 0 ? { synonyms: [...synonyms] } : {}
|
|
14225
|
+
};
|
|
14226
|
+
}
|
|
14227
|
+
function sameFeedSettings(held, next) {
|
|
14228
|
+
const a = held.settings;
|
|
14229
|
+
const b = next.settings;
|
|
14230
|
+
return held.synonymsGeneration === next.synonymsGeneration && a?.language === b?.language && a?.tokenizer === b?.tokenizer && a?.stopWords === b?.stopWords && JSON.stringify(a?.stopWordList ?? null) === JSON.stringify(b?.stopWordList ?? null) && (a?.bm25Revision ?? 0) === (b?.bm25Revision ?? 0);
|
|
14231
|
+
}
|
|
14232
|
+
function changeBetween(held, next) {
|
|
14233
|
+
const before = held === void 0 ? null : syncStateOfSettings(held.settings);
|
|
14234
|
+
if (held === void 0 || before === null) return null;
|
|
14235
|
+
if (next === null || next.indexUuid !== held.indexUuid) return "dropped";
|
|
14236
|
+
const after = syncStateOfSettings(next.settings);
|
|
14237
|
+
if (after === null) return "off";
|
|
14238
|
+
if (after.epoch !== before.epoch || next.serving.partitionCount !== held.serving.partitionCount) return "epoch";
|
|
14239
|
+
return sameFeedSettings(held, next) ? null : "settings";
|
|
14240
|
+
}
|
|
14241
|
+
function changesChangeBetween(held, next) {
|
|
14242
|
+
const before = held === void 0 ? null : changesStateOfSettings(held.settings);
|
|
14243
|
+
if (held === void 0 || before === null) return null;
|
|
14244
|
+
if (next === null || next.indexUuid !== held.indexUuid) return "dropped";
|
|
14245
|
+
const after = changesStateOfSettings(next.settings);
|
|
14246
|
+
if (after === null) return "changes-off";
|
|
14247
|
+
if (after.epoch !== before.epoch || next.serving.partitionCount !== held.serving.partitionCount) return "epoch";
|
|
14248
|
+
return before.previousDocument && !after.previousDocument ? "previous-document-off" : null;
|
|
14249
|
+
}
|
|
14250
|
+
|
|
14251
|
+
// src/distribution/cluster-node/device-sync/changes-export.ts
|
|
14252
|
+
function exportUnsupported(indexName) {
|
|
14253
|
+
return new NarsilError(
|
|
14254
|
+
ErrorCodes.CLUSTER_OPERATION_UNSUPPORTED,
|
|
14255
|
+
`A cluster node exports the documents of an index only while change capture is on, and index "${indexName}" has it off`,
|
|
14256
|
+
{ operation: "exportDocuments", indexName }
|
|
14257
|
+
);
|
|
14258
|
+
}
|
|
14259
|
+
async function exportClusterDocuments(deps, indexName) {
|
|
14260
|
+
refuseHiddenGeneration(indexName);
|
|
14261
|
+
validateIndexName(indexName);
|
|
14262
|
+
const held = await deps.view.current(indexName);
|
|
14263
|
+
const state = held === null ? null : changesStateOfSettings(held.settings);
|
|
14264
|
+
if (held === null || state === null) throw exportUnsupported(indexName);
|
|
14265
|
+
if (deps.store === null) {
|
|
14266
|
+
throw new NarsilError(
|
|
14267
|
+
ErrorCodes.CHANGES_BACKUP_STORE_REQUIRED,
|
|
14268
|
+
"Change capture in a cluster needs a backup store, because a cluster node builds a document export from a backup",
|
|
14269
|
+
{ indexName }
|
|
14270
|
+
);
|
|
14271
|
+
}
|
|
14272
|
+
const { partitionCount } = held.serving;
|
|
14273
|
+
const backup = await deps.catalog.qualifying(deps.store, indexName, partitionCount, state.epoch);
|
|
14274
|
+
if (backup === null) {
|
|
14275
|
+
await deps.catalog.startBackup(indexName, CHANGES_BACKUP_TAKER);
|
|
14276
|
+
throw new NarsilError(
|
|
14277
|
+
ErrorCodes.CHANGES_EXPORT_PREPARING,
|
|
14278
|
+
`The cluster is taking the backup that the export of index "${indexName}" needs. Ask again later`,
|
|
14279
|
+
{ indexName, retryAfterMs: CHANGES_EXPORT_RETRY_AFTER_MS }
|
|
14280
|
+
);
|
|
14281
|
+
}
|
|
14282
|
+
const { archive, release } = await deps.spools.acquire(deps.store, indexName, backup);
|
|
14283
|
+
try {
|
|
14284
|
+
const header = await decodeIndexConfiguration(archive.config);
|
|
14285
|
+
const positions = Array.from({ length: partitionCount }, (_, partitionId) => ({
|
|
14286
|
+
partitionId,
|
|
14287
|
+
seqNo: backup.partitions.find((partition) => partition.partitionId === partitionId)?.lastSeqNo ?? 0
|
|
14288
|
+
}));
|
|
14289
|
+
return documentExportStream({ ...header, syncEpoch: state.epoch }, archive.checkpoint(release), {
|
|
14290
|
+
epoch: state.epoch,
|
|
14291
|
+
positions
|
|
14292
|
+
});
|
|
14293
|
+
} catch (error) {
|
|
14294
|
+
release();
|
|
14295
|
+
throw error;
|
|
14296
|
+
}
|
|
14297
|
+
}
|
|
14298
|
+
|
|
14299
|
+
// src/distribution/cluster-node/device-sync/changes-settings.ts
|
|
14300
|
+
function indexNotFound(indexName) {
|
|
14301
|
+
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
14302
|
+
indexName
|
|
14303
|
+
});
|
|
14304
|
+
}
|
|
14305
|
+
function registeredNodesBelowChangeCapture(nodes) {
|
|
14306
|
+
return nodes.filter((node) => !reachesVersion(node.version, CHANGE_CAPTURE_SPEC_VERSION)).map((node) => node.nodeId);
|
|
14307
|
+
}
|
|
14308
|
+
function changesNotReady(olderNodes, indexName) {
|
|
14309
|
+
return new NarsilError(
|
|
14310
|
+
ErrorCodes.CHANGES_NOT_READY,
|
|
14311
|
+
`The cluster has ${olderNodes.length} node(s) below specification version ${CHANGE_CAPTURE_SPEC_VERSION}, which keep no history for change capture. Try again once every node registers that version`,
|
|
14312
|
+
{ indexName, nodeIds: [...olderNodes].sort(), minimumVersion: CHANGE_CAPTURE_SPEC_VERSION }
|
|
14313
|
+
);
|
|
14314
|
+
}
|
|
14315
|
+
async function requireChangeCaptureNodes(deps, indexName) {
|
|
14316
|
+
const olderNodes = registeredNodesBelowChangeCapture(await deps.coordinator.listNodes());
|
|
14317
|
+
if (olderNodes.length > 0) throw changesNotReady(olderNodes, indexName);
|
|
14318
|
+
}
|
|
14319
|
+
function storedSetting(setting) {
|
|
14320
|
+
return {
|
|
14321
|
+
enabled: true,
|
|
14322
|
+
...setting.previousDocument === true ? { previousDocument: true } : {},
|
|
14323
|
+
...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
|
|
14324
|
+
...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {}
|
|
14325
|
+
};
|
|
14326
|
+
}
|
|
14327
|
+
function nextChangesSettings(current, setting) {
|
|
14328
|
+
const { changes: _changes, ...others } = current;
|
|
14329
|
+
const syncOn = current.sync?.enabled === true && current.syncEpoch !== void 0;
|
|
14330
|
+
const changesOn = current.changes?.enabled === true && current.syncEpoch !== void 0;
|
|
14331
|
+
if (!setting.enabled) {
|
|
14332
|
+
if (current.changes === void 0) return null;
|
|
14333
|
+
return syncOn ? others : {};
|
|
14334
|
+
}
|
|
14335
|
+
if (syncOn || changesOn) return { ...others, changes: storedSetting(setting) };
|
|
14336
|
+
return { ...others, changes: storedSetting(setting), syncActive: false, syncEpoch: newSyncEpoch() };
|
|
14337
|
+
}
|
|
14338
|
+
async function updateClusterChanges(deps, indexName, value) {
|
|
14339
|
+
refuseHiddenGeneration(indexName);
|
|
14340
|
+
validateIndexName(indexName);
|
|
14341
|
+
const setting = requireChangesSetting(value);
|
|
14342
|
+
if (setting.enabled) {
|
|
14343
|
+
if (!deps.hasDurability()) {
|
|
14344
|
+
throw new NarsilError(
|
|
14345
|
+
ErrorCodes.CHANGES_DURABILITY_REQUIRED,
|
|
14346
|
+
`Index "${indexName}" needs write-ahead log durability on every node for change capture, because the feed reads the log`,
|
|
14347
|
+
{ indexName }
|
|
14348
|
+
);
|
|
14349
|
+
}
|
|
14350
|
+
if (!deps.hasBackupStore) {
|
|
14351
|
+
throw new NarsilError(
|
|
14352
|
+
ErrorCodes.CHANGES_BACKUP_STORE_REQUIRED,
|
|
14353
|
+
"Change capture in a cluster needs a backup store, because a cluster node builds a document export from a backup",
|
|
14354
|
+
{ indexName }
|
|
14355
|
+
);
|
|
14356
|
+
}
|
|
14357
|
+
await requireChangeCaptureNodes(deps, indexName);
|
|
14358
|
+
}
|
|
14359
|
+
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
14360
|
+
const current = await readIndexMetadataSnapshot(deps.coordinator, indexName);
|
|
14361
|
+
if (current === null) throw indexNotFound(indexName);
|
|
14362
|
+
const next = nextChangesSettings(current.settings ?? {}, setting);
|
|
14363
|
+
if (next === null) return;
|
|
14364
|
+
if (await deps.coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, metadataWithSync(current, next))) {
|
|
14365
|
+
await deps.view.refresh(indexName);
|
|
14366
|
+
return;
|
|
14367
|
+
}
|
|
14368
|
+
}
|
|
14369
|
+
throw new NarsilError(
|
|
14370
|
+
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
14371
|
+
`Index '${indexName}' kept changing while its changes setting was replaced ${MAX_SETTINGS_REPLACEMENT_ATTEMPTS} times over, so the new setting was not stored. Retry the request`,
|
|
14372
|
+
{ reason: SETTINGS_CONTENTION_REASON, indexName, attempts: MAX_SETTINGS_REPLACEMENT_ATTEMPTS }
|
|
14373
|
+
);
|
|
14374
|
+
}
|
|
14375
|
+
async function readClusterChanges(deps, indexName) {
|
|
14376
|
+
refuseHiddenGeneration(indexName);
|
|
14377
|
+
validateIndexName(indexName);
|
|
14378
|
+
const snapshot = await readIndexMetadataSnapshot(deps.coordinator, indexName);
|
|
14379
|
+
if (snapshot === null) throw indexNotFound(indexName);
|
|
14380
|
+
return changesStatusOfSettings(snapshot.settings);
|
|
14381
|
+
}
|
|
14382
|
+
|
|
14383
|
+
// src/distribution/cluster-node/device-sync/changes-source.ts
|
|
14384
|
+
function createClusterChangeSource(deps) {
|
|
14385
|
+
const { feed, view, coordinator } = deps;
|
|
14386
|
+
async function heldSchema(indexName) {
|
|
14387
|
+
const held = view.held(indexName);
|
|
14388
|
+
if (held === void 0) throw changesNotEnabled(indexName);
|
|
14389
|
+
const { servingName } = held.serving;
|
|
14390
|
+
const stored = await coordinator.getSchema(servingName);
|
|
14391
|
+
if (stored === null) throw changesNotEnabled(indexName);
|
|
14392
|
+
return { servingName, schema: validateSchema(stored) };
|
|
14393
|
+
}
|
|
14394
|
+
return {
|
|
14395
|
+
changesState(indexName) {
|
|
14396
|
+
const held = view.held(indexName);
|
|
14397
|
+
const state = changesStateOfSettings(held?.settings);
|
|
14398
|
+
if (held === void 0 || state === null) return null;
|
|
14399
|
+
return {
|
|
14400
|
+
epoch: state.epoch,
|
|
14401
|
+
partitionCount: held.serving.partitionCount,
|
|
14402
|
+
previousDocument: state.previousDocument
|
|
14403
|
+
};
|
|
14404
|
+
},
|
|
14405
|
+
async documentTest(indexName, filter) {
|
|
14406
|
+
const { servingName, schema } = await heldSchema(indexName);
|
|
14407
|
+
const valid = validChangeFilter(filter, schema);
|
|
14408
|
+
const config = await getClusterIndexConfig(coordinator, servingName, schema);
|
|
14409
|
+
return createDocumentTest(
|
|
14410
|
+
{
|
|
14411
|
+
schema,
|
|
14412
|
+
language: getLanguage(config.language ?? "english"),
|
|
14413
|
+
insertOptions: resolvePartitionInsertOptions(config, resolveIndexAnalysis(config)),
|
|
14414
|
+
vectorFieldPaths: getVectorFieldPaths(schema)
|
|
14415
|
+
},
|
|
14416
|
+
valid
|
|
14417
|
+
);
|
|
14418
|
+
},
|
|
14419
|
+
async vectorFieldPaths(indexName) {
|
|
14420
|
+
const { schema } = await heldSchema(indexName);
|
|
14421
|
+
return getVectorFieldPaths(schema);
|
|
14422
|
+
},
|
|
14423
|
+
bounds: (indexName, partitionId, freshness) => feed.bounds(indexName, partitionId, freshness),
|
|
14424
|
+
read: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => feed.read(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
|
|
14425
|
+
bytesBehind: (indexName, seqNos) => feed.bytesBehind(indexName, seqNos),
|
|
14426
|
+
pendingBeyondHead: (indexName, partitionId, head) => feed.pendingBeyondHead(indexName, partitionId, head),
|
|
14427
|
+
listen: (onChange) => feed.listen(onChange),
|
|
14428
|
+
watchIndex: (indexName) => feed.watchIndex?.(indexName) ?? (() => void 0)
|
|
14429
|
+
};
|
|
14430
|
+
}
|
|
14431
|
+
|
|
13688
14432
|
// src/distribution/cluster-node/reshard/leadership.ts
|
|
13689
14433
|
async function leadingAssignment2(deps, indexName, partitionId) {
|
|
13690
14434
|
let assignment = (await deps.coordinator.getAllocation(indexName))?.assignments.get(partitionId);
|
|
@@ -14029,56 +14773,8 @@ function createClusterFeedSource(deps) {
|
|
|
14029
14773
|
};
|
|
14030
14774
|
}
|
|
14031
14775
|
|
|
14032
|
-
// src/distribution/cluster-node/device-sync/state.ts
|
|
14033
|
-
function syncStateOfSettings(settings) {
|
|
14034
|
-
const setting = settings?.sync;
|
|
14035
|
-
if (setting === void 0 || !setting.enabled || settings?.syncEpoch === void 0) return null;
|
|
14036
|
-
return {
|
|
14037
|
-
...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
|
|
14038
|
-
...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {},
|
|
14039
|
-
active: settings?.syncActive === true,
|
|
14040
|
-
epoch: settings.syncEpoch
|
|
14041
|
-
};
|
|
14042
|
-
}
|
|
14043
|
-
function syncStatusOfSettings(settings) {
|
|
14044
|
-
const state = syncStateOfSettings(settings);
|
|
14045
|
-
if (state === null) return { enabled: false, active: false, epoch: null };
|
|
14046
|
-
return { enabled: true, ...state };
|
|
14047
|
-
}
|
|
14048
|
-
function analysisRevisionOf(language) {
|
|
14049
|
-
if (!hasLanguage(language)) return void 0;
|
|
14050
|
-
return getLanguage(language).revision;
|
|
14051
|
-
}
|
|
14052
|
-
function feedSettingsOf(settings, synonyms) {
|
|
14053
|
-
const language = settings?.language ?? getLanguage("english").name;
|
|
14054
|
-
const revision = analysisRevisionOf(language);
|
|
14055
|
-
return {
|
|
14056
|
-
language,
|
|
14057
|
-
...settings?.tokenizer !== void 0 ? { tokenizer: settings.tokenizer } : {},
|
|
14058
|
-
...settings?.stopWords !== void 0 ? { stopWords: settings.stopWords } : {},
|
|
14059
|
-
...settings?.stopWordList !== void 0 ? { stopWordList: settings.stopWordList } : {},
|
|
14060
|
-
...revision !== void 0 ? { analysisRevision: revision } : {},
|
|
14061
|
-
...settings?.bm25 !== void 0 ? { bm25: settings.bm25 } : {},
|
|
14062
|
-
...synonyms.length > 0 ? { synonyms: [...synonyms] } : {}
|
|
14063
|
-
};
|
|
14064
|
-
}
|
|
14065
|
-
function sameFeedSettings(held, next) {
|
|
14066
|
-
const a = held.settings;
|
|
14067
|
-
const b = next.settings;
|
|
14068
|
-
return held.synonymsGeneration === next.synonymsGeneration && a?.language === b?.language && a?.tokenizer === b?.tokenizer && a?.stopWords === b?.stopWords && JSON.stringify(a?.stopWordList ?? null) === JSON.stringify(b?.stopWordList ?? null) && (a?.bm25Revision ?? 0) === (b?.bm25Revision ?? 0);
|
|
14069
|
-
}
|
|
14070
|
-
function changeBetween(held, next) {
|
|
14071
|
-
const before = held === void 0 ? null : syncStateOfSettings(held.settings);
|
|
14072
|
-
if (held === void 0 || before === null) return null;
|
|
14073
|
-
if (next === null || next.indexUuid !== held.indexUuid) return "dropped";
|
|
14074
|
-
const after = syncStateOfSettings(next.settings);
|
|
14075
|
-
if (after === null) return "off";
|
|
14076
|
-
if (after.epoch !== before.epoch || next.serving.partitionCount !== held.serving.partitionCount) return "epoch";
|
|
14077
|
-
return sameFeedSettings(held, next) ? null : "settings";
|
|
14078
|
-
}
|
|
14079
|
-
|
|
14080
14776
|
// src/distribution/cluster-node/device-sync/settings.ts
|
|
14081
|
-
function
|
|
14777
|
+
function indexNotFound2(indexName) {
|
|
14082
14778
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
14083
14779
|
indexName
|
|
14084
14780
|
});
|
|
@@ -14093,7 +14789,7 @@ function syncNotReady(subject, olderNodes, indexName) {
|
|
|
14093
14789
|
function registeredNodesBelowDeviceSync(nodes) {
|
|
14094
14790
|
return nodes.filter((node) => !reachesVersion(node.version, DEVICE_SYNC_SPEC_VERSION)).map((node) => node.nodeId);
|
|
14095
14791
|
}
|
|
14096
|
-
function
|
|
14792
|
+
function storedSetting2(setting) {
|
|
14097
14793
|
return {
|
|
14098
14794
|
enabled: true,
|
|
14099
14795
|
...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
|
|
@@ -14102,12 +14798,19 @@ function storedSetting(setting) {
|
|
|
14102
14798
|
}
|
|
14103
14799
|
function nextSyncSettings(current, setting) {
|
|
14104
14800
|
const wasOn = current.sync?.enabled === true && current.syncEpoch !== void 0;
|
|
14105
|
-
|
|
14106
|
-
if (!
|
|
14801
|
+
const capture = current.changes?.enabled === true && current.syncEpoch !== void 0 ? { changes: current.changes, syncEpoch: current.syncEpoch } : null;
|
|
14802
|
+
if (!setting.enabled) {
|
|
14803
|
+
if (!wasOn && current.sync === void 0) return null;
|
|
14804
|
+
return capture ?? {};
|
|
14805
|
+
}
|
|
14806
|
+
if (!wasOn) {
|
|
14807
|
+
return capture === null ? { sync: storedSetting2(setting), syncActive: false, syncEpoch: newSyncEpoch() } : { ...capture, sync: storedSetting2(setting), syncActive: false };
|
|
14808
|
+
}
|
|
14107
14809
|
return {
|
|
14108
|
-
sync:
|
|
14810
|
+
sync: storedSetting2(setting),
|
|
14109
14811
|
...current.syncActive !== void 0 ? { syncActive: current.syncActive } : {},
|
|
14110
|
-
...current.syncEpoch !== void 0 ? { syncEpoch: current.syncEpoch } : {}
|
|
14812
|
+
...current.syncEpoch !== void 0 ? { syncEpoch: current.syncEpoch } : {},
|
|
14813
|
+
...current.changes !== void 0 ? { changes: current.changes } : {}
|
|
14111
14814
|
};
|
|
14112
14815
|
}
|
|
14113
14816
|
async function updateClusterSync(deps, indexName, value) {
|
|
@@ -14134,7 +14837,7 @@ async function updateClusterSync(deps, indexName, value) {
|
|
|
14134
14837
|
}
|
|
14135
14838
|
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
14136
14839
|
const current = await readIndexMetadataSnapshot(deps.coordinator, indexName);
|
|
14137
|
-
if (current === null) throw
|
|
14840
|
+
if (current === null) throw indexNotFound2(indexName);
|
|
14138
14841
|
if (setting.enabled) requireNoQuerySuggestions(suggestionStateOfSettings(current.settings), indexName);
|
|
14139
14842
|
const next = nextSyncSettings(current.settings ?? {}, setting);
|
|
14140
14843
|
if (next === null) return;
|
|
@@ -14153,19 +14856,24 @@ async function readClusterSync(deps, indexName) {
|
|
|
14153
14856
|
refuseHiddenGeneration(indexName);
|
|
14154
14857
|
validateIndexName(indexName);
|
|
14155
14858
|
const snapshot = await readIndexMetadataSnapshot(deps.coordinator, indexName);
|
|
14156
|
-
if (snapshot === null) throw
|
|
14859
|
+
if (snapshot === null) throw indexNotFound2(indexName);
|
|
14157
14860
|
return syncStatusOfSettings(snapshot.settings);
|
|
14158
14861
|
}
|
|
14159
14862
|
async function markClusterSyncActive(deps, indexName) {
|
|
14160
14863
|
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
14161
14864
|
const current = await readIndexMetadataSnapshot(deps.coordinator, indexName);
|
|
14162
|
-
if (current === null) throw
|
|
14865
|
+
if (current === null) throw indexNotFound2(indexName);
|
|
14163
14866
|
const settings = current.settings ?? {};
|
|
14164
14867
|
if (settings.sync?.enabled !== true || settings.syncEpoch === void 0) {
|
|
14165
14868
|
throw new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index '${indexName}' has device sync off`, { indexName });
|
|
14166
14869
|
}
|
|
14167
14870
|
if (settings.syncActive === true) return;
|
|
14168
|
-
const next = metadataWithSync(current, {
|
|
14871
|
+
const next = metadataWithSync(current, {
|
|
14872
|
+
sync: settings.sync,
|
|
14873
|
+
syncActive: true,
|
|
14874
|
+
syncEpoch: settings.syncEpoch,
|
|
14875
|
+
...settings.changes !== void 0 ? { changes: settings.changes } : {}
|
|
14876
|
+
});
|
|
14169
14877
|
if (await deps.coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) {
|
|
14170
14878
|
await deps.view.refresh(indexName);
|
|
14171
14879
|
return;
|
|
@@ -14177,6 +14885,60 @@ async function markClusterSyncActive(deps, indexName) {
|
|
|
14177
14885
|
{ reason: SETTINGS_CONTENTION_REASON, indexName, attempts: MAX_SETTINGS_REPLACEMENT_ATTEMPTS }
|
|
14178
14886
|
);
|
|
14179
14887
|
}
|
|
14888
|
+
|
|
14889
|
+
// src/distribution/cluster-node/device-sync/snapshots.ts
|
|
14890
|
+
function syncOff(indexName) {
|
|
14891
|
+
return new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index "${indexName}" has device sync off`, { indexName });
|
|
14892
|
+
}
|
|
14893
|
+
function preparing(indexName) {
|
|
14894
|
+
return new NarsilError(
|
|
14895
|
+
ErrorCodes.SYNC_SNAPSHOT_PREPARING,
|
|
14896
|
+
`The cluster is taking the backup that the snapshot of index "${indexName}" needs. Ask again later`,
|
|
14897
|
+
{ indexName, retryAfterMs: DEVICE_SYNC_SNAPSHOT_RETRY_AFTER_MS }
|
|
14898
|
+
);
|
|
14899
|
+
}
|
|
14900
|
+
function createClusterSnapshots(deps) {
|
|
14901
|
+
const { catalog, spools } = deps;
|
|
14902
|
+
function requireStore2(indexName) {
|
|
14903
|
+
if (deps.store === null) {
|
|
14904
|
+
throw new NarsilError(
|
|
14905
|
+
ErrorCodes.SYNC_BACKUP_STORE_REQUIRED,
|
|
14906
|
+
`Device sync in a cluster needs a backup store, because a cluster node takes a device's first copy from a backup`,
|
|
14907
|
+
{ indexName }
|
|
14908
|
+
);
|
|
14909
|
+
}
|
|
14910
|
+
return deps.store;
|
|
14911
|
+
}
|
|
14912
|
+
return {
|
|
14913
|
+
async snapshot(indexName) {
|
|
14914
|
+
let held = await deps.settings.view.current(indexName);
|
|
14915
|
+
const state = held === null ? null : syncStateOfSettings(held.settings);
|
|
14916
|
+
if (held === null || state === null) throw syncOff(indexName);
|
|
14917
|
+
if (!state.active) {
|
|
14918
|
+
await markClusterSyncActive(deps.settings, indexName);
|
|
14919
|
+
held = deps.settings.view.held(indexName) ?? held;
|
|
14920
|
+
}
|
|
14921
|
+
const store = requireStore2(indexName);
|
|
14922
|
+
const backup = await catalog.qualifying(store, indexName, held.serving.partitionCount, state.epoch);
|
|
14923
|
+
if (backup === null) {
|
|
14924
|
+
await catalog.startBackup(indexName, SYNC_BACKUP_TAKER);
|
|
14925
|
+
throw preparing(indexName);
|
|
14926
|
+
}
|
|
14927
|
+
const { archive, release } = await spools.acquire(store, indexName, backup);
|
|
14928
|
+
return { entityTag: archive.entityTag, layout: archive.layout, release };
|
|
14929
|
+
},
|
|
14930
|
+
async snapshotBytes(indexName) {
|
|
14931
|
+
const held = deps.settings.view.held(indexName);
|
|
14932
|
+
const state = held === void 0 ? null : syncStateOfSettings(held.settings);
|
|
14933
|
+
if (held === void 0 || state === null || deps.store === null) return 0;
|
|
14934
|
+
const archive = await spools.chosen(indexName);
|
|
14935
|
+
if (archive !== null) return archive.layout.byteLength;
|
|
14936
|
+
const [newest] = await catalog.matching(deps.store, indexName, held.serving.partitionCount, state.epoch);
|
|
14937
|
+
if (newest === void 0) return 0;
|
|
14938
|
+
return estimatedArchiveBytes(storedFilesOf(newest));
|
|
14939
|
+
}
|
|
14940
|
+
};
|
|
14941
|
+
}
|
|
14180
14942
|
function damaged2(backupName, reason) {
|
|
14181
14943
|
return new NarsilError(ErrorCodes.BACKUP_DAMAGED, `Backup '${backupName}' ${reason}, so a device can't load it`, {
|
|
14182
14944
|
backupName
|
|
@@ -14317,13 +15079,17 @@ async function spoolBackupArchive(store, backup, spoolDirectory) {
|
|
|
14317
15079
|
const prefix = `${sourceIndexName}/`;
|
|
14318
15080
|
const ordered = manifestFileRefs(manifest);
|
|
14319
15081
|
const byKey = new Map(files.map((file) => [file.ref.key, file]));
|
|
15082
|
+
const pathOf = (ref) => {
|
|
15083
|
+
const file = byKey.get(ref.key);
|
|
15084
|
+
const path = file === void 0 ? void 0 : pathByDigest.get(file.listed.sha256);
|
|
15085
|
+
if (path === void 0) throw damaged2(backup.name, `lost the spooled copy of "${ref.key}"`);
|
|
15086
|
+
return path;
|
|
15087
|
+
};
|
|
14320
15088
|
const layout = archiveLayout(
|
|
14321
15089
|
config,
|
|
14322
15090
|
packed,
|
|
14323
15091
|
ordered.map((ref) => {
|
|
14324
|
-
const
|
|
14325
|
-
const path = file === void 0 ? void 0 : pathByDigest.get(file.listed.sha256);
|
|
14326
|
-
if (path === void 0) throw damaged2(backup.name, `lost the spooled copy of "${ref.key}"`);
|
|
15092
|
+
const path = pathOf(ref);
|
|
14327
15093
|
return {
|
|
14328
15094
|
name: ref.key.slice(prefix.length),
|
|
14329
15095
|
bytes: ref.bytes,
|
|
@@ -14331,109 +15097,45 @@ async function spoolBackupArchive(store, backup, spoolDirectory) {
|
|
|
14331
15097
|
};
|
|
14332
15098
|
})
|
|
14333
15099
|
);
|
|
14334
|
-
|
|
15100
|
+
const inPartitionOrder = {
|
|
15101
|
+
...manifest,
|
|
15102
|
+
partitions: [...manifest.partitions].sort((a, b) => a.partitionId - b.partitionId)
|
|
15103
|
+
};
|
|
15104
|
+
const checkpoint = (release) => ({
|
|
15105
|
+
manifestBytes: packed,
|
|
15106
|
+
manifest: inPartitionOrder,
|
|
15107
|
+
readFile: (ref) => readSpooled2(pathOf(ref), 0, ref.bytes),
|
|
15108
|
+
readRange: (ref, offset, length) => readSpooled2(pathOf(ref), offset, length),
|
|
15109
|
+
release
|
|
15110
|
+
});
|
|
15111
|
+
return { backupName: backup.name, entityTag, layout, config, checkpoint, dispose };
|
|
14335
15112
|
} catch (error) {
|
|
14336
15113
|
await dispose();
|
|
14337
15114
|
throw error;
|
|
14338
15115
|
}
|
|
14339
15116
|
}
|
|
14340
15117
|
|
|
14341
|
-
// src/distribution/cluster-node/device-sync/
|
|
14342
|
-
function
|
|
14343
|
-
return
|
|
14344
|
-
}
|
|
14345
|
-
function syncOff(indexName) {
|
|
14346
|
-
return new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index "${indexName}" has device sync off`, { indexName });
|
|
14347
|
-
}
|
|
14348
|
-
function preparing(indexName) {
|
|
14349
|
-
return new NarsilError(
|
|
14350
|
-
ErrorCodes.SYNC_SNAPSHOT_PREPARING,
|
|
14351
|
-
`The cluster is taking the backup that the snapshot of index "${indexName}" needs. Ask again later`,
|
|
14352
|
-
{ indexName, retryAfterMs: DEVICE_SYNC_SNAPSHOT_RETRY_AFTER_MS }
|
|
14353
|
-
);
|
|
15118
|
+
// src/distribution/cluster-node/device-sync/spool-cache.ts
|
|
15119
|
+
function disposeOf(entry) {
|
|
15120
|
+
return entry.ready.then((archive) => archive.dispose()).catch(() => void 0);
|
|
14354
15121
|
}
|
|
14355
|
-
function
|
|
14356
|
-
const manifests = /* @__PURE__ */ new Map();
|
|
15122
|
+
function createSpoolCache(spoolDirectory, guard) {
|
|
14357
15123
|
const spools = /* @__PURE__ */ new Map();
|
|
14358
|
-
const
|
|
14359
|
-
const backingUp = /* @__PURE__ */ new Map();
|
|
15124
|
+
const chosenNames = /* @__PURE__ */ new Map();
|
|
14360
15125
|
let closed = false;
|
|
14361
|
-
function requireStore2(indexName) {
|
|
14362
|
-
if (deps.store === null) {
|
|
14363
|
-
throw new NarsilError(
|
|
14364
|
-
ErrorCodes.SYNC_BACKUP_STORE_REQUIRED,
|
|
14365
|
-
`Device sync in a cluster needs a backup store, because a cluster node takes a device's first copy from a backup`,
|
|
14366
|
-
{ indexName }
|
|
14367
|
-
);
|
|
14368
|
-
}
|
|
14369
|
-
return deps.store;
|
|
14370
|
-
}
|
|
14371
|
-
async function manifestOf(store, name) {
|
|
14372
|
-
if (manifests.has(name)) return manifests.get(name) ?? null;
|
|
14373
|
-
let read;
|
|
14374
|
-
try {
|
|
14375
|
-
read = await readBackupManifest(store, name);
|
|
14376
|
-
} catch (error) {
|
|
14377
|
-
if (!(error instanceof NarsilError)) throw error;
|
|
14378
|
-
if (error.code !== ErrorCodes.BACKUP_DAMAGED && error.code !== ErrorCodes.BACKUP_NOT_FOUND) throw error;
|
|
14379
|
-
read = null;
|
|
14380
|
-
}
|
|
14381
|
-
if (manifests.size >= CACHED_BACKUP_MANIFEST_LIMIT) {
|
|
14382
|
-
const oldest = manifests.keys().next().value;
|
|
14383
|
-
if (oldest !== void 0) manifests.delete(oldest);
|
|
14384
|
-
}
|
|
14385
|
-
manifests.set(name, read);
|
|
14386
|
-
return read;
|
|
14387
|
-
}
|
|
14388
|
-
async function backupsOf(indexName, held, epoch) {
|
|
14389
|
-
const store = requireStore2(indexName);
|
|
14390
|
-
const prefix = syncBackupPrefix(indexName);
|
|
14391
|
-
const names = (await listBackupNames(store)).filter(
|
|
14392
|
-
(name) => name.startsWith(prefix) || !name.startsWith(DEVICE_SYNC_BACKUP_NAME_PREFIX)
|
|
14393
|
-
);
|
|
14394
|
-
const matching = [];
|
|
14395
|
-
for (const name of names) {
|
|
14396
|
-
const manifest = await manifestOf(store, name);
|
|
14397
|
-
if (manifest !== null && manifest.indexName === indexName && manifest.syncEpoch === epoch && manifest.partitionCount === held.serving.partitionCount) {
|
|
14398
|
-
matching.push(manifest);
|
|
14399
|
-
}
|
|
14400
|
-
}
|
|
14401
|
-
return matching.sort((a, b) => b.completedAt - a.completedAt || b.startedAt - a.startedAt);
|
|
14402
|
-
}
|
|
14403
|
-
async function qualifyingBackup(indexName, held, epoch) {
|
|
14404
|
-
const candidates = await backupsOf(indexName, held, epoch);
|
|
14405
|
-
if (candidates.length === 0) return null;
|
|
14406
|
-
const oldest = [];
|
|
14407
|
-
for (let partitionId = 0; partitionId < held.serving.partitionCount; partitionId++) {
|
|
14408
|
-
oldest.push((await deps.bounds(indexName, partitionId)).oldestSeqNo);
|
|
14409
|
-
}
|
|
14410
|
-
return candidates.find(
|
|
14411
|
-
(backup) => backup.partitions.every((partition) => partition.lastSeqNo >= (oldest[partition.partitionId] ?? 0) - 1)
|
|
14412
|
-
) ?? null;
|
|
14413
|
-
}
|
|
14414
|
-
async function startBackup(indexName) {
|
|
14415
|
-
if (backingUp.has(indexName) || closed) return;
|
|
14416
|
-
for (const job of (await readBackupJobs(deps.coordinator)).values()) {
|
|
14417
|
-
if (job.kind === "backup" && job.status === "running" && job.indexName === indexName) return;
|
|
14418
|
-
}
|
|
14419
|
-
if (backingUp.has(indexName)) return;
|
|
14420
|
-
const backupName = `${syncBackupPrefix(indexName)}${generateId()}`;
|
|
14421
|
-
const run = requestSyncBackup(deps.requests, indexName, backupName).catch(deps.onError).finally(() => backingUp.delete(indexName));
|
|
14422
|
-
backingUp.set(indexName, run);
|
|
14423
|
-
}
|
|
14424
15126
|
function evictIdle(now3) {
|
|
14425
15127
|
for (const [name, entry] of spools) {
|
|
14426
15128
|
if (entry.users > 0) continue;
|
|
14427
|
-
const superseded =
|
|
15129
|
+
const superseded = chosenNames.get(entry.indexName) !== name;
|
|
14428
15130
|
if (!superseded && now3 - entry.lastUsedAt < DEVICE_SYNC_ARCHIVE_IDLE_MS) continue;
|
|
14429
15131
|
spools.delete(name);
|
|
14430
|
-
void entry
|
|
15132
|
+
void disposeOf(entry);
|
|
14431
15133
|
}
|
|
14432
15134
|
}
|
|
14433
15135
|
function spoolOf(store, indexName, backup) {
|
|
14434
15136
|
let entry = spools.get(backup.name);
|
|
14435
15137
|
if (entry === void 0) {
|
|
14436
|
-
const ready = spoolBackupArchive(store, backup,
|
|
15138
|
+
const ready = spoolBackupArchive(store, backup, spoolDirectory);
|
|
14437
15139
|
entry = { indexName, ready, users: 0, lastUsedAt: Date.now() };
|
|
14438
15140
|
spools.set(backup.name, entry);
|
|
14439
15141
|
const created = entry;
|
|
@@ -14444,23 +15146,10 @@ function createClusterSnapshots(deps) {
|
|
|
14444
15146
|
return entry;
|
|
14445
15147
|
}
|
|
14446
15148
|
return {
|
|
14447
|
-
async
|
|
14448
|
-
|
|
14449
|
-
|
|
14450
|
-
|
|
14451
|
-
if (!state.active) {
|
|
14452
|
-
await markClusterSyncActive(deps.settings, indexName);
|
|
14453
|
-
held = deps.settings.view.held(indexName) ?? held;
|
|
14454
|
-
}
|
|
14455
|
-
const store = requireStore2(indexName);
|
|
14456
|
-
const backup = await qualifyingBackup(indexName, held, state.epoch);
|
|
14457
|
-
if (backup === null) {
|
|
14458
|
-
await startBackup(indexName);
|
|
14459
|
-
throw preparing(indexName);
|
|
14460
|
-
}
|
|
14461
|
-
chosen.set(indexName, backup.name);
|
|
14462
|
-
const now3 = Date.now();
|
|
14463
|
-
evictIdle(now3);
|
|
15149
|
+
async acquire(store, indexName, backup) {
|
|
15150
|
+
guard();
|
|
15151
|
+
chosenNames.set(indexName, backup.name);
|
|
15152
|
+
evictIdle(Date.now());
|
|
14464
15153
|
const entry = spoolOf(store, indexName, backup);
|
|
14465
15154
|
entry.users += 1;
|
|
14466
15155
|
let archive;
|
|
@@ -14472,35 +15161,27 @@ function createClusterSnapshots(deps) {
|
|
|
14472
15161
|
}
|
|
14473
15162
|
let released = false;
|
|
14474
15163
|
return {
|
|
14475
|
-
|
|
14476
|
-
layout: archive.layout,
|
|
15164
|
+
archive,
|
|
14477
15165
|
release() {
|
|
14478
15166
|
if (released) return;
|
|
14479
15167
|
released = true;
|
|
14480
15168
|
entry.users -= 1;
|
|
14481
15169
|
entry.lastUsedAt = Date.now();
|
|
15170
|
+
if (closed && entry.users === 0) void disposeOf(entry);
|
|
14482
15171
|
}
|
|
14483
15172
|
};
|
|
14484
15173
|
},
|
|
14485
|
-
async
|
|
14486
|
-
const
|
|
14487
|
-
|
|
14488
|
-
|
|
14489
|
-
const spooled = spools.get(chosen.get(indexName) ?? "");
|
|
14490
|
-
if (spooled !== void 0) {
|
|
14491
|
-
const archive = await spooled.ready.catch(() => null);
|
|
14492
|
-
if (archive !== null) return archive.layout.byteLength;
|
|
14493
|
-
}
|
|
14494
|
-
const [newest] = await backupsOf(indexName, held, state.epoch);
|
|
14495
|
-
if (newest === void 0) return 0;
|
|
14496
|
-
return estimatedArchiveBytes(storedFilesOf(newest));
|
|
15174
|
+
async chosen(indexName) {
|
|
15175
|
+
const spooled = spools.get(chosenNames.get(indexName) ?? "");
|
|
15176
|
+
if (spooled === void 0) return null;
|
|
15177
|
+
return spooled.ready.catch(() => null);
|
|
14497
15178
|
},
|
|
14498
15179
|
async close() {
|
|
14499
15180
|
closed = true;
|
|
14500
|
-
|
|
14501
|
-
const entries = [...spools.values()];
|
|
15181
|
+
const idle = [...spools.values()].filter((entry) => entry.users === 0);
|
|
14502
15182
|
spools.clear();
|
|
14503
|
-
|
|
15183
|
+
chosenNames.clear();
|
|
15184
|
+
await Promise.all(idle.map(disposeOf));
|
|
14504
15185
|
}
|
|
14505
15186
|
};
|
|
14506
15187
|
}
|
|
@@ -14542,11 +15223,16 @@ function createDeviceSyncView(deps) {
|
|
|
14542
15223
|
}
|
|
14543
15224
|
}
|
|
14544
15225
|
async function adoptOnLocalCopies(indexName, next) {
|
|
14545
|
-
const
|
|
15226
|
+
const states = {
|
|
15227
|
+
sync: next === null ? null : syncStateOfSettings(next.settings),
|
|
15228
|
+
changes: next === null ? null : changesStateOfSettings(next.settings)
|
|
15229
|
+
};
|
|
14546
15230
|
const local = new Set(engine.listIndexes().map((index) => index.name));
|
|
14547
15231
|
const servingName = next?.serving.servingName ?? indexName;
|
|
14548
|
-
if (local.has(servingName)) await engine.
|
|
14549
|
-
if (servingName !== indexName && local.has(indexName))
|
|
15232
|
+
if (local.has(servingName)) await engine.adoptEpochStates(servingName, states);
|
|
15233
|
+
if (servingName !== indexName && local.has(indexName)) {
|
|
15234
|
+
await engine.adoptEpochStates(indexName, { sync: null, changes: null });
|
|
15235
|
+
}
|
|
14550
15236
|
}
|
|
14551
15237
|
async function reread(indexName) {
|
|
14552
15238
|
const previous = held.get(indexName);
|
|
@@ -14554,8 +15240,12 @@ function createDeviceSyncView(deps) {
|
|
|
14554
15240
|
if (next === null) held.delete(indexName);
|
|
14555
15241
|
else held.set(indexName, next);
|
|
14556
15242
|
await adoptOnLocalCopies(indexName, next);
|
|
14557
|
-
const
|
|
14558
|
-
|
|
15243
|
+
const kinds = /* @__PURE__ */ new Set();
|
|
15244
|
+
const syncChange = changeBetween(previous, next);
|
|
15245
|
+
const changesChange = changesChangeBetween(previous, next);
|
|
15246
|
+
if (syncChange !== null) kinds.add(syncChange);
|
|
15247
|
+
if (changesChange !== null) kinds.add(changesChange);
|
|
15248
|
+
for (const kind of kinds) emit({ kind, indexName });
|
|
14559
15249
|
return next;
|
|
14560
15250
|
}
|
|
14561
15251
|
function refresh(indexName) {
|
|
@@ -14653,7 +15343,7 @@ function createDeviceSyncView(deps) {
|
|
|
14653
15343
|
|
|
14654
15344
|
// src/distribution/cluster-node/device-sync/index.ts
|
|
14655
15345
|
function createClusterDeviceSync(deps) {
|
|
14656
|
-
const { config, coordinator, engine, onError } = deps;
|
|
15346
|
+
const { config, coordinator, engine, onError, guard } = deps;
|
|
14657
15347
|
const view = createDeviceSyncView({ coordinator, engine, onError });
|
|
14658
15348
|
const store = config.backups === void 0 ? null : openBackupStore(config.backups);
|
|
14659
15349
|
const settings = {
|
|
@@ -14662,38 +15352,61 @@ function createClusterDeviceSync(deps) {
|
|
|
14662
15352
|
hasBackupStore: store !== null,
|
|
14663
15353
|
hasDurability: () => engine.syncStorage() !== null
|
|
14664
15354
|
};
|
|
14665
|
-
const requests = { config, nodeId: deps.nodeId, guard
|
|
15355
|
+
const requests = { config, nodeId: deps.nodeId, guard, store };
|
|
15356
|
+
const spools = createSpoolCache(join(deps.spoolDirectory, "device-sync"), guard);
|
|
14666
15357
|
let snapshots = null;
|
|
15358
|
+
let feed = null;
|
|
15359
|
+
let changeSource = null;
|
|
14667
15360
|
let serving = null;
|
|
15361
|
+
function feedSource() {
|
|
15362
|
+
feed ??= createClusterFeedSource({
|
|
15363
|
+
nodeId: deps.nodeId,
|
|
15364
|
+
writeDeps: deps.writeDeps,
|
|
15365
|
+
config,
|
|
15366
|
+
coordinator,
|
|
15367
|
+
engine,
|
|
15368
|
+
view,
|
|
15369
|
+
refreshAllocation: deps.refreshAllocation,
|
|
15370
|
+
snapshotBytes: (indexName) => snapshots?.snapshotBytes(indexName) ?? Promise.resolve(0)
|
|
15371
|
+
});
|
|
15372
|
+
return feed;
|
|
15373
|
+
}
|
|
15374
|
+
const catalog = createBackupCatalog({
|
|
15375
|
+
coordinator,
|
|
15376
|
+
requests,
|
|
15377
|
+
bounds: (indexName, partitionId) => feedSource().bounds(indexName, partitionId),
|
|
15378
|
+
onError
|
|
15379
|
+
});
|
|
15380
|
+
function changesSource() {
|
|
15381
|
+
changeSource ??= createClusterChangeSource({ feed: feedSource(), view, coordinator });
|
|
15382
|
+
return changeSource;
|
|
15383
|
+
}
|
|
15384
|
+
async function admitChanges(indexName) {
|
|
15385
|
+
await requireChangeCaptureNodes(settings, indexName);
|
|
15386
|
+
await view.current(indexName);
|
|
15387
|
+
}
|
|
15388
|
+
async function* follow(indexName, options) {
|
|
15389
|
+
guard();
|
|
15390
|
+
await admitChanges(indexName);
|
|
15391
|
+
yield* followSourceChanges({ source: changesSource(), guard }, indexName, options);
|
|
15392
|
+
}
|
|
14668
15393
|
return {
|
|
14669
15394
|
view,
|
|
14670
|
-
|
|
14671
|
-
|
|
15395
|
+
operations: (track) => ({
|
|
15396
|
+
updateSync: (indexName, setting) => track(indexName, () => updateClusterSync(settings, indexName, setting)),
|
|
15397
|
+
getSync: (indexName) => track(indexName, () => readClusterSync(settings, indexName)),
|
|
15398
|
+
updateChanges: (indexName, setting) => track(indexName, () => updateClusterChanges(settings, indexName, setting)),
|
|
15399
|
+
getChanges: (indexName) => track(indexName, () => readClusterChanges(settings, indexName)),
|
|
15400
|
+
followChanges: (indexName, options) => follow(indexName, options),
|
|
15401
|
+
exportDocuments: (indexName) => track(indexName, () => exportClusterDocuments({ view, store, catalog, spools }, indexName))
|
|
15402
|
+
}),
|
|
14672
15403
|
serving() {
|
|
14673
15404
|
if (serving !== null) return serving;
|
|
14674
|
-
const
|
|
14675
|
-
nodeId: deps.nodeId,
|
|
14676
|
-
writeDeps: deps.writeDeps,
|
|
14677
|
-
config,
|
|
14678
|
-
coordinator,
|
|
14679
|
-
engine,
|
|
14680
|
-
view,
|
|
14681
|
-
refreshAllocation: deps.refreshAllocation,
|
|
14682
|
-
snapshotBytes: (indexName) => snapshots?.snapshotBytes(indexName) ?? Promise.resolve(0)
|
|
14683
|
-
});
|
|
14684
|
-
const built = createClusterSnapshots({
|
|
14685
|
-
coordinator,
|
|
14686
|
-
settings,
|
|
14687
|
-
store,
|
|
14688
|
-
requests,
|
|
14689
|
-
spoolDirectory: join(deps.spoolDirectory, "device-sync"),
|
|
14690
|
-
bounds: (indexName, partitionId) => feed.bounds(indexName, partitionId),
|
|
14691
|
-
onError
|
|
14692
|
-
});
|
|
15405
|
+
const built = createClusterSnapshots({ settings, store, catalog, spools });
|
|
14693
15406
|
snapshots = built;
|
|
14694
15407
|
serving = {
|
|
14695
15408
|
snapshot: (indexName) => built.snapshot(indexName),
|
|
14696
|
-
feed,
|
|
15409
|
+
feed: feedSource(),
|
|
14697
15410
|
async admitFeed(indexName) {
|
|
14698
15411
|
const olderNodes = registeredNodesBelowDeviceSync(await coordinator.listNodes());
|
|
14699
15412
|
if (olderNodes.length > 0) throw syncNotReady("The cluster has", olderNodes, indexName);
|
|
@@ -14702,10 +15415,12 @@ function createClusterDeviceSync(deps) {
|
|
|
14702
15415
|
};
|
|
14703
15416
|
return serving;
|
|
14704
15417
|
},
|
|
15418
|
+
changesServing: () => ({ source: changesSource(), admitFeed: admitChanges }),
|
|
14705
15419
|
start: () => view.start(),
|
|
14706
15420
|
async stop() {
|
|
14707
15421
|
view.stop();
|
|
14708
|
-
await
|
|
15422
|
+
await catalog.close();
|
|
15423
|
+
await spools.close();
|
|
14709
15424
|
}
|
|
14710
15425
|
};
|
|
14711
15426
|
}
|
|
@@ -14734,10 +15449,6 @@ function createAllocationErrorForwarder(forwardOnError) {
|
|
|
14734
15449
|
}
|
|
14735
15450
|
|
|
14736
15451
|
// src/distribution/replication/log.ts
|
|
14737
|
-
function estimateEntrySize(entry) {
|
|
14738
|
-
return REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
14739
|
-
}
|
|
14740
|
-
var computeChecksum = computeEntryChecksum;
|
|
14741
15452
|
function ownedDocument(document) {
|
|
14742
15453
|
if (document === null || document.byteOffset === 0 && document.byteLength === document.buffer.byteLength) {
|
|
14743
15454
|
return document;
|
|
@@ -14782,7 +15493,7 @@ function createReplicationLog(partitionId, config) {
|
|
|
14782
15493
|
}
|
|
14783
15494
|
function evictOldEntries() {
|
|
14784
15495
|
while (liveCount() > 0 && totalSizeBytes > retentionBytes) {
|
|
14785
|
-
totalSizeBytes -=
|
|
15496
|
+
totalSizeBytes -= entryByteEstimate(entries[headIndex]);
|
|
14786
15497
|
entries[headIndex] = null;
|
|
14787
15498
|
headIndex += 1;
|
|
14788
15499
|
}
|
|
@@ -14794,7 +15505,7 @@ function createReplicationLog(partitionId, config) {
|
|
|
14794
15505
|
}
|
|
14795
15506
|
}
|
|
14796
15507
|
function assertEntryFitsRetention(entry) {
|
|
14797
|
-
const entrySize =
|
|
15508
|
+
const entrySize = entryByteEstimate(entry);
|
|
14798
15509
|
if (entrySize > retentionBytes) {
|
|
14799
15510
|
throw new NarsilError(
|
|
14800
15511
|
ErrorCodes.REPLICATION_LOG_FULL,
|
|
@@ -14805,12 +15516,13 @@ function createReplicationLog(partitionId, config) {
|
|
|
14805
15516
|
}
|
|
14806
15517
|
function storeEntry(appended) {
|
|
14807
15518
|
const entry = { ...appended, document: ownedDocument(appended.document) };
|
|
15519
|
+
if (appended.previousDocument !== void 0) entry.previousDocument = ownedDocument(appended.previousDocument);
|
|
14808
15520
|
entries.push(entry);
|
|
14809
15521
|
if (entry.seqNo >= localLogEnd) {
|
|
14810
15522
|
localLogEnd = entry.seqNo;
|
|
14811
15523
|
localLogEndPrimaryTerm = entry.primaryTerm;
|
|
14812
15524
|
}
|
|
14813
|
-
totalSizeBytes +=
|
|
15525
|
+
totalSizeBytes += entryByteEstimate(entry);
|
|
14814
15526
|
evictOldEntries();
|
|
14815
15527
|
return { ...entry };
|
|
14816
15528
|
}
|
|
@@ -14823,16 +15535,7 @@ function createReplicationLog(partitionId, config) {
|
|
|
14823
15535
|
return void 0;
|
|
14824
15536
|
}
|
|
14825
15537
|
function verifyEntryChecksum(entry) {
|
|
14826
|
-
|
|
14827
|
-
seqNo: entry.seqNo,
|
|
14828
|
-
primaryTerm: entry.primaryTerm,
|
|
14829
|
-
operation: entry.operation,
|
|
14830
|
-
partitionId: entry.partitionId,
|
|
14831
|
-
indexName: entry.indexName,
|
|
14832
|
-
documentId: entry.documentId,
|
|
14833
|
-
document: entry.document
|
|
14834
|
-
});
|
|
14835
|
-
return expected === entry.checksum;
|
|
15538
|
+
return computeEntryChecksum(entry) === entry.checksum;
|
|
14836
15539
|
}
|
|
14837
15540
|
return {
|
|
14838
15541
|
assertFits: assertEntryFitsRetention,
|
|
@@ -14840,18 +15543,18 @@ function createReplicationLog(partitionId, config) {
|
|
|
14840
15543
|
assertEntryFitsRetention(partial);
|
|
14841
15544
|
const seqNo = nextSeqNo;
|
|
14842
15545
|
nextSeqNo += 1;
|
|
14843
|
-
|
|
14844
|
-
|
|
14845
|
-
|
|
14846
|
-
|
|
14847
|
-
|
|
14848
|
-
|
|
14849
|
-
|
|
14850
|
-
|
|
14851
|
-
|
|
14852
|
-
|
|
14853
|
-
|
|
14854
|
-
|
|
15546
|
+
return storeEntry(
|
|
15547
|
+
buildEntry({
|
|
15548
|
+
seqNo,
|
|
15549
|
+
primaryTerm: partial.primaryTerm,
|
|
15550
|
+
operation: partial.operation,
|
|
15551
|
+
partitionId,
|
|
15552
|
+
indexName: partial.indexName,
|
|
15553
|
+
documentId: partial.documentId,
|
|
15554
|
+
document: partial.document,
|
|
15555
|
+
previousDocument: partial.previousDocument
|
|
15556
|
+
})
|
|
15557
|
+
);
|
|
14855
15558
|
},
|
|
14856
15559
|
appendCommitted(entry) {
|
|
14857
15560
|
if (!Number.isSafeInteger(entry.seqNo) || entry.seqNo < 1) {
|
|
@@ -15009,6 +15712,138 @@ function forgetDroppedCopy(state, indexName) {
|
|
|
15009
15712
|
forgetCatchUpIndex(state.catchUp(), indexName);
|
|
15010
15713
|
}
|
|
15011
15714
|
|
|
15715
|
+
// src/engine/memory/table-model.ts
|
|
15716
|
+
function capacityFor(entries) {
|
|
15717
|
+
if (entries <= MAP_TABLE_MIN_CAPACITY) return MAP_TABLE_MIN_CAPACITY;
|
|
15718
|
+
return 2 ** Math.ceil(Math.log2(entries));
|
|
15719
|
+
}
|
|
15720
|
+
function largestCapacityFor(entries) {
|
|
15721
|
+
return Math.max(capacityFor(entries), 2 ** Math.floor(Math.log2(MAP_TABLE_SHRINK_DIVISOR * (entries + 1))));
|
|
15722
|
+
}
|
|
15723
|
+
function resetModel(model, shape, deleted) {
|
|
15724
|
+
model.deleted = deleted;
|
|
15725
|
+
model.resetEntries = shape.largest;
|
|
15726
|
+
model.resetDeleted = deleted;
|
|
15727
|
+
model.resetSmallestCapacity = capacityFor(shape.smallest);
|
|
15728
|
+
model.resetLargestCapacity = largestCapacityFor(shape.largest);
|
|
15729
|
+
model.addedAtLeast = 0;
|
|
15730
|
+
model.addedAtMost = 0;
|
|
15731
|
+
model.deletedSinceRehashBecamePossible = null;
|
|
15732
|
+
}
|
|
15733
|
+
function createTableModel(entries, churn, deleted, shape) {
|
|
15734
|
+
const model = {
|
|
15735
|
+
entries,
|
|
15736
|
+
churn,
|
|
15737
|
+
deleted,
|
|
15738
|
+
resetEntries: 0,
|
|
15739
|
+
resetDeleted: 0,
|
|
15740
|
+
resetSmallestCapacity: 0,
|
|
15741
|
+
resetLargestCapacity: 0,
|
|
15742
|
+
addedAtLeast: 0,
|
|
15743
|
+
addedAtMost: 0,
|
|
15744
|
+
deletedSinceRehashBecamePossible: null
|
|
15745
|
+
};
|
|
15746
|
+
resetModel(model, shape, deleted);
|
|
15747
|
+
return model;
|
|
15748
|
+
}
|
|
15749
|
+
function observeTable(model, entries, churn, shape) {
|
|
15750
|
+
const churned = Math.max(0, churn - model.churn);
|
|
15751
|
+
const change = entries - model.entries;
|
|
15752
|
+
const deleted = Math.max(churned, -change);
|
|
15753
|
+
const lowest = Math.max(0, Math.min(model.entries, entries) - deleted);
|
|
15754
|
+
model.entries = entries;
|
|
15755
|
+
model.churn = churn;
|
|
15756
|
+
model.deleted += deleted;
|
|
15757
|
+
model.addedAtLeast += Math.max(0, change);
|
|
15758
|
+
model.addedAtMost += Math.max(0, change) + churned;
|
|
15759
|
+
if (entries === 0) {
|
|
15760
|
+
resetModel(model, shape, Math.min(model.deleted, MAP_TABLE_MIN_CAPACITY));
|
|
15761
|
+
return;
|
|
15762
|
+
}
|
|
15763
|
+
if (shape.shards !== 1) return;
|
|
15764
|
+
if (model.deletedSinceRehashBecamePossible !== null) {
|
|
15765
|
+
model.deletedSinceRehashBecamePossible += deleted;
|
|
15766
|
+
} else if (model.resetEntries + model.resetDeleted + model.addedAtMost >= model.resetSmallestCapacity || lowest < model.resetLargestCapacity / MAP_TABLE_SHRINK_DIVISOR) {
|
|
15767
|
+
model.deletedSinceRehashBecamePossible = deleted;
|
|
15768
|
+
}
|
|
15769
|
+
if (model.addedAtLeast > model.resetLargestCapacity - model.resetEntries || entries < model.resetSmallestCapacity / MAP_TABLE_SHRINK_DIVISOR) {
|
|
15770
|
+
resetModel(model, shape, model.deletedSinceRehashBecamePossible ?? model.deleted);
|
|
15771
|
+
}
|
|
15772
|
+
}
|
|
15773
|
+
function tableGrowthBytes(model, shape, added) {
|
|
15774
|
+
if (added <= 0) return 0;
|
|
15775
|
+
const largestCapacity = largestCapacityFor(shape.largest);
|
|
15776
|
+
let grownBytes = 0;
|
|
15777
|
+
for (let capacity = capacityFor(shape.smallest); capacity <= largestCapacity; capacity *= 2) {
|
|
15778
|
+
const lastOccupied = shape.largest + added - 1;
|
|
15779
|
+
const deletedSlots = Math.max(0, Math.min(model.deleted, capacity - shape.largest));
|
|
15780
|
+
if (lastOccupied + deletedSlots < capacity) continue;
|
|
15781
|
+
const entriesAfter = shape.largest + added;
|
|
15782
|
+
const rebuildsAtSize = deletedSlots >= capacity / 2;
|
|
15783
|
+
const doubles = lastOccupied + Math.min(deletedSlots, capacity / 2 - 1) >= capacity;
|
|
15784
|
+
const rebuiltBytes = rebuildsAtSize ? allocatedTableBytes(capacity, entriesAfter) : 0;
|
|
15785
|
+
const doubledBytes = doubles ? allocatedTableBytes(capacity * 2, entriesAfter) : 0;
|
|
15786
|
+
grownBytes = Math.max(grownBytes, rebuiltBytes, doubledBytes);
|
|
15787
|
+
}
|
|
15788
|
+
return shape.shards * grownBytes;
|
|
15789
|
+
}
|
|
15790
|
+
function allocatedTableBytes(firstCapacity, entriesAfter) {
|
|
15791
|
+
let capacity = firstCapacity;
|
|
15792
|
+
let bytes = MAP_TABLE_HEADER_BYTES + MAP_TABLE_SLOT_BYTES * capacity;
|
|
15793
|
+
while (entriesAfter > capacity) {
|
|
15794
|
+
capacity *= 2;
|
|
15795
|
+
bytes += MAP_TABLE_HEADER_BYTES + MAP_TABLE_SLOT_BYTES * capacity;
|
|
15796
|
+
}
|
|
15797
|
+
return bytes;
|
|
15798
|
+
}
|
|
15799
|
+
|
|
15800
|
+
// src/engine/memory/document-tables.ts
|
|
15801
|
+
function plainShapeOf(entries) {
|
|
15802
|
+
return { shards: 1, smallest: entries, largest: entries };
|
|
15803
|
+
}
|
|
15804
|
+
function createDocumentTableGrowth(core, heldTableSizes) {
|
|
15805
|
+
const modelsByIndex = /* @__PURE__ */ new Map();
|
|
15806
|
+
return (indexName, documents, pendingDocuments) => {
|
|
15807
|
+
const manager = core.executor.getManager(indexName);
|
|
15808
|
+
if (manager === void 0 || manager === null) {
|
|
15809
|
+
modelsByIndex.delete(indexName);
|
|
15810
|
+
return 0;
|
|
15811
|
+
}
|
|
15812
|
+
const previous = modelsByIndex.get(indexName);
|
|
15813
|
+
const models = /* @__PURE__ */ new Map();
|
|
15814
|
+
modelsByIndex.set(indexName, models);
|
|
15815
|
+
const churn = documentChurnOf(indexName);
|
|
15816
|
+
let bytes = 0;
|
|
15817
|
+
const measure = (key, entries, shape, followsRemoves) => {
|
|
15818
|
+
let model = previous?.get(key);
|
|
15819
|
+
if (model === void 0) {
|
|
15820
|
+
model = createTableModel(entries, churn, followsRemoves ? churn : Number.POSITIVE_INFINITY, shape);
|
|
15821
|
+
} else {
|
|
15822
|
+
observeTable(model, entries, churn, shape);
|
|
15823
|
+
}
|
|
15824
|
+
models.set(key, model);
|
|
15825
|
+
if (documents <= 0) return;
|
|
15826
|
+
bytes += tableGrowthBytes(model, shape, pendingDocuments + documents) - tableGrowthBytes(model, shape, pendingDocuments);
|
|
15827
|
+
};
|
|
15828
|
+
const located = manager.countDocuments();
|
|
15829
|
+
measure("located", located, shardEntryRangeOf(located), true);
|
|
15830
|
+
for (const partition of manager.getAllPartitions()) {
|
|
15831
|
+
const entries = (isCompositePartition(partition) ? partition.live : partition).count();
|
|
15832
|
+
measure(`partition:${partition.partitionId}`, entries, shardEntryRangeOf(entries), true);
|
|
15833
|
+
}
|
|
15834
|
+
for (const [field, vectorIndex] of manager.getVectorIndexes()) {
|
|
15835
|
+
vectorIndex.visitDocumentTables(
|
|
15836
|
+
(table, entries, followsRemoves) => measure(`vector:${field}:${table}`, entries, shardEntryRangeOf(entries), followsRemoves)
|
|
15837
|
+
);
|
|
15838
|
+
}
|
|
15839
|
+
heldTableSizes?.(
|
|
15840
|
+
indexName,
|
|
15841
|
+
(partitionId, entries) => measure(`held:${partitionId}`, entries, plainShapeOf(entries), true)
|
|
15842
|
+
);
|
|
15843
|
+
return bytes;
|
|
15844
|
+
};
|
|
15845
|
+
}
|
|
15846
|
+
|
|
15012
15847
|
// src/engine/memory/write-analysis.ts
|
|
15013
15848
|
function writeAnalysisOf(core, indexName) {
|
|
15014
15849
|
const entry = core.indexRegistry.get(indexName);
|
|
@@ -15031,27 +15866,38 @@ function vectorGrowthBytesOf(core, indexName, documents) {
|
|
|
15031
15866
|
}
|
|
15032
15867
|
|
|
15033
15868
|
// src/engine/sync/adopt.ts
|
|
15034
|
-
function
|
|
15869
|
+
function sameSync(held, next) {
|
|
15035
15870
|
if (held === null || next === null) return held === next;
|
|
15036
15871
|
return held.epoch === next.epoch && held.active === next.active && held.historyMaxAgeMs === next.historyMaxAgeMs && held.historyMaxBytes === next.historyMaxBytes;
|
|
15037
15872
|
}
|
|
15038
|
-
|
|
15873
|
+
function sameChanges(held, next) {
|
|
15874
|
+
if (held === null || next === null) return held === next;
|
|
15875
|
+
return held.epoch === next.epoch && held.previousDocument === next.previousDocument && held.historyMaxAgeMs === next.historyMaxAgeMs && held.historyMaxBytes === next.historyMaxBytes;
|
|
15876
|
+
}
|
|
15877
|
+
function epochOf(states) {
|
|
15878
|
+
return states.sync?.epoch ?? states.changes?.epoch ?? null;
|
|
15879
|
+
}
|
|
15880
|
+
async function adoptIndexEpochStates(core, indexName, next) {
|
|
15039
15881
|
return oneSyncChangeAtATime(core.indexRegistry, indexName, async () => {
|
|
15040
15882
|
const entry = core.indexRegistry.get(indexName);
|
|
15041
15883
|
if (entry === void 0) return false;
|
|
15042
|
-
const held = entry.sync;
|
|
15043
|
-
if (
|
|
15884
|
+
const held = { sync: entry.sync, changes: entry.changes };
|
|
15885
|
+
if (sameSync(held.sync, next.sync) && sameChanges(held.changes, next.changes)) return false;
|
|
15044
15886
|
const storage = core.durability?.manager.syncStorage?.();
|
|
15045
|
-
const
|
|
15046
|
-
|
|
15047
|
-
|
|
15887
|
+
const heldEpoch = epochOf(held);
|
|
15888
|
+
const nextEpoch = epochOf(next);
|
|
15889
|
+
if (heldEpoch !== null && nextEpoch !== null && heldEpoch !== nextEpoch) await storage?.release(indexName);
|
|
15890
|
+
entry.sync = next.sync;
|
|
15891
|
+
entry.changes = next.changes;
|
|
15048
15892
|
try {
|
|
15049
15893
|
await core.durability?.manager.persistMetadata(indexName);
|
|
15050
15894
|
} catch (error) {
|
|
15051
|
-
entry.sync = held;
|
|
15895
|
+
entry.sync = held.sync;
|
|
15896
|
+
entry.changes = held.changes;
|
|
15052
15897
|
throw error;
|
|
15053
15898
|
}
|
|
15054
|
-
if (next === null) await storage?.release(indexName);
|
|
15899
|
+
if (next.sync === null && held.sync !== null) await storage?.release(indexName, next.changes !== null);
|
|
15900
|
+
else if (nextEpoch === null && heldEpoch !== null) await storage?.release(indexName);
|
|
15055
15901
|
return true;
|
|
15056
15902
|
});
|
|
15057
15903
|
}
|
|
@@ -15127,7 +15973,7 @@ async function openOnce(core, indexName, partitionId) {
|
|
|
15127
15973
|
if (directory === void 0) throw durabilityRequired(indexName, partitionId);
|
|
15128
15974
|
if ((checkpoint.manifest.analysisGeneration ?? 0) !== entry.analysisGeneration) return null;
|
|
15129
15975
|
const config = await encodeIndexConfiguration(entry);
|
|
15130
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
15976
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-ES4Z5SM4.mjs');
|
|
15131
15977
|
const archive = await openPagedPartitionArchive(checkpoint, directory, indexName, partitionId, manager);
|
|
15132
15978
|
handedOver = true;
|
|
15133
15979
|
try {
|
|
@@ -15289,8 +16135,10 @@ function createClusterWriteRecords() {
|
|
|
15289
16135
|
}
|
|
15290
16136
|
function applied(entry) {
|
|
15291
16137
|
const held = partition(entry.indexName, entry.partitionId);
|
|
16138
|
+
const heldBefore = held.table.count;
|
|
15292
16139
|
if (entry.operation === "INDEX") held.table.set(entry.documentId, entry.seqNo, entry.primaryTerm);
|
|
15293
16140
|
else held.table.remove(entry.documentId);
|
|
16141
|
+
if (entry.operation !== "INDEX" || held.table.count === heldBefore) noteDocumentChurn(entry.indexName, 1);
|
|
15294
16142
|
held.history.noteEntry(entry.seqNo, entry.primaryTerm);
|
|
15295
16143
|
}
|
|
15296
16144
|
function recordOfDocument(indexName, partitionId, documentId) {
|
|
@@ -15326,6 +16174,11 @@ function createClusterWriteRecords() {
|
|
|
15326
16174
|
applied,
|
|
15327
16175
|
replayed: applied,
|
|
15328
16176
|
partition,
|
|
16177
|
+
visitTableSizes(indexName, visit) {
|
|
16178
|
+
const partitions = indexes.get(indexName);
|
|
16179
|
+
if (partitions === void 0) return;
|
|
16180
|
+
for (const [partitionId, held] of partitions) visit(partitionId, held.table.count);
|
|
16181
|
+
},
|
|
15329
16182
|
adoptPartition(indexName, partitionId, records) {
|
|
15330
16183
|
Object.assign(partition(indexName, partitionId), records);
|
|
15331
16184
|
},
|
|
@@ -15835,7 +16688,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
|
|
|
15835
16688
|
);
|
|
15836
16689
|
}
|
|
15837
16690
|
try {
|
|
15838
|
-
const { openPagedPartitionArchive } = await import('./paged-partition-archive-
|
|
16691
|
+
const { openPagedPartitionArchive } = await import('./paged-partition-archive-ES4Z5SM4.mjs');
|
|
15839
16692
|
const archive = await openPagedPartitionArchive(checkpoint2, directory, indexName, partitionId, manager);
|
|
15840
16693
|
return {
|
|
15841
16694
|
stream: archiveStream({ ...archive, config, sourceIndexName: indexName }),
|
|
@@ -16050,7 +16903,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
|
|
|
16050
16903
|
if (directory !== null) {
|
|
16051
16904
|
const durable = core.durability?.manager;
|
|
16052
16905
|
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
16053
|
-
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-
|
|
16906
|
+
const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-ES4Z5SM4.mjs');
|
|
16054
16907
|
await removeReceivedFilesNothingReads(
|
|
16055
16908
|
directory,
|
|
16056
16909
|
core.requireManager(indexName),
|
|
@@ -16197,8 +17050,7 @@ async function createClusterLocalEngine(config, hooks) {
|
|
|
16197
17050
|
stampIndexUuid: (indexName, indexUuid) => stampIndexUuid(core, indexName, indexUuid),
|
|
16198
17051
|
applyStoredSynonyms: (indexName, load) => synonymModule().applyLoaded(core, indexName, load),
|
|
16199
17052
|
applyStoredBm25: (indexName, load) => applyStoredIndexBm25(core, indexName, load),
|
|
16200
|
-
|
|
16201
|
-
adoptSyncState: (indexName, state) => adoptIndexSyncState(core, indexName, state),
|
|
17053
|
+
adoptEpochStates: (indexName, states) => adoptIndexEpochStates(core, indexName, states),
|
|
16202
17054
|
syncStorage: () => core.durability?.manager.syncStorage?.() ?? null,
|
|
16203
17055
|
historyStorage: () => core.durability?.manager.historyStorage?.() ?? null,
|
|
16204
17056
|
emitEvent: (name, payload) => emitEngineEvent(core.eventHandlers, name, payload),
|
|
@@ -16221,6 +17073,10 @@ async function createClusterLocalEngine(config, hooks) {
|
|
|
16221
17073
|
},
|
|
16222
17074
|
writeAnalysisOf: (indexName) => writeAnalysisOf(core, indexName),
|
|
16223
17075
|
vectorGrowthBytesOf: (indexName, documents) => vectorGrowthBytesOf(core, indexName, documents),
|
|
17076
|
+
documentTableGrowthBytesOf: createDocumentTableGrowth(
|
|
17077
|
+
core,
|
|
17078
|
+
(indexName, visit) => writeRecords.visitTableSizes(indexName, visit)
|
|
17079
|
+
),
|
|
16224
17080
|
restartLocalLog: (indexName, partitionId, position) => restartLocalLog(core, indexName, partitionId, position),
|
|
16225
17081
|
isolateCopyFailures: (onFailed) => core.durability?.manager.isolatePartitionFailures?.(onFailed),
|
|
16226
17082
|
emptyPartitionCopy: (indexName, partitionId) => emptyPartitionCopy(core, indexName, partitionId, writeRecords),
|
|
@@ -16310,7 +17166,7 @@ function createIndexSettingsResultMessage(result, sourceNodeId, requestId) {
|
|
|
16310
17166
|
}
|
|
16311
17167
|
|
|
16312
17168
|
// src/distribution/cluster-node/synonym-sets.ts
|
|
16313
|
-
function
|
|
17169
|
+
function indexNotFound3(indexName) {
|
|
16314
17170
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
16315
17171
|
indexName
|
|
16316
17172
|
});
|
|
@@ -16318,7 +17174,7 @@ function indexNotFound2(indexName) {
|
|
|
16318
17174
|
async function readClusterSynonyms(coordinator, indexName) {
|
|
16319
17175
|
validateIndexName(indexName);
|
|
16320
17176
|
const state = await readIndexState(coordinator, await servingNameOf(coordinator, indexName));
|
|
16321
|
-
if (state === null) throw
|
|
17177
|
+
if (state === null) throw indexNotFound3(indexName);
|
|
16322
17178
|
return state.synonyms;
|
|
16323
17179
|
}
|
|
16324
17180
|
async function requireSettledPartitionCount(coordinator, indexName, servingName) {
|
|
@@ -16352,7 +17208,7 @@ async function swapStoredSet(coordinator, indexName, first, ref) {
|
|
|
16352
17208
|
const { indexUuid } = first.record;
|
|
16353
17209
|
let current = first;
|
|
16354
17210
|
for (let attempt = 1; ; attempt++) {
|
|
16355
|
-
if (current === null || current.record.indexUuid !== indexUuid) throw
|
|
17211
|
+
if (current === null || current.record.indexUuid !== indexUuid) throw indexNotFound3(indexName);
|
|
16356
17212
|
const replaced = current.settings?.synonyms ?? null;
|
|
16357
17213
|
const next = metadataWithSynonymSet(current, ref);
|
|
16358
17214
|
if (await coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) return replaced;
|
|
@@ -16404,7 +17260,7 @@ async function replaceClusterSynonyms(deps, indexName, synonyms) {
|
|
|
16404
17260
|
const servingName = await servingNameOf(coordinator, indexName);
|
|
16405
17261
|
await requireSettledPartitionCount(coordinator, indexName, servingName);
|
|
16406
17262
|
const first = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
16407
|
-
if (first === null) throw
|
|
17263
|
+
if (first === null) throw indexNotFound3(indexName);
|
|
16408
17264
|
const encoded2 = preparedSet(first, servingName, synonyms);
|
|
16409
17265
|
const { indexUuid } = first.record;
|
|
16410
17266
|
const ref = encoded2 === null ? null : await storeSynonymSet(coordinator, indexUuid, encoded2);
|
|
@@ -16532,6 +17388,7 @@ function bindNodeSources(node, sources) {
|
|
|
16532
17388
|
});
|
|
16533
17389
|
bindRerankWindowSource(node, () => rerankWindowCapOf(sources.engine));
|
|
16534
17390
|
bindSyncServing(node, sources.syncServing);
|
|
17391
|
+
bindChangesServing(node, sources.changesServing);
|
|
16535
17392
|
}
|
|
16536
17393
|
|
|
16537
17394
|
// src/distribution/cluster-node/node-identity.ts
|
|
@@ -16824,6 +17681,8 @@ function createAllocationWatcherState() {
|
|
|
16824
17681
|
pendingPartitions: /* @__PURE__ */ new Set(),
|
|
16825
17682
|
pendingTables: /* @__PURE__ */ new Map(),
|
|
16826
17683
|
restartWaiters: /* @__PURE__ */ new Map(),
|
|
17684
|
+
unfedReplicaTimer: null,
|
|
17685
|
+
unfedResyncs: /* @__PURE__ */ new Set(),
|
|
16827
17686
|
retainedPartitions: /* @__PURE__ */ new Set(),
|
|
16828
17687
|
indexesSeededFromRecord: /* @__PURE__ */ new Set(),
|
|
16829
17688
|
indexUuids: /* @__PURE__ */ new Map(),
|
|
@@ -16840,6 +17699,11 @@ function stopAllocationWatcher(state) {
|
|
|
16840
17699
|
clearTimeout(state.debounceTimer);
|
|
16841
17700
|
state.debounceTimer = null;
|
|
16842
17701
|
}
|
|
17702
|
+
if (state.unfedReplicaTimer !== null) {
|
|
17703
|
+
clearInterval(state.unfedReplicaTimer);
|
|
17704
|
+
state.unfedReplicaTimer = null;
|
|
17705
|
+
}
|
|
17706
|
+
state.unfedResyncs.clear();
|
|
16843
17707
|
for (const [timer, resolve] of state.restartWaiters) {
|
|
16844
17708
|
clearTimeout(timer);
|
|
16845
17709
|
resolve();
|
|
@@ -16972,7 +17836,8 @@ function processAllocationChange(state, config, table) {
|
|
|
16972
17836
|
partitionId,
|
|
16973
17837
|
isPrimary,
|
|
16974
17838
|
primaryNodeId: assignment.primary,
|
|
16975
|
-
primaryTerm: assignment.primaryTerm
|
|
17839
|
+
primaryTerm: assignment.primaryTerm,
|
|
17840
|
+
inSync: assignment.inSyncSet.includes(nodeId)
|
|
16976
17841
|
});
|
|
16977
17842
|
if (servesPartition(assignment, nodeId)) {
|
|
16978
17843
|
state.pendingPartitions.delete(key);
|
|
@@ -17159,6 +18024,30 @@ function removeDroppedPartitions(state, config, table, currentKeys) {
|
|
|
17159
18024
|
}
|
|
17160
18025
|
}
|
|
17161
18026
|
|
|
18027
|
+
// src/distribution/cluster/node-lifecycle/allocation-watcher/unfed-replicas.ts
|
|
18028
|
+
function startUnfedReplicaChecks(state, config) {
|
|
18029
|
+
const replicaUnfedForMs = config.replicaUnfedForMs;
|
|
18030
|
+
if (replicaUnfedForMs === void 0 || state.unfedReplicaTimer !== null) return;
|
|
18031
|
+
const timer = setInterval(
|
|
18032
|
+
() => resyncUnfedReplicas(state, config, replicaUnfedForMs),
|
|
18033
|
+
UNFED_REPLICA_CHECK_INTERVAL_MS
|
|
18034
|
+
);
|
|
18035
|
+
timer.unref?.();
|
|
18036
|
+
state.unfedReplicaTimer = timer;
|
|
18037
|
+
}
|
|
18038
|
+
function resyncUnfedReplicas(state, config, replicaUnfedForMs) {
|
|
18039
|
+
if (state.stopped) return;
|
|
18040
|
+
for (const [key, tracked] of state.trackedPartitions) {
|
|
18041
|
+
if (tracked.isPrimary || tracked.inSync || tracked.primaryNodeId === null) continue;
|
|
18042
|
+
if (state.activeBootstraps.has(key) || state.unfedResyncs.has(key)) continue;
|
|
18043
|
+
if (replicaUnfedForMs(tracked.indexName, tracked.partitionId) < UNFED_REPLICA_RESYNC_MS) continue;
|
|
18044
|
+
state.unfedResyncs.add(key);
|
|
18045
|
+
void restartBootstrapWhileOutOfSync(state, config, tracked.indexName, tracked.partitionId).finally(() => {
|
|
18046
|
+
state.unfedResyncs.delete(key);
|
|
18047
|
+
});
|
|
18048
|
+
}
|
|
18049
|
+
}
|
|
18050
|
+
|
|
17162
18051
|
// src/distribution/cluster/node-lifecycle/allocation-watcher/index.ts
|
|
17163
18052
|
async function startAllocationWatcher(state, config) {
|
|
17164
18053
|
const unwatchAllocation = await config.coordinator.watchAllocation((event) => {
|
|
@@ -17168,6 +18057,7 @@ async function startAllocationWatcher(state, config) {
|
|
|
17168
18057
|
scheduleProcessAllocation(state, config, event.table);
|
|
17169
18058
|
});
|
|
17170
18059
|
state.unwatchAllocation = unwatchAllocation;
|
|
18060
|
+
startUnfedReplicaChecks(state, config);
|
|
17171
18061
|
}
|
|
17172
18062
|
function scheduleProcessAllocation(state, config, table) {
|
|
17173
18063
|
const pending = state.pendingTables.get(table.indexName);
|
|
@@ -17566,7 +18456,14 @@ function createNodeLifecycle(deps) {
|
|
|
17566
18456
|
bootstrapMaxRetries: DEFAULT_NODE_LIFECYCLE_CONFIG.bootstrapMaxRetries,
|
|
17567
18457
|
allocationDebounceMs: DEFAULT_NODE_LIFECYCLE_CONFIG.allocationDebounceMs,
|
|
17568
18458
|
nodeHeartbeatIntervalMs: DEFAULT_NODE_LIFECYCLE_CONFIG.nodeHeartbeatIntervalMs,
|
|
17569
|
-
onBootstrapPartition: async (indexName, partitionId, primaryNodeId) =>
|
|
18459
|
+
onBootstrapPartition: async (indexName, partitionId, primaryNodeId) => {
|
|
18460
|
+
const feedClock = deps.writeDeps.replicaFeedClock;
|
|
18461
|
+
feedClock?.noteFed(indexName, partitionId);
|
|
18462
|
+
const synced = await deps.copyFailures.awaitReset(indexName, partitionId) && await deps.bootstrapPartition(indexName, partitionId, primaryNodeId);
|
|
18463
|
+
if (synced) feedClock?.noteFed(indexName, partitionId);
|
|
18464
|
+
return synced;
|
|
18465
|
+
},
|
|
18466
|
+
replicaUnfedForMs: (indexName, partitionId) => deps.writeDeps.replicaFeedClock?.unfedForMs(indexName, partitionId) ?? 0,
|
|
17570
18467
|
onLeadingTerm: (indexName, partitionId, assignment) => startLeadingTerm(deps.writeDeps, indexName, partitionId, assignment),
|
|
17571
18468
|
onHoldPartition: (indexName, partitionId) => {
|
|
17572
18469
|
deps.trackHeldPartitionWrite(engine.recordHeldPartition(indexName, partitionId));
|
|
@@ -17575,6 +18472,7 @@ function createNodeLifecycle(deps) {
|
|
|
17575
18472
|
onRemovePartition: (indexName, partitionId) => {
|
|
17576
18473
|
const removedIndexUuid = engine.indexUuidOf(indexName);
|
|
17577
18474
|
forgetReshardPartition(deps.reshard, indexName, partitionId);
|
|
18475
|
+
deps.writeDeps.replicaFeedClock?.forget(indexName, partitionId);
|
|
17578
18476
|
clearBootstrapSyncIndex(deps.bootstrapSyncState, indexName, partitionId);
|
|
17579
18477
|
deps.replicationLogs.delete(replicationLogKey(indexName, partitionId));
|
|
17580
18478
|
deps.trackHeldPartitionWrite(engine.forgetHeldPartition(indexName, partitionId));
|
|
@@ -17588,7 +18486,7 @@ function createNodeLifecycle(deps) {
|
|
|
17588
18486
|
}
|
|
17589
18487
|
|
|
17590
18488
|
// src/distribution/cluster-node/analysis-change/request.ts
|
|
17591
|
-
function
|
|
18489
|
+
function indexNotFound4(indexName) {
|
|
17592
18490
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
17593
18491
|
indexName
|
|
17594
18492
|
});
|
|
@@ -17609,10 +18507,10 @@ async function requireTargetAcceptable(coordinator, indexName, target) {
|
|
|
17609
18507
|
}
|
|
17610
18508
|
async function writePending(coordinator, indexName, target) {
|
|
17611
18509
|
const schema = await coordinator.getSchema(indexName);
|
|
17612
|
-
if (schema === null) throw
|
|
18510
|
+
if (schema === null) throw indexNotFound4(indexName);
|
|
17613
18511
|
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
17614
18512
|
const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
17615
|
-
if (snapshot === null) throw
|
|
18513
|
+
if (snapshot === null) throw indexNotFound4(indexName);
|
|
17616
18514
|
const running = snapshot.settings?.pendingAnalysis;
|
|
17617
18515
|
if (running !== void 0 && asksForPending(running, target)) return running;
|
|
17618
18516
|
const current = indexConfigFromSettings(schema, snapshot.settings, indexName);
|
|
@@ -17658,7 +18556,7 @@ async function changeClusterAnalysis(request) {
|
|
|
17658
18556
|
validateIndexName(servingName);
|
|
17659
18557
|
const schema = await coordinator.getSchema(servingName);
|
|
17660
18558
|
const metadata = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
17661
|
-
if (schema === null || metadata === null) throw
|
|
18559
|
+
if (schema === null || metadata === null) throw indexNotFound4(publicName);
|
|
17662
18560
|
const target = resolveAnalysisTarget(
|
|
17663
18561
|
publicName,
|
|
17664
18562
|
indexConfigFromSettings(schema, metadata.settings, servingName),
|
|
@@ -17686,7 +18584,7 @@ async function changeClusterAnalysis(request) {
|
|
|
17686
18584
|
for (; ; ) {
|
|
17687
18585
|
guard();
|
|
17688
18586
|
const snapshot = await readIndexMetadataSnapshot(coordinator, servingName);
|
|
17689
|
-
if (snapshot === null) throw
|
|
18587
|
+
if (snapshot === null) throw indexNotFound4(publicName);
|
|
17690
18588
|
if (outcomeOf(snapshot, pending, publicName) === "published") break;
|
|
17691
18589
|
await pause(ANALYSIS_CHANGE_POLL_INTERVAL_MS);
|
|
17692
18590
|
}
|
|
@@ -17695,7 +18593,7 @@ async function changeClusterAnalysis(request) {
|
|
|
17695
18593
|
}
|
|
17696
18594
|
|
|
17697
18595
|
// src/distribution/cluster-node/bm25-change.ts
|
|
17698
|
-
function
|
|
18596
|
+
function indexNotFound5(indexName) {
|
|
17699
18597
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
17700
18598
|
indexName
|
|
17701
18599
|
});
|
|
@@ -17703,14 +18601,14 @@ function indexNotFound4(indexName) {
|
|
|
17703
18601
|
async function replaceClusterBm25(coordinator, view, indexName, value) {
|
|
17704
18602
|
validateIndexName(indexName);
|
|
17705
18603
|
const schema = await coordinator.getSchema(indexName);
|
|
17706
|
-
if (schema === null) throw
|
|
18604
|
+
if (schema === null) throw indexNotFound5(indexName);
|
|
17707
18605
|
validateBM25Params(value, schema, ErrorCodes.CONFIG_INVALID);
|
|
17708
18606
|
const bm25 = storedBM25Params(value);
|
|
17709
18607
|
const olderNodes = registeredNodesBelowBm25Tuning(await coordinator.listNodes());
|
|
17710
18608
|
if (olderNodes.length > 0) throw bm25NotReady("The cluster has", olderNodes, indexName);
|
|
17711
18609
|
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
17712
18610
|
const current = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
17713
|
-
if (current === null) throw
|
|
18611
|
+
if (current === null) throw indexNotFound5(indexName);
|
|
17714
18612
|
const revision = (current.settings?.bm25Revision ?? 0) + 1;
|
|
17715
18613
|
const next = metadataWithBm25(current, bm25, revision);
|
|
17716
18614
|
if (await coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) {
|
|
@@ -19378,7 +20276,7 @@ function otherCopiesByNode(allocation, readBy) {
|
|
|
19378
20276
|
}
|
|
19379
20277
|
return byNode;
|
|
19380
20278
|
}
|
|
19381
|
-
function
|
|
20279
|
+
function answerOf2(deps, indexName, nodeId, partitionIds) {
|
|
19382
20280
|
if (nodeId === deps.nodeId) return localCountAnswer(deps.engine, indexName, partitionIds);
|
|
19383
20281
|
const message = createCountMessage({ indexName, partitionIds }, deps.nodeId);
|
|
19384
20282
|
return sendReadRequest(deps, nodeId, message, indexName, validateCountResultPayload);
|
|
@@ -19396,7 +20294,7 @@ async function countsOfEveryCopy(deps, indexName, allocation, readCopies, readBy
|
|
|
19396
20294
|
const copies = /* @__PURE__ */ new Map();
|
|
19397
20295
|
for (const [partitionId, entry] of readCopies) copies.set(partitionId, [entry]);
|
|
19398
20296
|
const requests = [...otherCopiesByNode(allocation, readBy)].map(async ([nodeId, partitionIds]) => {
|
|
19399
|
-
const answer2 = await
|
|
20297
|
+
const answer2 = await answerOf2(deps, indexName, nodeId, partitionIds);
|
|
19400
20298
|
return { partitionIds, answer: answer2 };
|
|
19401
20299
|
});
|
|
19402
20300
|
let answers;
|
|
@@ -19855,7 +20753,7 @@ async function queryCluster(deps, indexName, params, settingName) {
|
|
|
19855
20753
|
(reader) => searchCluster(deps, indexName, settingName, params, allocation, reader, false)
|
|
19856
20754
|
);
|
|
19857
20755
|
}
|
|
19858
|
-
const { rerankClusterQuery } = await import('./rerank-
|
|
20756
|
+
const { rerankClusterQuery } = await import('./rerank-ELMGHQ3T.mjs');
|
|
19859
20757
|
return withRankedReader(
|
|
19860
20758
|
deps,
|
|
19861
20759
|
indexName,
|
|
@@ -20575,7 +21473,8 @@ function createNodeMemoryReporting(options) {
|
|
|
20575
21473
|
collections: () => monitor.collections(),
|
|
20576
21474
|
refusedAfterCollection: (figures) => monitor.refusedAfterCollection(figures),
|
|
20577
21475
|
analysisOf: (indexName) => options.engine.writeAnalysisOf(indexName),
|
|
20578
|
-
vectorGrowthBytesOf: (indexName, documents) => options.engine.vectorGrowthBytesOf(indexName, documents)
|
|
21476
|
+
vectorGrowthBytesOf: (indexName, documents) => options.engine.vectorGrowthBytesOf(indexName, documents),
|
|
21477
|
+
documentTableGrowthBytesOf: (indexName, documents, pending) => options.engine.documentTableGrowthBytesOf(indexName, documents, pending)
|
|
20579
21478
|
});
|
|
20580
21479
|
const heapLimitBytes = readMemoryFigures()?.heapLimitBytes ?? Number.POSITIVE_INFINITY;
|
|
20581
21480
|
const writePressure = createWritePressure({
|
|
@@ -20625,6 +21524,36 @@ function createNodeMemoryReporting(options) {
|
|
|
20625
21524
|
};
|
|
20626
21525
|
}
|
|
20627
21526
|
|
|
21527
|
+
// src/distribution/cluster-node/replica-feed-clock.ts
|
|
21528
|
+
function feedKey(indexName, partitionId) {
|
|
21529
|
+
return `${indexName}:${partitionId}`;
|
|
21530
|
+
}
|
|
21531
|
+
function createReplicaFeedClock() {
|
|
21532
|
+
const fedAt = /* @__PURE__ */ new Map();
|
|
21533
|
+
return {
|
|
21534
|
+
noteFed(indexName, partitionId) {
|
|
21535
|
+
fedAt.set(feedKey(indexName, partitionId), Date.now());
|
|
21536
|
+
},
|
|
21537
|
+
unfedForMs(indexName, partitionId) {
|
|
21538
|
+
const key = feedKey(indexName, partitionId);
|
|
21539
|
+
const at = fedAt.get(key);
|
|
21540
|
+
if (at !== void 0) return Date.now() - at;
|
|
21541
|
+
fedAt.set(key, Date.now());
|
|
21542
|
+
return 0;
|
|
21543
|
+
},
|
|
21544
|
+
forget(indexName, partitionId) {
|
|
21545
|
+
fedAt.delete(feedKey(indexName, partitionId));
|
|
21546
|
+
}
|
|
21547
|
+
};
|
|
21548
|
+
}
|
|
21549
|
+
|
|
21550
|
+
// src/distribution/cluster-node/started-at.ts
|
|
21551
|
+
var latestStartedAtMs = 0;
|
|
21552
|
+
function nextStartedAt() {
|
|
21553
|
+
latestStartedAtMs = Math.max(Date.now(), latestStartedAtMs + 1);
|
|
21554
|
+
return new Date(latestStartedAtMs).toISOString();
|
|
21555
|
+
}
|
|
21556
|
+
|
|
20628
21557
|
// src/distribution/cluster-node/write-routing/numbering.ts
|
|
20629
21558
|
function createPrimaryNumbering(onNumbered) {
|
|
20630
21559
|
const scopes = /* @__PURE__ */ new Map();
|
|
@@ -20645,7 +21574,8 @@ function createPrimaryNumbering(onNumbered) {
|
|
|
20645
21574
|
partitionId: record.partitionId,
|
|
20646
21575
|
indexName: record.indexName,
|
|
20647
21576
|
documentId: record.documentId,
|
|
20648
|
-
document: record.document
|
|
21577
|
+
document: record.document,
|
|
21578
|
+
previousDocument: record.previousDocument
|
|
20649
21579
|
});
|
|
20650
21580
|
scope.entries.push(entry);
|
|
20651
21581
|
onNumbered?.(entry);
|
|
@@ -20702,7 +21632,7 @@ function createNodeWiring(input) {
|
|
|
20702
21632
|
address: config.address,
|
|
20703
21633
|
roles: [...input.roles],
|
|
20704
21634
|
capacity: config.capacity ?? DEFAULT_CAPACITY,
|
|
20705
|
-
startedAt: (
|
|
21635
|
+
startedAt: nextStartedAt(),
|
|
20706
21636
|
version: SPEC_VERSION
|
|
20707
21637
|
};
|
|
20708
21638
|
const memoryReporting = createNodeMemoryReporting({
|
|
@@ -20738,7 +21668,8 @@ function createNodeWiring(input) {
|
|
|
20738
21668
|
memoryView: memoryReporting.view,
|
|
20739
21669
|
writePressure: memoryReporting.writePressure,
|
|
20740
21670
|
writeProjection: memoryReporting.writeProjection,
|
|
20741
|
-
reshard: input.reshard
|
|
21671
|
+
reshard: input.reshard,
|
|
21672
|
+
replicaFeedClock: createReplicaFeedClock()
|
|
20742
21673
|
};
|
|
20743
21674
|
return {
|
|
20744
21675
|
registration,
|
|
@@ -21830,25 +22761,57 @@ function decideSyncTier(log, replicaLastSeqNo, replicaLastPrimaryTerm) {
|
|
|
21830
22761
|
const primaryTerm = log.termAt(replicaLastSeqNo);
|
|
21831
22762
|
return primaryTerm !== void 0 && primaryTerm === replicaLastPrimaryTerm ? "incremental" : "snapshot";
|
|
21832
22763
|
}
|
|
22764
|
+
function invalidSyncRequest(reason) {
|
|
22765
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, `Invalid SyncRequestPayload: ${reason}`);
|
|
22766
|
+
}
|
|
22767
|
+
function isNonNegativeInteger(value) {
|
|
22768
|
+
return typeof value === "number" && Number.isInteger(value) && value >= 0;
|
|
22769
|
+
}
|
|
22770
|
+
function optionalNonEmptyString(value, field) {
|
|
22771
|
+
if (value === void 0 || value === null) {
|
|
22772
|
+
return void 0;
|
|
22773
|
+
}
|
|
22774
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
22775
|
+
throw invalidSyncRequest(`"${field}" must be a non-empty string or absent`);
|
|
22776
|
+
}
|
|
22777
|
+
return value;
|
|
22778
|
+
}
|
|
22779
|
+
function decodeSyncRequest(payload) {
|
|
22780
|
+
let decoded;
|
|
22781
|
+
try {
|
|
22782
|
+
decoded = decode(payload);
|
|
22783
|
+
} catch (error) {
|
|
22784
|
+
throw invalidSyncRequest(`the payload is not valid MessagePack (${error instanceof Error ? error.message : error})`);
|
|
22785
|
+
}
|
|
22786
|
+
return validateSyncRequest(decoded);
|
|
22787
|
+
}
|
|
21833
22788
|
function validateSyncRequest(decoded) {
|
|
21834
22789
|
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
21835
|
-
throw
|
|
22790
|
+
throw invalidSyncRequest("expected an object");
|
|
21836
22791
|
}
|
|
21837
22792
|
const record = decoded;
|
|
21838
22793
|
if (typeof record.indexName !== "string") {
|
|
21839
|
-
throw
|
|
22794
|
+
throw invalidSyncRequest('"indexName" must be a string');
|
|
21840
22795
|
}
|
|
21841
|
-
if (
|
|
21842
|
-
throw
|
|
22796
|
+
if (!isNonNegativeInteger(record.partitionId)) {
|
|
22797
|
+
throw invalidSyncRequest('"partitionId" must be a non-negative integer');
|
|
21843
22798
|
}
|
|
21844
|
-
if (
|
|
21845
|
-
throw
|
|
22799
|
+
if (!isNonNegativeInteger(record.lastSeqNo)) {
|
|
22800
|
+
throw invalidSyncRequest('"lastSeqNo" must be a non-negative integer');
|
|
21846
22801
|
}
|
|
21847
|
-
if (
|
|
21848
|
-
throw
|
|
22802
|
+
if (!isNonNegativeInteger(record.lastPrimaryTerm)) {
|
|
22803
|
+
throw invalidSyncRequest('"lastPrimaryTerm" must be a non-negative integer');
|
|
21849
22804
|
}
|
|
21850
|
-
|
|
21851
|
-
|
|
22805
|
+
const indexUuid = optionalNonEmptyString(record.indexUuid, "indexUuid");
|
|
22806
|
+
const syncSession = optionalNonEmptyString(record.syncSession, "syncSession");
|
|
22807
|
+
return {
|
|
22808
|
+
indexName: record.indexName,
|
|
22809
|
+
partitionId: record.partitionId,
|
|
22810
|
+
lastSeqNo: record.lastSeqNo,
|
|
22811
|
+
lastPrimaryTerm: record.lastPrimaryTerm,
|
|
22812
|
+
...indexUuid === void 0 ? {} : { indexUuid },
|
|
22813
|
+
...syncSession === void 0 ? {} : { syncSession }
|
|
22814
|
+
};
|
|
21852
22815
|
}
|
|
21853
22816
|
|
|
21854
22817
|
// src/distribution/cluster-node/request-identity.ts
|
|
@@ -21912,7 +22875,7 @@ function createSingleResponseSink(respond) {
|
|
|
21912
22875
|
sink.closed = false;
|
|
21913
22876
|
return sink;
|
|
21914
22877
|
}
|
|
21915
|
-
async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build, metadata) {
|
|
22878
|
+
async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build, metadata, syncSession) {
|
|
21916
22879
|
const totalBytes = build.totalBytes;
|
|
21917
22880
|
const checksum = build.checksum;
|
|
21918
22881
|
const header = {
|
|
@@ -21922,7 +22885,7 @@ async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build
|
|
|
21922
22885
|
indexName,
|
|
21923
22886
|
checksum
|
|
21924
22887
|
};
|
|
21925
|
-
const startPayload = { header, totalBytes };
|
|
22888
|
+
const startPayload = { header, totalBytes, syncSession };
|
|
21926
22889
|
const startBytes = encode(startPayload);
|
|
21927
22890
|
assertMessageWithinLimit(startBytes, "SNAPSHOT_START");
|
|
21928
22891
|
await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_START, nodeId, requestId, startBytes);
|
|
@@ -22120,7 +23083,7 @@ async function streamBuiltSnapshot(message, sink, indexName, deps, options, buil
|
|
|
22120
23083
|
lastSeqNo: build.lastSeqNo ?? options.metadata.lastSeqNo,
|
|
22121
23084
|
primaryTerm: build.primaryTerm ?? options.metadata.primaryTerm
|
|
22122
23085
|
};
|
|
22123
|
-
await streamSnapshotToReplica(sink, deps.nodeId, message.requestId, indexName, build, metadata);
|
|
23086
|
+
await streamSnapshotToReplica(sink, deps.nodeId, message.requestId, indexName, build, metadata, options.syncSession);
|
|
22124
23087
|
await options.afterSnapshot(sink, build);
|
|
22125
23088
|
sink.closed = true;
|
|
22126
23089
|
} finally {
|
|
@@ -22188,7 +23151,7 @@ async function streamValidatedSnapshotRequest(message, sink, indexName, deps, op
|
|
|
22188
23151
|
|
|
22189
23152
|
// src/distribution/cluster-node/message-handler/sync.ts
|
|
22190
23153
|
async function handleSyncRequestMessage(message, respond, deps) {
|
|
22191
|
-
const request =
|
|
23154
|
+
const request = decodeSyncRequest(message.payload);
|
|
22192
23155
|
const authResult = await authorizeSnapshotRequest(deps.coordinator, request.indexName, message.sourceId);
|
|
22193
23156
|
if (authResult.outcome === "denied") {
|
|
22194
23157
|
throw new NarsilError(authResult.code, authResult.reason, {
|
|
@@ -22258,6 +23221,7 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
22258
23221
|
try {
|
|
22259
23222
|
const log = deps.writeDeps.getReplicationLog(request.indexName, request.partitionId);
|
|
22260
23223
|
const tier = decideSyncTier(log, request.lastSeqNo, request.lastPrimaryTerm);
|
|
23224
|
+
const syncSession = currentSyncSessionOf(request, message.sourceId, assignment, deps);
|
|
22261
23225
|
if (tier === "incremental") {
|
|
22262
23226
|
if (fromReplica) {
|
|
22263
23227
|
recordReplicaPosition(
|
|
@@ -22266,10 +23230,11 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
22266
23230
|
request.partitionId,
|
|
22267
23231
|
message.sourceId,
|
|
22268
23232
|
request.lastSeqNo,
|
|
22269
|
-
log.localLogEnd
|
|
23233
|
+
log.localLogEnd,
|
|
23234
|
+
request.syncSession !== void 0
|
|
22270
23235
|
);
|
|
22271
23236
|
}
|
|
22272
|
-
await sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1));
|
|
23237
|
+
await sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1), syncSession);
|
|
22273
23238
|
return;
|
|
22274
23239
|
}
|
|
22275
23240
|
if (fromReplica) {
|
|
@@ -22294,6 +23259,7 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
22294
23259
|
lastSeqNo: snapshotSeqNo,
|
|
22295
23260
|
primaryTerm: snapshotPrimaryTerm
|
|
22296
23261
|
},
|
|
23262
|
+
syncSession,
|
|
22297
23263
|
buildSnapshot: async () => {
|
|
22298
23264
|
const archive = await deps.engine.archiveReplicationPartition(request.indexName, request.partitionId, () => {
|
|
22299
23265
|
snapshotPrimaryTerm = log.termAt(log.localLogEnd) ?? 0;
|
|
@@ -22308,7 +23274,7 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
22308
23274
|
},
|
|
22309
23275
|
afterSnapshot: async (trailingSink, build) => {
|
|
22310
23276
|
const snapshotEnd = build.lastSeqNo ?? snapshotSeqNo;
|
|
22311
|
-
await sendSyncEntriesResponse(message, trailingSink, deps, log.getEntriesFrom(snapshotEnd + 1));
|
|
23277
|
+
await sendSyncEntriesResponse(message, trailingSink, deps, log.getEntriesFrom(snapshotEnd + 1), syncSession);
|
|
22312
23278
|
}
|
|
22313
23279
|
}
|
|
22314
23280
|
);
|
|
@@ -22316,10 +23282,26 @@ async function handleSyncRequestMessage(message, respond, deps) {
|
|
|
22316
23282
|
release();
|
|
22317
23283
|
}
|
|
22318
23284
|
}
|
|
22319
|
-
|
|
23285
|
+
function currentSyncSessionOf(request, sourceNodeId, assignment, deps) {
|
|
23286
|
+
const { indexName, partitionId, syncSession } = request;
|
|
23287
|
+
const catchUp = deps.writeDeps.catchUp;
|
|
23288
|
+
if (syncSession === void 0) {
|
|
23289
|
+
return openSyncSession(catchUp, indexName, partitionId, sourceNodeId, assignment.primaryTerm, assignment.replicas);
|
|
23290
|
+
}
|
|
23291
|
+
if (isCurrentSyncSession(catchUp, indexName, partitionId, sourceNodeId, syncSession, assignment.primaryTerm)) {
|
|
23292
|
+
return syncSession;
|
|
23293
|
+
}
|
|
23294
|
+
throw new NarsilError(
|
|
23295
|
+
ErrorCodes.REPLICATION_SYNC_FAILED,
|
|
23296
|
+
`Node '${sourceNodeId}' reported a sync of partition ${partitionId} of index '${indexName}' that is no longer its current sync, so it must open a new one`,
|
|
23297
|
+
{ indexName, partitionId, sourceNodeId }
|
|
23298
|
+
);
|
|
23299
|
+
}
|
|
23300
|
+
async function sendSyncEntriesResponse(message, respond, deps, entries, syncSession) {
|
|
22320
23301
|
const payload = {
|
|
22321
23302
|
entries,
|
|
22322
|
-
isLast: true
|
|
23303
|
+
isLast: true,
|
|
23304
|
+
syncSession
|
|
22323
23305
|
};
|
|
22324
23306
|
await respond({
|
|
22325
23307
|
type: ReplicationMessageTypes.SYNC_ENTRIES,
|
|
@@ -22471,6 +23453,7 @@ async function applyEntriesFromPrimary(entries, sourceNodeId, deps, previousPrim
|
|
|
22471
23453
|
const current = await resolveValidatedAssignment(first, sourceNodeId, deps);
|
|
22472
23454
|
noteAcceptedPrimaryTerm(deps.writeDeps.getReplicationLog(first.indexName, first.partitionId), current.primaryTerm);
|
|
22473
23455
|
await applyEntriesInLogOrder(first, entries, deps, previousPrimaryTerm);
|
|
23456
|
+
deps.writeDeps.replicaFeedClock?.noteFed(first.indexName, first.partitionId);
|
|
22474
23457
|
});
|
|
22475
23458
|
}
|
|
22476
23459
|
async function applyEntriesInLogOrder(first, entries, deps, previousPrimaryTerm) {
|
|
@@ -22494,6 +23477,19 @@ async function applyEntriesInLogOrder(first, entries, deps, previousPrimaryTerm)
|
|
|
22494
23477
|
}
|
|
22495
23478
|
continue;
|
|
22496
23479
|
}
|
|
23480
|
+
if (entry.seqNo <= tailSeqNo && log.termAt(entry.seqNo) === entry.primaryTerm) {
|
|
23481
|
+
if (log.verifyChecksum(entry)) continue;
|
|
23482
|
+
await applyRun2(run, log, deps);
|
|
23483
|
+
throw new NarsilError(ErrorCodes.REPLICATION_ENTRY_CORRUPT, `Invalid replication entry ${entry.seqNo}`, {
|
|
23484
|
+
indexName: entry.indexName,
|
|
23485
|
+
partitionId: entry.partitionId,
|
|
23486
|
+
seqNo: entry.seqNo
|
|
23487
|
+
});
|
|
23488
|
+
}
|
|
23489
|
+
if (entry.seqNo <= tailSeqNo && entry.seqNo < (log.oldestSeqNo ?? entry.seqNo + 1)) {
|
|
23490
|
+
await acknowledgeEvictedEntry(entry, run, log, deps);
|
|
23491
|
+
continue;
|
|
23492
|
+
}
|
|
22497
23493
|
const heldTermBefore = run.at(-1)?.primaryTerm ?? (tailSeqNo === 0 ? 0 : log.termAt(tailSeqNo));
|
|
22498
23494
|
if (entry.seqNo === tailSeqNo + 1 && expectedTermBefore !== void 0 && heldTermBefore !== void 0 && heldTermBefore !== expectedTermBefore) {
|
|
22499
23495
|
await applyRun2(run, log, deps);
|
|
@@ -22539,6 +23535,20 @@ async function applyEntriesInLogOrder(first, entries, deps, previousPrimaryTerm)
|
|
|
22539
23535
|
}
|
|
22540
23536
|
await applyRun2(run, log, deps);
|
|
22541
23537
|
}
|
|
23538
|
+
async function acknowledgeEvictedEntry(entry, run, log, deps) {
|
|
23539
|
+
const held = deps.engine.writeRecords.heldEntry(entry.indexName, entry.partitionId, entry.seqNo, entry.primaryTerm);
|
|
23540
|
+
const details = { indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo };
|
|
23541
|
+
if (held === "held" && log.verifyChecksum(entry)) return;
|
|
23542
|
+
await applyRun2(run, log, deps);
|
|
23543
|
+
if (held === "held") {
|
|
23544
|
+
throw new NarsilError(ErrorCodes.REPLICATION_ENTRY_CORRUPT, `Invalid replication entry ${entry.seqNo}`, details);
|
|
23545
|
+
}
|
|
23546
|
+
throw new NarsilError(
|
|
23547
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
23548
|
+
held === "absent" ? `This copy holds sequence number ${entry.seqNo} under another term than replication entry ${entry.seqNo}` : `Replication entry ${entry.seqNo} lies below the oldest entry that this copy retains, and its term history no longer covers that sequence number`,
|
|
23549
|
+
details
|
|
23550
|
+
);
|
|
23551
|
+
}
|
|
22542
23552
|
async function applyRun2(run, log, deps) {
|
|
22543
23553
|
if (run.length === 0) return;
|
|
22544
23554
|
const { applied, error } = await deps.engine.applyReplicationEntries(run);
|
|
@@ -23068,7 +24078,7 @@ async function deleteClusterBuildRecord(coordinator, indexName) {
|
|
|
23068
24078
|
}
|
|
23069
24079
|
|
|
23070
24080
|
// src/distribution/cluster-node/query-suggestions/host.ts
|
|
23071
|
-
function
|
|
24081
|
+
function indexNotFound6(indexName) {
|
|
23072
24082
|
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
|
|
23073
24083
|
indexName
|
|
23074
24084
|
});
|
|
@@ -23077,7 +24087,7 @@ async function updateClusterState(deps, indexName, change) {
|
|
|
23077
24087
|
const { coordinator } = deps.config;
|
|
23078
24088
|
for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
|
|
23079
24089
|
const current = await readIndexMetadataSnapshot(coordinator, indexName);
|
|
23080
|
-
if (current === null) throw
|
|
24090
|
+
if (current === null) throw indexNotFound6(indexName);
|
|
23081
24091
|
await deps.view.refresh(indexName);
|
|
23082
24092
|
const next = change(suggestionStateOfSettings(current.settings));
|
|
23083
24093
|
if (await coordinator.compareAndSet(
|
|
@@ -23097,7 +24107,7 @@ async function updateClusterState(deps, indexName, change) {
|
|
|
23097
24107
|
}
|
|
23098
24108
|
async function createSuggestionsIndex(deps, name, source, language) {
|
|
23099
24109
|
const sourceIndex = deps.view.get(source.index);
|
|
23100
|
-
if (sourceIndex === null) throw
|
|
24110
|
+
if (sourceIndex === null) throw indexNotFound6(source.index);
|
|
23101
24111
|
const config = { schema: { ...SUGGESTIONS_SCHEMA }, language, stopWords: /* @__PURE__ */ new Set() };
|
|
23102
24112
|
try {
|
|
23103
24113
|
await deps.createMarkedIndex(
|
|
@@ -23628,6 +24638,9 @@ function clusterNodeEngine(node, options) {
|
|
|
23628
24638
|
getSync(indexName) {
|
|
23629
24639
|
return node.getSync(indexName);
|
|
23630
24640
|
},
|
|
24641
|
+
updateChanges: (indexName, setting) => node.updateChanges(indexName, setting),
|
|
24642
|
+
getChanges: (indexName) => node.getChanges(indexName),
|
|
24643
|
+
followChanges: (indexName, followOptions) => node.followChanges(indexName, followOptions),
|
|
23631
24644
|
async getSynonyms(indexName) {
|
|
23632
24645
|
return node.getSynonyms(indexName);
|
|
23633
24646
|
},
|
|
@@ -23689,7 +24702,7 @@ function clusterNodeEngine(node, options) {
|
|
|
23689
24702
|
getPartitionStats: () => unsupported("getPartitionStats"),
|
|
23690
24703
|
snapshot: () => unsupported("snapshot"),
|
|
23691
24704
|
restore: () => unsupported("restore"),
|
|
23692
|
-
exportDocuments: () =>
|
|
24705
|
+
exportDocuments: (indexName) => node.exportDocuments(indexName),
|
|
23693
24706
|
importDocumentExport: () => unsupported("importDocumentExport"),
|
|
23694
24707
|
rebalance: (indexName, targetPartitionCount) => node.rebalance(indexName, targetPartitionCount),
|
|
23695
24708
|
updatePartitionConfig: () => unsupported("updatePartitionConfig"),
|
|
@@ -23706,6 +24719,8 @@ function clusterNodeEngine(node, options) {
|
|
|
23706
24719
|
bindRerankWindowSource(engine, () => rerankWindowCapOf(node));
|
|
23707
24720
|
const syncServing = syncServingSourceOf(node);
|
|
23708
24721
|
if (syncServing !== void 0) bindSyncServing(engine, syncServing);
|
|
24722
|
+
const changesServing = changesServingSourceOf(node);
|
|
24723
|
+
if (changesServing !== void 0) bindChangesServing(engine, changesServing);
|
|
23709
24724
|
const suggestions = boundQuerySuggestionsOf(node);
|
|
23710
24725
|
if (suggestions !== void 0) bindQuerySuggestions(engine, suggestions);
|
|
23711
24726
|
return engine;
|
|
@@ -24020,8 +25035,7 @@ async function createClusterNode(callerConfig) {
|
|
|
24020
25035
|
isOrphaned: (indexName) => orphanedIndexes.has(indexName),
|
|
24021
25036
|
suggestionHooks: suggestions.hooks
|
|
24022
25037
|
}),
|
|
24023
|
-
|
|
24024
|
-
getSync: (indexName) => trackOp(indexName, () => deviceSync.getSync(indexName)),
|
|
25038
|
+
...deviceSync.operations(trackOp),
|
|
24025
25039
|
cluster,
|
|
24026
25040
|
async start() {
|
|
24027
25041
|
guardShutdown();
|
|
@@ -24104,7 +25118,8 @@ async function createClusterNode(callerConfig) {
|
|
|
24104
25118
|
coordinator: writeDeps.coordinator,
|
|
24105
25119
|
engine,
|
|
24106
25120
|
memoryView: memoryReporting.view,
|
|
24107
|
-
syncServing: deviceSync.serving
|
|
25121
|
+
syncServing: deviceSync.serving,
|
|
25122
|
+
changesServing: deviceSync.changesServing
|
|
24108
25123
|
});
|
|
24109
25124
|
suggestions.bind(node, { engine: clusterNodeEngine(node), dropIndex: (name) => node.dropIndex(name) });
|
|
24110
25125
|
return node;
|