@delali/narsil 0.3.1-next.185 → 0.3.1-next.193

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/README.md +1 -0
  2. package/dist/adapters/filesystem.mjs +2 -2
  3. package/dist/adapters/gcs.mjs +2 -2
  4. package/dist/adapters/indexeddb.mjs +1 -1
  5. package/dist/adapters/s3.mjs +2 -2
  6. package/dist/browser/chunk-4E5APHTM.mjs +1459 -0
  7. package/dist/browser/{chunk-4ZICWS4V.mjs → chunk-QAOVAYAG.mjs} +1 -1
  8. package/dist/browser/{chunk-CPSHS46N.mjs → chunk-UQG4HUGQ.mjs} +82 -50
  9. package/dist/browser/{chunk-2Q45HI5H.mjs → chunk-YINZFPLT.mjs} +7 -0
  10. package/dist/browser/{module-XVLHJ7T7.mjs → module-ZZTNG5JZ.mjs} +26 -3
  11. package/dist/browser/{optional-writes-ZTBHYQMN.mjs → optional-writes-J5BHKEQL.mjs} +21 -8
  12. package/dist/browser/{storage-leaf-4XMCITJ6.mjs → storage-leaf-N3JMQ6CF.mjs} +14569 -13747
  13. package/dist/{capabilities-DAHXphRn.d.ts → capabilities-CLnQHIqV.d.ts} +51 -9
  14. package/dist/{chunk-KLVMK3R5.mjs → chunk-2CKDCC65.mjs} +1 -1
  15. package/dist/{chunk-CODGHJVW.mjs → chunk-2HVQYOOD.mjs} +40 -12
  16. package/dist/{chunk-M5I64AQW.mjs → chunk-3N5GUXBI.mjs} +33 -33
  17. package/dist/{chunk-DU23J5JG.mjs → chunk-5CIHQIA4.mjs} +6 -6
  18. package/dist/{react-native/chunks/chunk-ZUFOYZ5P.mjs → chunk-62X472AC.mjs} +1 -1
  19. package/dist/{chunk-QJGDB3WR.mjs → chunk-6GHX35ZM.mjs} +2 -2
  20. package/dist/{chunk-G3XRHAJA.mjs → chunk-6MC6BCXK.mjs} +3 -3
  21. package/dist/{chunk-X5NHWOXT.mjs → chunk-7P5YWDRQ.mjs} +2 -2
  22. package/dist/{chunk-MHBG33R5.mjs → chunk-7X4IIREW.mjs} +76 -46
  23. package/dist/{chunk-F4OOBPKA.mjs → chunk-AOYQTX6W.mjs} +1 -1
  24. package/dist/{chunk-D6FYUTFN.mjs → chunk-ASZLNVWW.mjs} +1 -1
  25. package/dist/{chunk-LR6JJVML.mjs → chunk-D2EJUM4A.mjs} +764 -699
  26. package/dist/{chunk-FETZYCH7.mjs → chunk-F6F4G322.mjs} +26 -1
  27. package/dist/{chunk-2YWWHMVC.mjs → chunk-FGOGT273.mjs} +2 -2
  28. package/dist/{chunk-5BUEBIXR.mjs → chunk-GLXLSPLO.mjs} +18 -5
  29. package/dist/{chunk-IE3Q62QD.mjs → chunk-GR5GMFJJ.mjs} +1 -1
  30. package/dist/{chunk-Y7QUFIGV.mjs → chunk-GXED6R3Q.mjs} +2 -2
  31. package/dist/{chunk-BVZWWMOU.mjs → chunk-HHYQMGPW.mjs} +2 -2
  32. package/dist/{chunk-SL53DHVE.mjs → chunk-HSA3H3VW.mjs} +121 -68
  33. package/dist/{chunk-YM2HI5N5.mjs → chunk-IL5DRCNU.mjs} +2 -2
  34. package/dist/{chunk-JDG6CCXM.mjs → chunk-JENWA4WX.mjs} +2 -2
  35. package/dist/{chunk-2GONI5HL.mjs → chunk-JXINPXRS.mjs} +5 -4
  36. package/dist/chunk-K5GNK5GE.mjs +602 -0
  37. package/dist/{chunk-WLFOLAQI.mjs → chunk-KBU3ZFJN.mjs} +2 -2
  38. package/dist/{chunk-EDEE7ETQ.mjs → chunk-KOA6T2IC.mjs} +3 -3
  39. package/dist/{chunk-N3M4NPML.mjs → chunk-M2Z5WVWW.mjs} +1 -1
  40. package/dist/{chunk-EWGIF6FQ.mjs → chunk-NJJN4FMM.mjs} +1 -1
  41. package/dist/{chunk-C7YQHCSO.mjs → chunk-P4F4UWLC.mjs} +1 -1
  42. package/dist/{chunk-VYSWVQCE.mjs → chunk-PAPQGC6I.mjs} +2 -2
  43. package/dist/{chunk-I2WJCRIH.mjs → chunk-R2PJKSNL.mjs} +1 -1
  44. package/dist/{chunk-6SZQETSC.mjs → chunk-RORNUNXW.mjs} +5 -2
  45. package/dist/{chunk-FBQYC5OX.mjs → chunk-SB5HCIIO.mjs} +16 -13
  46. package/dist/{chunk-CYCVPTTR.mjs → chunk-SCZI7COG.mjs} +2 -2
  47. package/dist/{chunk-O337MDRJ.mjs → chunk-SHES4VP5.mjs} +38 -13
  48. package/dist/{chunk-H4QYQ5DE.mjs → chunk-SIIDIGKI.mjs} +2 -2
  49. package/dist/{chunk-A3OAFUZI.mjs → chunk-SM2SYVIX.mjs} +2 -2
  50. package/dist/{chunk-II6XF6M4.mjs → chunk-SMLUZGSI.mjs} +9 -8
  51. package/dist/{chunk-2KCCQK4R.mjs → chunk-TSE2VWPX.mjs} +2 -2
  52. package/dist/{chunk-WSIU6YXQ.mjs → chunk-YPEJ6ODL.mjs} +1732 -743
  53. package/dist/{chunk-7WXGUTWK.mjs → chunk-Z6SIJH5O.mjs} +17 -6
  54. package/dist/{chunk-Q3ZZH5Z7.mjs → chunk-ZKOQR7EA.mjs} +7 -0
  55. package/dist/{chunk-CKEOTSF5.mjs → chunk-ZS2YUB2L.mjs} +6 -3
  56. package/dist/{chunk-FCQMZDRI.mjs → chunk-ZVZYJIIP.mjs} +3 -3
  57. package/dist/{client-DpqtT2XG.d.ts → client-uB0kp2yT.d.ts} +88 -5
  58. package/dist/client.d.ts +7 -7
  59. package/dist/client.mjs +3 -1265
  60. package/dist/{config-D9WR3MhO.d.ts → config-BvHQ-C7U.d.ts} +3 -3
  61. package/dist/{config-C687-reW.d.ts → config-PqgNY1H7.d.ts} +1 -1
  62. package/dist/{core-leaf-NEUM7PU3.mjs → core-leaf-TAR3GJVE.mjs} +3 -3
  63. package/dist/{direct-executor-37IH3ISS.mjs → direct-executor-WSZALYCV.mjs} +8 -8
  64. package/dist/distribution/coordinator/etcd.d.ts +1 -1
  65. package/dist/distribution/coordinator/etcd.mjs +35 -15
  66. package/dist/distribution/coordinator/in-memory.d.ts +1 -1
  67. package/dist/distribution/coordinator/in-memory.mjs +16 -6
  68. package/dist/distribution/transport/grpc.mjs +2 -2
  69. package/dist/distribution/transport/tcp.mjs +2 -2
  70. package/dist/distribution.d.ts +70 -9
  71. package/dist/distribution.mjs +1695 -680
  72. package/dist/{durability-config-NWU2G2SJ.mjs → durability-config-VWS2K24E.mjs} +1 -1
  73. package/dist/{react-native/chunks/embed-4LBSQD7U.mjs → embed-4PHHKGBK.mjs} +1 -1
  74. package/dist/embeddings/openai.mjs +2 -2
  75. package/dist/{engine-De6JzTW8.d.ts → engine-qjAxJSKk.d.ts} +206 -7
  76. package/dist/{index-CBUl4z3E.d.ts → index-BaQIAnk0.d.ts} +137 -111
  77. package/dist/index.browser.mjs +3 -3
  78. package/dist/index.d.ts +4 -4
  79. package/dist/index.mjs +8 -8
  80. package/dist/index.react-native.mjs +9 -9
  81. package/dist/{long-query-thread-DIPJBMV7.mjs → long-query-thread-N2DNPEA3.mjs} +21 -21
  82. package/dist/{module-Z3CL4GJ4.mjs → module-UQFIJQMG.mjs} +33 -13
  83. package/dist/{msgpack-leaf-7KBYOVTF.mjs → msgpack-leaf-H5ZYW7S7.mjs} +4 -4
  84. package/dist/{optional-reads-ENKMVQ4Q.mjs → optional-reads-5HFMF2QZ.mjs} +2 -2
  85. package/dist/{optional-writes-YIVJMQ4D.mjs → optional-writes-CGDBJBHQ.mjs} +31 -18
  86. package/dist/{paged-partition-archive-IZ6U7DVN.mjs → paged-partition-archive-UK44CVXG.mjs} +20 -20
  87. package/dist/partial-words.browser.mjs +3 -3
  88. package/dist/partial-words.mjs +8 -8
  89. package/dist/pattern.browser.mjs +2 -2
  90. package/dist/pattern.mjs +3 -3
  91. package/dist/persistence/durability/checkpoint-worker.d.ts +7 -0
  92. package/dist/persistence/durability/checkpoint-worker.mjs +18 -18
  93. package/dist/{phrase-module-ZY3TLTQA.mjs → phrase-module-EFESWOCR.mjs} +2 -2
  94. package/dist/{query-actions-7EOKT3WD.mjs → query-actions-7BL3YZDW.mjs} +8 -8
  95. package/dist/query-suggestions/merge-worker.mjs +11 -11
  96. package/dist/query-suggestions.browser.mjs +3 -3
  97. package/dist/query-suggestions.d.ts +1 -1
  98. package/dist/query-suggestions.mjs +9 -9
  99. package/dist/react-native/chunks/{chunk-TQ26BKBH.mjs → chunk-43R4GCM7.mjs} +7 -0
  100. package/dist/react-native/chunks/{chunk-CBU4M3EU.mjs → chunk-63DZ26X5.mjs} +277 -75
  101. package/dist/react-native/chunks/{chunk-BN6FUB7N.mjs → chunk-AN4HKLB4.mjs} +1 -1
  102. package/dist/react-native/chunks/{chunk-T5HF3FW3.mjs → chunk-BNFGA7TY.mjs} +3 -3
  103. package/dist/react-native/chunks/{chunk-PVMEHBQO.mjs → chunk-CW2ZFXSE.mjs} +1 -1
  104. package/dist/react-native/chunks/{chunk-NLEFMPY3.mjs → chunk-G434H5WG.mjs} +15 -4
  105. package/dist/react-native/chunks/{chunk-SK3DPAI4.mjs → chunk-HTBY4GON.mjs} +2 -2
  106. package/dist/react-native/chunks/{chunk-ZQPSXN6C.mjs → chunk-IP3QLETQ.mjs} +5 -5
  107. package/dist/react-native/chunks/{chunk-7TPO4FLE.mjs → chunk-LOKGE4OQ.mjs} +3 -3
  108. package/dist/react-native/chunks/{chunk-ZDDJVR7X.mjs → chunk-MRH36RN3.mjs} +2 -2
  109. package/dist/react-native/chunks/{chunk-YOTP2FPE.mjs → chunk-UO44MA5M.mjs} +1 -1
  110. package/dist/{chunk-4JBTALUC.mjs → react-native/chunks/chunk-UYAZLGEM.mjs} +1 -1
  111. package/dist/react-native/chunks/{chunk-WFQ6YQZA.mjs → chunk-UYKIV2KT.mjs} +1 -1
  112. package/dist/react-native/chunks/{chunk-IEY4DARA.mjs → chunk-XHYOQ4K4.mjs} +2 -2
  113. package/dist/react-native/chunks/{chunk-OZ3XEM6D.mjs → chunk-ZUJCUPVS.mjs} +1 -1
  114. package/dist/react-native/chunks/{core-leaf-TPBRRWVN.mjs → core-leaf-5PVST5PR.mjs} +4 -4
  115. package/dist/react-native/chunks/{durability-config-RVTWSXPH.mjs → durability-config-FCSD4MXY.mjs} +1 -1
  116. package/dist/{embed-2QMYCD2Q.mjs → react-native/chunks/embed-HA2I7RCF.mjs} +1 -1
  117. package/dist/react-native/chunks/{gzip.react-native-DLO4WW27.mjs → gzip.react-native-CVM4NQG2.mjs} +2 -2
  118. package/dist/react-native/chunks/{module-COLNJNML.mjs → module-QRCYB7QV.mjs} +32 -9
  119. package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +7 -0
  120. package/dist/react-native/chunks/{optional-reads-74HYI4DO.mjs → optional-reads-AFPO2FEB.mjs} +3 -3
  121. package/dist/react-native/chunks/{optional-writes-QETIM6BI.mjs → optional-writes-GC6JWMAF.mjs} +31 -18
  122. package/dist/react-native/chunks/{phrase-module-UC4YGEBB.mjs → phrase-module-IGYBK7XH.mjs} +3 -3
  123. package/dist/react-native/chunks/{resolve-vector-text-FU7D4QP5.mjs → resolve-vector-text-GE2CRJKI.mjs} +1 -1
  124. package/dist/react-native/chunks/{restore-queue-3E2MJIJS.mjs → restore-queue-2ACONS4W.mjs} +12 -12
  125. package/dist/react-native/chunks/{segment-codec-leaf-IE4DXUZV.mjs → segment-codec-leaf-SSIWXVUG.mjs} +10 -10
  126. package/dist/react-native/chunks/{segment-merge-leaf-62LQ5WZB.mjs → segment-merge-leaf-I2WDFE6O.mjs} +11 -11
  127. package/dist/react-native/chunks/{storage-leaf-LNOLKWZB.mjs → storage-leaf-YXXMG4VR.mjs} +14727 -13915
  128. package/dist/react-native/chunks/{store-leaf-Y7CBOPFZ.mjs → store-leaf-FS7A4PZA.mjs} +11 -11
  129. package/dist/react-native/chunks/{suggestions-leaf-PSUGL6MI.mjs → suggestions-leaf-PSFCT5EX.mjs} +11 -11
  130. package/dist/react-native/chunks/{worker-copies-MW5O5U3D.mjs → worker-copies-QRMISMEL.mjs} +10 -10
  131. package/dist/react-native/partial-words.mjs +9 -9
  132. package/dist/react-native/pattern.mjs +5 -5
  133. package/dist/react-native/query-suggestions.mjs +9 -9
  134. package/dist/react-native/sync.mjs +433 -51
  135. package/dist/react-native/synonyms.mjs +9 -9
  136. package/dist/react.d.ts +8 -8
  137. package/dist/react.mjs +7 -0
  138. package/dist/{rerank-KTV5L7MZ.mjs → rerank-4GFC26FD.mjs} +8 -8
  139. package/dist/rerankers/http.mjs +2 -2
  140. package/dist/{resolve-vector-text-SD6BEMKZ.mjs → resolve-vector-text-OD4J3WHF.mjs} +2 -2
  141. package/dist/{restore-queue-ES7TDJ5V.mjs → restore-queue-E4EIUYVV.mjs} +11 -11
  142. package/dist/{segment-codec-leaf-DO75VKEV.mjs → segment-codec-leaf-MP6DILVE.mjs} +9 -9
  143. package/dist/{segment-merge-leaf-JGND6LG5.mjs → segment-merge-leaf-ED2NMUC5.mjs} +9 -9
  144. package/dist/server.d.ts +13 -9
  145. package/dist/server.mjs +875 -348
  146. package/dist/{storage-leaf-ZI3H2WJY.mjs → storage-leaf-HQFO37W7.mjs} +5234 -5835
  147. package/dist/{store-leaf-YIH36Y5J.mjs → store-leaf-PL2G7RM5.mjs} +10 -10
  148. package/dist/{suggestions-leaf-FP5HSTFP.mjs → suggestions-leaf-I3MEHACP.mjs} +12 -12
  149. package/dist/sync/filesystem.d.ts +2 -2
  150. package/dist/sync/filesystem.mjs +16 -16
  151. package/dist/sync/indexeddb.browser.mjs +1 -1
  152. package/dist/sync/indexeddb.d.ts +2 -2
  153. package/dist/sync/indexeddb.mjs +3 -3
  154. package/dist/sync.browser.mjs +35 -1108
  155. package/dist/sync.d.ts +3 -3
  156. package/dist/sync.mjs +430 -51
  157. package/dist/synonyms.browser.mjs +3 -3
  158. package/dist/synonyms.mjs +8 -8
  159. package/dist/{tasks-X2IJQc8J.d.ts → tasks-C65V7lIr.d.ts} +3 -3
  160. package/dist/thread-LTFI3RK5.mjs +31 -0
  161. package/dist/{types-D0YasjoX.d.ts → types-CpCg-Gq-.d.ts} +1 -1
  162. package/dist/vector/search-worker.mjs +5 -5
  163. package/dist/{worker-copies-WYPRCNB7.mjs → worker-copies-P7YWXEP5.mjs} +11 -11
  164. package/dist/workers/entry.mjs +14 -14
  165. package/openapi.json +144 -0
  166. package/package.json +5 -7
  167. package/dist/chunk-URVH6BLO.mjs +0 -217
  168. package/dist/react-native/chunks/msgpack-leaf-GTKLF6GJ.mjs +0 -7
  169. package/dist/thread-NQRNW3IH.mjs +0 -31
  170. /package/dist/{chunk-CV3Q2EKO.mjs → chunk-S5QMWECL.mjs} +0 -0
@@ -1,55 +1,55 @@
1
- import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-FCQMZDRI.mjs';
2
- import { emptySerializablePartition } from './chunk-A3OAFUZI.mjs';
3
- import { applyStoredIndexBm25 } from './chunk-BVZWWMOU.mjs';
4
- import { keyBytesOf, querySuggestionsOf, createQuerySuggestionsRuntime, publicBuildOf, buildRecordToWire, buildRecordFromWire, SUGGESTIONS_SCHEMA } from './chunk-DU23J5JG.mjs';
5
- import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, archiveStream, buildArchiveFromMemory, indexMetadataOf, applyIndexEntry, applyDeleteEntry, resolveAnalysisTarget, requireConfigurationWithinSizeLimit, memoryCheckpointStore, createByteReader, writeArchiveFile, namedSettingsOf, createDetachedIndex, loadPartitionSegments, holdsSettings, configOfNamedSettings, closePartitionWriter, requireRestoreMemory, archiveFootprint, storedBytesOfSegments, validateReplicationEntry, readSegmentManifest, replaySyncedWal, nestedSchemaOf } from './chunk-WSIU6YXQ.mjs';
6
- import { readVectorFile } from './chunk-II6XF6M4.mjs';
7
- import { addBackgroundAdmission, workWaitingForMemory, resolveHybridFusion, fuseRankings } from './chunk-Y7QUFIGV.mjs';
1
+ import { routableAllocation, resolvePartitionId, resolveNodeTargets, resolvePrimaryAssignment, requireAssignedPrimary, assertSufficientActiveReplicas, getInSyncReplicaTargets } from './chunk-ZVZYJIIP.mjs';
2
+ import { emptySerializablePartition } from './chunk-SM2SYVIX.mjs';
3
+ import { applyStoredIndexBm25 } from './chunk-HHYQMGPW.mjs';
4
+ import { keyBytesOf, querySuggestionsOf, createQuerySuggestionsRuntime, publicBuildOf, buildRecordToWire, buildRecordFromWire, SUGGESTIONS_SCHEMA } from './chunk-5CIHQIA4.mjs';
5
+ import { stampAnalysisGeneration, holdsNamedSettings, dropHeldAnalysisChange, releaseHeldAnalysisChange, heldAnalysisIsBuilt, buildHeldAnalysisChange, archiveStream, buildArchiveFromMemory, indexMetadataOf, applyIndexEntry, applyDeleteEntry, documentExportStream, followSourceChanges, requireChangesSetting, resolveAnalysisTarget, requireConfigurationWithinSizeLimit, memoryCheckpointStore, createByteReader, writeArchiveFile, namedSettingsOf, createDetachedIndex, loadPartitionSegments, changesStatusOf, holdsSettings, configOfNamedSettings, closePartitionWriter, requireRestoreMemory, archiveFootprint, storedBytesOfSegments, validateReplicationEntry, readSegmentManifest, replaySyncedWal, nestedSchemaOf } from './chunk-YPEJ6ODL.mjs';
6
+ import { readVectorFile } from './chunk-SMLUZGSI.mjs';
7
+ import { addBackgroundAdmission, workWaitingForMemory, resolveHybridFusion, fuseRankings } from './chunk-GXED6R3Q.mjs';
8
8
  import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
9
- import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, storedFilesOf, BACKUP_MANIFEST_READS_AT_ONCE, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, SYNC_BACKUP_TAKER, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, readRestorePlacement, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-5BUEBIXR.mjs';
10
- import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, suggestionStateOfSettings, indexConfigKey, metadataWithSync, indexConfigFromSettings, metadataWithBm25, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, isRecord2, decodeIndexSettings, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, storedSynonymSetOf, RESHARD_FORWARD_BATCH_DOCUMENTS, RESHARD_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, SYNC_EPOCH_CAS_ATTEMPTS, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS, metadataWithSuggestionState } from './chunk-CODGHJVW.mjs';
11
- import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, refuseEncodeWithoutRoom, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, MAX_SOURCE_ID_LENGTH, ADMISSION_TIMEOUT_MS, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, OUT_OF_STEP_REMOVAL_RETRY_INITIAL_DELAY_MS, OUT_OF_STEP_REMOVAL_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-2GONI5HL.mjs';
12
- export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-2GONI5HL.mjs';
13
- import { encodeIndexConfiguration, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireSyncSetting, requireNoQuerySuggestions, decodeIndexConfiguration, INDEX_CONFIGURATION_ENVELOPE_VERSION } from './chunk-URVH6BLO.mjs';
14
- import { restoredConfigFields, restoredEmbedding } from './chunk-N3M4NPML.mjs';
15
- import { estimatedArchiveBytes, readSnapshotArchive, archiveLayout, openManifestPayload, sourceIndexNameOf, listedKeysOf, decodeManifestPayload } from './chunk-7WXGUTWK.mjs';
9
+ import { bindEngineWriteBlock, bindIndexReads, syncServingSourceOf, bindSyncServing, changesServingSourceOf, bindChangesServing, validateBackupConfig, BACKUP_SWEEP_INTERVAL_MS, bindWriteBlockLookup, backupKeys, validateBackupName, CHANGES_BACKUP_TAKER, BACKUP_MANIFEST_READS_AT_ONCE, storedFilesOf, decodeBackupManifest, BACKUP_JOBS_KEY, BACKUP_DELETES_AT_ONCE, BACKUP_PROGRESS_REPORT_INTERVAL_MS, BACKUP_PARTITION_RESULT_RETENTION_MS, BACKUP_JOB_POLL_INTERVAL_MS, BACKUP_FILES_AT_ONCE, isCount, storeWithoutSaveIfAbsent, CLUSTER_UUID_KEY, isSha256Hex, BACKUP_PUBLISH_RETRY_MS, isBackupTaker, readRestorePlacement, SYNC_BACKUP_TAKER, BACKUP_JOB_CAS_ATTEMPTS, BACKUP_JOB_CAS_RETRY_MS, MAX_CLUSTER_UUID_BYTES, MAX_FINISHED_BACKUP_JOBS, BACKUP_FILE_CHUNK_BYTES, encodeBackupManifest, SYNC_BACKUPS_KEPT, BACKUP_PARTITIONS_AT_ONCE, BACKUP_PARTITION_POLL_INTERVAL_MS, BACKUP_PRIMARY_WAIT_MS, readBackupPartition, readStoredFile } from './chunk-GLXLSPLO.mjs';
10
+ import { DEFAULT_NODE_LIFECYCLE_CONFIG, DEFAULT_CONTROLLER_CONFIG, readIndexMetadataSnapshot, indexMetadataSnapshotOf, getIndexMetadata, getClusterIndexConfig, withPartitionCount, validateIndexName, indexConfigKey, metadataWithSync, suggestionStateOfSettings, indexConfigFromSettings, metadataWithBm25, readIndexState, storeSynonymSet, deleteSynonymSet, indexSettingsOf, putIndexMetadata, isValidInteger, isRecord, HIDDEN_GENERATION_PREFIX, isRecord2, decodeIndexSettings, metadataWithSynonymSet, RESHARD_RECORD_CAS_ATTEMPTS, indexUuidOfStoredMetadata, storedSynonymSetOf, RESHARD_FORWARD_BATCH_DOCUMENTS, RESHARD_RETRY_DELAY_MS, TEARDOWN_CAS_ATTEMPTS, RESHARD_RECORD_CHECK_INTERVAL_MS, RESHARD_POLL_INTERVAL_MS, ALLOCATION_RETRY_DELAY_MS, TEARDOWN_RETRY_ROUNDS, ANALYSIS_CHANGE_RETRY_MS, putIndexReplicationFactor, ALLOCATION_CAS_ATTEMPTS, TEARDOWN_RETRY_DELAY_MS, RESHARD_CONTROLLER_POLL_INTERVAL_MS, MAX_RESHARD_CUTOVER_ATTEMPTS, ANALYSIS_CHANGE_POLL_INTERVAL_MS, RECOVERY_CAS_ATTEMPTS, ESTIMATED_PARTITION_BYTES, REBALANCE_THRESHOLD, putGenerationMetadata, RESHARD_CUTOVER_LAG_ENTRIES, ANALYSIS_REBUILD_RESEND_MS, RESHARD_CUTOVER_TIMEOUT_MS, PARTITION_STORES_TIMEOUT_MS, SYNC_EPOCH_CAS_ATTEMPTS, syncSettingsOfSnapshot, epochFeaturesOn, LEADERSHIP_IMBALANCE_THRESHOLD, BOOTSTRAP_CAS_ATTEMPTS, metadataWithSuggestionState, UNFED_REPLICA_CHECK_INTERVAL_MS, UNFED_REPLICA_RESYNC_MS } from './chunk-2HVQYOOD.mjs';
11
+ import { MAX_WAIT_FOR_ACTIVE_REPLICAS, MIN_CONTROLLER_LEASE_TTL_MS, MAX_NODE_LEFT_DELAY_MS, DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, DEFAULT_MAX_STREAMS_PER_INDEX, DEFAULT_CAPACITY, CATCH_UP_TICK_MS, MEMORY_SAMPLE_INTERVAL_MS, createWritePressure, WRITE_MEMORY_SAMPLE_MAX_AGE_MS, DEFAULT_IN_FLIGHT_WRITE_FRACTION, MAX_SETTINGS_REPLACEMENT_ATTEMPTS, SETTINGS_CONTENTION_REASON, MAX_DROP_ACKNOWLEDGE_WAIT_MS, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR, DEFAULT_CREATE_INDEX_WAIT_MS, DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS, DROP_ACKNOWLEDGE_POLL_INTERVAL_MS, COORDINATOR_REQUEST_BUCKET_BOUNDS_SECONDS, COPY_FAILURE_ATTEMPTS, SEARCH_CONTEXT_REAPER_INTERVAL_MS, DECOMMISSION_RECHECK_INTERVAL_MS, refuseEncodeWithoutRoom, answerRefusalOf, withAdmittedWrite, estimateForwardedBytes, PARTITION_BYTES_INDEX_OVERHEAD_BYTES, MAX_SOURCE_ID_LENGTH, ADMISSION_TIMEOUT_MS, PARTITION_BYTES_ENTRY_BYTES, PARTITION_BYTES_REPORT_BUDGET_BYTES, CATCH_UP_FEED_WINDOW_MS, MAX_CATCH_UP_IN_FLIGHT_BYTES, SYNCED_POSITION_REPORT_TIMEOUT_MS, SNAPSHOT_CHUNK_YIELD_INTERVAL_MS, ANSWER_DECODE_HEAP_BYTES_PER_PAYLOAD_BYTE, ADMISSION_RETRIES_AFTER_IN_SYNC_CHANGE, REGISTRATION_LEASE_DRIFT_SHARE, REGISTRATION_LEASE_CONFIRM_TIMEOUT_MS, WRITE_APPLY_YIELD_INTERVAL_MS, ROLLBACK_RETRY_INITIAL_DELAY_MS, ROLLBACK_RETRY_DELAY_CAP_MS, OUT_OF_STEP_REMOVAL_RETRY_INITIAL_DELAY_MS, OUT_OF_STEP_REMOVAL_RETRY_DELAY_CAP_MS, FETCH_GROUP_DOCUMENT_IDS, COPY_FAILURE_RETRY_DELAY_MS, SEARCH_CONTEXT_ENTRY_HEAP_BYTES, CLEAR_PAGE_SIZE, estimateDocumentsBytes, estimateDocumentBytes, PRIMARY_CHANGE_RETRY_WINDOW_MS, PRIMARY_CHANGE_POLL_INTERVAL_MS, PARTITION_WRITE_CONCURRENCY, FORWARD_RETRY_INITIAL_DELAY_MS, FORWARD_RETRY_DELAY_CAP_MS } from './chunk-JXINPXRS.mjs';
12
+ export { DEFAULT_CAPACITY, DEFAULT_PARTITION_COUNT, DEFAULT_REPLICATION_FACTOR } from './chunk-JXINPXRS.mjs';
13
+ import { encodeIndexConfiguration, MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, decodeIndexConfiguration, requireSyncSetting, requireNoQuerySuggestions, validChangeFilter, createDocumentTest, INDEX_CONFIGURATION_ENVELOPE_VERSION, changesNotEnabled } from './chunk-K5GNK5GE.mjs';
14
+ import { restoredConfigFields, restoredEmbedding } from './chunk-M2Z5WVWW.mjs';
15
+ import { readSnapshotArchive, archiveLayout, openManifestPayload, sourceIndexNameOf, listedKeysOf, decodeManifestPayload, estimatedArchiveBytes } from './chunk-Z6SIJH5O.mjs';
16
16
  import { CONTROLLER_LEASE_KEY } from './chunk-44W3CBNG.mjs';
17
17
  import './chunk-T4HMSZCX.mjs';
18
- import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, optionalIndexUuid, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, buildEntry, createInsyncRemoveMessage, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, createInsyncAddMessage, validateAckPayload, checkpointLastSeqNo, checkpointPrimaryTerm, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-LR6JJVML.mjs';
19
- import { directoryCheckpointStore, readListedFile, encodeSegmentManifest } from './chunk-M5I64AQW.mjs';
18
+ import { createForwardBatchResultMessage, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, DEFAULT_LOG_RETENTION_BYTES, buildEntry, optionalIndexUuid, entryByteEstimate, computeEntryChecksum, createPrimaryResignMessage, validateInsyncConfirmPayload, createInsyncRemoveMessage, SNAPSHOT_CHUNK_SIZE, createEntryBatchMessage, createInsyncConfirmMessage, createEntryMessage, validateAckPayload, createInsyncAddMessage, checkpointLastSeqNo, checkpointPrimaryTerm, INSYNC_CAS_ATTEMPTS, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage } from './chunk-D2EJUM4A.mjs';
19
+ import { directoryCheckpointStore, readListedFile, encodeSegmentManifest } from './chunk-3N5GUXBI.mjs';
20
20
  import { TransportErrorCodes, ClusterMessageTypes, ReplicationMessageTypes, SuggestionMessageTypes, SyncMessageTypes, QueryMessageTypes, TransportError, DEFAULT_TRANSPORT_CONFIG } from './chunk-5FLRXF73.mjs';
21
21
  import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET } from './chunk-JDDXV6UF.mjs';
22
- import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-2KCCQK4R.mjs';
22
+ import { manifestFileRefs, segmentKey, SEGMENT_MANIFEST_VERSION, vectorFileKey } from './chunk-TSE2VWPX.mjs';
23
23
  import { getPath, getFs } from './chunk-HLPRHOSO.mjs';
24
24
  import './chunk-MSIN4NM3.mjs';
25
- import './chunk-CV3Q2EKO.mjs';
25
+ import './chunk-S5QMWECL.mjs';
26
26
  import './chunk-LNGE7SLY.mjs';
27
- import './chunk-I2WJCRIH.mjs';
28
- import './chunk-4JBTALUC.mjs';
29
- import { deserializePartition } from './chunk-H4QYQ5DE.mjs';
30
- import { decodeVectorFilePayload, bytesToVectors, isDead, MIN_EF_SEARCH, MAX_EF_SEARCH, MIN_OVERSAMPLE } from './chunk-WLFOLAQI.mjs';
31
- import './chunk-2YWWHMVC.mjs';
32
- import { RestoredHistory } from './chunk-G3XRHAJA.mjs';
27
+ import './chunk-R2PJKSNL.mjs';
28
+ import './chunk-62X472AC.mjs';
29
+ import { deserializePartition } from './chunk-SIIDIGKI.mjs';
30
+ import { decodeVectorFilePayload, bytesToVectors, isDead, MIN_EF_SEARCH, MAX_EF_SEARCH, MIN_OVERSAMPLE } from './chunk-KBU3ZFJN.mjs';
31
+ import './chunk-FGOGT273.mjs';
32
+ import { RestoredHistory } from './chunk-6MC6BCXK.mjs';
33
33
  import { oneSyncChangeAtATime } from './chunk-V6EUBAIT.mjs';
34
- import { historyFilesOf, removeHistoryFolder, QUERY_HISTORY_FOLDER, RESTORE_STAGING_INFIX, openChecksummedEnvelope, historyFolderOf, listHistoryFolder, historyFileOf, COUNTS_STREAM_CHUNK_BYTES, BUILD_WRITE_BATCH, restoreStagingPrefix, crc32Init, crc32Final, crc32Update, MAX_COUNTS_STREAM_KEY_BYTES, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, storedStateOf } from './chunk-EDEE7ETQ.mjs';
34
+ import { historyFilesOf, removeHistoryFolder, QUERY_HISTORY_FOLDER, RESTORE_STAGING_INFIX, openChecksummedEnvelope, historyFolderOf, listHistoryFolder, historyFileOf, COUNTS_STREAM_CHUNK_BYTES, BUILD_WRITE_BATCH, restoreStagingPrefix, crc32Init, crc32Final, crc32Update, MAX_COUNTS_STREAM_KEY_BYTES, MAX_HISTORY_FILE_BYTES, packChecksummedEnvelope, storedStateOf } from './chunk-KOA6T2IC.mjs';
35
35
  import { boundQuerySuggestionsOf, bindQuerySuggestions } from './chunk-S2POFTLN.mjs';
36
- import { oversampledShardSize, runtimeHost, bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, importOptionalReads, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, emitEngineEvent, synonymModule, createEngineIndex, generateId, getLanguage, resolveIndexAnalysis, memoryFractionsOf, createMemoryMonitor, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, bm25SetsValue, prepareConfiguredSynonyms, storedArrayPositionGap, updateDocumentVectors, arrayPositionsAreStale, fnv1a, readRecentMemoryFigures, normalizeSort, sortModesOf, resolveProjection, requireLoadedOptionalReads, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, validateBM25Shape, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, hasLanguage, FIELD_FILTER_OPERATORS, yieldToEventLoop, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, customTokensOf, visitTokens, clampRowCount, now, sortSignatureOf, listBindingOf, requireMatchingCursor, decodePageCursor, encodePageCursor, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, clampLimit, requireWithinResultWindow, providedDocId, holdsQuotedText, queryBindingOf, engineCoreOf, sortSignatureEntry, fnv1aUint32s, loadRerankRuntime, mergePartitionStats, mergeEntries, regionsOverlap, compareScores } from './chunk-SL53DHVE.mjs';
37
- import { encodeDocument, encodeDocumentReply } from './chunk-KLVMK3R5.mjs';
38
- import { collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, onRefusalAfterCollection, reservedHeapBytes, reserveHeapBytes } from './chunk-F4OOBPKA.mjs';
36
+ import { oversampledShardSize, runtimeHost, bindRerankWindowSource, rerankWindowCapOf, validateMemoryFractions, requireKnownConfig, importOptionalReads, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, emitEngineEvent, synonymModule, createEngineIndex, generateId, documentChurnOf, isCompositePartition, getLanguage, resolveIndexAnalysis, memoryFractionsOf, createMemoryMonitor, readMemoryFigures, FULL_MEMORY_COLLECTION_INTERVAL_MS, validateBM25Params, storedBM25Params, requireSurfaceFormsForPartialFields, bm25SetsValue, prepareConfiguredSynonyms, storedArrayPositionGap, updateDocumentVectors, arrayPositionsAreStale, fnv1a, readRecentMemoryFigures, normalizeSort, sortModesOf, resolveProjection, noteDocumentChurn, MAP_TABLE_MIN_CAPACITY, MAP_TABLE_SHRINK_DIVISOR, requireLoadedOptionalReads, applyProjection, FILTER_EXPRESSION_KEYS, filterTestCount, setNestedValue, validateBM25Shape, CASE_FOLD_EXPANSION_FACTOR, PATTERN_RUN_HEAP_BYTES, ENUM_VALUE_HEAP_BYTES, DOCUMENT_VALUE_HEAP_BYTES, DOCUMENT_HEAP_BYTES_PER_ESTIMATED_BYTE, DOCUMENT_PROPERTY_HEAP_BYTES, VECTOR_DOCUMENT_HEAP_BYTES, VECTOR_COMPONENT_HEAP_BYTES, hasLanguage, FIELD_FILTER_OPERATORS, yieldToEventLoop, NORMALIZATION_EXPANSION_FACTOR, TERM_UNIT_HEAP_BYTES, customTokensOf, visitTokens, getVectorFieldPaths, resolvePartitionInsertOptions, clampRowCount, now, sortSignatureOf, listBindingOf, requireMatchingCursor, decodePageCursor, encodePageCursor, DISTINCT_TERM_HEAP_BYTES, PARTIAL_SPELLING_HEAP_BYTES, clampLimit, requireWithinResultWindow, providedDocId, MAP_TABLE_HEADER_BYTES, MAP_TABLE_SLOT_BYTES, holdsQuotedText, queryBindingOf, engineCoreOf, sortSignatureEntry, fnv1aUint32s, loadRerankRuntime, mergePartitionStats, mergeEntries, regionsOverlap, compareScores } from './chunk-HSA3H3VW.mjs';
37
+ import { encodeDocument, encodeDocumentReply } from './chunk-2CKDCC65.mjs';
38
+ import { collectYoungGenerationBeforeRefusal, collectGarbageBeforeRefusal, onRefusalAfterCollection, reservedHeapBytes, reserveHeapBytes } from './chunk-AOYQTX6W.mjs';
39
39
  import { decode } from './chunk-GXCDWLQI.mjs';
40
40
  import { newSyncEpoch } from './chunk-BUMREPLU.mjs';
41
41
  import './chunk-7T4LAYT4.mjs';
42
- import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE, buildStoredValues } from './chunk-YM2HI5N5.mjs';
43
- import { loadMessagePack, validateSchema, requireCreatableSchema, schemaFieldsOf, getNestedValue, MAX_NESTING_DEPTH, MAX_STORED_VECTOR_DIMENSION, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-X5NHWOXT.mjs';
42
+ import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, isPatternTolerance, MAX_FILTER_ARRAY_SIZE, buildStoredValues } from './chunk-IL5DRCNU.mjs';
43
+ import { loadMessagePack, validateSchema, requireCreatableSchema, schemaFieldsOf, getNestedValue, MAX_NESTING_DEPTH, MAX_STORED_VECTOR_DIMENSION, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-7P5YWDRQ.mjs';
44
44
  import './chunk-U2BKYEC7.mjs';
45
- import './chunk-6SZQETSC.mjs';
45
+ import { shardEntryRangeOf } from './chunk-RORNUNXW.mjs';
46
46
  import { collectAndReturnPages } from './chunk-UPDNNOTG.mjs';
47
47
  import { BATCH_CHUNK_SIZE, MAX_SORT_FIELDS, MAX_DOC_ID_LENGTH, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, DEFAULT_PAGE_SIZE, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, HYBRID_RESCORE_WINDOW, DEFAULT_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT } from './chunk-3ECIRFS5.mjs';
48
- import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback } from './chunk-D6FYUTFN.mjs';
48
+ import { nativeSearchCoreIsEnabled, loadNativeCore, noteFallback } from './chunk-ASZLNVWW.mjs';
49
49
  import './chunk-ZXX5GZEX.mjs';
50
- import { joinChunks, readWhole } from './chunk-IE3Q62QD.mjs';
50
+ import { joinChunks, readWhole } from './chunk-GR5GMFJJ.mjs';
51
51
  import { MAX_DOCUMENT_ENCODING_DEPTH } from './chunk-4PY4JYMY.mjs';
52
- import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-Q3ZZH5Z7.mjs';
52
+ import { ErrorCodes, NarsilError, describeError, ServerErrorCodes } from './chunk-ZKOQR7EA.mjs';
53
53
  import { holdsUnpairedSurrogate, compareCodePoints, readSortField, isSortMode, SORT_MODES, exceedsCodePoints, toComparableSortValue } from './chunk-TGPKS2UM.mjs';
54
54
  import { encode } from '@msgpack/msgpack';
55
55
  import { randomUUID, createHash } from 'crypto';
@@ -225,12 +225,13 @@ async function clearRebalanceProgress(coordinator, indexName, partitionId) {
225
225
  }
226
226
 
227
227
  // src/distribution/cluster/version.ts
228
- var SPEC_VERSION = "1.4";
228
+ var SPEC_VERSION = "1.5";
229
229
  var PHRASE_SEARCH_SPEC_VERSION = "1.1";
230
230
  var BM25_TUNING_SPEC_VERSION = "1.1";
231
231
  var BACKUP_SPEC_VERSION = "1.2";
232
232
  var DEVICE_SYNC_SPEC_VERSION = "1.3";
233
233
  var QUERY_SUGGESTIONS_SPEC_VERSION = "1.4";
234
+ var CHANGE_CAPTURE_SPEC_VERSION = "1.5";
234
235
  var VERSION_PATTERN = /^(\d+)\.(\d+)$/;
235
236
  function versionPartsOf(version) {
236
237
  const match = VERSION_PATTERN.exec(version);
@@ -3203,6 +3204,50 @@ function chunkByBudget(items, budget) {
3203
3204
  return chunks;
3204
3205
  }
3205
3206
 
3207
+ // src/distribution/cluster/in-sync-version.ts
3208
+ function inSyncVersionOf(assignment) {
3209
+ return assignment.inSyncVersion ?? 0;
3210
+ }
3211
+ function holdSameMembers(left, right) {
3212
+ const leftMembers = new Set(left);
3213
+ const rightMembers = new Set(right);
3214
+ if (leftMembers.size !== rightMembers.size) return false;
3215
+ for (const nodeId of rightMembers) {
3216
+ if (!leftMembers.has(nodeId)) return false;
3217
+ }
3218
+ return true;
3219
+ }
3220
+ function dropsAReplica(previous, next) {
3221
+ const nextReplicas = new Set(next.replicas);
3222
+ return previous.replicas.some((nodeId) => !nextReplicas.has(nodeId));
3223
+ }
3224
+ function stampedInSyncVersion(previous, next) {
3225
+ if (previous === void 0) return inSyncVersionOf(next);
3226
+ const stored = inSyncVersionOf(previous);
3227
+ const unchanged = holdSameMembers(previous.inSyncSet, next.inSyncSet) && !dropsAReplica(previous, next);
3228
+ const raised = unchanged ? stored : stored + 1;
3229
+ return Math.max(raised, inSyncVersionOf(next));
3230
+ }
3231
+ function withInSyncVersions(previous, next) {
3232
+ let stamped = null;
3233
+ for (const [partitionId, assignment] of next.assignments) {
3234
+ const version = stampedInSyncVersion(previous?.assignments.get(partitionId), assignment);
3235
+ if (version === inSyncVersionOf(assignment)) continue;
3236
+ stamped ??= new Map(next.assignments);
3237
+ stamped.set(partitionId, { ...assignment, inSyncVersion: version });
3238
+ }
3239
+ return stamped === null ? next : { ...next, assignments: stamped };
3240
+ }
3241
+ function putAllocationOver(coordinator, indexName, previous, next, writerNodeId, replicaRegistration) {
3242
+ return coordinator.putAllocation(
3243
+ indexName,
3244
+ withInSyncVersions(previous, next),
3245
+ previous === null ? null : previous.version,
3246
+ writerNodeId,
3247
+ replicaRegistration
3248
+ );
3249
+ }
3250
+
3206
3251
  // src/distribution/cluster/table-identity.ts
3207
3252
  function belongsToAnotherIndex(table, indexUuid) {
3208
3253
  return table.indexUuid !== void 0 && indexUuid !== void 0 && table.indexUuid !== indexUuid;
@@ -3217,16 +3262,10 @@ async function rewriteForCurrentIndex(coordinator, table) {
3217
3262
  return stampedWith(table, indexUuid);
3218
3263
  }
3219
3264
 
3220
- // src/distribution/replication/constants.ts
3221
- var REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES = 40;
3222
- var DEFAULT_LOG_RETENTION_BYTES = 268435456;
3223
- var INSYNC_CAS_ATTEMPTS = 5;
3224
- var SNAPSHOT_CHUNK_SIZE = 65536;
3225
-
3226
3265
  // src/distribution/replication/insync.ts
3227
3266
  var REFUSE = { outcome: "refuse" };
3228
3267
  var ACCEPT = { outcome: "accept" };
3229
- async function updateInsyncSet(request, coordinator, controllerNodeId, decide) {
3268
+ async function updateInsyncSet(request, coordinator, controllerNodeId, decide, replicaRegistration) {
3230
3269
  const refused = {
3231
3270
  indexName: request.indexName,
3232
3271
  partitionId: request.partitionId,
@@ -3261,7 +3300,14 @@ async function updateInsyncSet(request, coordinator, controllerNodeId, decide) {
3261
3300
  if (rewritten === null) {
3262
3301
  return refused;
3263
3302
  }
3264
- const written = await coordinator.putAllocation(request.indexName, rewritten, table.version, controllerNodeId);
3303
+ const written = await putAllocationOver(
3304
+ coordinator,
3305
+ request.indexName,
3306
+ table,
3307
+ rewritten,
3308
+ controllerNodeId,
3309
+ replicaRegistration
3310
+ );
3265
3311
  if (written) {
3266
3312
  return { ...refused, accepted: true };
3267
3313
  }
@@ -3286,7 +3332,7 @@ function handleInsyncRemoval(payload, coordinator, controllerNodeId) {
3286
3332
  return REFUSE;
3287
3333
  }
3288
3334
  if (!assignment.inSyncSet.includes(payload.replicaNodeId)) {
3289
- return ACCEPT;
3335
+ return { outcome: "write", assignment: { ...assignment, inSyncVersion: inSyncVersionOf(assignment) + 1 } };
3290
3336
  }
3291
3337
  return {
3292
3338
  outcome: "write",
@@ -3305,16 +3351,21 @@ async function requestInsyncAdmission(payload, controllerNodeId, transport, sour
3305
3351
  }
3306
3352
  async function handleInsyncAdmission(payload, coordinator, controllerNodeId) {
3307
3353
  const registered = await coordinator.listNodes();
3308
- if (!registered.some((node) => node.nodeId === payload.replicaNodeId)) {
3354
+ const registration = registered.find((node) => node.nodeId === payload.replicaNodeId);
3355
+ if (registration === void 0 || registration.startedAt !== payload.replicaStartedAt) {
3309
3356
  return { indexName: payload.indexName, partitionId: payload.partitionId, accepted: false };
3310
3357
  }
3311
- return updateInsyncSet(payload, coordinator, controllerNodeId, (assignment) => {
3358
+ const replicaRegistration = { nodeId: payload.replicaNodeId, startedAt: payload.replicaStartedAt };
3359
+ const decide = (assignment) => {
3312
3360
  if (assignment.state !== "ACTIVE") {
3313
3361
  return REFUSE;
3314
3362
  }
3315
3363
  if (!assignment.replicas.includes(payload.replicaNodeId)) {
3316
3364
  return REFUSE;
3317
3365
  }
3366
+ if (payload.inSyncVersion !== inSyncVersionOf(assignment)) {
3367
+ return REFUSE;
3368
+ }
3318
3369
  if (payload.appliedSeqNo < assignment.commitPoint || payload.appliedSeqNo < payload.commitPoint) {
3319
3370
  return REFUSE;
3320
3371
  }
@@ -3331,7 +3382,8 @@ async function handleInsyncAdmission(payload, coordinator, controllerNodeId) {
3331
3382
  commitPoint
3332
3383
  }
3333
3384
  };
3334
- });
3385
+ };
3386
+ return updateInsyncSet(payload, coordinator, controllerNodeId, decide, replicaRegistration);
3335
3387
  }
3336
3388
 
3337
3389
  // src/distribution/replication/primary.ts
@@ -3344,7 +3396,7 @@ async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeI
3344
3396
  }
3345
3397
  async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3, indexUuid, previousPrimaryTerm) {
3346
3398
  if (entries.length === 0) {
3347
- return { acknowledged: [], failed: [] };
3399
+ return { acknowledged: [], failed: [], refusedAsInvalid: [] };
3348
3400
  }
3349
3401
  if (entries.length === 1) {
3350
3402
  return replicateToReplicas(
@@ -3364,13 +3416,14 @@ async function replicateBatchToReplicas(entries, inSyncReplicas, transport, sour
3364
3416
  async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport, resolveNodeTargets3) {
3365
3417
  const uniqueReplicas = [...new Set(inSyncReplicas)];
3366
3418
  if (uniqueReplicas.length === 0) {
3367
- return { acknowledged: [], failed: [] };
3419
+ return { acknowledged: [], failed: [], refusedAsInvalid: [] };
3368
3420
  }
3369
3421
  const sendResults = await Promise.allSettled(
3370
3422
  uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
3371
3423
  );
3372
3424
  const acknowledged = [];
3373
3425
  const failed2 = [];
3426
+ const refusedAsInvalid = [];
3374
3427
  for (let i = 0; i < sendResults.length; i++) {
3375
3428
  const result = sendResults[i];
3376
3429
  const replicaNodeId = uniqueReplicas[i];
@@ -3386,9 +3439,10 @@ async function sendAndCollectAcks(message, ackEntry, inSyncReplicas, transport,
3386
3439
  acknowledged.push(replicaNodeId);
3387
3440
  } else {
3388
3441
  failed2.push(replicaNodeId);
3442
+ if (refusesAsInvalid(response)) refusedAsInvalid.push(replicaNodeId);
3389
3443
  }
3390
3444
  }
3391
- return { acknowledged, failed: failed2 };
3445
+ return { acknowledged, failed: failed2, refusedAsInvalid };
3392
3446
  }
3393
3447
  async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
3394
3448
  const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
@@ -3406,6 +3460,15 @@ async function sendToReplica(transport, replicaNodeId, message, resolveNodeTarge
3406
3460
  }
3407
3461
  throw lastError instanceof Error ? lastError : new Error(String(lastError));
3408
3462
  }
3463
+ function refusesAsInvalid(response) {
3464
+ if (!response.type.endsWith(".error")) return false;
3465
+ try {
3466
+ const payload = decode(response.payload);
3467
+ return isRecord(payload) && payload.code === ErrorCodes.REPLICATION_ENTRY_INVALID;
3468
+ } catch (_) {
3469
+ return false;
3470
+ }
3471
+ }
3409
3472
  function isMatchingAck(response, entry) {
3410
3473
  if (response.type !== ReplicationMessageTypes.ACK) {
3411
3474
  return false;
@@ -3425,6 +3488,7 @@ function partitionKey(indexName, partitionId) {
3425
3488
  function createCatchUpState() {
3426
3489
  return {
3427
3490
  cursors: /* @__PURE__ */ new Map(),
3491
+ syncSessions: /* @__PURE__ */ new Map(),
3428
3492
  pendingAdmissions: /* @__PURE__ */ new Map(),
3429
3493
  admittedReplicas: /* @__PURE__ */ new Map(),
3430
3494
  admissionsToWithdraw: /* @__PURE__ */ new Map(),
@@ -3451,17 +3515,26 @@ function noteAcknowledgedPosition(state, key, replicaNodeId, appliedSeqNo) {
3451
3515
  acknowledged.set(replicaNodeId, appliedSeqNo);
3452
3516
  state.acknowledgedSeqNos.set(key, acknowledged);
3453
3517
  }
3454
- function recordReplicaPosition(state, indexName, partitionId, replicaNodeId, appliedSeqNo, logEnd) {
3518
+ function recordReplicaPosition(state, indexName, partitionId, replicaNodeId, appliedSeqNo, logEnd, confirmsPosition) {
3455
3519
  const key = partitionKey(indexName, partitionId);
3456
3520
  const reported = Math.min(appliedSeqNo, logEnd);
3457
3521
  const partition = state.cursors.get(key) ?? /* @__PURE__ */ new Map();
3458
3522
  const existing = partition.get(replicaNodeId);
3459
3523
  if (existing === void 0) {
3460
- partition.set(replicaNodeId, { appliedSeqNo: reported, busy: false, syncEpoch: 0, feed: null });
3461
- } else if (reported < existing.appliedSeqNo) {
3462
- openFreshSyncSession(existing, reported);
3524
+ partition.set(replicaNodeId, {
3525
+ appliedSeqNo: reported,
3526
+ confirmed: confirmsPosition,
3527
+ busy: false,
3528
+ syncEpoch: 0,
3529
+ feed: null
3530
+ });
3463
3531
  } else {
3464
- existing.appliedSeqNo = reported;
3532
+ if (!confirmsPosition || reported < existing.appliedSeqNo) {
3533
+ openFreshSyncSession(existing, reported);
3534
+ } else {
3535
+ existing.appliedSeqNo = reported;
3536
+ }
3537
+ existing.confirmed = confirmsPosition;
3465
3538
  }
3466
3539
  state.cursors.set(key, partition);
3467
3540
  noteAcknowledgedPosition(state, key, replicaNodeId, reported);
@@ -3486,7 +3559,10 @@ function forgetReplica(state, indexName, partitionId, replicaNodeId) {
3486
3559
  state.cursors.delete(key);
3487
3560
  }
3488
3561
  }
3489
- clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
3562
+ const pending = pendingAdmissionOf(state, indexName, partitionId, replicaNodeId);
3563
+ if (pending !== void 0 && pending.answer === null) {
3564
+ clearPendingAdmission(state, indexName, partitionId, replicaNodeId, pending);
3565
+ }
3490
3566
  }
3491
3567
  function forgetKeysOfIndex(entries, indexName) {
3492
3568
  for (const key of [...entries.keys()]) {
@@ -3499,6 +3575,7 @@ function forgetCatchUpIndex(state, indexName) {
3499
3575
  for (const cursor of replicas.values()) cursor.feed?.cancel();
3500
3576
  state.cursors.delete(key);
3501
3577
  }
3578
+ forgetKeysOfIndex(state.syncSessions, indexName);
3502
3579
  forgetKeysOfIndex(state.pendingAdmissions, indexName);
3503
3580
  forgetKeysOfIndex(state.admittedReplicas, indexName);
3504
3581
  forgetKeysOfIndex(state.admissionsToWithdraw, indexName);
@@ -3521,28 +3598,18 @@ function feedsReplica(state, indexName, partitionId, replicaNodeId) {
3521
3598
  }
3522
3599
  function getPendingAdmissions(state, indexName, partitionId) {
3523
3600
  const pending = state.pendingAdmissions.get(partitionKey(indexName, partitionId));
3524
- return pending === void 0 ? [] : [...pending];
3601
+ return pending === void 0 ? [] : [...pending.keys()];
3525
3602
  }
3526
- function recordAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId, primaryTerm) {
3527
- const key = partitionKey(indexName, partitionId);
3528
- const owed = state.admissionsToWithdraw.get(key) ?? /* @__PURE__ */ new Map();
3529
- owed.set(replicaNodeId, { primaryTerm, withdrawing: false });
3530
- state.admissionsToWithdraw.set(key, owed);
3531
- }
3532
- function forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId) {
3533
- const key = partitionKey(indexName, partitionId);
3534
- const owed = state.admissionsToWithdraw.get(key);
3535
- if (owed === void 0) return;
3536
- owed.delete(replicaNodeId);
3537
- if (owed.size === 0) state.admissionsToWithdraw.delete(key);
3603
+ function pendingAdmissionOf(state, indexName, partitionId, replicaNodeId) {
3604
+ return state.pendingAdmissions.get(partitionKey(indexName, partitionId))?.get(replicaNodeId);
3538
3605
  }
3539
3606
  function markPendingAdmission(state, indexName, partitionId, replicaNodeId) {
3540
3607
  const key = partitionKey(indexName, partitionId);
3541
- const pending = state.pendingAdmissions.get(key) ?? /* @__PURE__ */ new Set();
3608
+ const pending = state.pendingAdmissions.get(key) ?? /* @__PURE__ */ new Map();
3542
3609
  if (pending.has(replicaNodeId)) {
3543
3610
  return false;
3544
3611
  }
3545
- pending.add(replicaNodeId);
3612
+ pending.set(replicaNodeId, { failedWrite: false, unanswered: false, answer: null });
3546
3613
  state.pendingAdmissions.set(key, pending);
3547
3614
  forgetRemovedReplica(state, indexName, partitionId, replicaNodeId);
3548
3615
  advanceAdmissionGeneration(state, indexName, partitionId, replicaNodeId);
@@ -3570,21 +3637,10 @@ function forgetRemovedReplica(state, indexName, partitionId, replicaNodeId) {
3570
3637
  removed.delete(replicaNodeId);
3571
3638
  if (removed.size === 0) state.removedReplicas.delete(key);
3572
3639
  }
3573
- function recordRemovedReplica(state, indexName, partitionId, replicaNodeId, primaryTerm) {
3574
- const key = partitionKey(indexName, partitionId);
3575
- const removed = state.removedReplicas.get(key) ?? /* @__PURE__ */ new Map();
3576
- removed.set(replicaNodeId, primaryTerm);
3577
- state.removedReplicas.set(key, removed);
3578
- }
3579
- function withoutRemovedReplicas(state, indexName, partitionId, primaryTerm, targets) {
3580
- const removed = state.removedReplicas.get(partitionKey(indexName, partitionId));
3581
- if (removed === void 0) return targets;
3582
- return targets.filter((replicaNodeId) => removed.get(replicaNodeId) !== primaryTerm);
3583
- }
3584
- function clearPendingAdmission(state, indexName, partitionId, replicaNodeId) {
3640
+ function clearPendingAdmission(state, indexName, partitionId, replicaNodeId, claim) {
3585
3641
  const key = partitionKey(indexName, partitionId);
3586
3642
  const pending = state.pendingAdmissions.get(key);
3587
- if (pending === void 0) {
3643
+ if (pending === void 0 || claim !== void 0 && pending.get(replicaNodeId) !== claim) {
3588
3644
  return;
3589
3645
  }
3590
3646
  pending.delete(replicaNodeId);
@@ -3618,6 +3674,47 @@ function keepAcknowledgementsOf(state, ledPartitionKeys) {
3618
3674
  }
3619
3675
 
3620
3676
  // src/distribution/cluster-node/catch-up/admitted.ts
3677
+ function admissionToWithdrawOf(state, indexName, partitionId, replicaNodeId) {
3678
+ return state.admissionsToWithdraw.get(partitionKey(indexName, partitionId))?.get(replicaNodeId);
3679
+ }
3680
+ function withUnansweredAdmissions(state, indexName, partitionId, localNodeId, targets) {
3681
+ const pending = state.pendingAdmissions.get(partitionKey(indexName, partitionId));
3682
+ if (pending === void 0) return targets;
3683
+ const unanswered = [...pending].filter(([replicaNodeId, admission]) => admission.unanswered && replicaNodeId !== localNodeId).map(([replicaNodeId]) => replicaNodeId);
3684
+ return [.../* @__PURE__ */ new Set([...targets, ...unanswered])];
3685
+ }
3686
+ function settleFencedAdmission(state, indexName, partitionId, replicaNodeId, fenced) {
3687
+ if (fenced.pending?.unanswered === true) {
3688
+ clearPendingAdmission(state, indexName, partitionId, replicaNodeId, fenced.pending);
3689
+ }
3690
+ if (fenced.owed !== void 0 && admissionToWithdrawOf(state, indexName, partitionId, replicaNodeId) === fenced.owed) {
3691
+ forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId);
3692
+ }
3693
+ }
3694
+ function recordAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId, primaryTerm) {
3695
+ const key = partitionKey(indexName, partitionId);
3696
+ const owed = state.admissionsToWithdraw.get(key) ?? /* @__PURE__ */ new Map();
3697
+ owed.set(replicaNodeId, { primaryTerm, withdrawing: false });
3698
+ state.admissionsToWithdraw.set(key, owed);
3699
+ }
3700
+ function forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId) {
3701
+ const key = partitionKey(indexName, partitionId);
3702
+ const owed = state.admissionsToWithdraw.get(key);
3703
+ if (owed === void 0) return;
3704
+ owed.delete(replicaNodeId);
3705
+ if (owed.size === 0) state.admissionsToWithdraw.delete(key);
3706
+ }
3707
+ function recordRemovedReplica(state, indexName, partitionId, replicaNodeId, primaryTerm) {
3708
+ const key = partitionKey(indexName, partitionId);
3709
+ const removed = state.removedReplicas.get(key) ?? /* @__PURE__ */ new Map();
3710
+ removed.set(replicaNodeId, primaryTerm);
3711
+ state.removedReplicas.set(key, removed);
3712
+ }
3713
+ function withoutRemovedReplicas(state, indexName, partitionId, primaryTerm, targets) {
3714
+ const removed = state.removedReplicas.get(partitionKey(indexName, partitionId));
3715
+ if (removed === void 0) return targets;
3716
+ return targets.filter((replicaNodeId) => removed.get(replicaNodeId) !== primaryTerm);
3717
+ }
3621
3718
  function recordAdmittedReplica(state, indexName, partitionId, replicaNodeId, primaryTerm, generation) {
3622
3719
  const key = partitionKey(indexName, partitionId);
3623
3720
  const admitted = state.admittedReplicas.get(key) ?? /* @__PURE__ */ new Map();
@@ -3652,6 +3749,371 @@ function admittedBeyondTable(state, indexName, partitionId, assignment, generati
3652
3749
  return beyond;
3653
3750
  }
3654
3751
 
3752
+ // src/distribution/cluster-node/catch-up/feed.ts
3753
+ function inSyncSignature(assignment) {
3754
+ return [...assignment.inSyncSet].sort().join("\n");
3755
+ }
3756
+ function createCatchUpFeed(assignment, replicaStartedAt) {
3757
+ let release = () => {
3758
+ };
3759
+ const whenCancelled = new Promise((resolve) => {
3760
+ release = resolve;
3761
+ });
3762
+ const feed = {
3763
+ primaryTerm: assignment.primaryTerm,
3764
+ inSyncSet: inSyncSignature(assignment),
3765
+ replicaStartedAt,
3766
+ whenCancelled,
3767
+ cancelled: false,
3768
+ cancel: () => {
3769
+ feed.cancelled = true;
3770
+ release();
3771
+ }
3772
+ };
3773
+ return feed;
3774
+ }
3775
+ function feedIsStale(feed, assignment, registered, replicaNodeId) {
3776
+ const replicaLeft = registered !== null && registered.get(replicaNodeId) !== feed.replicaStartedAt;
3777
+ return replicaLeft || assignment.primaryTerm !== feed.primaryTerm || inSyncSignature(assignment) !== feed.inSyncSet;
3778
+ }
3779
+ function untilCancelled(feed, work, fallback) {
3780
+ if (feed.cancelled) {
3781
+ work.catch(() => {
3782
+ });
3783
+ return Promise.resolve(fallback);
3784
+ }
3785
+ return Promise.race([work, feed.whenCancelled.then(() => fallback)]);
3786
+ }
3787
+
3788
+ // src/distribution/cluster-node/catch-up/batches.ts
3789
+ function batchBytes(entries) {
3790
+ let total = 0;
3791
+ for (const entry of entries) {
3792
+ total += entryByteEstimate(entry);
3793
+ }
3794
+ return total;
3795
+ }
3796
+ function nextWireBatch(state, pending) {
3797
+ const chunks = chunkByBudget(pending, {
3798
+ ...WIRE_BATCH_BUDGET,
3799
+ payloadBytesOf: (entry) => entry.document?.byteLength ?? 0,
3800
+ breaksRun: (entry, previous) => entry.seqNo !== previous.seqNo + 1
3801
+ });
3802
+ const batch = chunks[0] ?? [];
3803
+ if (batch.length === 0) {
3804
+ return [];
3805
+ }
3806
+ const available = MAX_CATCH_UP_IN_FLIGHT_BYTES - state.inFlightBytes;
3807
+ return batchBytes(batch) > available ? [] : batch;
3808
+ }
3809
+ async function sendBatch(state, deps, batch, replicaNodeId, cursor, feed) {
3810
+ const reservedBytes = batchBytes(batch);
3811
+ const sendingEpoch = cursor.syncEpoch;
3812
+ state.inFlightBytes += reservedBytes;
3813
+ const delivery = (async () => {
3814
+ const first = batch[0];
3815
+ const indexName = first?.indexName;
3816
+ const table = indexName === void 0 ? null : await deps.coordinator.getAllocation(indexName);
3817
+ if (feed.cancelled) {
3818
+ return false;
3819
+ }
3820
+ const result = await replicateBatchToReplicas(
3821
+ batch,
3822
+ [replicaNodeId],
3823
+ deps.transport,
3824
+ deps.nodeId,
3825
+ deps.resolveNodeTargets,
3826
+ table?.indexUuid,
3827
+ first === void 0 ? void 0 : previousTermOf(deps.getReplicationLog(first.indexName, first.partitionId), first.seqNo)
3828
+ );
3829
+ return result.acknowledged.includes(replicaNodeId);
3830
+ })().finally(() => {
3831
+ state.inFlightBytes -= reservedBytes;
3832
+ });
3833
+ const acknowledged = await untilCancelled(feed, delivery, false);
3834
+ if (!acknowledged || feed.cancelled || cursor.syncEpoch !== sendingEpoch) {
3835
+ return false;
3836
+ }
3837
+ cursor.appliedSeqNo = Math.max(cursor.appliedSeqNo, batch[batch.length - 1].seqNo);
3838
+ cursor.confirmed = true;
3839
+ return true;
3840
+ }
3841
+ async function sendHeldEntryAgain(state, deps, entry, replicaNodeId) {
3842
+ const reservedBytes = entryByteEstimate(entry);
3843
+ if (reservedBytes > MAX_CATCH_UP_IN_FLIGHT_BYTES - state.inFlightBytes) return "unconfirmed";
3844
+ state.inFlightBytes += reservedBytes;
3845
+ try {
3846
+ const table = await deps.coordinator.getAllocation(entry.indexName);
3847
+ const result = await replicateBatchToReplicas(
3848
+ [entry],
3849
+ [replicaNodeId],
3850
+ deps.transport,
3851
+ deps.nodeId,
3852
+ deps.resolveNodeTargets,
3853
+ table?.indexUuid,
3854
+ previousTermOf(deps.getReplicationLog(entry.indexName, entry.partitionId), entry.seqNo)
3855
+ );
3856
+ if (result.acknowledged.includes(replicaNodeId)) return "acknowledged";
3857
+ return result.refusedAsInvalid.includes(replicaNodeId) ? "refused-as-invalid" : "unconfirmed";
3858
+ } finally {
3859
+ state.inFlightBytes -= reservedBytes;
3860
+ }
3861
+ }
3862
+ function validatePartitionStoresPayload(decoded) {
3863
+ if (!isRecord(decoded)) {
3864
+ return null;
3865
+ }
3866
+ if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
3867
+ return null;
3868
+ }
3869
+ return { indexName: decoded.indexName };
3870
+ }
3871
+ function validatePartitionStoresResultPayload(decoded) {
3872
+ if (!isRecord(decoded)) {
3873
+ return null;
3874
+ }
3875
+ if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
3876
+ return null;
3877
+ }
3878
+ if (decoded.indexUuid !== null && typeof decoded.indexUuid !== "string") {
3879
+ return null;
3880
+ }
3881
+ if (!Array.isArray(decoded.partitionIds)) {
3882
+ return null;
3883
+ }
3884
+ const partitionIds = [];
3885
+ for (const partitionId of decoded.partitionIds) {
3886
+ if (typeof partitionId !== "number" || !Number.isInteger(partitionId) || partitionId < 0) {
3887
+ return null;
3888
+ }
3889
+ partitionIds.push(partitionId);
3890
+ }
3891
+ return { indexName: decoded.indexName, indexUuid: decoded.indexUuid, partitionIds };
3892
+ }
3893
+ function createPartitionStoresMessage(payload, sourceNodeId, requestId) {
3894
+ return {
3895
+ type: ClusterMessageTypes.PARTITION_STORES,
3896
+ sourceId: sourceNodeId,
3897
+ requestId,
3898
+ payload: encode(payload)
3899
+ };
3900
+ }
3901
+ function createPartitionStoresResultMessage(result, sourceNodeId, requestId) {
3902
+ return {
3903
+ type: ClusterMessageTypes.PARTITION_STORES,
3904
+ sourceId: sourceNodeId,
3905
+ requestId,
3906
+ payload: encode(result)
3907
+ };
3908
+ }
3909
+
3910
+ // src/distribution/cluster-node/catch-up/listed-copy.ts
3911
+ function answerOf(payload) {
3912
+ try {
3913
+ return validatePartitionStoresResultPayload(decode(payload));
3914
+ } catch (_) {
3915
+ return null;
3916
+ }
3917
+ }
3918
+ async function replicaListsPartition(deps, indexName, partitionId, replicaNodeId) {
3919
+ const table = await deps.coordinator.getAllocation(indexName);
3920
+ const message = createPartitionStoresMessage({ indexName }, deps.nodeId, generateId());
3921
+ for (const target of await resolveNodeTargets(replicaNodeId, deps)) {
3922
+ let payload;
3923
+ try {
3924
+ payload = (await deps.transport.send(target, message)).payload;
3925
+ } catch (_) {
3926
+ continue;
3927
+ }
3928
+ const answer2 = answerOf(payload);
3929
+ if (answer2 === null || answer2.indexName !== indexName || answer2.indexUuid === null) return false;
3930
+ if (table?.indexUuid !== void 0 && answer2.indexUuid !== table.indexUuid) return false;
3931
+ return answer2.partitionIds.includes(partitionId);
3932
+ }
3933
+ return false;
3934
+ }
3935
+
3936
+ // src/distribution/cluster-node/catch-up/admission.ts
3937
+ async function sendToController(payload, controllerNodeId, deps) {
3938
+ const targets = await resolveNodeTargets(controllerNodeId, deps);
3939
+ let sentWithoutAnswer = false;
3940
+ for (const target of targets) {
3941
+ try {
3942
+ const result = await requestInsyncAdmission(payload, target, deps.transport, deps.nodeId);
3943
+ if (result.accepted) return "accepted";
3944
+ return sentWithoutAnswer ? "unanswered" : "refused";
3945
+ } catch (_) {
3946
+ sentWithoutAnswer = true;
3947
+ }
3948
+ }
3949
+ return "unanswered";
3950
+ }
3951
+ function withTimeout(work, timeoutMs, fallback) {
3952
+ return new Promise((resolve) => {
3953
+ const timer = setTimeout(() => resolve(fallback), timeoutMs);
3954
+ timer.unref?.();
3955
+ work.then((value) => {
3956
+ clearTimeout(timer);
3957
+ resolve(value);
3958
+ }).catch(() => {
3959
+ clearTimeout(timer);
3960
+ resolve(fallback);
3961
+ });
3962
+ });
3963
+ }
3964
+ async function withdrawAdmission(removal, controllerNodeId, deps) {
3965
+ const targets = await resolveNodeTargets(controllerNodeId, deps);
3966
+ for (const target of targets) {
3967
+ try {
3968
+ const result = await requestInsyncRemoval(
3969
+ removal.indexName,
3970
+ removal.partitionId,
3971
+ removal.replicaNodeId,
3972
+ removal.primaryTerm,
3973
+ target,
3974
+ deps.transport,
3975
+ deps.nodeId
3976
+ );
3977
+ if (result.accepted) {
3978
+ return true;
3979
+ }
3980
+ } catch (_) {
3981
+ }
3982
+ }
3983
+ return false;
3984
+ }
3985
+ async function registeredStartedAt(deps, replicaNodeId) {
3986
+ try {
3987
+ const registered = await deps.coordinator.listNodes();
3988
+ return registered.find((node) => node.nodeId === replicaNodeId)?.startedAt ?? null;
3989
+ } catch (_) {
3990
+ return null;
3991
+ }
3992
+ }
3993
+ function claimHolds(claim, payload) {
3994
+ const current = pendingAdmissionOf(claim.state, payload.indexName, payload.partitionId, payload.replicaNodeId);
3995
+ return current === claim.pending && !claim.pending.failedWrite && claim.cursor.syncEpoch === claim.proposedEpoch;
3996
+ }
3997
+ async function changedInSyncVersion(claim, payload) {
3998
+ const { deps } = claim;
3999
+ let assignment;
4000
+ try {
4001
+ const table = await (deps.readLiveAllocation?.(payload.indexName) ?? deps.coordinator.getAllocation(payload.indexName));
4002
+ assignment = table?.assignments.get(payload.partitionId);
4003
+ } catch (_) {
4004
+ return null;
4005
+ }
4006
+ if (assignment === void 0) return null;
4007
+ const inSyncVersion = inSyncVersionOf(assignment);
4008
+ const qualifies = inSyncVersion !== payload.inSyncVersion && assignment.primary === deps.nodeId && assignment.primaryTerm === payload.primaryTerm && assignment.state === "ACTIVE" && assignment.replicas.includes(payload.replicaNodeId) && !assignment.inSyncSet.includes(payload.replicaNodeId) && payload.appliedSeqNo >= assignment.commitPoint;
4009
+ return qualifies ? inSyncVersion : null;
4010
+ }
4011
+ async function provePositionAfterRead(claim, request) {
4012
+ const { state, cursor, deps } = claim;
4013
+ if (request.appliedSeqNo === 0) return true;
4014
+ const entry = deps.getReplicationLog(request.indexName, request.partitionId).getEntry(request.appliedSeqNo);
4015
+ const provingEpoch = cursor.syncEpoch;
4016
+ if (entry === void 0) {
4017
+ const listed = await replicaListsPartition(
4018
+ deps,
4019
+ request.indexName,
4020
+ request.partitionId,
4021
+ request.replicaNodeId
4022
+ ).catch(() => false);
4023
+ return listed && cursor.syncEpoch === provingEpoch;
4024
+ }
4025
+ const answer2 = await sendHeldEntryAgain(state, deps, entry, request.replicaNodeId).catch(
4026
+ () => "unconfirmed"
4027
+ );
4028
+ if (answer2 === "refused-as-invalid" && cursor.syncEpoch === provingEpoch) {
4029
+ forgetReplica(state, request.indexName, request.partitionId, request.replicaNodeId);
4030
+ }
4031
+ return answer2 === "acknowledged" && cursor.syncEpoch === provingEpoch;
4032
+ }
4033
+ async function askController(claim, payload, controllerNodeId) {
4034
+ let request = payload;
4035
+ let retries = 0;
4036
+ while (claimHolds(claim, request)) {
4037
+ const replicaStartedAt = await registeredStartedAt(claim.deps, request.replicaNodeId);
4038
+ if (replicaStartedAt === null) return "refused";
4039
+ if (!await provePositionAfterRead(claim, request) || !claimHolds(claim, request)) return "refused";
4040
+ const answer2 = await withTimeout(
4041
+ sendToController({ ...request, replicaStartedAt }, controllerNodeId, claim.deps),
4042
+ ADMISSION_TIMEOUT_MS,
4043
+ "unanswered"
4044
+ );
4045
+ if (answer2 !== "refused" || retries >= ADMISSION_RETRIES_AFTER_IN_SYNC_CHANGE) return answer2;
4046
+ const inSyncVersion = await changedInSyncVersion(claim, request);
4047
+ if (inSyncVersion === null) return answer2;
4048
+ request = { ...request, inSyncVersion };
4049
+ retries += 1;
4050
+ }
4051
+ return "refused";
4052
+ }
4053
+ async function proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
4054
+ if (claimAdmission(state, deps, indexName, partitionId, replicaNodeId, cursor)) {
4055
+ await requestClaimedAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
4056
+ }
4057
+ }
4058
+ function claimAdmission(state, deps, indexName, partitionId, replicaNodeId, cursor) {
4059
+ const log = deps.getReplicationLog(indexName, partitionId);
4060
+ if (!cursor.confirmed || cursor.appliedSeqNo < log.commitPoint) {
4061
+ return false;
4062
+ }
4063
+ return markPendingAdmission(state, indexName, partitionId, replicaNodeId);
4064
+ }
4065
+ async function requestClaimedAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
4066
+ const pending = pendingAdmissionOf(state, indexName, partitionId, replicaNodeId);
4067
+ if (pending === void 0) return;
4068
+ let settle = () => {
4069
+ };
4070
+ pending.answer = new Promise((resolve) => {
4071
+ settle = resolve;
4072
+ });
4073
+ const claim = { state, deps, cursor, proposedEpoch: cursor.syncEpoch, pending };
4074
+ try {
4075
+ const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
4076
+ if (controllerNodeId === null) {
4077
+ return;
4078
+ }
4079
+ const payload = {
4080
+ indexName,
4081
+ partitionId,
4082
+ replicaNodeId,
4083
+ primaryTerm: assignment.primaryTerm,
4084
+ appliedSeqNo: cursor.appliedSeqNo,
4085
+ commitPoint: deps.getReplicationLog(indexName, partitionId).commitPoint,
4086
+ inSyncVersion: inSyncVersionOf(assignment)
4087
+ };
4088
+ const answer2 = await askController(claim, payload, controllerNodeId);
4089
+ pending.unanswered = answer2 === "unanswered";
4090
+ settle(answer2);
4091
+ if (answer2 === "accepted" && pending.failedWrite) {
4092
+ recordAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId, assignment.primaryTerm);
4093
+ return;
4094
+ }
4095
+ const outlived = answer2 === "accepted" && cursor.syncEpoch !== claim.proposedEpoch;
4096
+ const withdrawn = outlived && await withTimeout(withdrawAdmission(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
4097
+ if (answer2 === "unanswered" || outlived && !withdrawn) {
4098
+ recordAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId, assignment.primaryTerm);
4099
+ } else if (answer2 === "accepted" && !outlived) {
4100
+ forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId);
4101
+ forgetRemovedReplica(state, indexName, partitionId, replicaNodeId);
4102
+ recordAdmittedReplica(
4103
+ state,
4104
+ indexName,
4105
+ partitionId,
4106
+ replicaNodeId,
4107
+ assignment.primaryTerm,
4108
+ advanceAdmissionGeneration(state, indexName, partitionId, replicaNodeId)
4109
+ );
4110
+ }
4111
+ } finally {
4112
+ settle("refused");
4113
+ if (!pending.unanswered) clearPendingAdmission(state, indexName, partitionId, replicaNodeId, pending);
4114
+ }
4115
+ }
4116
+
3655
4117
  // src/distribution/cluster-node/write-routing/replication.ts
3656
4118
  async function assertPrimaryWriteAuthority(entry, deps) {
3657
4119
  const table = await deps.coordinator.getAllocation(entry.indexName);
@@ -3765,7 +4227,13 @@ function replicaTargetsOf(entry, assignment, deps) {
3765
4227
  entry.indexName,
3766
4228
  entry.partitionId,
3767
4229
  entry.primaryTerm,
3768
- getInSyncReplicaTargets(assignment, deps.nodeId, [...pending, ...admitted])
4230
+ withUnansweredAdmissions(
4231
+ deps.catchUp,
4232
+ entry.indexName,
4233
+ entry.partitionId,
4234
+ deps.nodeId,
4235
+ getInSyncReplicaTargets(assignment, deps.nodeId, [...pending, ...admitted])
4236
+ )
3769
4237
  );
3770
4238
  return { admitted, targets };
3771
4239
  }
@@ -3815,21 +4283,70 @@ async function replicateEntry(entry, assignment, deps, indexUuid) {
3815
4283
  )
3816
4284
  );
3817
4285
  recordAcknowledgements(deps.catchUp, entry.indexName, entry.partitionId, result.acknowledged, entry.seqNo);
3818
- const inSyncFailures = cancelAdmissionsForFailures(entry, result.failed, assignment, admitted, deps);
3819
- await removeFailedReplicasFromInsync(entry, inSyncFailures, deps);
4286
+ await removeInSyncFailures(entry, result.failed, assignment, admitted, deps);
3820
4287
  await assertPrimaryWriteAuthority(entry, deps);
3821
4288
  deps.getReplicationLog(entry.indexName, entry.partitionId).advanceCommitPoint(entry.seqNo);
3822
4289
  }
3823
- function cancelAdmissionsForFailures(entry, failedReplicas, assignment, admitted, deps) {
4290
+ async function inSyncFailuresOf(entry, failedReplicas, assignment, admitted, deps) {
3824
4291
  const inSyncFailures = [];
4292
+ const fenced = [];
4293
+ const answers = [];
4294
+ const admittedNow = admittedBeyondTable(deps.catchUp, entry.indexName, entry.partitionId, assignment);
3825
4295
  for (const replicaNodeId of failedReplicas) {
3826
- if (assignment.inSyncSet.includes(replicaNodeId) || admitted.includes(replicaNodeId)) {
4296
+ if (assignment.inSyncSet.includes(replicaNodeId) || admitted.includes(replicaNodeId) || admittedNow.includes(replicaNodeId)) {
4297
+ inSyncFailures.push(replicaNodeId);
4298
+ continue;
4299
+ }
4300
+ const pending = pendingAdmissionOf(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId);
4301
+ if (pending === void 0) continue;
4302
+ if (pending.unanswered) {
3827
4303
  inSyncFailures.push(replicaNodeId);
4304
+ const owed = admissionToWithdrawOf(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId);
4305
+ fenced.push({ replicaNodeId, pending, owed });
3828
4306
  continue;
3829
4307
  }
3830
- clearPendingAdmission(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId);
4308
+ pending.failedWrite = true;
4309
+ if (pending.answer === null) {
4310
+ clearPendingAdmission(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId, pending);
4311
+ } else {
4312
+ answers.push({ replicaNodeId, answer: pending.answer });
4313
+ }
3831
4314
  }
3832
- return inSyncFailures;
4315
+ const timeoutMs = replicationTimeoutOf(deps);
4316
+ const settled = await Promise.all(
4317
+ answers.map(async ({ replicaNodeId, answer: answer2 }) => ({
4318
+ replicaNodeId,
4319
+ answer: await withTimeout(answer2, timeoutMs, "unanswered")
4320
+ }))
4321
+ );
4322
+ const unanswered = [];
4323
+ for (const { replicaNodeId, answer: answer2 } of settled) {
4324
+ if (answer2 === "accepted") inSyncFailures.push(replicaNodeId);
4325
+ if (answer2 === "unanswered") unanswered.push(replicaNodeId);
4326
+ }
4327
+ return { inSyncFailures, unanswered, fenced };
4328
+ }
4329
+ async function removeInSyncFailures(entry, failedReplicas, assignment, admitted, deps) {
4330
+ const { inSyncFailures, unanswered, fenced } = await inSyncFailuresOf(
4331
+ entry,
4332
+ failedReplicas,
4333
+ assignment,
4334
+ admitted,
4335
+ deps
4336
+ );
4337
+ await removeFailedReplicasFromInsync(entry, inSyncFailures, deps);
4338
+ for (const { replicaNodeId, pending, owed } of fenced) {
4339
+ settleFencedAdmission(deps.catchUp, entry.indexName, entry.partitionId, replicaNodeId, { pending, owed });
4340
+ }
4341
+ if (unanswered.length === 0) return;
4342
+ await deps.refreshAllocation?.(entry.indexName);
4343
+ await assertPrimaryWriteAuthority(entry, deps);
4344
+ const timeoutMs = replicationTimeoutOf(deps);
4345
+ throw new NarsilError(
4346
+ ErrorCodes.REPLICATION_INSYNC_REMOVAL_FAILED,
4347
+ `The controller did not answer the admission of ${unanswered.map((nodeId) => `'${nodeId}'`).join(", ")} to index '${entry.indexName}' partition ${entry.partitionId} within ${timeoutMs}ms after a failed write, so the primary cannot tell whether to remove it`,
4348
+ { indexName: entry.indexName, partitionId: entry.partitionId, unansweredReplicas: unanswered, timeoutMs }
4349
+ );
3833
4350
  }
3834
4351
  async function replicateEntryBatch(entries, assignment, deps, indexUuid) {
3835
4352
  if (entries.length === 0) {
@@ -3852,8 +4369,7 @@ async function replicateEntryBatch(entries, assignment, deps, indexUuid) {
3852
4369
  )
3853
4370
  );
3854
4371
  recordAcknowledgements(deps.catchUp, lastEntry.indexName, lastEntry.partitionId, result.acknowledged, lastEntry.seqNo);
3855
- const inSyncFailures = cancelAdmissionsForFailures(lastEntry, result.failed, assignment, admitted, deps);
3856
- await removeFailedReplicasFromInsync(lastEntry, inSyncFailures, deps);
4372
+ await removeInSyncFailures(lastEntry, result.failed, assignment, admitted, deps);
3857
4373
  await assertPrimaryWriteAuthority(lastEntry, deps);
3858
4374
  deps.getReplicationLog(lastEntry.indexName, lastEntry.partitionId).advanceCommitPoint(lastEntry.seqNo);
3859
4375
  }
@@ -5304,15 +5820,28 @@ function processRoomIn(figures, fractions) {
5304
5820
  }
5305
5821
  function createWriteProjection(options) {
5306
5822
  const bounds = /* @__PURE__ */ new Map();
5823
+ const pendingDocuments = /* @__PURE__ */ new Map();
5307
5824
  const { refusal: refusal2 } = options;
5825
+ function settlePending(indexName, count2) {
5826
+ const remaining = (pendingDocuments.get(indexName) ?? 0) - count2;
5827
+ if (remaining > 0) pendingDocuments.set(indexName, remaining);
5828
+ else pendingDocuments.delete(indexName);
5829
+ }
5308
5830
  function fittingCount(indexName, figures, groupBounds, start, mostDocuments, countExactly) {
5309
5831
  const end = Math.min(groupBounds.length, start + mostDocuments);
5310
5832
  if (figures === null) return end - start;
5311
5833
  const reserved = reservedHeapBytes();
5312
5834
  const heapRoom = heapRoomIn(figures, options.fractions) - reserved;
5313
5835
  const processRoom = processRoomIn(figures, options.fractions) - reserved;
5314
- const mostGrowth = options.vectorGrowthBytesOf(indexName, end - start);
5315
- const fits = (next, count3) => next <= heapRoom && (next + mostGrowth <= processRoom || next + options.vectorGrowthBytesOf(indexName, count3 + 1) <= processRoom);
5836
+ const pending = pendingDocuments.get(indexName) ?? 0;
5837
+ const mostTableGrowth = options.documentTableGrowthBytesOf(indexName, end - start, pending);
5838
+ const mostVectorGrowth = options.vectorGrowthBytesOf(indexName, end - start);
5839
+ const fits = (next, count3) => {
5840
+ const most = next + mostTableGrowth;
5841
+ if (most <= heapRoom && most + mostVectorGrowth <= processRoom) return true;
5842
+ const withTables = next + options.documentTableGrowthBytesOf(indexName, count3 + 1, pending);
5843
+ return withTables <= heapRoom && withTables + options.vectorGrowthBytesOf(indexName, count3 + 1) <= processRoom;
5844
+ };
5316
5845
  let taken = 0;
5317
5846
  let count2 = 0;
5318
5847
  while (start + count2 < end) {
@@ -5327,10 +5856,22 @@ function createWriteProjection(options) {
5327
5856
  return count2;
5328
5857
  }
5329
5858
  function reserve(indexName, groupBounds, start, count2) {
5330
- let bytes = count2 === 0 ? 0 : options.vectorGrowthBytesOf(indexName, count2);
5859
+ if (count2 === 0) return UNPROJECTED;
5860
+ const pending = pendingDocuments.get(indexName) ?? 0;
5861
+ let bytes = options.vectorGrowthBytesOf(indexName, count2) + options.documentTableGrowthBytesOf(indexName, count2, pending);
5331
5862
  for (let index = start; index < start + count2; index++) bytes += groupBounds[index];
5332
5863
  const reservation = reserveHeapBytes(bytes);
5333
- return { count: count2, release: () => reservation.release() };
5864
+ pendingDocuments.set(indexName, pending + count2);
5865
+ let released = false;
5866
+ return {
5867
+ count: count2,
5868
+ release: () => {
5869
+ if (released) return;
5870
+ released = true;
5871
+ reservation.release();
5872
+ settlePending(indexName, count2);
5873
+ }
5874
+ };
5334
5875
  }
5335
5876
  function blockBytesOf(figures) {
5336
5877
  return Math.floor(
@@ -6955,7 +7496,7 @@ async function runBackupCreation(context, request) {
6955
7496
  const indexName = jobIndexNameOf(context);
6956
7497
  const generation = await servingNameOf(coordinator, indexName);
6957
7498
  const named = (await readIndexMetadataSnapshot(coordinator, indexName))?.settings;
6958
- const syncEpoch = named?.sync?.enabled === true ? named.syncEpoch : void 0;
7499
+ const syncEpoch = epochFeaturesOn(named) ? named?.syncEpoch : void 0;
6959
7500
  const metadata = await requireIndexMetadata(context, indexName, generation);
6960
7501
  const clusterUuid = await clusterUuidOf(coordinator);
6961
7502
  await reportJobProgress(context, 0, metadata.partitionCount);
@@ -7255,9 +7796,9 @@ async function runBackupRestore(context, request) {
7255
7796
  }
7256
7797
 
7257
7798
  // src/distribution/backups/sync-retention.ts
7258
- async function surplusSyncBackups(store, indexName) {
7799
+ async function surplusTakenBackups(store, indexName, takenBy) {
7259
7800
  const taken = (await readEveryReadableManifest(store)).filter(
7260
- (manifest) => manifest.indexName === indexName && manifest.takenBy === SYNC_BACKUP_TAKER
7801
+ (manifest) => manifest.indexName === indexName && manifest.takenBy === takenBy
7261
7802
  );
7262
7803
  taken.sort((a, b) => b.completedAt - a.completedAt || b.startedAt - a.startedAt);
7263
7804
  return taken.slice(SYNC_BACKUPS_KEPT).map((manifest) => manifest.name);
@@ -7337,8 +7878,10 @@ async function startJob(deps, store, job, run, afterJob) {
7337
7878
  }
7338
7879
  function takenByOf(request) {
7339
7880
  if (request.takenBy === void 0) return void 0;
7340
- if (request.takenBy !== SYNC_BACKUP_TAKER) throw malformed3("backup", `"takenBy" must be "${SYNC_BACKUP_TAKER}"`);
7341
- return SYNC_BACKUP_TAKER;
7881
+ if (!isBackupTaker(request.takenBy)) {
7882
+ throw malformed3("backup", `"takenBy" must be "${SYNC_BACKUP_TAKER}" or "${CHANGES_BACKUP_TAKER}"`);
7883
+ }
7884
+ return request.takenBy;
7342
7885
  }
7343
7886
  function queryHistoryOf(request) {
7344
7887
  if (request.queryHistory === void 0) return false;
@@ -7348,8 +7891,8 @@ function queryHistoryOf(request) {
7348
7891
  function quietRefusal(error) {
7349
7892
  return error instanceof NarsilError && (error.code === ErrorCodes.BACKUP_IN_PROGRESS || error.code === ErrorCodes.BACKUP_NOT_FOUND);
7350
7893
  }
7351
- async function deleteSurplusSyncBackups(deps, store, indexName) {
7352
- for (const backupName of await surplusSyncBackups(store, indexName)) {
7894
+ async function deleteSurplusTakenBackups(deps, store, indexName, takenBy) {
7895
+ for (const backupName of await surplusTakenBackups(store, indexName, takenBy)) {
7353
7896
  if (!deps.isActive()) return;
7354
7897
  try {
7355
7898
  await admitDeletion(deps.config.coordinator, store, backupName);
@@ -7385,7 +7928,7 @@ async function acceptRequest(deps, kind, message) {
7385
7928
  const takenBy = takenByOf(request);
7386
7929
  const queryHistory = queryHistoryOf(request);
7387
7930
  await admitBackup(coordinator, store, backupName, indexName);
7388
- const afterJob = takenBy === void 0 ? void 0 : () => deleteSurplusSyncBackups(deps, store, indexName);
7931
+ const afterJob = takenBy === void 0 ? void 0 : () => deleteSurplusTakenBackups(deps, store, indexName, takenBy);
7389
7932
  return jobIdOf(
7390
7933
  startJob(
7391
7934
  deps,
@@ -7915,8 +8458,8 @@ async function runJob(deps, request, options) {
7915
8458
  function requestBackup(deps, indexName, backupName, options) {
7916
8459
  return backUp(deps, indexName, backupName, options?.queryHistory === true ? { queryHistory: true } : {}, options);
7917
8460
  }
7918
- async function requestSyncBackup(deps, indexName, backupName) {
7919
- await backUp(deps, indexName, backupName, { takenBy: SYNC_BACKUP_TAKER });
8461
+ async function requestTakenBackup(deps, indexName, backupName, takenBy) {
8462
+ await backUp(deps, indexName, backupName, { takenBy });
7920
8463
  }
7921
8464
  async function backUp(deps, indexName, backupName, extra, options) {
7922
8465
  const store = requireBackupStore(deps.store);
@@ -8228,7 +8771,7 @@ function createAnalysisView(deps) {
8228
8771
  answerAt(indexName, requested, read) {
8229
8772
  const generation = engine.analysisGenerationOf(indexName);
8230
8773
  if (generation === void 0) return read();
8231
- const readAtRequested = (answerOf2) => answerOf2().then((answer2) => {
8774
+ const readAtRequested = (answerOf3) => answerOf3().then((answer2) => {
8232
8775
  const after = engine.analysisGenerationOf(indexName) ?? requested;
8233
8776
  if (after !== requested) throw generationMissing(indexName, requested, after);
8234
8777
  return answer2;
@@ -8681,6 +9224,11 @@ function applyStartPayload(state, record) {
8681
9224
  fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "totalBytes must be a non-negative integer");
8682
9225
  return;
8683
9226
  }
9227
+ const syncSession = record.syncSession;
9228
+ if (syncSession !== void 0 && syncSession !== null && (typeof syncSession !== "string" || syncSession === "")) {
9229
+ fail(state, ErrorCodes.SNAPSHOT_SYNC_HEADER_INVALID, "syncSession must be a non-empty string or absent");
9230
+ return;
9231
+ }
8684
9232
  try {
8685
9233
  state.spool = openSnapshotSpoolWriter(state.expectation.spoolDirectory);
8686
9234
  } catch (err) {
@@ -8689,6 +9237,7 @@ function applyStartPayload(state, record) {
8689
9237
  }
8690
9238
  state.header = header;
8691
9239
  state.totalBytes = start.totalBytes;
9240
+ state.syncSession = typeof syncSession === "string" ? syncSession : null;
8692
9241
  }
8693
9242
  function applyChunkPayload(state, payload) {
8694
9243
  if (state.totalBytes === null || state.header === null) {
@@ -8837,7 +9386,8 @@ function createSnapshotStreamState(expectation) {
8837
9386
  expectedNextOffset: 0,
8838
9387
  endPayload: null,
8839
9388
  failure: null,
8840
- firstFrameSeen: false
9389
+ firstFrameSeen: false,
9390
+ syncSession: null
8841
9391
  };
8842
9392
  }
8843
9393
  function handleIncomingSnapshotRecord(state, record) {
@@ -8953,7 +9503,8 @@ var TRANSIENT_CODES = /* @__PURE__ */ new Set([
8953
9503
  var TRANSIENT_PRIMARY_CODES = /* @__PURE__ */ new Set([
8954
9504
  ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED,
8955
9505
  ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED,
8956
- ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE
9506
+ ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
9507
+ ErrorCodes.REPLICATION_SYNC_FAILED
8957
9508
  ]);
8958
9509
  function isTransientFailure(code, details) {
8959
9510
  if (TRANSIENT_CODES.has(code)) {
@@ -9133,6 +9684,7 @@ function createLiveSyncFrameState(indexName, partitionId, spoolDirectory) {
9133
9684
  return {
9134
9685
  snapshotState: createSnapshotStreamState({ indexName, partitionId, spoolDirectory }),
9135
9686
  syncEntries: [],
9687
+ syncSession: null,
9136
9688
  sawSnapshotFrame: false,
9137
9689
  error: null
9138
9690
  };
@@ -9165,7 +9717,8 @@ function handleLiveSyncFrame(state, frame) {
9165
9717
  state.error = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "received sync_entries before snapshot_end");
9166
9718
  return;
9167
9719
  }
9168
- state.syncEntries.push(entriesPayload);
9720
+ if (!noteSyncSession(state, entriesPayload.syncSession)) return;
9721
+ state.syncEntries.push({ entries: entriesPayload.entries, isLast: entriesPayload.isLast });
9169
9722
  return;
9170
9723
  }
9171
9724
  if (state.syncEntries.length > 0 && !state.sawSnapshotFrame) {
@@ -9179,7 +9732,22 @@ function handleLiveSyncFrame(state, frame) {
9179
9732
  handleIncomingSnapshotRecord(state.snapshotState, record);
9180
9733
  if (state.snapshotState.failure !== null) {
9181
9734
  state.error = streamFailureToError(state.snapshotState.failure);
9735
+ return;
9736
+ }
9737
+ noteSyncSession(state, state.snapshotState.syncSession);
9738
+ }
9739
+ function noteSyncSession(state, syncSession) {
9740
+ if (syncSession === null) return true;
9741
+ if (state.syncSession === null) {
9742
+ state.syncSession = syncSession;
9743
+ return true;
9182
9744
  }
9745
+ if (state.syncSession === syncSession) return true;
9746
+ state.error = new NarsilError(
9747
+ ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
9748
+ "one sync answer carried two different sync sessions"
9749
+ );
9750
+ return false;
9183
9751
  }
9184
9752
  function decodeSyncEntriesRecord(record) {
9185
9753
  if (!("entries" in record) && !("isLast" in record)) {
@@ -9191,6 +9759,13 @@ function decodeSyncEntriesRecord(record) {
9191
9759
  "Invalid SyncEntriesPayload: expected entries array and isLast boolean"
9192
9760
  );
9193
9761
  }
9762
+ const syncSession = record.syncSession ?? null;
9763
+ if (syncSession !== null && (typeof syncSession !== "string" || syncSession === "")) {
9764
+ return new NarsilError(
9765
+ ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
9766
+ "Invalid SyncEntriesPayload: syncSession must be a non-empty string or absent"
9767
+ );
9768
+ }
9194
9769
  const entries = [];
9195
9770
  for (const candidate of record.entries) {
9196
9771
  try {
@@ -9199,7 +9774,7 @@ function decodeSyncEntriesRecord(record) {
9199
9774
  return new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, err instanceof Error ? err.message : String(err));
9200
9775
  }
9201
9776
  }
9202
- return { entries, isLast: record.isLast };
9777
+ return { entries, isLast: record.isLast, syncSession };
9203
9778
  }
9204
9779
  function streamFailureToError(failure) {
9205
9780
  return new NarsilError(failure.code, failure.message, failure.details);
@@ -9434,7 +10009,7 @@ async function withRoutedIndexUuid(payload, deps) {
9434
10009
  const indexUuid = (await deps.coordinator.getAllocation(payload.indexName))?.indexUuid;
9435
10010
  return indexUuid === void 0 ? payload : { ...payload, indexUuid };
9436
10011
  }
9437
- async function reportSyncedPosition(indexName, partitionId, target, deadline, deps) {
10012
+ async function reportSyncedPosition(indexName, partitionId, target, syncSession, deadline, deps) {
9438
10013
  const logState = readLocalLogState(indexName, partitionId, deps);
9439
10014
  const details = { indexName, partitionId, target };
9440
10015
  const budgetMs = Math.min(SYNCED_POSITION_REPORT_TIMEOUT_MS, deadline - Date.now());
@@ -9449,7 +10024,8 @@ async function reportSyncedPosition(indexName, partitionId, target, deadline, de
9449
10024
  indexName,
9450
10025
  partitionId,
9451
10026
  lastSeqNo: logState.lastSeqNo,
9452
- lastPrimaryTerm: logState.lastPrimaryTerm
10027
+ lastPrimaryTerm: logState.lastPrimaryTerm,
10028
+ ...syncSession === null ? {} : { syncSession }
9453
10029
  };
9454
10030
  const request = {
9455
10031
  type: ReplicationMessageTypes.SYNC_REQUEST,
@@ -9481,6 +10057,9 @@ async function reportSyncedPosition(indexName, partitionId, target, deadline, de
9481
10057
  } finally {
9482
10058
  abandonSnapshotStream(frameState.snapshotState);
9483
10059
  }
10060
+ if (frameState.error !== null) {
10061
+ return frameState.error;
10062
+ }
9484
10063
  if (frameState.sawSnapshotFrame) {
9485
10064
  return new NarsilError(
9486
10065
  ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED,
@@ -9488,7 +10067,7 @@ async function reportSyncedPosition(indexName, partitionId, target, deadline, de
9488
10067
  details
9489
10068
  );
9490
10069
  }
9491
- return frameState.error;
10070
+ return null;
9492
10071
  }
9493
10072
  async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNodeId, targets, coordinatorConfig, partitionCount, deadline, deps, abortCheck) {
9494
10073
  let lastError = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "no live sync attempts were made", {
@@ -9523,7 +10102,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
9523
10102
  )
9524
10103
  };
9525
10104
  }
9526
- const attempt = await syncFromTarget(
10105
+ const { attempt, syncSession } = await syncFromTarget(
9527
10106
  state,
9528
10107
  entry,
9529
10108
  indexName,
@@ -9535,8 +10114,7 @@ async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNo
9535
10114
  deps,
9536
10115
  abortCheck
9537
10116
  );
9538
- const movedThePosition = attempt.ok && (attempt.tier === "snapshot" || attempt.entriesApplied > 0);
9539
- const failure = !attempt.ok ? attempt.error : movedThePosition ? await reportSyncedPosition(indexName, partitionId, target, deadline, deps) : null;
10117
+ const failure = attempt.ok ? await reportSyncedPosition(indexName, partitionId, target, syncSession, deadline, deps) : attempt.error;
9540
10118
  if (failure === null) {
9541
10119
  return attempt;
9542
10120
  }
@@ -9554,7 +10132,7 @@ async function syncFromTarget(state, entry, indexName, partitionId, target, coor
9554
10132
  const spoolDirectory = deps.snapshotSpoolDirectory ?? defaultSpoolDirectory();
9555
10133
  const frameState = createLiveSyncFrameState(indexName, partitionId, spoolDirectory);
9556
10134
  try {
9557
- return await syncFrameStateFromTarget(
10135
+ const attempt = await syncFrameStateFromTarget(
9558
10136
  state,
9559
10137
  entry,
9560
10138
  indexName,
@@ -9567,6 +10145,7 @@ async function syncFromTarget(state, entry, indexName, partitionId, target, coor
9567
10145
  abortCheck,
9568
10146
  frameState
9569
10147
  );
10148
+ return { attempt, syncSession: frameState.syncSession };
9570
10149
  } finally {
9571
10150
  abandonSnapshotStream(frameState.snapshotState);
9572
10151
  }
@@ -9869,157 +10448,39 @@ async function executeBootstrapSync(state, entry, indexName, partitionId, primar
9869
10448
  );
9870
10449
  }
9871
10450
 
9872
- // src/distribution/cluster-node/catch-up/admission.ts
9873
- async function sendToController(payload, controllerNodeId, deps) {
9874
- const targets = await resolveNodeTargets(controllerNodeId, deps);
9875
- let sentWithoutAnswer = false;
9876
- for (const target of targets) {
9877
- try {
9878
- const result = await requestInsyncAdmission(payload, target, deps.transport, deps.nodeId);
9879
- if (result.accepted) return "accepted";
9880
- return sentWithoutAnswer ? "unanswered" : "refused";
9881
- } catch (_) {
9882
- sentWithoutAnswer = true;
9883
- }
9884
- }
9885
- return "unanswered";
9886
- }
9887
- function withTimeout(work, timeoutMs, fallback) {
9888
- return new Promise((resolve) => {
9889
- const timer = setTimeout(() => resolve(fallback), timeoutMs);
9890
- timer.unref?.();
9891
- work.then((value) => {
9892
- clearTimeout(timer);
9893
- resolve(value);
9894
- }).catch(() => {
9895
- clearTimeout(timer);
9896
- resolve(fallback);
9897
- });
9898
- });
9899
- }
9900
- async function withdrawAdmission(removal, controllerNodeId, deps) {
9901
- const targets = await resolveNodeTargets(controllerNodeId, deps);
9902
- for (const target of targets) {
9903
- try {
9904
- const result = await requestInsyncRemoval(
9905
- removal.indexName,
9906
- removal.partitionId,
9907
- removal.replicaNodeId,
9908
- removal.primaryTerm,
9909
- target,
9910
- deps.transport,
9911
- deps.nodeId
9912
- );
9913
- if (result.accepted) {
9914
- return true;
9915
- }
9916
- } catch (_) {
9917
- }
9918
- }
9919
- return false;
9920
- }
9921
- async function proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
9922
- if (claimAdmission(state, deps, indexName, partitionId, replicaNodeId, cursor)) {
9923
- await requestClaimedAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
9924
- }
9925
- }
9926
- function claimAdmission(state, deps, indexName, partitionId, replicaNodeId, cursor) {
9927
- const log = deps.getReplicationLog(indexName, partitionId);
9928
- if (cursor.appliedSeqNo < log.commitPoint) {
9929
- return false;
9930
- }
9931
- return markPendingAdmission(state, indexName, partitionId, replicaNodeId);
9932
- }
9933
- async function requestClaimedAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor) {
9934
- const log = deps.getReplicationLog(indexName, partitionId);
9935
- const proposedEpoch = cursor.syncEpoch;
9936
- try {
9937
- const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
9938
- if (controllerNodeId === null) {
9939
- return;
9940
- }
9941
- const payload = {
9942
- indexName,
9943
- partitionId,
9944
- replicaNodeId,
9945
- primaryTerm: assignment.primaryTerm,
9946
- appliedSeqNo: cursor.appliedSeqNo,
9947
- commitPoint: log.commitPoint
9948
- };
9949
- const answer2 = await withTimeout(
9950
- sendToController(payload, controllerNodeId, deps),
9951
- ADMISSION_TIMEOUT_MS,
9952
- "unanswered"
9953
- );
9954
- const outlived = answer2 === "accepted" && cursor.syncEpoch !== proposedEpoch;
9955
- const withdrawn = outlived && await withTimeout(withdrawAdmission(payload, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
9956
- if (answer2 === "unanswered" || outlived && !withdrawn) {
9957
- recordAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId, assignment.primaryTerm);
9958
- } else if (answer2 === "accepted" && !outlived) {
9959
- forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId);
9960
- forgetRemovedReplica(state, indexName, partitionId, replicaNodeId);
9961
- recordAdmittedReplica(
9962
- state,
9963
- indexName,
9964
- partitionId,
9965
- replicaNodeId,
9966
- assignment.primaryTerm,
9967
- advanceAdmissionGeneration(state, indexName, partitionId, replicaNodeId)
9968
- );
9969
- }
9970
- } finally {
9971
- clearPendingAdmission(state, indexName, partitionId, replicaNodeId);
10451
+ // src/distribution/cluster-node/catch-up/sessions.ts
10452
+ function openSyncSession(state, indexName, partitionId, replicaNodeId, primaryTerm, assignedNodeIds) {
10453
+ const key = partitionKey(indexName, partitionId);
10454
+ const sessions = state.syncSessions.get(key) ?? /* @__PURE__ */ new Map();
10455
+ for (const [nodeId, session] of sessions) {
10456
+ if (session.primaryTerm !== primaryTerm || !assignedNodeIds.includes(nodeId)) sessions.delete(nodeId);
9972
10457
  }
10458
+ const id = generateId();
10459
+ sessions.set(replicaNodeId, { id, primaryTerm });
10460
+ state.syncSessions.set(key, sessions);
10461
+ return id;
9973
10462
  }
9974
-
9975
- // src/distribution/cluster-node/catch-up/feed.ts
9976
- function inSyncSignature(assignment) {
9977
- return [...assignment.inSyncSet].sort().join("\n");
9978
- }
9979
- function createCatchUpFeed(assignment, replicaStartedAt) {
9980
- let release = () => {
9981
- };
9982
- const whenCancelled = new Promise((resolve) => {
9983
- release = resolve;
9984
- });
9985
- const feed = {
9986
- primaryTerm: assignment.primaryTerm,
9987
- inSyncSet: inSyncSignature(assignment),
9988
- replicaStartedAt,
9989
- whenCancelled,
9990
- cancelled: false,
9991
- cancel: () => {
9992
- feed.cancelled = true;
9993
- release();
9994
- }
9995
- };
9996
- return feed;
9997
- }
9998
- function feedIsStale(feed, assignment, registered, replicaNodeId) {
9999
- const replicaLeft = registered !== null && registered.get(replicaNodeId) !== feed.replicaStartedAt;
10000
- return replicaLeft || assignment.primaryTerm !== feed.primaryTerm || inSyncSignature(assignment) !== feed.inSyncSet;
10463
+ function isCurrentSyncSession(state, indexName, partitionId, replicaNodeId, syncSession, primaryTerm) {
10464
+ const session = state.syncSessions.get(partitionKey(indexName, partitionId))?.get(replicaNodeId);
10465
+ return session !== void 0 && session.id === syncSession && session.primaryTerm === primaryTerm;
10001
10466
  }
10002
- function untilCancelled(feed, work, fallback) {
10003
- if (feed.cancelled) {
10004
- work.catch(() => {
10005
- });
10006
- return Promise.resolve(fallback);
10007
- }
10008
- return Promise.race([work, feed.whenCancelled.then(() => fallback)]);
10467
+ function forgetSyncSessions(state, indexName, partitionId) {
10468
+ state.syncSessions.delete(partitionKey(indexName, partitionId));
10009
10469
  }
10010
10470
 
10011
10471
  // src/distribution/cluster-node/catch-up/withdrawals.ts
10012
10472
  async function withdrawAppliedAdmission(state, deps, indexName, partitionId, replicaNodeId, owed) {
10013
10473
  owed.withdrawing = true;
10474
+ const pending = pendingAdmissionOf(state, indexName, partitionId, replicaNodeId);
10014
10475
  try {
10015
10476
  const controllerNodeId = await deps.coordinator.getLeaseHolder(CONTROLLER_LEASE_KEY);
10016
10477
  if (controllerNodeId === null) return;
10017
10478
  const removal = { indexName, partitionId, replicaNodeId, primaryTerm: owed.primaryTerm };
10018
10479
  const withdrawn = await withTimeout(withdrawAdmission(removal, controllerNodeId, deps), ADMISSION_TIMEOUT_MS, false);
10019
- if (!withdrawn || state.admissionsToWithdraw.get(partitionKey(indexName, partitionId))?.get(replicaNodeId) !== owed) {
10020
- return;
10021
- }
10022
- forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId);
10480
+ if (!withdrawn) return;
10481
+ const stillOwed = admissionToWithdrawOf(state, indexName, partitionId, replicaNodeId) === owed;
10482
+ settleFencedAdmission(state, indexName, partitionId, replicaNodeId, { pending, owed });
10483
+ if (!stillOwed) return;
10023
10484
  forgetAdmittedReplica(state, indexName, partitionId, replicaNodeId);
10024
10485
  if (!getPendingAdmissions(state, indexName, partitionId).includes(replicaNodeId)) {
10025
10486
  recordRemovedReplica(state, indexName, partitionId, replicaNodeId, owed.primaryTerm);
@@ -10031,14 +10492,14 @@ async function withdrawAppliedAdmission(state, deps, indexName, partitionId, rep
10031
10492
  function withdrawAppliedAdmissions(state, deps, indexName, partitionId, assignment) {
10032
10493
  const owedByReplica = state.admissionsToWithdraw.get(partitionKey(indexName, partitionId));
10033
10494
  if (owedByReplica === void 0) return [];
10034
- const pending = getPendingAdmissions(state, indexName, partitionId);
10035
10495
  const started = [];
10036
10496
  for (const [replicaNodeId, owed] of [...owedByReplica]) {
10037
- if (owed.primaryTerm !== assignment.primaryTerm || !assignment.replicas.includes(replicaNodeId)) {
10038
- forgetAdmissionToWithdraw(state, indexName, partitionId, replicaNodeId);
10497
+ const pending = pendingAdmissionOf(state, indexName, partitionId, replicaNodeId);
10498
+ if (owed.primaryTerm !== assignment.primaryTerm) {
10499
+ settleFencedAdmission(state, indexName, partitionId, replicaNodeId, { pending, owed });
10039
10500
  continue;
10040
10501
  }
10041
- if (owed.withdrawing || pending.includes(replicaNodeId) || !assignment.inSyncSet.includes(replicaNodeId)) continue;
10502
+ if (owed.withdrawing || pending !== void 0 && !pending.unanswered) continue;
10042
10503
  started.push(withdrawAppliedAdmission(state, deps, indexName, partitionId, replicaNodeId, owed).catch(() => {
10043
10504
  }));
10044
10505
  }
@@ -10046,16 +10507,6 @@ function withdrawAppliedAdmissions(state, deps, indexName, partitionId, assignme
10046
10507
  }
10047
10508
 
10048
10509
  // src/distribution/cluster-node/catch-up/pump.ts
10049
- function entryBytes(entry) {
10050
- return REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
10051
- }
10052
- function batchBytes(entries) {
10053
- let total = 0;
10054
- for (const entry of entries) {
10055
- total += entryBytes(entry);
10056
- }
10057
- return total;
10058
- }
10059
10510
  function replicaHasFallenOutOfRetention(log, cursor) {
10060
10511
  const oldest = log.oldestSeqNo;
10061
10512
  if (oldest === void 0) {
@@ -10063,19 +10514,6 @@ function replicaHasFallenOutOfRetention(log, cursor) {
10063
10514
  }
10064
10515
  return cursor.appliedSeqNo + 1 < oldest;
10065
10516
  }
10066
- function nextWireBatch(state, pending) {
10067
- const chunks = chunkByBudget(pending, {
10068
- ...WIRE_BATCH_BUDGET,
10069
- payloadBytesOf: (entry) => entry.document?.byteLength ?? 0,
10070
- breaksRun: (entry, previous) => entry.seqNo !== previous.seqNo + 1
10071
- });
10072
- const batch = chunks[0] ?? [];
10073
- if (batch.length === 0) {
10074
- return [];
10075
- }
10076
- const available = MAX_CATCH_UP_IN_FLIGHT_BYTES - state.inFlightBytes;
10077
- return batchBytes(batch) > available ? [] : batch;
10078
- }
10079
10517
  async function pumpReplica(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor, replicaStartedAt) {
10080
10518
  if (cursor.busy) {
10081
10519
  return;
@@ -10102,6 +10540,7 @@ async function feedReplica(state, deps, indexName, partitionId, assignment, repl
10102
10540
  return;
10103
10541
  }
10104
10542
  if (cursor.appliedSeqNo >= log.localLogEnd) {
10543
+ if (!cursor.confirmed) return;
10105
10544
  const proposal = proposeAdmission(state, deps, indexName, partitionId, assignment, replicaNodeId, cursor);
10106
10545
  await untilCancelled(feed, proposal, void 0);
10107
10546
  return;
@@ -10152,37 +10591,6 @@ async function closeGapInPartitionQueue(state, deps, indexName, partitionId, ass
10152
10591
  );
10153
10592
  }
10154
10593
  }
10155
- async function sendBatch(state, deps, batch, replicaNodeId, cursor, feed) {
10156
- const reservedBytes = batchBytes(batch);
10157
- const sendingEpoch = cursor.syncEpoch;
10158
- state.inFlightBytes += reservedBytes;
10159
- const delivery = (async () => {
10160
- const first = batch[0];
10161
- const indexName = first?.indexName;
10162
- const table = indexName === void 0 ? null : await deps.coordinator.getAllocation(indexName);
10163
- if (feed.cancelled) {
10164
- return false;
10165
- }
10166
- const result = await replicateBatchToReplicas(
10167
- batch,
10168
- [replicaNodeId],
10169
- deps.transport,
10170
- deps.nodeId,
10171
- deps.resolveNodeTargets,
10172
- table?.indexUuid,
10173
- first === void 0 ? void 0 : previousTermOf(deps.getReplicationLog(first.indexName, first.partitionId), first.seqNo)
10174
- );
10175
- return result.acknowledged.includes(replicaNodeId);
10176
- })().finally(() => {
10177
- state.inFlightBytes -= reservedBytes;
10178
- });
10179
- const acknowledged = await untilCancelled(feed, delivery, false);
10180
- if (!acknowledged || feed.cancelled || cursor.syncEpoch !== sendingEpoch) {
10181
- return false;
10182
- }
10183
- cursor.appliedSeqNo = Math.max(cursor.appliedSeqNo, batch[batch.length - 1].seqNo);
10184
- return true;
10185
- }
10186
10594
  function cancelFeeds(replicas) {
10187
10595
  for (const cursor of replicas.values()) {
10188
10596
  cursor.feed?.cancel();
@@ -10206,6 +10614,7 @@ async function pumpPartition(state, deps, key, replicas, started, registered) {
10206
10614
  if (assignment === void 0 || assignment.primary !== deps.nodeId) {
10207
10615
  cancelFeeds(replicas);
10208
10616
  state.cursors.delete(key);
10617
+ forgetSyncSessions(state, indexName, partitionId);
10209
10618
  state.pendingAdmissions.delete(key);
10210
10619
  state.admissionsToWithdraw.delete(key);
10211
10620
  return;
@@ -11285,6 +11694,7 @@ function cloneAssignments(assignments) {
11285
11694
  primary: assignment.primary,
11286
11695
  replicas: [...assignment.replicas],
11287
11696
  inSyncSet: [...assignment.inSyncSet],
11697
+ ...assignment.inSyncVersion !== void 0 ? { inSyncVersion: assignment.inSyncVersion } : {},
11288
11698
  commitPoint: assignment.commitPoint,
11289
11699
  state: assignment.state,
11290
11700
  primaryTerm: assignment.primaryTerm,
@@ -11550,6 +11960,37 @@ function finishDepartureRun(state, run, onExpiry) {
11550
11960
  state.departureTimer.unref?.();
11551
11961
  }
11552
11962
 
11963
+ // src/distribution/cluster/controller/event-loop/previous-document-placement.ts
11964
+ async function recordsPreviousDocuments(coordinator, indexName) {
11965
+ try {
11966
+ const changes = (await readIndexMetadataSnapshot(coordinator, indexName))?.settings?.changes;
11967
+ return changes?.enabled === true && changes.previousDocument === true;
11968
+ } catch {
11969
+ return true;
11970
+ }
11971
+ }
11972
+ async function namesRecordingPreviousDocuments(coordinator, indexNames) {
11973
+ const recording = /* @__PURE__ */ new Set();
11974
+ for (const indexName of indexNames) {
11975
+ if (isHiddenGeneration(indexName) || !await recordsPreviousDocuments(coordinator, indexName)) continue;
11976
+ recording.add(indexName);
11977
+ const generation = (await coordinator.getAllocation(indexName))?.generation;
11978
+ if (generation !== void 0) recording.add(generation);
11979
+ const record = await readRebalanceRecord(coordinator, indexName);
11980
+ if (record !== null) {
11981
+ recording.add(record.sourceGeneration);
11982
+ recording.add(record.targetGeneration);
11983
+ }
11984
+ }
11985
+ return recording;
11986
+ }
11987
+ async function nodesForEachIndex(coordinator, nodes, indexNames) {
11988
+ const capable = nodes.filter((node) => reachesVersion(node.version, CHANGE_CAPTURE_SPEC_VERSION));
11989
+ if (capable.length === nodes.length) return () => nodes;
11990
+ const recording = await namesRecordingPreviousDocuments(coordinator, indexNames);
11991
+ return (indexName) => recording.has(indexName) ? capable : nodes;
11992
+ }
11993
+
11553
11994
  // src/distribution/cluster/controller/event-loop/state.ts
11554
11995
  function createEventLoopState(initialIndexNames, nodeLeftDelayMs = DEFAULT_CONTROLLER_CONFIG.nodeLeftDelayMs) {
11555
11996
  return {
@@ -11708,53 +12149,6 @@ async function tearDownAllocation(indexName, coordinator, state, isActive, round
11708
12149
  { indexName, attempts: TEARDOWN_CAS_ATTEMPTS * TEARDOWN_RETRY_ROUNDS }
11709
12150
  );
11710
12151
  }
11711
- function validatePartitionStoresPayload(decoded) {
11712
- if (!isRecord(decoded)) {
11713
- return null;
11714
- }
11715
- if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
11716
- return null;
11717
- }
11718
- return { indexName: decoded.indexName };
11719
- }
11720
- function validatePartitionStoresResultPayload(decoded) {
11721
- if (!isRecord(decoded)) {
11722
- return null;
11723
- }
11724
- if (typeof decoded.indexName !== "string" || decoded.indexName.length === 0) {
11725
- return null;
11726
- }
11727
- if (decoded.indexUuid !== null && typeof decoded.indexUuid !== "string") {
11728
- return null;
11729
- }
11730
- if (!Array.isArray(decoded.partitionIds)) {
11731
- return null;
11732
- }
11733
- const partitionIds = [];
11734
- for (const partitionId of decoded.partitionIds) {
11735
- if (typeof partitionId !== "number" || !Number.isInteger(partitionId) || partitionId < 0) {
11736
- return null;
11737
- }
11738
- partitionIds.push(partitionId);
11739
- }
11740
- return { indexName: decoded.indexName, indexUuid: decoded.indexUuid, partitionIds };
11741
- }
11742
- function createPartitionStoresMessage(payload, sourceNodeId, requestId) {
11743
- return {
11744
- type: ClusterMessageTypes.PARTITION_STORES,
11745
- sourceId: sourceNodeId,
11746
- requestId,
11747
- payload: encode(payload)
11748
- };
11749
- }
11750
- function createPartitionStoresResultMessage(result, sourceNodeId, requestId) {
11751
- return {
11752
- type: ClusterMessageTypes.PARTITION_STORES,
11753
- sourceId: sourceNodeId,
11754
- requestId,
11755
- payload: encode(result)
11756
- };
11757
- }
11758
12152
 
11759
12153
  // src/distribution/cluster/controller/event-loop/unassigned-recovery.ts
11760
12154
  function liveHoldersOfUnassignedPartitions(table, liveNodeIds) {
@@ -11917,7 +12311,7 @@ async function writeRecoveryOutcome(deps, indexName, liveNodeIds, answers) {
11917
12311
  if (next === null) {
11918
12312
  return false;
11919
12313
  }
11920
- const written = await deps.coordinator.putAllocation(indexName, next, table.version, deps.controllerNodeId);
12314
+ const written = await putAllocationOver(deps.coordinator, indexName, table, next, deps.controllerNodeId);
11921
12315
  if (written) {
11922
12316
  return anyPromoted;
11923
12317
  }
@@ -11996,7 +12390,7 @@ async function runAllocatorForIndex(coordinator, writerNodeId, indexName, nodes,
11996
12390
  return;
11997
12391
  }
11998
12392
  const table = stampedWith(result.table, metadata.indexUuid);
11999
- if (await coordinator.putAllocation(indexName, table, storedTable?.version ?? null, writerNodeId)) {
12393
+ if (await putAllocationOver(coordinator, indexName, storedTable, table, writerNodeId)) {
12000
12394
  return;
12001
12395
  }
12002
12396
  if (attempt + 1 < ALLOCATION_CAS_ATTEMPTS) {
@@ -12035,6 +12429,7 @@ async function runAllocatorForAllIndexes(state, coordinator, isActive, onError)
12035
12429
  if (!holdsDataNode(nodes) || !isActive()) {
12036
12430
  return;
12037
12431
  }
12432
+ const nodesFor = await nodesForEachIndex(coordinator, nodes, state.knownIndexes);
12038
12433
  const departures = startDepartureRun(new Set(nodes.map((node) => node.nodeId)));
12039
12434
  const retainedFor = (table) => retainedNodesFor(state, departures, table);
12040
12435
  let anyIndexFailed = false;
@@ -12043,8 +12438,10 @@ async function runAllocatorForAllIndexes(state, coordinator, isActive, onError)
12043
12438
  return;
12044
12439
  }
12045
12440
  try {
12046
- await runAllocatorForIndex(coordinator, writerNodeId, indexName, nodes, isActive, retainedFor);
12047
- await recoverIndexPartitions(state, coordinator, indexName, nodes, isActive, retainedFor);
12441
+ const placeable = nodesFor(indexName);
12442
+ if (!holdsDataNode(placeable)) continue;
12443
+ await runAllocatorForIndex(coordinator, writerNodeId, indexName, placeable, isActive, retainedFor);
12444
+ await recoverIndexPartitions(state, coordinator, indexName, placeable, isActive, retainedFor);
12048
12445
  } catch (error) {
12049
12446
  anyIndexFailed = true;
12050
12447
  if (onError !== void 0) {
@@ -12086,7 +12483,10 @@ async function allocateIndexNow(state, coordinator, indexName, isActive, onError
12086
12483
  if (holdsDataNode(nodes) && isActive()) {
12087
12484
  const departures = startDepartureRun(new Set(nodes.map((node) => node.nodeId)));
12088
12485
  const retainedFor = (table) => retainedNodesFor(state, departures, table);
12089
- await runAllocatorForIndex(coordinator, writerNodeId, indexName, nodes, isActive, retainedFor);
12486
+ const placeable = (await nodesForEachIndex(coordinator, nodes, state.knownIndexes))(indexName);
12487
+ if (holdsDataNode(placeable)) {
12488
+ await runAllocatorForIndex(coordinator, writerNodeId, indexName, placeable, isActive, retainedFor);
12489
+ }
12090
12490
  }
12091
12491
  } finally {
12092
12492
  if (isActive()) {
@@ -12116,8 +12516,12 @@ async function handleSchemaCreated(indexName, coordinator, writerNodeId, knownIn
12116
12516
  return;
12117
12517
  }
12118
12518
  const currentTable = ownTable === null || ownTable.assignments.size === 0 ? null : ownTable;
12519
+ const placeable = (await nodesForEachIndex(coordinator, nodes, knownIndexes))(indexName);
12520
+ if (!holdsDataNode(placeable) || !isActive()) {
12521
+ return;
12522
+ }
12119
12523
  const result = allocate(
12120
- nodes,
12524
+ placeable,
12121
12525
  currentTable,
12122
12526
  indexName,
12123
12527
  metadata.partitionCount,
@@ -12127,16 +12531,16 @@ async function handleSchemaCreated(indexName, coordinator, writerNodeId, knownIn
12127
12531
  if (!isActive()) {
12128
12532
  return;
12129
12533
  }
12130
- const expectedVersion = storedTable !== null ? storedTable.version : null;
12131
- if (await coordinator.putAllocation(
12534
+ if (await putAllocationOver(
12535
+ coordinator,
12132
12536
  indexName,
12537
+ storedTable,
12133
12538
  stampedWith(result.table, metadata.indexUuid),
12134
- expectedVersion,
12135
12539
  writerNodeId
12136
12540
  )) {
12137
12541
  return;
12138
12542
  }
12139
- await runAllocatorForIndex(coordinator, writerNodeId, indexName, nodes, isActive);
12543
+ await runAllocatorForIndex(coordinator, writerNodeId, indexName, placeable, isActive);
12140
12544
  }
12141
12545
  async function handleSchemaEvent(event, coordinator, state, isActive, onError) {
12142
12546
  if (event.type === "schema_created") {
@@ -12207,7 +12611,7 @@ function metadataPublishingAnalysis(snapshot, pending) {
12207
12611
  ...pending.tokenizer !== void 0 ? { tokenizer: pending.tokenizer } : {},
12208
12612
  ...pending.stopWords !== void 0 ? { stopWords: pending.stopWords } : {},
12209
12613
  ...pending.stopWordList !== void 0 ? { stopWordList: pending.stopWordList } : {},
12210
- ...isRecord2(rest.sync) && rest.sync.enabled === true ? { syncActive: false, syncEpoch: newSyncEpoch() } : {}
12614
+ ...isRecord2(rest.sync) && rest.sync.enabled === true || isRecord2(rest.changes) && rest.changes.enabled === true ? { syncActive: false, syncEpoch: newSyncEpoch() } : {}
12211
12615
  });
12212
12616
  }
12213
12617
  function metadataWithAnalysisFailure(snapshot, failure) {
@@ -12465,7 +12869,7 @@ async function processBootstrapComplete(payload, coordinator, controllerNodeId)
12465
12869
  if (rewritten === null) {
12466
12870
  return false;
12467
12871
  }
12468
- const written = await coordinator.putAllocation(payload.indexName, rewritten, table.version, controllerNodeId);
12872
+ const written = await putAllocationOver(coordinator, payload.indexName, table, rewritten, controllerNodeId);
12469
12873
  if (written) {
12470
12874
  return true;
12471
12875
  }
@@ -12509,7 +12913,19 @@ function validateInsyncAddPayload(decoded) {
12509
12913
  if (!isValidInteger(decoded.commitPoint) || decoded.commitPoint < 0) {
12510
12914
  return null;
12511
12915
  }
12512
- return { ...base, appliedSeqNo: decoded.appliedSeqNo, commitPoint: decoded.commitPoint };
12916
+ if (!isValidInteger(decoded.inSyncVersion) || decoded.inSyncVersion < 0) {
12917
+ return null;
12918
+ }
12919
+ if (typeof decoded.replicaStartedAt !== "string" || decoded.replicaStartedAt.length === 0) {
12920
+ return null;
12921
+ }
12922
+ return {
12923
+ ...base,
12924
+ appliedSeqNo: decoded.appliedSeqNo,
12925
+ commitPoint: decoded.commitPoint,
12926
+ inSyncVersion: decoded.inSyncVersion,
12927
+ replicaStartedAt: decoded.replicaStartedAt
12928
+ };
12513
12929
  }
12514
12930
  function decodeMessage(message) {
12515
12931
  try {
@@ -12644,7 +13060,7 @@ async function resign(payload, sourceId, coordinator, controllerNodeId) {
12644
13060
  assignments.set(payload.partitionId, failedOver(assignment, sourceId, registered));
12645
13061
  const rewritten = await rewriteForCurrentIndex(coordinator, { ...table, version: table.version + 1, assignments });
12646
13062
  if (rewritten === null) return false;
12647
- if (await coordinator.putAllocation(payload.indexName, rewritten, table.version, controllerNodeId)) return true;
13063
+ if (await putAllocationOver(coordinator, payload.indexName, table, rewritten, controllerNodeId)) return true;
12648
13064
  }
12649
13065
  return false;
12650
13066
  }
@@ -12751,9 +13167,9 @@ async function renewClusterSyncEpoch(coordinator, indexName) {
12751
13167
  for (let attempt = 1; attempt <= SYNC_EPOCH_CAS_ATTEMPTS; attempt++) {
12752
13168
  const current = await readIndexMetadataSnapshot(coordinator, indexName);
12753
13169
  if (current === null) return false;
12754
- const sync = current.settings?.sync;
12755
- if (sync?.enabled !== true) return true;
12756
- const next = metadataWithSync(current, { sync, syncActive: false, syncEpoch: newSyncEpoch() });
13170
+ const held = syncSettingsOfSnapshot(current);
13171
+ if (!epochFeaturesOn(held)) return true;
13172
+ const next = metadataWithSync(current, { ...held, syncActive: false, syncEpoch: newSyncEpoch() });
12757
13173
  if (await coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) return true;
12758
13174
  }
12759
13175
  return false;
@@ -13682,9 +14098,337 @@ var DEVICE_SYNC_HEAD_POLL_MS = 1e3;
13682
14098
  var DEVICE_SYNC_BOUNDS_REUSE_MS = 250;
13683
14099
  var DEVICE_SYNC_ARCHIVE_IDLE_MS = 6e5;
13684
14100
  var DEVICE_SYNC_SNAPSHOT_RETRY_AFTER_MS = 5e3;
13685
- var DEVICE_SYNC_BACKUP_NAME_PREFIX = "sync-";
14101
+ var CHANGES_EXPORT_RETRY_AFTER_MS = 5e3;
13686
14102
  var CACHED_BACKUP_MANIFEST_LIMIT = 4096;
13687
14103
 
14104
+ // src/distribution/cluster-node/device-sync/backup-catalog.ts
14105
+ var TAKEN_BACKUP_NAME = new RegExp(`^(${SYNC_BACKUP_TAKER}|${CHANGES_BACKUP_TAKER})-[0-9a-f]{32}-`);
14106
+ function takenBackupPrefix(takenBy, indexName) {
14107
+ return `${takenBy}-${sha256Hex(new TextEncoder().encode(indexName)).slice(0, 32)}-`;
14108
+ }
14109
+ function mayHoldIndex(name, indexName) {
14110
+ const taken = TAKEN_BACKUP_NAME.exec(name);
14111
+ if (taken === null) return true;
14112
+ const taker = taken[1] === CHANGES_BACKUP_TAKER ? CHANGES_BACKUP_TAKER : SYNC_BACKUP_TAKER;
14113
+ return name.startsWith(takenBackupPrefix(taker, indexName));
14114
+ }
14115
+ function createBackupCatalog(deps) {
14116
+ const manifests = /* @__PURE__ */ new Map();
14117
+ const backingUp = /* @__PURE__ */ new Map();
14118
+ let closed = false;
14119
+ async function manifestOf(store, name) {
14120
+ if (manifests.has(name)) return manifests.get(name) ?? null;
14121
+ let read;
14122
+ try {
14123
+ read = await readBackupManifest(store, name);
14124
+ } catch (error) {
14125
+ if (!(error instanceof NarsilError)) throw error;
14126
+ if (error.code !== ErrorCodes.BACKUP_DAMAGED && error.code !== ErrorCodes.BACKUP_NOT_FOUND) throw error;
14127
+ read = null;
14128
+ }
14129
+ if (manifests.size >= CACHED_BACKUP_MANIFEST_LIMIT) {
14130
+ const oldest = manifests.keys().next().value;
14131
+ if (oldest !== void 0) manifests.delete(oldest);
14132
+ }
14133
+ manifests.set(name, read);
14134
+ return read;
14135
+ }
14136
+ async function matching(store, indexName, partitionCount, epoch) {
14137
+ const names = (await listBackupNames(store)).filter((name) => mayHoldIndex(name, indexName));
14138
+ const found = [];
14139
+ for (const name of names) {
14140
+ const manifest = await manifestOf(store, name);
14141
+ if (manifest !== null && manifest.indexName === indexName && manifest.syncEpoch === epoch && manifest.partitionCount === partitionCount) {
14142
+ found.push(manifest);
14143
+ }
14144
+ }
14145
+ return found.sort((a, b) => b.completedAt - a.completedAt || b.startedAt - a.startedAt);
14146
+ }
14147
+ return {
14148
+ matching,
14149
+ async qualifying(store, indexName, partitionCount, epoch) {
14150
+ const candidates = await matching(store, indexName, partitionCount, epoch);
14151
+ if (candidates.length === 0) return null;
14152
+ const oldest = [];
14153
+ for (let partitionId = 0; partitionId < partitionCount; partitionId++) {
14154
+ oldest.push((await deps.bounds(indexName, partitionId)).oldestSeqNo);
14155
+ }
14156
+ const qualifies = (backup) => {
14157
+ const covered = new Set(backup.partitions.map((partition) => partition.partitionId));
14158
+ if (covered.size !== partitionCount || oldest.some((_, partitionId) => !covered.has(partitionId))) return false;
14159
+ return backup.partitions.every((partition) => partition.lastSeqNo >= (oldest[partition.partitionId] ?? 0) - 1);
14160
+ };
14161
+ return candidates.find(qualifies) ?? null;
14162
+ },
14163
+ async startBackup(indexName, takenBy) {
14164
+ const key = `${takenBy}\0${indexName}`;
14165
+ if (backingUp.has(key) || closed) return;
14166
+ for (const job of (await readBackupJobs(deps.coordinator)).values()) {
14167
+ if (job.kind === "backup" && job.status === "running" && job.indexName === indexName) return;
14168
+ }
14169
+ if (backingUp.has(key) || closed) return;
14170
+ const backupName = `${takenBackupPrefix(takenBy, indexName)}${generateId()}`;
14171
+ const run = requestTakenBackup(deps.requests, indexName, backupName, takenBy).catch(deps.onError).finally(() => backingUp.delete(key));
14172
+ backingUp.set(key, run);
14173
+ },
14174
+ async close() {
14175
+ closed = true;
14176
+ await Promise.all([...backingUp.values()]);
14177
+ }
14178
+ };
14179
+ }
14180
+
14181
+ // src/distribution/cluster-node/device-sync/state.ts
14182
+ function syncStateOfSettings(settings) {
14183
+ const setting = settings?.sync;
14184
+ if (setting === void 0 || !setting.enabled || settings?.syncEpoch === void 0) return null;
14185
+ return {
14186
+ ...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
14187
+ ...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {},
14188
+ active: settings?.syncActive === true,
14189
+ epoch: settings.syncEpoch
14190
+ };
14191
+ }
14192
+ function changesStateOfSettings(settings) {
14193
+ const setting = settings?.changes;
14194
+ if (setting === void 0 || !setting.enabled || settings?.syncEpoch === void 0) return null;
14195
+ return {
14196
+ previousDocument: setting.previousDocument === true,
14197
+ ...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
14198
+ ...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {},
14199
+ epoch: settings.syncEpoch
14200
+ };
14201
+ }
14202
+ function changesStatusOfSettings(settings) {
14203
+ return changesStatusOf(changesStateOfSettings(settings));
14204
+ }
14205
+ function syncStatusOfSettings(settings) {
14206
+ const state = syncStateOfSettings(settings);
14207
+ if (state === null) return { enabled: false, active: false, epoch: null };
14208
+ return { enabled: true, ...state };
14209
+ }
14210
+ function analysisRevisionOf(language) {
14211
+ if (!hasLanguage(language)) return void 0;
14212
+ return getLanguage(language).revision;
14213
+ }
14214
+ function feedSettingsOf(settings, synonyms) {
14215
+ const language = settings?.language ?? getLanguage("english").name;
14216
+ const revision = analysisRevisionOf(language);
14217
+ return {
14218
+ language,
14219
+ ...settings?.tokenizer !== void 0 ? { tokenizer: settings.tokenizer } : {},
14220
+ ...settings?.stopWords !== void 0 ? { stopWords: settings.stopWords } : {},
14221
+ ...settings?.stopWordList !== void 0 ? { stopWordList: settings.stopWordList } : {},
14222
+ ...revision !== void 0 ? { analysisRevision: revision } : {},
14223
+ ...settings?.bm25 !== void 0 ? { bm25: settings.bm25 } : {},
14224
+ ...synonyms.length > 0 ? { synonyms: [...synonyms] } : {}
14225
+ };
14226
+ }
14227
+ function sameFeedSettings(held, next) {
14228
+ const a = held.settings;
14229
+ const b = next.settings;
14230
+ return held.synonymsGeneration === next.synonymsGeneration && a?.language === b?.language && a?.tokenizer === b?.tokenizer && a?.stopWords === b?.stopWords && JSON.stringify(a?.stopWordList ?? null) === JSON.stringify(b?.stopWordList ?? null) && (a?.bm25Revision ?? 0) === (b?.bm25Revision ?? 0);
14231
+ }
14232
+ function changeBetween(held, next) {
14233
+ const before = held === void 0 ? null : syncStateOfSettings(held.settings);
14234
+ if (held === void 0 || before === null) return null;
14235
+ if (next === null || next.indexUuid !== held.indexUuid) return "dropped";
14236
+ const after = syncStateOfSettings(next.settings);
14237
+ if (after === null) return "off";
14238
+ if (after.epoch !== before.epoch || next.serving.partitionCount !== held.serving.partitionCount) return "epoch";
14239
+ return sameFeedSettings(held, next) ? null : "settings";
14240
+ }
14241
+ function changesChangeBetween(held, next) {
14242
+ const before = held === void 0 ? null : changesStateOfSettings(held.settings);
14243
+ if (held === void 0 || before === null) return null;
14244
+ if (next === null || next.indexUuid !== held.indexUuid) return "dropped";
14245
+ const after = changesStateOfSettings(next.settings);
14246
+ if (after === null) return "changes-off";
14247
+ if (after.epoch !== before.epoch || next.serving.partitionCount !== held.serving.partitionCount) return "epoch";
14248
+ return before.previousDocument && !after.previousDocument ? "previous-document-off" : null;
14249
+ }
14250
+
14251
+ // src/distribution/cluster-node/device-sync/changes-export.ts
14252
+ function exportUnsupported(indexName) {
14253
+ return new NarsilError(
14254
+ ErrorCodes.CLUSTER_OPERATION_UNSUPPORTED,
14255
+ `A cluster node exports the documents of an index only while change capture is on, and index "${indexName}" has it off`,
14256
+ { operation: "exportDocuments", indexName }
14257
+ );
14258
+ }
14259
+ async function exportClusterDocuments(deps, indexName) {
14260
+ refuseHiddenGeneration(indexName);
14261
+ validateIndexName(indexName);
14262
+ const held = await deps.view.current(indexName);
14263
+ const state = held === null ? null : changesStateOfSettings(held.settings);
14264
+ if (held === null || state === null) throw exportUnsupported(indexName);
14265
+ if (deps.store === null) {
14266
+ throw new NarsilError(
14267
+ ErrorCodes.CHANGES_BACKUP_STORE_REQUIRED,
14268
+ "Change capture in a cluster needs a backup store, because a cluster node builds a document export from a backup",
14269
+ { indexName }
14270
+ );
14271
+ }
14272
+ const { partitionCount } = held.serving;
14273
+ const backup = await deps.catalog.qualifying(deps.store, indexName, partitionCount, state.epoch);
14274
+ if (backup === null) {
14275
+ await deps.catalog.startBackup(indexName, CHANGES_BACKUP_TAKER);
14276
+ throw new NarsilError(
14277
+ ErrorCodes.CHANGES_EXPORT_PREPARING,
14278
+ `The cluster is taking the backup that the export of index "${indexName}" needs. Ask again later`,
14279
+ { indexName, retryAfterMs: CHANGES_EXPORT_RETRY_AFTER_MS }
14280
+ );
14281
+ }
14282
+ const { archive, release } = await deps.spools.acquire(deps.store, indexName, backup);
14283
+ try {
14284
+ const header = await decodeIndexConfiguration(archive.config);
14285
+ const positions = Array.from({ length: partitionCount }, (_, partitionId) => ({
14286
+ partitionId,
14287
+ seqNo: backup.partitions.find((partition) => partition.partitionId === partitionId)?.lastSeqNo ?? 0
14288
+ }));
14289
+ return documentExportStream({ ...header, syncEpoch: state.epoch }, archive.checkpoint(release), {
14290
+ epoch: state.epoch,
14291
+ positions
14292
+ });
14293
+ } catch (error) {
14294
+ release();
14295
+ throw error;
14296
+ }
14297
+ }
14298
+
14299
+ // src/distribution/cluster-node/device-sync/changes-settings.ts
14300
+ function indexNotFound(indexName) {
14301
+ return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
14302
+ indexName
14303
+ });
14304
+ }
14305
+ function registeredNodesBelowChangeCapture(nodes) {
14306
+ return nodes.filter((node) => !reachesVersion(node.version, CHANGE_CAPTURE_SPEC_VERSION)).map((node) => node.nodeId);
14307
+ }
14308
+ function changesNotReady(olderNodes, indexName) {
14309
+ return new NarsilError(
14310
+ ErrorCodes.CHANGES_NOT_READY,
14311
+ `The cluster has ${olderNodes.length} node(s) below specification version ${CHANGE_CAPTURE_SPEC_VERSION}, which keep no history for change capture. Try again once every node registers that version`,
14312
+ { indexName, nodeIds: [...olderNodes].sort(), minimumVersion: CHANGE_CAPTURE_SPEC_VERSION }
14313
+ );
14314
+ }
14315
+ async function requireChangeCaptureNodes(deps, indexName) {
14316
+ const olderNodes = registeredNodesBelowChangeCapture(await deps.coordinator.listNodes());
14317
+ if (olderNodes.length > 0) throw changesNotReady(olderNodes, indexName);
14318
+ }
14319
+ function storedSetting(setting) {
14320
+ return {
14321
+ enabled: true,
14322
+ ...setting.previousDocument === true ? { previousDocument: true } : {},
14323
+ ...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
14324
+ ...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {}
14325
+ };
14326
+ }
14327
+ function nextChangesSettings(current, setting) {
14328
+ const { changes: _changes, ...others } = current;
14329
+ const syncOn = current.sync?.enabled === true && current.syncEpoch !== void 0;
14330
+ const changesOn = current.changes?.enabled === true && current.syncEpoch !== void 0;
14331
+ if (!setting.enabled) {
14332
+ if (current.changes === void 0) return null;
14333
+ return syncOn ? others : {};
14334
+ }
14335
+ if (syncOn || changesOn) return { ...others, changes: storedSetting(setting) };
14336
+ return { ...others, changes: storedSetting(setting), syncActive: false, syncEpoch: newSyncEpoch() };
14337
+ }
14338
+ async function updateClusterChanges(deps, indexName, value) {
14339
+ refuseHiddenGeneration(indexName);
14340
+ validateIndexName(indexName);
14341
+ const setting = requireChangesSetting(value);
14342
+ if (setting.enabled) {
14343
+ if (!deps.hasDurability()) {
14344
+ throw new NarsilError(
14345
+ ErrorCodes.CHANGES_DURABILITY_REQUIRED,
14346
+ `Index "${indexName}" needs write-ahead log durability on every node for change capture, because the feed reads the log`,
14347
+ { indexName }
14348
+ );
14349
+ }
14350
+ if (!deps.hasBackupStore) {
14351
+ throw new NarsilError(
14352
+ ErrorCodes.CHANGES_BACKUP_STORE_REQUIRED,
14353
+ "Change capture in a cluster needs a backup store, because a cluster node builds a document export from a backup",
14354
+ { indexName }
14355
+ );
14356
+ }
14357
+ await requireChangeCaptureNodes(deps, indexName);
14358
+ }
14359
+ for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
14360
+ const current = await readIndexMetadataSnapshot(deps.coordinator, indexName);
14361
+ if (current === null) throw indexNotFound(indexName);
14362
+ const next = nextChangesSettings(current.settings ?? {}, setting);
14363
+ if (next === null) return;
14364
+ if (await deps.coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, metadataWithSync(current, next))) {
14365
+ await deps.view.refresh(indexName);
14366
+ return;
14367
+ }
14368
+ }
14369
+ throw new NarsilError(
14370
+ ErrorCodes.CLUSTER_WRITE_BLOCKED,
14371
+ `Index '${indexName}' kept changing while its changes setting was replaced ${MAX_SETTINGS_REPLACEMENT_ATTEMPTS} times over, so the new setting was not stored. Retry the request`,
14372
+ { reason: SETTINGS_CONTENTION_REASON, indexName, attempts: MAX_SETTINGS_REPLACEMENT_ATTEMPTS }
14373
+ );
14374
+ }
14375
+ async function readClusterChanges(deps, indexName) {
14376
+ refuseHiddenGeneration(indexName);
14377
+ validateIndexName(indexName);
14378
+ const snapshot = await readIndexMetadataSnapshot(deps.coordinator, indexName);
14379
+ if (snapshot === null) throw indexNotFound(indexName);
14380
+ return changesStatusOfSettings(snapshot.settings);
14381
+ }
14382
+
14383
+ // src/distribution/cluster-node/device-sync/changes-source.ts
14384
+ function createClusterChangeSource(deps) {
14385
+ const { feed, view, coordinator } = deps;
14386
+ async function heldSchema(indexName) {
14387
+ const held = view.held(indexName);
14388
+ if (held === void 0) throw changesNotEnabled(indexName);
14389
+ const { servingName } = held.serving;
14390
+ const stored = await coordinator.getSchema(servingName);
14391
+ if (stored === null) throw changesNotEnabled(indexName);
14392
+ return { servingName, schema: validateSchema(stored) };
14393
+ }
14394
+ return {
14395
+ changesState(indexName) {
14396
+ const held = view.held(indexName);
14397
+ const state = changesStateOfSettings(held?.settings);
14398
+ if (held === void 0 || state === null) return null;
14399
+ return {
14400
+ epoch: state.epoch,
14401
+ partitionCount: held.serving.partitionCount,
14402
+ previousDocument: state.previousDocument
14403
+ };
14404
+ },
14405
+ async documentTest(indexName, filter) {
14406
+ const { servingName, schema } = await heldSchema(indexName);
14407
+ const valid = validChangeFilter(filter, schema);
14408
+ const config = await getClusterIndexConfig(coordinator, servingName, schema);
14409
+ return createDocumentTest(
14410
+ {
14411
+ schema,
14412
+ language: getLanguage(config.language ?? "english"),
14413
+ insertOptions: resolvePartitionInsertOptions(config, resolveIndexAnalysis(config)),
14414
+ vectorFieldPaths: getVectorFieldPaths(schema)
14415
+ },
14416
+ valid
14417
+ );
14418
+ },
14419
+ async vectorFieldPaths(indexName) {
14420
+ const { schema } = await heldSchema(indexName);
14421
+ return getVectorFieldPaths(schema);
14422
+ },
14423
+ bounds: (indexName, partitionId, freshness) => feed.bounds(indexName, partitionId, freshness),
14424
+ read: (indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes) => feed.read(indexName, partitionId, afterSeqNo, upToSeqNo, maxBytes),
14425
+ bytesBehind: (indexName, seqNos) => feed.bytesBehind(indexName, seqNos),
14426
+ pendingBeyondHead: (indexName, partitionId, head) => feed.pendingBeyondHead(indexName, partitionId, head),
14427
+ listen: (onChange) => feed.listen(onChange),
14428
+ watchIndex: (indexName) => feed.watchIndex?.(indexName) ?? (() => void 0)
14429
+ };
14430
+ }
14431
+
13688
14432
  // src/distribution/cluster-node/reshard/leadership.ts
13689
14433
  async function leadingAssignment2(deps, indexName, partitionId) {
13690
14434
  let assignment = (await deps.coordinator.getAllocation(indexName))?.assignments.get(partitionId);
@@ -14029,56 +14773,8 @@ function createClusterFeedSource(deps) {
14029
14773
  };
14030
14774
  }
14031
14775
 
14032
- // src/distribution/cluster-node/device-sync/state.ts
14033
- function syncStateOfSettings(settings) {
14034
- const setting = settings?.sync;
14035
- if (setting === void 0 || !setting.enabled || settings?.syncEpoch === void 0) return null;
14036
- return {
14037
- ...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
14038
- ...setting.historyMaxBytes !== void 0 ? { historyMaxBytes: setting.historyMaxBytes } : {},
14039
- active: settings?.syncActive === true,
14040
- epoch: settings.syncEpoch
14041
- };
14042
- }
14043
- function syncStatusOfSettings(settings) {
14044
- const state = syncStateOfSettings(settings);
14045
- if (state === null) return { enabled: false, active: false, epoch: null };
14046
- return { enabled: true, ...state };
14047
- }
14048
- function analysisRevisionOf(language) {
14049
- if (!hasLanguage(language)) return void 0;
14050
- return getLanguage(language).revision;
14051
- }
14052
- function feedSettingsOf(settings, synonyms) {
14053
- const language = settings?.language ?? getLanguage("english").name;
14054
- const revision = analysisRevisionOf(language);
14055
- return {
14056
- language,
14057
- ...settings?.tokenizer !== void 0 ? { tokenizer: settings.tokenizer } : {},
14058
- ...settings?.stopWords !== void 0 ? { stopWords: settings.stopWords } : {},
14059
- ...settings?.stopWordList !== void 0 ? { stopWordList: settings.stopWordList } : {},
14060
- ...revision !== void 0 ? { analysisRevision: revision } : {},
14061
- ...settings?.bm25 !== void 0 ? { bm25: settings.bm25 } : {},
14062
- ...synonyms.length > 0 ? { synonyms: [...synonyms] } : {}
14063
- };
14064
- }
14065
- function sameFeedSettings(held, next) {
14066
- const a = held.settings;
14067
- const b = next.settings;
14068
- return held.synonymsGeneration === next.synonymsGeneration && a?.language === b?.language && a?.tokenizer === b?.tokenizer && a?.stopWords === b?.stopWords && JSON.stringify(a?.stopWordList ?? null) === JSON.stringify(b?.stopWordList ?? null) && (a?.bm25Revision ?? 0) === (b?.bm25Revision ?? 0);
14069
- }
14070
- function changeBetween(held, next) {
14071
- const before = held === void 0 ? null : syncStateOfSettings(held.settings);
14072
- if (held === void 0 || before === null) return null;
14073
- if (next === null || next.indexUuid !== held.indexUuid) return "dropped";
14074
- const after = syncStateOfSettings(next.settings);
14075
- if (after === null) return "off";
14076
- if (after.epoch !== before.epoch || next.serving.partitionCount !== held.serving.partitionCount) return "epoch";
14077
- return sameFeedSettings(held, next) ? null : "settings";
14078
- }
14079
-
14080
14776
  // src/distribution/cluster-node/device-sync/settings.ts
14081
- function indexNotFound(indexName) {
14777
+ function indexNotFound2(indexName) {
14082
14778
  return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
14083
14779
  indexName
14084
14780
  });
@@ -14093,7 +14789,7 @@ function syncNotReady(subject, olderNodes, indexName) {
14093
14789
  function registeredNodesBelowDeviceSync(nodes) {
14094
14790
  return nodes.filter((node) => !reachesVersion(node.version, DEVICE_SYNC_SPEC_VERSION)).map((node) => node.nodeId);
14095
14791
  }
14096
- function storedSetting(setting) {
14792
+ function storedSetting2(setting) {
14097
14793
  return {
14098
14794
  enabled: true,
14099
14795
  ...setting.historyMaxAgeMs !== void 0 ? { historyMaxAgeMs: setting.historyMaxAgeMs } : {},
@@ -14102,12 +14798,19 @@ function storedSetting(setting) {
14102
14798
  }
14103
14799
  function nextSyncSettings(current, setting) {
14104
14800
  const wasOn = current.sync?.enabled === true && current.syncEpoch !== void 0;
14105
- if (!setting.enabled) return wasOn || current.sync !== void 0 ? {} : null;
14106
- if (!wasOn) return { sync: storedSetting(setting), syncActive: false, syncEpoch: newSyncEpoch() };
14801
+ const capture = current.changes?.enabled === true && current.syncEpoch !== void 0 ? { changes: current.changes, syncEpoch: current.syncEpoch } : null;
14802
+ if (!setting.enabled) {
14803
+ if (!wasOn && current.sync === void 0) return null;
14804
+ return capture ?? {};
14805
+ }
14806
+ if (!wasOn) {
14807
+ return capture === null ? { sync: storedSetting2(setting), syncActive: false, syncEpoch: newSyncEpoch() } : { ...capture, sync: storedSetting2(setting), syncActive: false };
14808
+ }
14107
14809
  return {
14108
- sync: storedSetting(setting),
14810
+ sync: storedSetting2(setting),
14109
14811
  ...current.syncActive !== void 0 ? { syncActive: current.syncActive } : {},
14110
- ...current.syncEpoch !== void 0 ? { syncEpoch: current.syncEpoch } : {}
14812
+ ...current.syncEpoch !== void 0 ? { syncEpoch: current.syncEpoch } : {},
14813
+ ...current.changes !== void 0 ? { changes: current.changes } : {}
14111
14814
  };
14112
14815
  }
14113
14816
  async function updateClusterSync(deps, indexName, value) {
@@ -14134,7 +14837,7 @@ async function updateClusterSync(deps, indexName, value) {
14134
14837
  }
14135
14838
  for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
14136
14839
  const current = await readIndexMetadataSnapshot(deps.coordinator, indexName);
14137
- if (current === null) throw indexNotFound(indexName);
14840
+ if (current === null) throw indexNotFound2(indexName);
14138
14841
  if (setting.enabled) requireNoQuerySuggestions(suggestionStateOfSettings(current.settings), indexName);
14139
14842
  const next = nextSyncSettings(current.settings ?? {}, setting);
14140
14843
  if (next === null) return;
@@ -14153,19 +14856,24 @@ async function readClusterSync(deps, indexName) {
14153
14856
  refuseHiddenGeneration(indexName);
14154
14857
  validateIndexName(indexName);
14155
14858
  const snapshot = await readIndexMetadataSnapshot(deps.coordinator, indexName);
14156
- if (snapshot === null) throw indexNotFound(indexName);
14859
+ if (snapshot === null) throw indexNotFound2(indexName);
14157
14860
  return syncStatusOfSettings(snapshot.settings);
14158
14861
  }
14159
14862
  async function markClusterSyncActive(deps, indexName) {
14160
14863
  for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
14161
14864
  const current = await readIndexMetadataSnapshot(deps.coordinator, indexName);
14162
- if (current === null) throw indexNotFound(indexName);
14865
+ if (current === null) throw indexNotFound2(indexName);
14163
14866
  const settings = current.settings ?? {};
14164
14867
  if (settings.sync?.enabled !== true || settings.syncEpoch === void 0) {
14165
14868
  throw new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index '${indexName}' has device sync off`, { indexName });
14166
14869
  }
14167
14870
  if (settings.syncActive === true) return;
14168
- const next = metadataWithSync(current, { sync: settings.sync, syncActive: true, syncEpoch: settings.syncEpoch });
14871
+ const next = metadataWithSync(current, {
14872
+ sync: settings.sync,
14873
+ syncActive: true,
14874
+ syncEpoch: settings.syncEpoch,
14875
+ ...settings.changes !== void 0 ? { changes: settings.changes } : {}
14876
+ });
14169
14877
  if (await deps.coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) {
14170
14878
  await deps.view.refresh(indexName);
14171
14879
  return;
@@ -14177,6 +14885,60 @@ async function markClusterSyncActive(deps, indexName) {
14177
14885
  { reason: SETTINGS_CONTENTION_REASON, indexName, attempts: MAX_SETTINGS_REPLACEMENT_ATTEMPTS }
14178
14886
  );
14179
14887
  }
14888
+
14889
+ // src/distribution/cluster-node/device-sync/snapshots.ts
14890
+ function syncOff(indexName) {
14891
+ return new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index "${indexName}" has device sync off`, { indexName });
14892
+ }
14893
+ function preparing(indexName) {
14894
+ return new NarsilError(
14895
+ ErrorCodes.SYNC_SNAPSHOT_PREPARING,
14896
+ `The cluster is taking the backup that the snapshot of index "${indexName}" needs. Ask again later`,
14897
+ { indexName, retryAfterMs: DEVICE_SYNC_SNAPSHOT_RETRY_AFTER_MS }
14898
+ );
14899
+ }
14900
+ function createClusterSnapshots(deps) {
14901
+ const { catalog, spools } = deps;
14902
+ function requireStore2(indexName) {
14903
+ if (deps.store === null) {
14904
+ throw new NarsilError(
14905
+ ErrorCodes.SYNC_BACKUP_STORE_REQUIRED,
14906
+ `Device sync in a cluster needs a backup store, because a cluster node takes a device's first copy from a backup`,
14907
+ { indexName }
14908
+ );
14909
+ }
14910
+ return deps.store;
14911
+ }
14912
+ return {
14913
+ async snapshot(indexName) {
14914
+ let held = await deps.settings.view.current(indexName);
14915
+ const state = held === null ? null : syncStateOfSettings(held.settings);
14916
+ if (held === null || state === null) throw syncOff(indexName);
14917
+ if (!state.active) {
14918
+ await markClusterSyncActive(deps.settings, indexName);
14919
+ held = deps.settings.view.held(indexName) ?? held;
14920
+ }
14921
+ const store = requireStore2(indexName);
14922
+ const backup = await catalog.qualifying(store, indexName, held.serving.partitionCount, state.epoch);
14923
+ if (backup === null) {
14924
+ await catalog.startBackup(indexName, SYNC_BACKUP_TAKER);
14925
+ throw preparing(indexName);
14926
+ }
14927
+ const { archive, release } = await spools.acquire(store, indexName, backup);
14928
+ return { entityTag: archive.entityTag, layout: archive.layout, release };
14929
+ },
14930
+ async snapshotBytes(indexName) {
14931
+ const held = deps.settings.view.held(indexName);
14932
+ const state = held === void 0 ? null : syncStateOfSettings(held.settings);
14933
+ if (held === void 0 || state === null || deps.store === null) return 0;
14934
+ const archive = await spools.chosen(indexName);
14935
+ if (archive !== null) return archive.layout.byteLength;
14936
+ const [newest] = await catalog.matching(deps.store, indexName, held.serving.partitionCount, state.epoch);
14937
+ if (newest === void 0) return 0;
14938
+ return estimatedArchiveBytes(storedFilesOf(newest));
14939
+ }
14940
+ };
14941
+ }
14180
14942
  function damaged2(backupName, reason) {
14181
14943
  return new NarsilError(ErrorCodes.BACKUP_DAMAGED, `Backup '${backupName}' ${reason}, so a device can't load it`, {
14182
14944
  backupName
@@ -14317,13 +15079,17 @@ async function spoolBackupArchive(store, backup, spoolDirectory) {
14317
15079
  const prefix = `${sourceIndexName}/`;
14318
15080
  const ordered = manifestFileRefs(manifest);
14319
15081
  const byKey = new Map(files.map((file) => [file.ref.key, file]));
15082
+ const pathOf = (ref) => {
15083
+ const file = byKey.get(ref.key);
15084
+ const path = file === void 0 ? void 0 : pathByDigest.get(file.listed.sha256);
15085
+ if (path === void 0) throw damaged2(backup.name, `lost the spooled copy of "${ref.key}"`);
15086
+ return path;
15087
+ };
14320
15088
  const layout = archiveLayout(
14321
15089
  config,
14322
15090
  packed,
14323
15091
  ordered.map((ref) => {
14324
- const file = byKey.get(ref.key);
14325
- const path = file === void 0 ? void 0 : pathByDigest.get(file.listed.sha256);
14326
- if (path === void 0) throw damaged2(backup.name, `lost the spooled copy of "${ref.key}"`);
15092
+ const path = pathOf(ref);
14327
15093
  return {
14328
15094
  name: ref.key.slice(prefix.length),
14329
15095
  bytes: ref.bytes,
@@ -14331,109 +15097,45 @@ async function spoolBackupArchive(store, backup, spoolDirectory) {
14331
15097
  };
14332
15098
  })
14333
15099
  );
14334
- return { backupName: backup.name, entityTag, layout, dispose };
15100
+ const inPartitionOrder = {
15101
+ ...manifest,
15102
+ partitions: [...manifest.partitions].sort((a, b) => a.partitionId - b.partitionId)
15103
+ };
15104
+ const checkpoint = (release) => ({
15105
+ manifestBytes: packed,
15106
+ manifest: inPartitionOrder,
15107
+ readFile: (ref) => readSpooled2(pathOf(ref), 0, ref.bytes),
15108
+ readRange: (ref, offset, length) => readSpooled2(pathOf(ref), offset, length),
15109
+ release
15110
+ });
15111
+ return { backupName: backup.name, entityTag, layout, config, checkpoint, dispose };
14335
15112
  } catch (error) {
14336
15113
  await dispose();
14337
15114
  throw error;
14338
15115
  }
14339
15116
  }
14340
15117
 
14341
- // src/distribution/cluster-node/device-sync/snapshots.ts
14342
- function syncBackupPrefix(indexName) {
14343
- return `${DEVICE_SYNC_BACKUP_NAME_PREFIX}${sha256Hex(new TextEncoder().encode(indexName)).slice(0, 32)}-`;
14344
- }
14345
- function syncOff(indexName) {
14346
- return new NarsilError(ErrorCodes.SYNC_NOT_ENABLED, `Index "${indexName}" has device sync off`, { indexName });
14347
- }
14348
- function preparing(indexName) {
14349
- return new NarsilError(
14350
- ErrorCodes.SYNC_SNAPSHOT_PREPARING,
14351
- `The cluster is taking the backup that the snapshot of index "${indexName}" needs. Ask again later`,
14352
- { indexName, retryAfterMs: DEVICE_SYNC_SNAPSHOT_RETRY_AFTER_MS }
14353
- );
15118
+ // src/distribution/cluster-node/device-sync/spool-cache.ts
15119
+ function disposeOf(entry) {
15120
+ return entry.ready.then((archive) => archive.dispose()).catch(() => void 0);
14354
15121
  }
14355
- function createClusterSnapshots(deps) {
14356
- const manifests = /* @__PURE__ */ new Map();
15122
+ function createSpoolCache(spoolDirectory, guard) {
14357
15123
  const spools = /* @__PURE__ */ new Map();
14358
- const chosen = /* @__PURE__ */ new Map();
14359
- const backingUp = /* @__PURE__ */ new Map();
15124
+ const chosenNames = /* @__PURE__ */ new Map();
14360
15125
  let closed = false;
14361
- function requireStore2(indexName) {
14362
- if (deps.store === null) {
14363
- throw new NarsilError(
14364
- ErrorCodes.SYNC_BACKUP_STORE_REQUIRED,
14365
- `Device sync in a cluster needs a backup store, because a cluster node takes a device's first copy from a backup`,
14366
- { indexName }
14367
- );
14368
- }
14369
- return deps.store;
14370
- }
14371
- async function manifestOf(store, name) {
14372
- if (manifests.has(name)) return manifests.get(name) ?? null;
14373
- let read;
14374
- try {
14375
- read = await readBackupManifest(store, name);
14376
- } catch (error) {
14377
- if (!(error instanceof NarsilError)) throw error;
14378
- if (error.code !== ErrorCodes.BACKUP_DAMAGED && error.code !== ErrorCodes.BACKUP_NOT_FOUND) throw error;
14379
- read = null;
14380
- }
14381
- if (manifests.size >= CACHED_BACKUP_MANIFEST_LIMIT) {
14382
- const oldest = manifests.keys().next().value;
14383
- if (oldest !== void 0) manifests.delete(oldest);
14384
- }
14385
- manifests.set(name, read);
14386
- return read;
14387
- }
14388
- async function backupsOf(indexName, held, epoch) {
14389
- const store = requireStore2(indexName);
14390
- const prefix = syncBackupPrefix(indexName);
14391
- const names = (await listBackupNames(store)).filter(
14392
- (name) => name.startsWith(prefix) || !name.startsWith(DEVICE_SYNC_BACKUP_NAME_PREFIX)
14393
- );
14394
- const matching = [];
14395
- for (const name of names) {
14396
- const manifest = await manifestOf(store, name);
14397
- if (manifest !== null && manifest.indexName === indexName && manifest.syncEpoch === epoch && manifest.partitionCount === held.serving.partitionCount) {
14398
- matching.push(manifest);
14399
- }
14400
- }
14401
- return matching.sort((a, b) => b.completedAt - a.completedAt || b.startedAt - a.startedAt);
14402
- }
14403
- async function qualifyingBackup(indexName, held, epoch) {
14404
- const candidates = await backupsOf(indexName, held, epoch);
14405
- if (candidates.length === 0) return null;
14406
- const oldest = [];
14407
- for (let partitionId = 0; partitionId < held.serving.partitionCount; partitionId++) {
14408
- oldest.push((await deps.bounds(indexName, partitionId)).oldestSeqNo);
14409
- }
14410
- return candidates.find(
14411
- (backup) => backup.partitions.every((partition) => partition.lastSeqNo >= (oldest[partition.partitionId] ?? 0) - 1)
14412
- ) ?? null;
14413
- }
14414
- async function startBackup(indexName) {
14415
- if (backingUp.has(indexName) || closed) return;
14416
- for (const job of (await readBackupJobs(deps.coordinator)).values()) {
14417
- if (job.kind === "backup" && job.status === "running" && job.indexName === indexName) return;
14418
- }
14419
- if (backingUp.has(indexName)) return;
14420
- const backupName = `${syncBackupPrefix(indexName)}${generateId()}`;
14421
- const run = requestSyncBackup(deps.requests, indexName, backupName).catch(deps.onError).finally(() => backingUp.delete(indexName));
14422
- backingUp.set(indexName, run);
14423
- }
14424
15126
  function evictIdle(now3) {
14425
15127
  for (const [name, entry] of spools) {
14426
15128
  if (entry.users > 0) continue;
14427
- const superseded = chosen.get(entry.indexName) !== name;
15129
+ const superseded = chosenNames.get(entry.indexName) !== name;
14428
15130
  if (!superseded && now3 - entry.lastUsedAt < DEVICE_SYNC_ARCHIVE_IDLE_MS) continue;
14429
15131
  spools.delete(name);
14430
- void entry.ready.then((archive) => archive.dispose()).catch(() => void 0);
15132
+ void disposeOf(entry);
14431
15133
  }
14432
15134
  }
14433
15135
  function spoolOf(store, indexName, backup) {
14434
15136
  let entry = spools.get(backup.name);
14435
15137
  if (entry === void 0) {
14436
- const ready = spoolBackupArchive(store, backup, deps.spoolDirectory);
15138
+ const ready = spoolBackupArchive(store, backup, spoolDirectory);
14437
15139
  entry = { indexName, ready, users: 0, lastUsedAt: Date.now() };
14438
15140
  spools.set(backup.name, entry);
14439
15141
  const created = entry;
@@ -14444,23 +15146,10 @@ function createClusterSnapshots(deps) {
14444
15146
  return entry;
14445
15147
  }
14446
15148
  return {
14447
- async snapshot(indexName) {
14448
- let held = await deps.settings.view.current(indexName);
14449
- const state = held === null ? null : syncStateOfSettings(held.settings);
14450
- if (held === null || state === null) throw syncOff(indexName);
14451
- if (!state.active) {
14452
- await markClusterSyncActive(deps.settings, indexName);
14453
- held = deps.settings.view.held(indexName) ?? held;
14454
- }
14455
- const store = requireStore2(indexName);
14456
- const backup = await qualifyingBackup(indexName, held, state.epoch);
14457
- if (backup === null) {
14458
- await startBackup(indexName);
14459
- throw preparing(indexName);
14460
- }
14461
- chosen.set(indexName, backup.name);
14462
- const now3 = Date.now();
14463
- evictIdle(now3);
15149
+ async acquire(store, indexName, backup) {
15150
+ guard();
15151
+ chosenNames.set(indexName, backup.name);
15152
+ evictIdle(Date.now());
14464
15153
  const entry = spoolOf(store, indexName, backup);
14465
15154
  entry.users += 1;
14466
15155
  let archive;
@@ -14472,35 +15161,27 @@ function createClusterSnapshots(deps) {
14472
15161
  }
14473
15162
  let released = false;
14474
15163
  return {
14475
- entityTag: archive.entityTag,
14476
- layout: archive.layout,
15164
+ archive,
14477
15165
  release() {
14478
15166
  if (released) return;
14479
15167
  released = true;
14480
15168
  entry.users -= 1;
14481
15169
  entry.lastUsedAt = Date.now();
15170
+ if (closed && entry.users === 0) void disposeOf(entry);
14482
15171
  }
14483
15172
  };
14484
15173
  },
14485
- async snapshotBytes(indexName) {
14486
- const held = deps.settings.view.held(indexName);
14487
- const state = held === void 0 ? null : syncStateOfSettings(held.settings);
14488
- if (held === void 0 || state === null || deps.store === null) return 0;
14489
- const spooled = spools.get(chosen.get(indexName) ?? "");
14490
- if (spooled !== void 0) {
14491
- const archive = await spooled.ready.catch(() => null);
14492
- if (archive !== null) return archive.layout.byteLength;
14493
- }
14494
- const [newest] = await backupsOf(indexName, held, state.epoch);
14495
- if (newest === void 0) return 0;
14496
- return estimatedArchiveBytes(storedFilesOf(newest));
15174
+ async chosen(indexName) {
15175
+ const spooled = spools.get(chosenNames.get(indexName) ?? "");
15176
+ if (spooled === void 0) return null;
15177
+ return spooled.ready.catch(() => null);
14497
15178
  },
14498
15179
  async close() {
14499
15180
  closed = true;
14500
- await Promise.all([...backingUp.values()]);
14501
- const entries = [...spools.values()];
15181
+ const idle = [...spools.values()].filter((entry) => entry.users === 0);
14502
15182
  spools.clear();
14503
- await Promise.all(entries.map((entry) => entry.ready.then((archive) => archive.dispose()).catch(() => void 0)));
15183
+ chosenNames.clear();
15184
+ await Promise.all(idle.map(disposeOf));
14504
15185
  }
14505
15186
  };
14506
15187
  }
@@ -14542,11 +15223,16 @@ function createDeviceSyncView(deps) {
14542
15223
  }
14543
15224
  }
14544
15225
  async function adoptOnLocalCopies(indexName, next) {
14545
- const state = next === null ? null : syncStateOfSettings(next.settings);
15226
+ const states = {
15227
+ sync: next === null ? null : syncStateOfSettings(next.settings),
15228
+ changes: next === null ? null : changesStateOfSettings(next.settings)
15229
+ };
14546
15230
  const local = new Set(engine.listIndexes().map((index) => index.name));
14547
15231
  const servingName = next?.serving.servingName ?? indexName;
14548
- if (local.has(servingName)) await engine.adoptSyncState(servingName, state);
14549
- if (servingName !== indexName && local.has(indexName)) await engine.adoptSyncState(indexName, null);
15232
+ if (local.has(servingName)) await engine.adoptEpochStates(servingName, states);
15233
+ if (servingName !== indexName && local.has(indexName)) {
15234
+ await engine.adoptEpochStates(indexName, { sync: null, changes: null });
15235
+ }
14550
15236
  }
14551
15237
  async function reread(indexName) {
14552
15238
  const previous = held.get(indexName);
@@ -14554,8 +15240,12 @@ function createDeviceSyncView(deps) {
14554
15240
  if (next === null) held.delete(indexName);
14555
15241
  else held.set(indexName, next);
14556
15242
  await adoptOnLocalCopies(indexName, next);
14557
- const change = changeBetween(previous, next);
14558
- if (change !== null) emit({ kind: change, indexName });
15243
+ const kinds = /* @__PURE__ */ new Set();
15244
+ const syncChange = changeBetween(previous, next);
15245
+ const changesChange = changesChangeBetween(previous, next);
15246
+ if (syncChange !== null) kinds.add(syncChange);
15247
+ if (changesChange !== null) kinds.add(changesChange);
15248
+ for (const kind of kinds) emit({ kind, indexName });
14559
15249
  return next;
14560
15250
  }
14561
15251
  function refresh(indexName) {
@@ -14653,7 +15343,7 @@ function createDeviceSyncView(deps) {
14653
15343
 
14654
15344
  // src/distribution/cluster-node/device-sync/index.ts
14655
15345
  function createClusterDeviceSync(deps) {
14656
- const { config, coordinator, engine, onError } = deps;
15346
+ const { config, coordinator, engine, onError, guard } = deps;
14657
15347
  const view = createDeviceSyncView({ coordinator, engine, onError });
14658
15348
  const store = config.backups === void 0 ? null : openBackupStore(config.backups);
14659
15349
  const settings = {
@@ -14662,38 +15352,61 @@ function createClusterDeviceSync(deps) {
14662
15352
  hasBackupStore: store !== null,
14663
15353
  hasDurability: () => engine.syncStorage() !== null
14664
15354
  };
14665
- const requests = { config, nodeId: deps.nodeId, guard: deps.guard, store };
15355
+ const requests = { config, nodeId: deps.nodeId, guard, store };
15356
+ const spools = createSpoolCache(join(deps.spoolDirectory, "device-sync"), guard);
14666
15357
  let snapshots = null;
15358
+ let feed = null;
15359
+ let changeSource = null;
14667
15360
  let serving = null;
15361
+ function feedSource() {
15362
+ feed ??= createClusterFeedSource({
15363
+ nodeId: deps.nodeId,
15364
+ writeDeps: deps.writeDeps,
15365
+ config,
15366
+ coordinator,
15367
+ engine,
15368
+ view,
15369
+ refreshAllocation: deps.refreshAllocation,
15370
+ snapshotBytes: (indexName) => snapshots?.snapshotBytes(indexName) ?? Promise.resolve(0)
15371
+ });
15372
+ return feed;
15373
+ }
15374
+ const catalog = createBackupCatalog({
15375
+ coordinator,
15376
+ requests,
15377
+ bounds: (indexName, partitionId) => feedSource().bounds(indexName, partitionId),
15378
+ onError
15379
+ });
15380
+ function changesSource() {
15381
+ changeSource ??= createClusterChangeSource({ feed: feedSource(), view, coordinator });
15382
+ return changeSource;
15383
+ }
15384
+ async function admitChanges(indexName) {
15385
+ await requireChangeCaptureNodes(settings, indexName);
15386
+ await view.current(indexName);
15387
+ }
15388
+ async function* follow(indexName, options) {
15389
+ guard();
15390
+ await admitChanges(indexName);
15391
+ yield* followSourceChanges({ source: changesSource(), guard }, indexName, options);
15392
+ }
14668
15393
  return {
14669
15394
  view,
14670
- updateSync: (indexName, setting) => updateClusterSync(settings, indexName, setting),
14671
- getSync: (indexName) => readClusterSync(settings, indexName),
15395
+ operations: (track) => ({
15396
+ updateSync: (indexName, setting) => track(indexName, () => updateClusterSync(settings, indexName, setting)),
15397
+ getSync: (indexName) => track(indexName, () => readClusterSync(settings, indexName)),
15398
+ updateChanges: (indexName, setting) => track(indexName, () => updateClusterChanges(settings, indexName, setting)),
15399
+ getChanges: (indexName) => track(indexName, () => readClusterChanges(settings, indexName)),
15400
+ followChanges: (indexName, options) => follow(indexName, options),
15401
+ exportDocuments: (indexName) => track(indexName, () => exportClusterDocuments({ view, store, catalog, spools }, indexName))
15402
+ }),
14672
15403
  serving() {
14673
15404
  if (serving !== null) return serving;
14674
- const feed = createClusterFeedSource({
14675
- nodeId: deps.nodeId,
14676
- writeDeps: deps.writeDeps,
14677
- config,
14678
- coordinator,
14679
- engine,
14680
- view,
14681
- refreshAllocation: deps.refreshAllocation,
14682
- snapshotBytes: (indexName) => snapshots?.snapshotBytes(indexName) ?? Promise.resolve(0)
14683
- });
14684
- const built = createClusterSnapshots({
14685
- coordinator,
14686
- settings,
14687
- store,
14688
- requests,
14689
- spoolDirectory: join(deps.spoolDirectory, "device-sync"),
14690
- bounds: (indexName, partitionId) => feed.bounds(indexName, partitionId),
14691
- onError
14692
- });
15405
+ const built = createClusterSnapshots({ settings, store, catalog, spools });
14693
15406
  snapshots = built;
14694
15407
  serving = {
14695
15408
  snapshot: (indexName) => built.snapshot(indexName),
14696
- feed,
15409
+ feed: feedSource(),
14697
15410
  async admitFeed(indexName) {
14698
15411
  const olderNodes = registeredNodesBelowDeviceSync(await coordinator.listNodes());
14699
15412
  if (olderNodes.length > 0) throw syncNotReady("The cluster has", olderNodes, indexName);
@@ -14702,10 +15415,12 @@ function createClusterDeviceSync(deps) {
14702
15415
  };
14703
15416
  return serving;
14704
15417
  },
15418
+ changesServing: () => ({ source: changesSource(), admitFeed: admitChanges }),
14705
15419
  start: () => view.start(),
14706
15420
  async stop() {
14707
15421
  view.stop();
14708
- await snapshots?.close();
15422
+ await catalog.close();
15423
+ await spools.close();
14709
15424
  }
14710
15425
  };
14711
15426
  }
@@ -14734,10 +15449,6 @@ function createAllocationErrorForwarder(forwardOnError) {
14734
15449
  }
14735
15450
 
14736
15451
  // src/distribution/replication/log.ts
14737
- function estimateEntrySize(entry) {
14738
- return REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
14739
- }
14740
- var computeChecksum = computeEntryChecksum;
14741
15452
  function ownedDocument(document) {
14742
15453
  if (document === null || document.byteOffset === 0 && document.byteLength === document.buffer.byteLength) {
14743
15454
  return document;
@@ -14782,7 +15493,7 @@ function createReplicationLog(partitionId, config) {
14782
15493
  }
14783
15494
  function evictOldEntries() {
14784
15495
  while (liveCount() > 0 && totalSizeBytes > retentionBytes) {
14785
- totalSizeBytes -= estimateEntrySize(entries[headIndex]);
15496
+ totalSizeBytes -= entryByteEstimate(entries[headIndex]);
14786
15497
  entries[headIndex] = null;
14787
15498
  headIndex += 1;
14788
15499
  }
@@ -14794,7 +15505,7 @@ function createReplicationLog(partitionId, config) {
14794
15505
  }
14795
15506
  }
14796
15507
  function assertEntryFitsRetention(entry) {
14797
- const entrySize = REPLICATION_ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
15508
+ const entrySize = entryByteEstimate(entry);
14798
15509
  if (entrySize > retentionBytes) {
14799
15510
  throw new NarsilError(
14800
15511
  ErrorCodes.REPLICATION_LOG_FULL,
@@ -14805,12 +15516,13 @@ function createReplicationLog(partitionId, config) {
14805
15516
  }
14806
15517
  function storeEntry(appended) {
14807
15518
  const entry = { ...appended, document: ownedDocument(appended.document) };
15519
+ if (appended.previousDocument !== void 0) entry.previousDocument = ownedDocument(appended.previousDocument);
14808
15520
  entries.push(entry);
14809
15521
  if (entry.seqNo >= localLogEnd) {
14810
15522
  localLogEnd = entry.seqNo;
14811
15523
  localLogEndPrimaryTerm = entry.primaryTerm;
14812
15524
  }
14813
- totalSizeBytes += estimateEntrySize(entry);
15525
+ totalSizeBytes += entryByteEstimate(entry);
14814
15526
  evictOldEntries();
14815
15527
  return { ...entry };
14816
15528
  }
@@ -14823,16 +15535,7 @@ function createReplicationLog(partitionId, config) {
14823
15535
  return void 0;
14824
15536
  }
14825
15537
  function verifyEntryChecksum(entry) {
14826
- const expected = computeChecksum({
14827
- seqNo: entry.seqNo,
14828
- primaryTerm: entry.primaryTerm,
14829
- operation: entry.operation,
14830
- partitionId: entry.partitionId,
14831
- indexName: entry.indexName,
14832
- documentId: entry.documentId,
14833
- document: entry.document
14834
- });
14835
- return expected === entry.checksum;
15538
+ return computeEntryChecksum(entry) === entry.checksum;
14836
15539
  }
14837
15540
  return {
14838
15541
  assertFits: assertEntryFitsRetention,
@@ -14840,18 +15543,18 @@ function createReplicationLog(partitionId, config) {
14840
15543
  assertEntryFitsRetention(partial);
14841
15544
  const seqNo = nextSeqNo;
14842
15545
  nextSeqNo += 1;
14843
- const entryWithoutChecksum = {
14844
- seqNo,
14845
- primaryTerm: partial.primaryTerm,
14846
- operation: partial.operation,
14847
- partitionId,
14848
- indexName: partial.indexName,
14849
- documentId: partial.documentId,
14850
- document: partial.document
14851
- };
14852
- const checksum = computeChecksum(entryWithoutChecksum);
14853
- const entry = { ...entryWithoutChecksum, checksum };
14854
- return storeEntry(entry);
15546
+ return storeEntry(
15547
+ buildEntry({
15548
+ seqNo,
15549
+ primaryTerm: partial.primaryTerm,
15550
+ operation: partial.operation,
15551
+ partitionId,
15552
+ indexName: partial.indexName,
15553
+ documentId: partial.documentId,
15554
+ document: partial.document,
15555
+ previousDocument: partial.previousDocument
15556
+ })
15557
+ );
14855
15558
  },
14856
15559
  appendCommitted(entry) {
14857
15560
  if (!Number.isSafeInteger(entry.seqNo) || entry.seqNo < 1) {
@@ -15009,6 +15712,138 @@ function forgetDroppedCopy(state, indexName) {
15009
15712
  forgetCatchUpIndex(state.catchUp(), indexName);
15010
15713
  }
15011
15714
 
15715
+ // src/engine/memory/table-model.ts
15716
+ function capacityFor(entries) {
15717
+ if (entries <= MAP_TABLE_MIN_CAPACITY) return MAP_TABLE_MIN_CAPACITY;
15718
+ return 2 ** Math.ceil(Math.log2(entries));
15719
+ }
15720
+ function largestCapacityFor(entries) {
15721
+ return Math.max(capacityFor(entries), 2 ** Math.floor(Math.log2(MAP_TABLE_SHRINK_DIVISOR * (entries + 1))));
15722
+ }
15723
+ function resetModel(model, shape, deleted) {
15724
+ model.deleted = deleted;
15725
+ model.resetEntries = shape.largest;
15726
+ model.resetDeleted = deleted;
15727
+ model.resetSmallestCapacity = capacityFor(shape.smallest);
15728
+ model.resetLargestCapacity = largestCapacityFor(shape.largest);
15729
+ model.addedAtLeast = 0;
15730
+ model.addedAtMost = 0;
15731
+ model.deletedSinceRehashBecamePossible = null;
15732
+ }
15733
+ function createTableModel(entries, churn, deleted, shape) {
15734
+ const model = {
15735
+ entries,
15736
+ churn,
15737
+ deleted,
15738
+ resetEntries: 0,
15739
+ resetDeleted: 0,
15740
+ resetSmallestCapacity: 0,
15741
+ resetLargestCapacity: 0,
15742
+ addedAtLeast: 0,
15743
+ addedAtMost: 0,
15744
+ deletedSinceRehashBecamePossible: null
15745
+ };
15746
+ resetModel(model, shape, deleted);
15747
+ return model;
15748
+ }
15749
+ function observeTable(model, entries, churn, shape) {
15750
+ const churned = Math.max(0, churn - model.churn);
15751
+ const change = entries - model.entries;
15752
+ const deleted = Math.max(churned, -change);
15753
+ const lowest = Math.max(0, Math.min(model.entries, entries) - deleted);
15754
+ model.entries = entries;
15755
+ model.churn = churn;
15756
+ model.deleted += deleted;
15757
+ model.addedAtLeast += Math.max(0, change);
15758
+ model.addedAtMost += Math.max(0, change) + churned;
15759
+ if (entries === 0) {
15760
+ resetModel(model, shape, Math.min(model.deleted, MAP_TABLE_MIN_CAPACITY));
15761
+ return;
15762
+ }
15763
+ if (shape.shards !== 1) return;
15764
+ if (model.deletedSinceRehashBecamePossible !== null) {
15765
+ model.deletedSinceRehashBecamePossible += deleted;
15766
+ } else if (model.resetEntries + model.resetDeleted + model.addedAtMost >= model.resetSmallestCapacity || lowest < model.resetLargestCapacity / MAP_TABLE_SHRINK_DIVISOR) {
15767
+ model.deletedSinceRehashBecamePossible = deleted;
15768
+ }
15769
+ if (model.addedAtLeast > model.resetLargestCapacity - model.resetEntries || entries < model.resetSmallestCapacity / MAP_TABLE_SHRINK_DIVISOR) {
15770
+ resetModel(model, shape, model.deletedSinceRehashBecamePossible ?? model.deleted);
15771
+ }
15772
+ }
15773
+ function tableGrowthBytes(model, shape, added) {
15774
+ if (added <= 0) return 0;
15775
+ const largestCapacity = largestCapacityFor(shape.largest);
15776
+ let grownBytes = 0;
15777
+ for (let capacity = capacityFor(shape.smallest); capacity <= largestCapacity; capacity *= 2) {
15778
+ const lastOccupied = shape.largest + added - 1;
15779
+ const deletedSlots = Math.max(0, Math.min(model.deleted, capacity - shape.largest));
15780
+ if (lastOccupied + deletedSlots < capacity) continue;
15781
+ const entriesAfter = shape.largest + added;
15782
+ const rebuildsAtSize = deletedSlots >= capacity / 2;
15783
+ const doubles = lastOccupied + Math.min(deletedSlots, capacity / 2 - 1) >= capacity;
15784
+ const rebuiltBytes = rebuildsAtSize ? allocatedTableBytes(capacity, entriesAfter) : 0;
15785
+ const doubledBytes = doubles ? allocatedTableBytes(capacity * 2, entriesAfter) : 0;
15786
+ grownBytes = Math.max(grownBytes, rebuiltBytes, doubledBytes);
15787
+ }
15788
+ return shape.shards * grownBytes;
15789
+ }
15790
+ function allocatedTableBytes(firstCapacity, entriesAfter) {
15791
+ let capacity = firstCapacity;
15792
+ let bytes = MAP_TABLE_HEADER_BYTES + MAP_TABLE_SLOT_BYTES * capacity;
15793
+ while (entriesAfter > capacity) {
15794
+ capacity *= 2;
15795
+ bytes += MAP_TABLE_HEADER_BYTES + MAP_TABLE_SLOT_BYTES * capacity;
15796
+ }
15797
+ return bytes;
15798
+ }
15799
+
15800
+ // src/engine/memory/document-tables.ts
15801
+ function plainShapeOf(entries) {
15802
+ return { shards: 1, smallest: entries, largest: entries };
15803
+ }
15804
+ function createDocumentTableGrowth(core, heldTableSizes) {
15805
+ const modelsByIndex = /* @__PURE__ */ new Map();
15806
+ return (indexName, documents, pendingDocuments) => {
15807
+ const manager = core.executor.getManager(indexName);
15808
+ if (manager === void 0 || manager === null) {
15809
+ modelsByIndex.delete(indexName);
15810
+ return 0;
15811
+ }
15812
+ const previous = modelsByIndex.get(indexName);
15813
+ const models = /* @__PURE__ */ new Map();
15814
+ modelsByIndex.set(indexName, models);
15815
+ const churn = documentChurnOf(indexName);
15816
+ let bytes = 0;
15817
+ const measure = (key, entries, shape, followsRemoves) => {
15818
+ let model = previous?.get(key);
15819
+ if (model === void 0) {
15820
+ model = createTableModel(entries, churn, followsRemoves ? churn : Number.POSITIVE_INFINITY, shape);
15821
+ } else {
15822
+ observeTable(model, entries, churn, shape);
15823
+ }
15824
+ models.set(key, model);
15825
+ if (documents <= 0) return;
15826
+ bytes += tableGrowthBytes(model, shape, pendingDocuments + documents) - tableGrowthBytes(model, shape, pendingDocuments);
15827
+ };
15828
+ const located = manager.countDocuments();
15829
+ measure("located", located, shardEntryRangeOf(located), true);
15830
+ for (const partition of manager.getAllPartitions()) {
15831
+ const entries = (isCompositePartition(partition) ? partition.live : partition).count();
15832
+ measure(`partition:${partition.partitionId}`, entries, shardEntryRangeOf(entries), true);
15833
+ }
15834
+ for (const [field, vectorIndex] of manager.getVectorIndexes()) {
15835
+ vectorIndex.visitDocumentTables(
15836
+ (table, entries, followsRemoves) => measure(`vector:${field}:${table}`, entries, shardEntryRangeOf(entries), followsRemoves)
15837
+ );
15838
+ }
15839
+ heldTableSizes?.(
15840
+ indexName,
15841
+ (partitionId, entries) => measure(`held:${partitionId}`, entries, plainShapeOf(entries), true)
15842
+ );
15843
+ return bytes;
15844
+ };
15845
+ }
15846
+
15012
15847
  // src/engine/memory/write-analysis.ts
15013
15848
  function writeAnalysisOf(core, indexName) {
15014
15849
  const entry = core.indexRegistry.get(indexName);
@@ -15031,27 +15866,38 @@ function vectorGrowthBytesOf(core, indexName, documents) {
15031
15866
  }
15032
15867
 
15033
15868
  // src/engine/sync/adopt.ts
15034
- function sameState(held, next) {
15869
+ function sameSync(held, next) {
15035
15870
  if (held === null || next === null) return held === next;
15036
15871
  return held.epoch === next.epoch && held.active === next.active && held.historyMaxAgeMs === next.historyMaxAgeMs && held.historyMaxBytes === next.historyMaxBytes;
15037
15872
  }
15038
- async function adoptIndexSyncState(core, indexName, next) {
15873
+ function sameChanges(held, next) {
15874
+ if (held === null || next === null) return held === next;
15875
+ return held.epoch === next.epoch && held.previousDocument === next.previousDocument && held.historyMaxAgeMs === next.historyMaxAgeMs && held.historyMaxBytes === next.historyMaxBytes;
15876
+ }
15877
+ function epochOf(states) {
15878
+ return states.sync?.epoch ?? states.changes?.epoch ?? null;
15879
+ }
15880
+ async function adoptIndexEpochStates(core, indexName, next) {
15039
15881
  return oneSyncChangeAtATime(core.indexRegistry, indexName, async () => {
15040
15882
  const entry = core.indexRegistry.get(indexName);
15041
15883
  if (entry === void 0) return false;
15042
- const held = entry.sync;
15043
- if (sameState(held, next)) return false;
15884
+ const held = { sync: entry.sync, changes: entry.changes };
15885
+ if (sameSync(held.sync, next.sync) && sameChanges(held.changes, next.changes)) return false;
15044
15886
  const storage = core.durability?.manager.syncStorage?.();
15045
- const anotherEpoch = held !== null && next !== null && held.epoch !== next.epoch;
15046
- if (anotherEpoch) await storage?.release(indexName);
15047
- entry.sync = next;
15887
+ const heldEpoch = epochOf(held);
15888
+ const nextEpoch = epochOf(next);
15889
+ if (heldEpoch !== null && nextEpoch !== null && heldEpoch !== nextEpoch) await storage?.release(indexName);
15890
+ entry.sync = next.sync;
15891
+ entry.changes = next.changes;
15048
15892
  try {
15049
15893
  await core.durability?.manager.persistMetadata(indexName);
15050
15894
  } catch (error) {
15051
- entry.sync = held;
15895
+ entry.sync = held.sync;
15896
+ entry.changes = held.changes;
15052
15897
  throw error;
15053
15898
  }
15054
- if (next === null) await storage?.release(indexName);
15899
+ if (next.sync === null && held.sync !== null) await storage?.release(indexName, next.changes !== null);
15900
+ else if (nextEpoch === null && heldEpoch !== null) await storage?.release(indexName);
15055
15901
  return true;
15056
15902
  });
15057
15903
  }
@@ -15127,7 +15973,7 @@ async function openOnce(core, indexName, partitionId) {
15127
15973
  if (directory === void 0) throw durabilityRequired(indexName, partitionId);
15128
15974
  if ((checkpoint.manifest.analysisGeneration ?? 0) !== entry.analysisGeneration) return null;
15129
15975
  const config = await encodeIndexConfiguration(entry);
15130
- const { openPagedPartitionArchive } = await import('./paged-partition-archive-IZ6U7DVN.mjs');
15976
+ const { openPagedPartitionArchive } = await import('./paged-partition-archive-UK44CVXG.mjs');
15131
15977
  const archive = await openPagedPartitionArchive(checkpoint, directory, indexName, partitionId, manager);
15132
15978
  handedOver = true;
15133
15979
  try {
@@ -15289,8 +16135,10 @@ function createClusterWriteRecords() {
15289
16135
  }
15290
16136
  function applied(entry) {
15291
16137
  const held = partition(entry.indexName, entry.partitionId);
16138
+ const heldBefore = held.table.count;
15292
16139
  if (entry.operation === "INDEX") held.table.set(entry.documentId, entry.seqNo, entry.primaryTerm);
15293
16140
  else held.table.remove(entry.documentId);
16141
+ if (entry.operation !== "INDEX" || held.table.count === heldBefore) noteDocumentChurn(entry.indexName, 1);
15294
16142
  held.history.noteEntry(entry.seqNo, entry.primaryTerm);
15295
16143
  }
15296
16144
  function recordOfDocument(indexName, partitionId, documentId) {
@@ -15326,6 +16174,11 @@ function createClusterWriteRecords() {
15326
16174
  applied,
15327
16175
  replayed: applied,
15328
16176
  partition,
16177
+ visitTableSizes(indexName, visit) {
16178
+ const partitions = indexes.get(indexName);
16179
+ if (partitions === void 0) return;
16180
+ for (const [partitionId, held] of partitions) visit(partitionId, held.table.count);
16181
+ },
15329
16182
  adoptPartition(indexName, partitionId, records) {
15330
16183
  Object.assign(partition(indexName, partitionId), records);
15331
16184
  },
@@ -15444,7 +16297,7 @@ async function publishPartitionCopy(core, indexName, partitionId, records, load)
15444
16297
  });
15445
16298
  for (const vector of current.getVectorIndexes().values()) vector.scheduleBuild();
15446
16299
  } finally {
15447
- core.executor.discardDetachedIndex(detached);
16300
+ await core.executor.discardDetachedIndex(detached);
15448
16301
  }
15449
16302
  }
15450
16303
 
@@ -15835,7 +16688,7 @@ async function archiveReplicationPartition(core, indexName, partitionId, lastSeq
15835
16688
  );
15836
16689
  }
15837
16690
  try {
15838
- const { openPagedPartitionArchive } = await import('./paged-partition-archive-IZ6U7DVN.mjs');
16691
+ const { openPagedPartitionArchive } = await import('./paged-partition-archive-UK44CVXG.mjs');
15839
16692
  const archive = await openPagedPartitionArchive(checkpoint2, directory, indexName, partitionId, manager);
15840
16693
  return {
15841
16694
  stream: archiveStream({ ...archive, config, sourceIndexName: indexName }),
@@ -16050,7 +16903,7 @@ async function restoreReplicationPartition(core, engine, indexName, partitionId,
16050
16903
  if (directory !== null) {
16051
16904
  const durable = core.durability?.manager;
16052
16905
  await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
16053
- const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-IZ6U7DVN.mjs');
16906
+ const { removeReceivedFilesNothingReads } = await import('./paged-partition-archive-UK44CVXG.mjs');
16054
16907
  await removeReceivedFilesNothingReads(
16055
16908
  directory,
16056
16909
  core.requireManager(indexName),
@@ -16197,8 +17050,7 @@ async function createClusterLocalEngine(config, hooks) {
16197
17050
  stampIndexUuid: (indexName, indexUuid) => stampIndexUuid(core, indexName, indexUuid),
16198
17051
  applyStoredSynonyms: (indexName, load) => synonymModule().applyLoaded(core, indexName, load),
16199
17052
  applyStoredBm25: (indexName, load) => applyStoredIndexBm25(core, indexName, load),
16200
- syncStateOf: (indexName) => core.indexRegistry.get(indexName)?.sync,
16201
- adoptSyncState: (indexName, state) => adoptIndexSyncState(core, indexName, state),
17053
+ adoptEpochStates: (indexName, states) => adoptIndexEpochStates(core, indexName, states),
16202
17054
  syncStorage: () => core.durability?.manager.syncStorage?.() ?? null,
16203
17055
  historyStorage: () => core.durability?.manager.historyStorage?.() ?? null,
16204
17056
  emitEvent: (name, payload) => emitEngineEvent(core.eventHandlers, name, payload),
@@ -16221,6 +17073,10 @@ async function createClusterLocalEngine(config, hooks) {
16221
17073
  },
16222
17074
  writeAnalysisOf: (indexName) => writeAnalysisOf(core, indexName),
16223
17075
  vectorGrowthBytesOf: (indexName, documents) => vectorGrowthBytesOf(core, indexName, documents),
17076
+ documentTableGrowthBytesOf: createDocumentTableGrowth(
17077
+ core,
17078
+ (indexName, visit) => writeRecords.visitTableSizes(indexName, visit)
17079
+ ),
16224
17080
  restartLocalLog: (indexName, partitionId, position) => restartLocalLog(core, indexName, partitionId, position),
16225
17081
  isolateCopyFailures: (onFailed) => core.durability?.manager.isolatePartitionFailures?.(onFailed),
16226
17082
  emptyPartitionCopy: (indexName, partitionId) => emptyPartitionCopy(core, indexName, partitionId, writeRecords),
@@ -16310,7 +17166,7 @@ function createIndexSettingsResultMessage(result, sourceNodeId, requestId) {
16310
17166
  }
16311
17167
 
16312
17168
  // src/distribution/cluster-node/synonym-sets.ts
16313
- function indexNotFound2(indexName) {
17169
+ function indexNotFound3(indexName) {
16314
17170
  return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
16315
17171
  indexName
16316
17172
  });
@@ -16318,7 +17174,7 @@ function indexNotFound2(indexName) {
16318
17174
  async function readClusterSynonyms(coordinator, indexName) {
16319
17175
  validateIndexName(indexName);
16320
17176
  const state = await readIndexState(coordinator, await servingNameOf(coordinator, indexName));
16321
- if (state === null) throw indexNotFound2(indexName);
17177
+ if (state === null) throw indexNotFound3(indexName);
16322
17178
  return state.synonyms;
16323
17179
  }
16324
17180
  async function requireSettledPartitionCount(coordinator, indexName, servingName) {
@@ -16352,7 +17208,7 @@ async function swapStoredSet(coordinator, indexName, first, ref) {
16352
17208
  const { indexUuid } = first.record;
16353
17209
  let current = first;
16354
17210
  for (let attempt = 1; ; attempt++) {
16355
- if (current === null || current.record.indexUuid !== indexUuid) throw indexNotFound2(indexName);
17211
+ if (current === null || current.record.indexUuid !== indexUuid) throw indexNotFound3(indexName);
16356
17212
  const replaced = current.settings?.synonyms ?? null;
16357
17213
  const next = metadataWithSynonymSet(current, ref);
16358
17214
  if (await coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) return replaced;
@@ -16404,7 +17260,7 @@ async function replaceClusterSynonyms(deps, indexName, synonyms) {
16404
17260
  const servingName = await servingNameOf(coordinator, indexName);
16405
17261
  await requireSettledPartitionCount(coordinator, indexName, servingName);
16406
17262
  const first = await readIndexMetadataSnapshot(coordinator, servingName);
16407
- if (first === null) throw indexNotFound2(indexName);
17263
+ if (first === null) throw indexNotFound3(indexName);
16408
17264
  const encoded2 = preparedSet(first, servingName, synonyms);
16409
17265
  const { indexUuid } = first.record;
16410
17266
  const ref = encoded2 === null ? null : await storeSynonymSet(coordinator, indexUuid, encoded2);
@@ -16532,6 +17388,7 @@ function bindNodeSources(node, sources) {
16532
17388
  });
16533
17389
  bindRerankWindowSource(node, () => rerankWindowCapOf(sources.engine));
16534
17390
  bindSyncServing(node, sources.syncServing);
17391
+ bindChangesServing(node, sources.changesServing);
16535
17392
  }
16536
17393
 
16537
17394
  // src/distribution/cluster-node/node-identity.ts
@@ -16824,6 +17681,8 @@ function createAllocationWatcherState() {
16824
17681
  pendingPartitions: /* @__PURE__ */ new Set(),
16825
17682
  pendingTables: /* @__PURE__ */ new Map(),
16826
17683
  restartWaiters: /* @__PURE__ */ new Map(),
17684
+ unfedReplicaTimer: null,
17685
+ unfedResyncs: /* @__PURE__ */ new Set(),
16827
17686
  retainedPartitions: /* @__PURE__ */ new Set(),
16828
17687
  indexesSeededFromRecord: /* @__PURE__ */ new Set(),
16829
17688
  indexUuids: /* @__PURE__ */ new Map(),
@@ -16840,6 +17699,11 @@ function stopAllocationWatcher(state) {
16840
17699
  clearTimeout(state.debounceTimer);
16841
17700
  state.debounceTimer = null;
16842
17701
  }
17702
+ if (state.unfedReplicaTimer !== null) {
17703
+ clearInterval(state.unfedReplicaTimer);
17704
+ state.unfedReplicaTimer = null;
17705
+ }
17706
+ state.unfedResyncs.clear();
16843
17707
  for (const [timer, resolve] of state.restartWaiters) {
16844
17708
  clearTimeout(timer);
16845
17709
  resolve();
@@ -16972,7 +17836,8 @@ function processAllocationChange(state, config, table) {
16972
17836
  partitionId,
16973
17837
  isPrimary,
16974
17838
  primaryNodeId: assignment.primary,
16975
- primaryTerm: assignment.primaryTerm
17839
+ primaryTerm: assignment.primaryTerm,
17840
+ inSync: assignment.inSyncSet.includes(nodeId)
16976
17841
  });
16977
17842
  if (servesPartition(assignment, nodeId)) {
16978
17843
  state.pendingPartitions.delete(key);
@@ -17159,6 +18024,30 @@ function removeDroppedPartitions(state, config, table, currentKeys) {
17159
18024
  }
17160
18025
  }
17161
18026
 
18027
+ // src/distribution/cluster/node-lifecycle/allocation-watcher/unfed-replicas.ts
18028
+ function startUnfedReplicaChecks(state, config) {
18029
+ const replicaUnfedForMs = config.replicaUnfedForMs;
18030
+ if (replicaUnfedForMs === void 0 || state.unfedReplicaTimer !== null) return;
18031
+ const timer = setInterval(
18032
+ () => resyncUnfedReplicas(state, config, replicaUnfedForMs),
18033
+ UNFED_REPLICA_CHECK_INTERVAL_MS
18034
+ );
18035
+ timer.unref?.();
18036
+ state.unfedReplicaTimer = timer;
18037
+ }
18038
+ function resyncUnfedReplicas(state, config, replicaUnfedForMs) {
18039
+ if (state.stopped) return;
18040
+ for (const [key, tracked] of state.trackedPartitions) {
18041
+ if (tracked.isPrimary || tracked.inSync || tracked.primaryNodeId === null) continue;
18042
+ if (state.activeBootstraps.has(key) || state.unfedResyncs.has(key)) continue;
18043
+ if (replicaUnfedForMs(tracked.indexName, tracked.partitionId) < UNFED_REPLICA_RESYNC_MS) continue;
18044
+ state.unfedResyncs.add(key);
18045
+ void restartBootstrapWhileOutOfSync(state, config, tracked.indexName, tracked.partitionId).finally(() => {
18046
+ state.unfedResyncs.delete(key);
18047
+ });
18048
+ }
18049
+ }
18050
+
17162
18051
  // src/distribution/cluster/node-lifecycle/allocation-watcher/index.ts
17163
18052
  async function startAllocationWatcher(state, config) {
17164
18053
  const unwatchAllocation = await config.coordinator.watchAllocation((event) => {
@@ -17168,6 +18057,7 @@ async function startAllocationWatcher(state, config) {
17168
18057
  scheduleProcessAllocation(state, config, event.table);
17169
18058
  });
17170
18059
  state.unwatchAllocation = unwatchAllocation;
18060
+ startUnfedReplicaChecks(state, config);
17171
18061
  }
17172
18062
  function scheduleProcessAllocation(state, config, table) {
17173
18063
  const pending = state.pendingTables.get(table.indexName);
@@ -17566,7 +18456,14 @@ function createNodeLifecycle(deps) {
17566
18456
  bootstrapMaxRetries: DEFAULT_NODE_LIFECYCLE_CONFIG.bootstrapMaxRetries,
17567
18457
  allocationDebounceMs: DEFAULT_NODE_LIFECYCLE_CONFIG.allocationDebounceMs,
17568
18458
  nodeHeartbeatIntervalMs: DEFAULT_NODE_LIFECYCLE_CONFIG.nodeHeartbeatIntervalMs,
17569
- onBootstrapPartition: async (indexName, partitionId, primaryNodeId) => await deps.copyFailures.awaitReset(indexName, partitionId) && deps.bootstrapPartition(indexName, partitionId, primaryNodeId),
18459
+ onBootstrapPartition: async (indexName, partitionId, primaryNodeId) => {
18460
+ const feedClock = deps.writeDeps.replicaFeedClock;
18461
+ feedClock?.noteFed(indexName, partitionId);
18462
+ const synced = await deps.copyFailures.awaitReset(indexName, partitionId) && await deps.bootstrapPartition(indexName, partitionId, primaryNodeId);
18463
+ if (synced) feedClock?.noteFed(indexName, partitionId);
18464
+ return synced;
18465
+ },
18466
+ replicaUnfedForMs: (indexName, partitionId) => deps.writeDeps.replicaFeedClock?.unfedForMs(indexName, partitionId) ?? 0,
17570
18467
  onLeadingTerm: (indexName, partitionId, assignment) => startLeadingTerm(deps.writeDeps, indexName, partitionId, assignment),
17571
18468
  onHoldPartition: (indexName, partitionId) => {
17572
18469
  deps.trackHeldPartitionWrite(engine.recordHeldPartition(indexName, partitionId));
@@ -17575,6 +18472,7 @@ function createNodeLifecycle(deps) {
17575
18472
  onRemovePartition: (indexName, partitionId) => {
17576
18473
  const removedIndexUuid = engine.indexUuidOf(indexName);
17577
18474
  forgetReshardPartition(deps.reshard, indexName, partitionId);
18475
+ deps.writeDeps.replicaFeedClock?.forget(indexName, partitionId);
17578
18476
  clearBootstrapSyncIndex(deps.bootstrapSyncState, indexName, partitionId);
17579
18477
  deps.replicationLogs.delete(replicationLogKey(indexName, partitionId));
17580
18478
  deps.trackHeldPartitionWrite(engine.forgetHeldPartition(indexName, partitionId));
@@ -17588,7 +18486,7 @@ function createNodeLifecycle(deps) {
17588
18486
  }
17589
18487
 
17590
18488
  // src/distribution/cluster-node/analysis-change/request.ts
17591
- function indexNotFound3(indexName) {
18489
+ function indexNotFound4(indexName) {
17592
18490
  return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
17593
18491
  indexName
17594
18492
  });
@@ -17609,10 +18507,10 @@ async function requireTargetAcceptable(coordinator, indexName, target) {
17609
18507
  }
17610
18508
  async function writePending(coordinator, indexName, target) {
17611
18509
  const schema = await coordinator.getSchema(indexName);
17612
- if (schema === null) throw indexNotFound3(indexName);
18510
+ if (schema === null) throw indexNotFound4(indexName);
17613
18511
  for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
17614
18512
  const snapshot = await readIndexMetadataSnapshot(coordinator, indexName);
17615
- if (snapshot === null) throw indexNotFound3(indexName);
18513
+ if (snapshot === null) throw indexNotFound4(indexName);
17616
18514
  const running = snapshot.settings?.pendingAnalysis;
17617
18515
  if (running !== void 0 && asksForPending(running, target)) return running;
17618
18516
  const current = indexConfigFromSettings(schema, snapshot.settings, indexName);
@@ -17658,7 +18556,7 @@ async function changeClusterAnalysis(request) {
17658
18556
  validateIndexName(servingName);
17659
18557
  const schema = await coordinator.getSchema(servingName);
17660
18558
  const metadata = await readIndexMetadataSnapshot(coordinator, servingName);
17661
- if (schema === null || metadata === null) throw indexNotFound3(publicName);
18559
+ if (schema === null || metadata === null) throw indexNotFound4(publicName);
17662
18560
  const target = resolveAnalysisTarget(
17663
18561
  publicName,
17664
18562
  indexConfigFromSettings(schema, metadata.settings, servingName),
@@ -17686,7 +18584,7 @@ async function changeClusterAnalysis(request) {
17686
18584
  for (; ; ) {
17687
18585
  guard();
17688
18586
  const snapshot = await readIndexMetadataSnapshot(coordinator, servingName);
17689
- if (snapshot === null) throw indexNotFound3(publicName);
18587
+ if (snapshot === null) throw indexNotFound4(publicName);
17690
18588
  if (outcomeOf(snapshot, pending, publicName) === "published") break;
17691
18589
  await pause(ANALYSIS_CHANGE_POLL_INTERVAL_MS);
17692
18590
  }
@@ -17695,7 +18593,7 @@ async function changeClusterAnalysis(request) {
17695
18593
  }
17696
18594
 
17697
18595
  // src/distribution/cluster-node/bm25-change.ts
17698
- function indexNotFound4(indexName) {
18596
+ function indexNotFound5(indexName) {
17699
18597
  return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
17700
18598
  indexName
17701
18599
  });
@@ -17703,14 +18601,14 @@ function indexNotFound4(indexName) {
17703
18601
  async function replaceClusterBm25(coordinator, view, indexName, value) {
17704
18602
  validateIndexName(indexName);
17705
18603
  const schema = await coordinator.getSchema(indexName);
17706
- if (schema === null) throw indexNotFound4(indexName);
18604
+ if (schema === null) throw indexNotFound5(indexName);
17707
18605
  validateBM25Params(value, schema, ErrorCodes.CONFIG_INVALID);
17708
18606
  const bm25 = storedBM25Params(value);
17709
18607
  const olderNodes = registeredNodesBelowBm25Tuning(await coordinator.listNodes());
17710
18608
  if (olderNodes.length > 0) throw bm25NotReady("The cluster has", olderNodes, indexName);
17711
18609
  for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
17712
18610
  const current = await readIndexMetadataSnapshot(coordinator, indexName);
17713
- if (current === null) throw indexNotFound4(indexName);
18611
+ if (current === null) throw indexNotFound5(indexName);
17714
18612
  const revision = (current.settings?.bm25Revision ?? 0) + 1;
17715
18613
  const next = metadataWithBm25(current, bm25, revision);
17716
18614
  if (await coordinator.compareAndSet(indexConfigKey(indexName), current.bytes, next)) {
@@ -19378,7 +20276,7 @@ function otherCopiesByNode(allocation, readBy) {
19378
20276
  }
19379
20277
  return byNode;
19380
20278
  }
19381
- function answerOf(deps, indexName, nodeId, partitionIds) {
20279
+ function answerOf2(deps, indexName, nodeId, partitionIds) {
19382
20280
  if (nodeId === deps.nodeId) return localCountAnswer(deps.engine, indexName, partitionIds);
19383
20281
  const message = createCountMessage({ indexName, partitionIds }, deps.nodeId);
19384
20282
  return sendReadRequest(deps, nodeId, message, indexName, validateCountResultPayload);
@@ -19396,7 +20294,7 @@ async function countsOfEveryCopy(deps, indexName, allocation, readCopies, readBy
19396
20294
  const copies = /* @__PURE__ */ new Map();
19397
20295
  for (const [partitionId, entry] of readCopies) copies.set(partitionId, [entry]);
19398
20296
  const requests = [...otherCopiesByNode(allocation, readBy)].map(async ([nodeId, partitionIds]) => {
19399
- const answer2 = await answerOf(deps, indexName, nodeId, partitionIds);
20297
+ const answer2 = await answerOf2(deps, indexName, nodeId, partitionIds);
19400
20298
  return { partitionIds, answer: answer2 };
19401
20299
  });
19402
20300
  let answers;
@@ -19855,7 +20753,7 @@ async function queryCluster(deps, indexName, params, settingName) {
19855
20753
  (reader) => searchCluster(deps, indexName, settingName, params, allocation, reader, false)
19856
20754
  );
19857
20755
  }
19858
- const { rerankClusterQuery } = await import('./rerank-KTV5L7MZ.mjs');
20756
+ const { rerankClusterQuery } = await import('./rerank-4GFC26FD.mjs');
19859
20757
  return withRankedReader(
19860
20758
  deps,
19861
20759
  indexName,
@@ -20575,7 +21473,8 @@ function createNodeMemoryReporting(options) {
20575
21473
  collections: () => monitor.collections(),
20576
21474
  refusedAfterCollection: (figures) => monitor.refusedAfterCollection(figures),
20577
21475
  analysisOf: (indexName) => options.engine.writeAnalysisOf(indexName),
20578
- vectorGrowthBytesOf: (indexName, documents) => options.engine.vectorGrowthBytesOf(indexName, documents)
21476
+ vectorGrowthBytesOf: (indexName, documents) => options.engine.vectorGrowthBytesOf(indexName, documents),
21477
+ documentTableGrowthBytesOf: (indexName, documents, pending) => options.engine.documentTableGrowthBytesOf(indexName, documents, pending)
20579
21478
  });
20580
21479
  const heapLimitBytes = readMemoryFigures()?.heapLimitBytes ?? Number.POSITIVE_INFINITY;
20581
21480
  const writePressure = createWritePressure({
@@ -20625,6 +21524,36 @@ function createNodeMemoryReporting(options) {
20625
21524
  };
20626
21525
  }
20627
21526
 
21527
+ // src/distribution/cluster-node/replica-feed-clock.ts
21528
+ function feedKey(indexName, partitionId) {
21529
+ return `${indexName}:${partitionId}`;
21530
+ }
21531
+ function createReplicaFeedClock() {
21532
+ const fedAt = /* @__PURE__ */ new Map();
21533
+ return {
21534
+ noteFed(indexName, partitionId) {
21535
+ fedAt.set(feedKey(indexName, partitionId), Date.now());
21536
+ },
21537
+ unfedForMs(indexName, partitionId) {
21538
+ const key = feedKey(indexName, partitionId);
21539
+ const at = fedAt.get(key);
21540
+ if (at !== void 0) return Date.now() - at;
21541
+ fedAt.set(key, Date.now());
21542
+ return 0;
21543
+ },
21544
+ forget(indexName, partitionId) {
21545
+ fedAt.delete(feedKey(indexName, partitionId));
21546
+ }
21547
+ };
21548
+ }
21549
+
21550
+ // src/distribution/cluster-node/started-at.ts
21551
+ var latestStartedAtMs = 0;
21552
+ function nextStartedAt() {
21553
+ latestStartedAtMs = Math.max(Date.now(), latestStartedAtMs + 1);
21554
+ return new Date(latestStartedAtMs).toISOString();
21555
+ }
21556
+
20628
21557
  // src/distribution/cluster-node/write-routing/numbering.ts
20629
21558
  function createPrimaryNumbering(onNumbered) {
20630
21559
  const scopes = /* @__PURE__ */ new Map();
@@ -20645,7 +21574,8 @@ function createPrimaryNumbering(onNumbered) {
20645
21574
  partitionId: record.partitionId,
20646
21575
  indexName: record.indexName,
20647
21576
  documentId: record.documentId,
20648
- document: record.document
21577
+ document: record.document,
21578
+ previousDocument: record.previousDocument
20649
21579
  });
20650
21580
  scope.entries.push(entry);
20651
21581
  onNumbered?.(entry);
@@ -20702,7 +21632,7 @@ function createNodeWiring(input) {
20702
21632
  address: config.address,
20703
21633
  roles: [...input.roles],
20704
21634
  capacity: config.capacity ?? DEFAULT_CAPACITY,
20705
- startedAt: (/* @__PURE__ */ new Date()).toISOString(),
21635
+ startedAt: nextStartedAt(),
20706
21636
  version: SPEC_VERSION
20707
21637
  };
20708
21638
  const memoryReporting = createNodeMemoryReporting({
@@ -20738,7 +21668,8 @@ function createNodeWiring(input) {
20738
21668
  memoryView: memoryReporting.view,
20739
21669
  writePressure: memoryReporting.writePressure,
20740
21670
  writeProjection: memoryReporting.writeProjection,
20741
- reshard: input.reshard
21671
+ reshard: input.reshard,
21672
+ replicaFeedClock: createReplicaFeedClock()
20742
21673
  };
20743
21674
  return {
20744
21675
  registration,
@@ -21830,25 +22761,57 @@ function decideSyncTier(log, replicaLastSeqNo, replicaLastPrimaryTerm) {
21830
22761
  const primaryTerm = log.termAt(replicaLastSeqNo);
21831
22762
  return primaryTerm !== void 0 && primaryTerm === replicaLastPrimaryTerm ? "incremental" : "snapshot";
21832
22763
  }
22764
+ function invalidSyncRequest(reason) {
22765
+ return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, `Invalid SyncRequestPayload: ${reason}`);
22766
+ }
22767
+ function isNonNegativeInteger(value) {
22768
+ return typeof value === "number" && Number.isInteger(value) && value >= 0;
22769
+ }
22770
+ function optionalNonEmptyString(value, field) {
22771
+ if (value === void 0 || value === null) {
22772
+ return void 0;
22773
+ }
22774
+ if (typeof value !== "string" || value.length === 0) {
22775
+ throw invalidSyncRequest(`"${field}" must be a non-empty string or absent`);
22776
+ }
22777
+ return value;
22778
+ }
22779
+ function decodeSyncRequest(payload) {
22780
+ let decoded;
22781
+ try {
22782
+ decoded = decode(payload);
22783
+ } catch (error) {
22784
+ throw invalidSyncRequest(`the payload is not valid MessagePack (${error instanceof Error ? error.message : error})`);
22785
+ }
22786
+ return validateSyncRequest(decoded);
22787
+ }
21833
22788
  function validateSyncRequest(decoded) {
21834
22789
  if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
21835
- throw new Error("Invalid SyncRequestPayload: expected an object");
22790
+ throw invalidSyncRequest("expected an object");
21836
22791
  }
21837
22792
  const record = decoded;
21838
22793
  if (typeof record.indexName !== "string") {
21839
- throw new Error('Invalid SyncRequestPayload: "indexName" must be a string');
22794
+ throw invalidSyncRequest('"indexName" must be a string');
21840
22795
  }
21841
- if (typeof record.partitionId !== "number" || !Number.isInteger(record.partitionId) || record.partitionId < 0) {
21842
- throw new Error('Invalid SyncRequestPayload: "partitionId" must be a non-negative integer');
22796
+ if (!isNonNegativeInteger(record.partitionId)) {
22797
+ throw invalidSyncRequest('"partitionId" must be a non-negative integer');
21843
22798
  }
21844
- if (typeof record.lastSeqNo !== "number" || !Number.isInteger(record.lastSeqNo) || record.lastSeqNo < 0) {
21845
- throw new Error('Invalid SyncRequestPayload: "lastSeqNo" must be a non-negative integer');
22799
+ if (!isNonNegativeInteger(record.lastSeqNo)) {
22800
+ throw invalidSyncRequest('"lastSeqNo" must be a non-negative integer');
21846
22801
  }
21847
- if (typeof record.lastPrimaryTerm !== "number" || !Number.isInteger(record.lastPrimaryTerm) || record.lastPrimaryTerm < 0) {
21848
- throw new Error('Invalid SyncRequestPayload: "lastPrimaryTerm" must be a non-negative integer');
22802
+ if (!isNonNegativeInteger(record.lastPrimaryTerm)) {
22803
+ throw invalidSyncRequest('"lastPrimaryTerm" must be a non-negative integer');
21849
22804
  }
21850
- optionalIndexUuid(record.indexUuid, "SyncRequestPayload");
21851
- return record;
22805
+ const indexUuid = optionalNonEmptyString(record.indexUuid, "indexUuid");
22806
+ const syncSession = optionalNonEmptyString(record.syncSession, "syncSession");
22807
+ return {
22808
+ indexName: record.indexName,
22809
+ partitionId: record.partitionId,
22810
+ lastSeqNo: record.lastSeqNo,
22811
+ lastPrimaryTerm: record.lastPrimaryTerm,
22812
+ ...indexUuid === void 0 ? {} : { indexUuid },
22813
+ ...syncSession === void 0 ? {} : { syncSession }
22814
+ };
21852
22815
  }
21853
22816
 
21854
22817
  // src/distribution/cluster-node/request-identity.ts
@@ -21912,7 +22875,7 @@ function createSingleResponseSink(respond) {
21912
22875
  sink.closed = false;
21913
22876
  return sink;
21914
22877
  }
21915
- async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build, metadata) {
22878
+ async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build, metadata, syncSession) {
21916
22879
  const totalBytes = build.totalBytes;
21917
22880
  const checksum = build.checksum;
21918
22881
  const header = {
@@ -21922,7 +22885,7 @@ async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build
21922
22885
  indexName,
21923
22886
  checksum
21924
22887
  };
21925
- const startPayload = { header, totalBytes };
22888
+ const startPayload = { header, totalBytes, syncSession };
21926
22889
  const startBytes = encode(startPayload);
21927
22890
  assertMessageWithinLimit(startBytes, "SNAPSHOT_START");
21928
22891
  await respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_START, nodeId, requestId, startBytes);
@@ -22120,7 +23083,7 @@ async function streamBuiltSnapshot(message, sink, indexName, deps, options, buil
22120
23083
  lastSeqNo: build.lastSeqNo ?? options.metadata.lastSeqNo,
22121
23084
  primaryTerm: build.primaryTerm ?? options.metadata.primaryTerm
22122
23085
  };
22123
- await streamSnapshotToReplica(sink, deps.nodeId, message.requestId, indexName, build, metadata);
23086
+ await streamSnapshotToReplica(sink, deps.nodeId, message.requestId, indexName, build, metadata, options.syncSession);
22124
23087
  await options.afterSnapshot(sink, build);
22125
23088
  sink.closed = true;
22126
23089
  } finally {
@@ -22188,7 +23151,7 @@ async function streamValidatedSnapshotRequest(message, sink, indexName, deps, op
22188
23151
 
22189
23152
  // src/distribution/cluster-node/message-handler/sync.ts
22190
23153
  async function handleSyncRequestMessage(message, respond, deps) {
22191
- const request = validateSyncRequest(decode(message.payload));
23154
+ const request = decodeSyncRequest(message.payload);
22192
23155
  const authResult = await authorizeSnapshotRequest(deps.coordinator, request.indexName, message.sourceId);
22193
23156
  if (authResult.outcome === "denied") {
22194
23157
  throw new NarsilError(authResult.code, authResult.reason, {
@@ -22258,6 +23221,7 @@ async function handleSyncRequestMessage(message, respond, deps) {
22258
23221
  try {
22259
23222
  const log = deps.writeDeps.getReplicationLog(request.indexName, request.partitionId);
22260
23223
  const tier = decideSyncTier(log, request.lastSeqNo, request.lastPrimaryTerm);
23224
+ const syncSession = currentSyncSessionOf(request, message.sourceId, assignment, deps);
22261
23225
  if (tier === "incremental") {
22262
23226
  if (fromReplica) {
22263
23227
  recordReplicaPosition(
@@ -22266,10 +23230,11 @@ async function handleSyncRequestMessage(message, respond, deps) {
22266
23230
  request.partitionId,
22267
23231
  message.sourceId,
22268
23232
  request.lastSeqNo,
22269
- log.localLogEnd
23233
+ log.localLogEnd,
23234
+ request.syncSession !== void 0
22270
23235
  );
22271
23236
  }
22272
- await sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1));
23237
+ await sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1), syncSession);
22273
23238
  return;
22274
23239
  }
22275
23240
  if (fromReplica) {
@@ -22294,6 +23259,7 @@ async function handleSyncRequestMessage(message, respond, deps) {
22294
23259
  lastSeqNo: snapshotSeqNo,
22295
23260
  primaryTerm: snapshotPrimaryTerm
22296
23261
  },
23262
+ syncSession,
22297
23263
  buildSnapshot: async () => {
22298
23264
  const archive = await deps.engine.archiveReplicationPartition(request.indexName, request.partitionId, () => {
22299
23265
  snapshotPrimaryTerm = log.termAt(log.localLogEnd) ?? 0;
@@ -22308,7 +23274,7 @@ async function handleSyncRequestMessage(message, respond, deps) {
22308
23274
  },
22309
23275
  afterSnapshot: async (trailingSink, build) => {
22310
23276
  const snapshotEnd = build.lastSeqNo ?? snapshotSeqNo;
22311
- await sendSyncEntriesResponse(message, trailingSink, deps, log.getEntriesFrom(snapshotEnd + 1));
23277
+ await sendSyncEntriesResponse(message, trailingSink, deps, log.getEntriesFrom(snapshotEnd + 1), syncSession);
22312
23278
  }
22313
23279
  }
22314
23280
  );
@@ -22316,10 +23282,26 @@ async function handleSyncRequestMessage(message, respond, deps) {
22316
23282
  release();
22317
23283
  }
22318
23284
  }
22319
- async function sendSyncEntriesResponse(message, respond, deps, entries) {
23285
+ function currentSyncSessionOf(request, sourceNodeId, assignment, deps) {
23286
+ const { indexName, partitionId, syncSession } = request;
23287
+ const catchUp = deps.writeDeps.catchUp;
23288
+ if (syncSession === void 0) {
23289
+ return openSyncSession(catchUp, indexName, partitionId, sourceNodeId, assignment.primaryTerm, assignment.replicas);
23290
+ }
23291
+ if (isCurrentSyncSession(catchUp, indexName, partitionId, sourceNodeId, syncSession, assignment.primaryTerm)) {
23292
+ return syncSession;
23293
+ }
23294
+ throw new NarsilError(
23295
+ ErrorCodes.REPLICATION_SYNC_FAILED,
23296
+ `Node '${sourceNodeId}' reported a sync of partition ${partitionId} of index '${indexName}' that is no longer its current sync, so it must open a new one`,
23297
+ { indexName, partitionId, sourceNodeId }
23298
+ );
23299
+ }
23300
+ async function sendSyncEntriesResponse(message, respond, deps, entries, syncSession) {
22320
23301
  const payload = {
22321
23302
  entries,
22322
- isLast: true
23303
+ isLast: true,
23304
+ syncSession
22323
23305
  };
22324
23306
  await respond({
22325
23307
  type: ReplicationMessageTypes.SYNC_ENTRIES,
@@ -22471,6 +23453,7 @@ async function applyEntriesFromPrimary(entries, sourceNodeId, deps, previousPrim
22471
23453
  const current = await resolveValidatedAssignment(first, sourceNodeId, deps);
22472
23454
  noteAcceptedPrimaryTerm(deps.writeDeps.getReplicationLog(first.indexName, first.partitionId), current.primaryTerm);
22473
23455
  await applyEntriesInLogOrder(first, entries, deps, previousPrimaryTerm);
23456
+ deps.writeDeps.replicaFeedClock?.noteFed(first.indexName, first.partitionId);
22474
23457
  });
22475
23458
  }
22476
23459
  async function applyEntriesInLogOrder(first, entries, deps, previousPrimaryTerm) {
@@ -22494,6 +23477,19 @@ async function applyEntriesInLogOrder(first, entries, deps, previousPrimaryTerm)
22494
23477
  }
22495
23478
  continue;
22496
23479
  }
23480
+ if (entry.seqNo <= tailSeqNo && log.termAt(entry.seqNo) === entry.primaryTerm) {
23481
+ if (log.verifyChecksum(entry)) continue;
23482
+ await applyRun2(run, log, deps);
23483
+ throw new NarsilError(ErrorCodes.REPLICATION_ENTRY_CORRUPT, `Invalid replication entry ${entry.seqNo}`, {
23484
+ indexName: entry.indexName,
23485
+ partitionId: entry.partitionId,
23486
+ seqNo: entry.seqNo
23487
+ });
23488
+ }
23489
+ if (entry.seqNo <= tailSeqNo && entry.seqNo < (log.oldestSeqNo ?? entry.seqNo + 1)) {
23490
+ await acknowledgeEvictedEntry(entry, run, log, deps);
23491
+ continue;
23492
+ }
22497
23493
  const heldTermBefore = run.at(-1)?.primaryTerm ?? (tailSeqNo === 0 ? 0 : log.termAt(tailSeqNo));
22498
23494
  if (entry.seqNo === tailSeqNo + 1 && expectedTermBefore !== void 0 && heldTermBefore !== void 0 && heldTermBefore !== expectedTermBefore) {
22499
23495
  await applyRun2(run, log, deps);
@@ -22539,6 +23535,20 @@ async function applyEntriesInLogOrder(first, entries, deps, previousPrimaryTerm)
22539
23535
  }
22540
23536
  await applyRun2(run, log, deps);
22541
23537
  }
23538
+ async function acknowledgeEvictedEntry(entry, run, log, deps) {
23539
+ const held = deps.engine.writeRecords.heldEntry(entry.indexName, entry.partitionId, entry.seqNo, entry.primaryTerm);
23540
+ const details = { indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo };
23541
+ if (held === "held" && log.verifyChecksum(entry)) return;
23542
+ await applyRun2(run, log, deps);
23543
+ if (held === "held") {
23544
+ throw new NarsilError(ErrorCodes.REPLICATION_ENTRY_CORRUPT, `Invalid replication entry ${entry.seqNo}`, details);
23545
+ }
23546
+ throw new NarsilError(
23547
+ ErrorCodes.REPLICATION_ENTRY_INVALID,
23548
+ held === "absent" ? `This copy holds sequence number ${entry.seqNo} under another term than replication entry ${entry.seqNo}` : `Replication entry ${entry.seqNo} lies below the oldest entry that this copy retains, and its term history no longer covers that sequence number`,
23549
+ details
23550
+ );
23551
+ }
22542
23552
  async function applyRun2(run, log, deps) {
22543
23553
  if (run.length === 0) return;
22544
23554
  const { applied, error } = await deps.engine.applyReplicationEntries(run);
@@ -23068,7 +24078,7 @@ async function deleteClusterBuildRecord(coordinator, indexName) {
23068
24078
  }
23069
24079
 
23070
24080
  // src/distribution/cluster-node/query-suggestions/host.ts
23071
- function indexNotFound5(indexName) {
24081
+ function indexNotFound6(indexName) {
23072
24082
  return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index '${indexName}' does not exist in the cluster`, {
23073
24083
  indexName
23074
24084
  });
@@ -23077,7 +24087,7 @@ async function updateClusterState(deps, indexName, change) {
23077
24087
  const { coordinator } = deps.config;
23078
24088
  for (let attempt = 1; attempt <= MAX_SETTINGS_REPLACEMENT_ATTEMPTS; attempt++) {
23079
24089
  const current = await readIndexMetadataSnapshot(coordinator, indexName);
23080
- if (current === null) throw indexNotFound5(indexName);
24090
+ if (current === null) throw indexNotFound6(indexName);
23081
24091
  await deps.view.refresh(indexName);
23082
24092
  const next = change(suggestionStateOfSettings(current.settings));
23083
24093
  if (await coordinator.compareAndSet(
@@ -23097,7 +24107,7 @@ async function updateClusterState(deps, indexName, change) {
23097
24107
  }
23098
24108
  async function createSuggestionsIndex(deps, name, source, language) {
23099
24109
  const sourceIndex = deps.view.get(source.index);
23100
- if (sourceIndex === null) throw indexNotFound5(source.index);
24110
+ if (sourceIndex === null) throw indexNotFound6(source.index);
23101
24111
  const config = { schema: { ...SUGGESTIONS_SCHEMA }, language, stopWords: /* @__PURE__ */ new Set() };
23102
24112
  try {
23103
24113
  await deps.createMarkedIndex(
@@ -23628,6 +24638,9 @@ function clusterNodeEngine(node, options) {
23628
24638
  getSync(indexName) {
23629
24639
  return node.getSync(indexName);
23630
24640
  },
24641
+ updateChanges: (indexName, setting) => node.updateChanges(indexName, setting),
24642
+ getChanges: (indexName) => node.getChanges(indexName),
24643
+ followChanges: (indexName, followOptions) => node.followChanges(indexName, followOptions),
23631
24644
  async getSynonyms(indexName) {
23632
24645
  return node.getSynonyms(indexName);
23633
24646
  },
@@ -23689,7 +24702,7 @@ function clusterNodeEngine(node, options) {
23689
24702
  getPartitionStats: () => unsupported("getPartitionStats"),
23690
24703
  snapshot: () => unsupported("snapshot"),
23691
24704
  restore: () => unsupported("restore"),
23692
- exportDocuments: () => unsupported("exportDocuments"),
24705
+ exportDocuments: (indexName) => node.exportDocuments(indexName),
23693
24706
  importDocumentExport: () => unsupported("importDocumentExport"),
23694
24707
  rebalance: (indexName, targetPartitionCount) => node.rebalance(indexName, targetPartitionCount),
23695
24708
  updatePartitionConfig: () => unsupported("updatePartitionConfig"),
@@ -23706,6 +24719,8 @@ function clusterNodeEngine(node, options) {
23706
24719
  bindRerankWindowSource(engine, () => rerankWindowCapOf(node));
23707
24720
  const syncServing = syncServingSourceOf(node);
23708
24721
  if (syncServing !== void 0) bindSyncServing(engine, syncServing);
24722
+ const changesServing = changesServingSourceOf(node);
24723
+ if (changesServing !== void 0) bindChangesServing(engine, changesServing);
23709
24724
  const suggestions = boundQuerySuggestionsOf(node);
23710
24725
  if (suggestions !== void 0) bindQuerySuggestions(engine, suggestions);
23711
24726
  return engine;
@@ -24020,8 +25035,7 @@ async function createClusterNode(callerConfig) {
24020
25035
  isOrphaned: (indexName) => orphanedIndexes.has(indexName),
24021
25036
  suggestionHooks: suggestions.hooks
24022
25037
  }),
24023
- updateSync: (indexName, setting) => trackOp(indexName, () => deviceSync.updateSync(indexName, setting)),
24024
- getSync: (indexName) => trackOp(indexName, () => deviceSync.getSync(indexName)),
25038
+ ...deviceSync.operations(trackOp),
24025
25039
  cluster,
24026
25040
  async start() {
24027
25041
  guardShutdown();
@@ -24104,7 +25118,8 @@ async function createClusterNode(callerConfig) {
24104
25118
  coordinator: writeDeps.coordinator,
24105
25119
  engine,
24106
25120
  memoryView: memoryReporting.view,
24107
- syncServing: deviceSync.serving
25121
+ syncServing: deviceSync.serving,
25122
+ changesServing: deviceSync.changesServing
24108
25123
  });
24109
25124
  suggestions.bind(node, { engine: clusterNodeEngine(node), dropIndex: (name) => node.dropIndex(name) });
24110
25125
  return node;