@delali/narsil 0.3.1-next.140 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,11 +1,12 @@
1
- import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, decodeSegmentManifest, readListedFile, adapterCheckpointStore, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, DIRECTORY_LOCK_ATTEMPTS, createSeqOwner, readSegmentManifest, snapshotBundleKey, writeMetadataEnvelope, reclaimUnlistedKeys, manifestReferencedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, MAX_SEGMENTS_PER_PARTITION, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, metadataConfigOf, durableMetadataConfigOf, buildEntry, readCurrentManifest, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, captureWholePartition, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-CGVBGHBX.mjs';
2
- import { applyDeleteEntry, applyIndexEntry } from './chunk-6BSB4N6B.mjs';
1
+ import { decodeSegmentManifest, metadataFromConfig, SINGLE_NODE_PRIMARY_TERM, readMetadataEnvelope, createSeqOwner, readSegmentManifest, writeMetadataEnvelope, reclaimUnlistedKeys, loadSegmentedCheckpoint, checkpointLastSeqNo, nextSegmentKey, isRecord, invalidManifest, deserializePayloadV2, unpackEnvelopeBytes, checkpointPrimaryTerm, frameRecord, isId, isUint32, isSequenceNumber, writeCheckpointSegments, partitionContentsOf, writeLiveVectors, commitCheckpointManifest, removeCheckpointGarbage, readWalSegments, durableRecordMissing, growPartitionsTo, writeSegmentHeader, VECTOR_FILE_ENVELOPE_VERSION, SEGMENT_HEADER_SIZE, documentsOfPayloadV2, readCommitMarker, nextMarkerSlot, encodeMarkerSlot, markerSlotOffset, metadataConfigOf, durableMetadataConfigOf, buildEntry, readCurrentManifest, captureWholePartition } from './chunk-KTOK275N.mjs';
2
+ import { DEFAULT_SEGMENT_MAX_BYTES, DEFAULT_CHECKPOINT_INTERVAL_MS, DEFAULT_CHECKPOINT_MUTATION_THRESHOLD, DEFAULT_COMPACTION_THRESHOLD, DEFAULT_ASYNC_FLUSH_INTERVAL_MS, WRITES_STALL_ABOVE_UNCHECKPOINTED_BYTES, directoryCheckpointStore, manifestKey, readListedFile, adapterCheckpointStore, DIRECTORY_LOCK_ATTEMPTS, snapshotBundleKey, manifestReferencedKeys, manifestFileRefs, DIRECTORY_LOCK_SETTLE_MS, readSegmentFile, MAX_SEGMENTS_PER_PARTITION, DIRECTORY_IDENTITY_CHECK_INTERVAL_MS, WHOLE_PARTITION_LARGEST_DOCUMENT_COUNT, WHOLE_PARTITION_MIN_CHANGED_SHARE, CHECKPOINT_TIMEOUT_RECOVERY_BACKOFF_MS, CHECKPOINT_WORKER_TIMEOUT_MS, CHECKPOINT_WORKER_IDLE_MS, LOG_RECORDS_PER_CHECKPOINT } from './chunk-MVM33ZUL.mjs';
3
+ import { applyDeleteEntry, applyIndexEntry } from './chunk-TCLEOZ4P.mjs';
3
4
  import { nodeThreadId, spawnNodeWorker } from './chunk-HOZW3PKP.mjs';
4
- import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-5LYWZJE2.mjs';
5
- import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment } from './chunk-ZAIZQOTM.mjs';
6
- import './chunk-26A5JFT2.mjs';
7
- import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-4MFH226Q.mjs';
5
+ import { decodeVectorIndexParts, VECTOR_FILE_MAX_VECTORS, deadBytesFor, countDead, decodeVectorFilePayload, decodeVectorGraphPayload, graphPartsOfEarlierPayload } from './chunk-LVY2LI35.mjs';
6
+ import { encodeDocument, fnv1a, requireValidFieldTypes, openChecksummedEnvelope, HEADER_SIZE, buildSegmentPayload, resolvePartitionInsertOptions, freezeSegmentShared, createSharedFrozenSegment, createRequestId } from './chunk-5DB2VDGI.mjs';
8
7
  import './chunk-JPTQZWJQ.mjs';
8
+ import './chunk-26A5JFT2.mjs';
9
+ import { compareCodePoints, MIN_M, MAX_FILE_BYTES, detectRuntime, resolveWorkerEntry } from './chunk-D33A5VMU.mjs';
9
10
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-MZI3TEOW.mjs';
10
11
  import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
11
12
  import { decode } from '@msgpack/msgpack';
@@ -183,6 +184,29 @@ async function lockDirectory(root) {
183
184
  };
184
185
  }
185
186
 
187
+ // src/persistence/durability/list-recursive.ts
188
+ async function listRecursive(dir, base, pathMod, fs) {
189
+ const results = [];
190
+ let entries;
191
+ try {
192
+ entries = await fs.readdir(dir, { withFileTypes: true });
193
+ } catch (err) {
194
+ if (err.code === "ENOENT") {
195
+ return results;
196
+ }
197
+ throw err;
198
+ }
199
+ for (const entry of entries) {
200
+ const full = pathMod.join(dir, entry.name);
201
+ if (entry.isDirectory()) {
202
+ results.push(...await listRecursive(full, base, pathMod, fs));
203
+ } else {
204
+ results.push(pathMod.relative(base, full).split(pathMod.sep).join("/"));
205
+ }
206
+ }
207
+ return results;
208
+ }
209
+
186
210
  // src/persistence/durability/durable-filesystem.ts
187
211
  function validateRelativeKey(key) {
188
212
  if (key.length === 0) {
@@ -480,8 +504,8 @@ function createDurableDirectory(root) {
480
504
  const fs = await getFs();
481
505
  await ensureDir(target.path, target.pathMod, fs);
482
506
  await fs.rename(source.path, target.path);
483
- await fsyncDirectory(target.pathMod.dirname(target.path), fs);
484
- await fsyncDirectory(source.pathMod.dirname(source.path), fs);
507
+ const directories = new Set([target.path, source.path].map((path) => target.pathMod.dirname(path)));
508
+ for (const movedIn of directories) await fsyncDirectory(movedIn, fs);
485
509
  },
486
510
  async list(prefix) {
487
511
  const fs = await getFs();
@@ -496,27 +520,6 @@ function createDurableDirectory(root) {
496
520
  }
497
521
  };
498
522
  }
499
- async function listRecursive(dir, base, pathMod, fs) {
500
- const results = [];
501
- let entries;
502
- try {
503
- entries = await fs.readdir(dir, { withFileTypes: true });
504
- } catch (err) {
505
- if (err.code === "ENOENT") {
506
- return results;
507
- }
508
- throw err;
509
- }
510
- for (const entry of entries) {
511
- const full = pathMod.join(dir, entry.name);
512
- if (entry.isDirectory()) {
513
- results.push(...await listRecursive(full, base, pathMod, fs));
514
- } else {
515
- results.push(pathMod.relative(base, full).split(pathMod.sep).join("/"));
516
- }
517
- }
518
- return results;
519
- }
520
523
 
521
524
  // src/persistence/durability/sha256.ts
522
525
  var ROUND_CONSTANTS = new Uint32Array([
@@ -785,6 +788,7 @@ function storageFailed(code, message, key, error) {
785
788
  }
786
789
  function directoryRestoreStorage(directory) {
787
790
  return {
791
+ directory,
788
792
  read: (key) => directory.read(key),
789
793
  write: (key, chunks) => directory.atomicWrite(key, chunks),
790
794
  writeJournal: (key, bytes) => directory.atomicWrite(key, bytes),
@@ -829,6 +833,25 @@ function pathUnder(path, root) {
829
833
  const next = path.charAt(root.length);
830
834
  return next === "" || next === "/" || next === "\\";
831
835
  }
836
+ function stagedDirectory(directory, staged, unstaged) {
837
+ const { readRange, byteLength } = directory;
838
+ return {
839
+ root: directory.root,
840
+ appendHandle: (key) => directory.appendHandle(staged(key)),
841
+ markerHandle: (key) => directory.markerHandle(staged(key)),
842
+ atomicWrite: (key, data) => directory.atomicWrite(staged(key), data),
843
+ syncDirectoryOf: (key) => directory.syncDirectoryOf(staged(key)),
844
+ read: (key) => directory.read(staged(key)),
845
+ remove: (key) => directory.remove(staged(key)),
846
+ move: (from, to) => directory.move(staged(from), staged(to)),
847
+ list: async (prefix) => (await directory.list(staged(prefix))).map(unstaged),
848
+ pathOf: (key) => directory.pathOf(staged(key)),
849
+ ...readRange === void 0 ? {} : {
850
+ readRange: (key, offset, length) => readRange.call(directory, staged(key), offset, length)
851
+ },
852
+ ...byteLength === void 0 ? {} : { byteLength: (key) => byteLength.call(directory, staged(key)) }
853
+ };
854
+ }
832
855
  function createRestoreStaging(storage, indexName, options = {}) {
833
856
  const stagingPrefix = newStagingPrefix();
834
857
  const indexRoot = `${indexName}/`;
@@ -850,6 +873,7 @@ function createRestoreStaging(storage, indexName, options = {}) {
850
873
  };
851
874
  const pathOf = storage.pathOf;
852
875
  if (pathOf !== void 0) store.pathOf = (key) => pathOf.call(storage, staged(key));
876
+ if (storage.directory !== void 0) store.directory = stagedDirectory(storage.directory, staged, unstaged);
853
877
  let retiredPrefix = null;
854
878
  async function finishOnStorage() {
855
879
  retiredPrefix = await finishJournal(storage, indexName, stagingPrefix);
@@ -969,7 +993,8 @@ function createArchiveCheckpointAccess(deps) {
969
993
  const whole = await readListedFile(store, ref.key, { bytes: ref.bytes, what: "checkpoint file" });
970
994
  return whole.slice(offset, offset + length);
971
995
  },
972
- release
996
+ release,
997
+ directory: deps.directory
973
998
  };
974
999
  } catch (error) {
975
1000
  release();
@@ -1062,6 +1087,13 @@ function formatSegmentStart(startSeqNo) {
1062
1087
  }
1063
1088
 
1064
1089
  // src/persistence/durability/recovery.ts
1090
+ async function removeLeftoverRewriteFiles(directory, names) {
1091
+ for (const indexName of names) {
1092
+ for (const place of ["rebuilt", "received"]) {
1093
+ for (const key of await directory.list(`${indexName}/${place}/`)) await directory.remove(key);
1094
+ }
1095
+ }
1096
+ }
1065
1097
  async function listPersistedIndexes(directory) {
1066
1098
  const metaKeys = await directory.list("");
1067
1099
  const names = [];
@@ -1081,6 +1113,7 @@ async function loadMetadata(directory, indexName) {
1081
1113
  return metadata;
1082
1114
  }
1083
1115
  async function replayWal(directory, indexName, partitionId, fromSeqNoExclusive, deps) {
1116
+ await directory.remove(nextSegmentKey(indexName, partitionId));
1084
1117
  const read = await readWalSegments(directory, indexName, partitionId);
1085
1118
  if (read === null) {
1086
1119
  return { highestSeqNo: fromSeqNoExclusive, highestPrimaryTerm: 0 };
@@ -1185,6 +1218,22 @@ async function makeEveryAppliedMutationDurable(indexState, failPartition2) {
1185
1218
  }
1186
1219
  }
1187
1220
  }
1221
+ async function openNewLogSegments(indexState, failPartition2) {
1222
+ const rolls = [...indexState.partitions].map(async ([partitionId, partition]) => {
1223
+ try {
1224
+ if (partition.failed === null) await partition.walWriter.prepareCheckpointRoll();
1225
+ } catch (error) {
1226
+ failPartition2(partitionId, partition, error);
1227
+ return;
1228
+ }
1229
+ const rolled = partition.appendChain.then(() => partition.failed === null ? partition.walWriter.rollForCheckpoint() : void 0).catch((error) => {
1230
+ failPartition2(partitionId, partition, error);
1231
+ });
1232
+ partition.appendChain = rolled;
1233
+ await rolled;
1234
+ });
1235
+ await Promise.all(rolls);
1236
+ }
1188
1237
  function aTargetFailed(indexState, targets) {
1189
1238
  return targets.some((target) => (indexState.partitions.get(target.partitionId)?.failed ?? null) !== null);
1190
1239
  }
@@ -1217,9 +1266,10 @@ function wholePartitionsWhereMostChanged(manager, rules) {
1217
1266
  return whole;
1218
1267
  };
1219
1268
  }
1220
- async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf) {
1269
+ async function captureCheckpoint(indexState, manager, vectorIndexes, wholePartitionsOf = noWholePartitions, priorCheckpointThatLimitsTheRecords, listedVectorFields = [], termHistoryOf, onCaptured) {
1221
1270
  for (const vectorIndex of vectorIndexes.values()) await vectorIndex.completeGraph();
1222
1271
  return whileNoMutationApplies([...indexState.partitions.values()], () => {
1272
+ onCaptured?.();
1223
1273
  const documentCount = manager.countDocuments();
1224
1274
  const targets = [];
1225
1275
  let recordsLeftInLog = 0;
@@ -1575,23 +1625,68 @@ async function writeOneBoundedCheckpoint(input) {
1575
1625
  const writeRecords = input.hooks.writeRecords ?? null;
1576
1626
  const { indexName } = input;
1577
1627
  const priorManifest = await readCurrentManifest(store, input.indexName);
1628
+ const textOnDisk = metadata.textStorage === "disk";
1629
+ const wholePartitionsOf = wholePartitionsWhereMostChanged(manager, {
1630
+ priorCheckpoint: priorManifest?.checkpoint ?? [],
1631
+ priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
1632
+ everyPartition: fromMemory,
1633
+ aWorkerCanSerialise: input.canOffload && !textOnDisk && checkpointWorkerIsUsable(),
1634
+ ...writeRecords === null ? {} : {
1635
+ captureRecords: (partitionId) => (segments) => writeRecords.captureSegments(indexName, partitionId, segments)
1636
+ }
1637
+ });
1638
+ let changesSinceCapture = null;
1578
1639
  const capture2 = await captureCheckpoint(
1579
1640
  input.indexState,
1580
1641
  manager,
1581
1642
  vectorIndexes,
1582
- wholePartitionsWhereMostChanged(manager, {
1583
- priorCheckpoint: priorManifest?.checkpoint ?? [],
1584
- priorPartitionIds: (priorManifest?.partitions ?? []).map((partition) => partition.partitionId),
1585
- everyPartition: fromMemory,
1586
- aWorkerCanSerialise: input.canOffload && checkpointWorkerIsUsable(),
1587
- ...writeRecords === null ? {} : {
1588
- captureRecords: (partitionId) => (segments2) => writeRecords.captureSegments(indexName, partitionId, segments2)
1589
- }
1590
- }),
1643
+ textOnDisk && !fromMemory ? void 0 : wholePartitionsOf,
1591
1644
  fromMemory ? void 0 : priorManifest?.checkpoint ?? [],
1592
1645
  priorManifest?.vectors ?? [],
1593
- writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo)
1646
+ writeRecords === null ? void 0 : (partitionId, lastSeqNo) => writeRecords.termHistoryOf(indexName, partitionId, lastSeqNo),
1647
+ textOnDisk ? () => {
1648
+ changesSinceCapture = manager.openCheckpointChangeLog();
1649
+ } : void 0
1594
1650
  );
1651
+ try {
1652
+ await openNewLogSegments(input.indexState, input.failPartition);
1653
+ return await finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, () => {
1654
+ return changesSinceCapture;
1655
+ });
1656
+ } finally {
1657
+ changesSinceCapture?.close();
1658
+ }
1659
+ }
1660
+ async function adoptWrittenSegments(input, manager, partitions, changes) {
1661
+ const { indexName, indexState, hooks } = input;
1662
+ const { actionPartitionsOf, closeUnusedSegments, prepareDiskAdoption, swapPreparedPartitions } = await import('./module-BUFNZMEU.mjs');
1663
+ const adoption = await prepareDiskAdoption(input.directory, manager, partitions);
1664
+ if (adoption === null) return [];
1665
+ const adopted = await whileNoMutationApplies([...indexState.partitions.values()], () => {
1666
+ if (changes.partitionsReplaced) return false;
1667
+ const changed = changes.takeWrittenDocIds();
1668
+ swapPreparedPartitions(manager, adoption.partitions, changed, (partitionId, segments) => {
1669
+ hooks.writeRecords?.adoptPagedSegments(indexName, partitionId, segments, changed);
1670
+ });
1671
+ hooks.queueDiskAdoption?.({
1672
+ type: "adoptDiskSegments",
1673
+ indexName,
1674
+ partitions: actionPartitionsOf(adoption),
1675
+ changed: [...changed],
1676
+ requestId: createRequestId()
1677
+ });
1678
+ return true;
1679
+ });
1680
+ if (!adopted) {
1681
+ closeUnusedSegments(adoption);
1682
+ return [];
1683
+ }
1684
+ await hooks.awaitCopiesOfIndex?.(indexName);
1685
+ return adoption.droppedKeys;
1686
+ }
1687
+ async function finishBoundedCheckpoint(input, manager, store, metadata, priorManifest, capture2, fromMemory, changesSinceCapture) {
1688
+ const vectorIndexes = input.hooks.getVectorIndexes(input.indexName);
1689
+ const writeRecords = input.hooks.writeRecords ?? null;
1595
1690
  const { targets, documentCount } = capture2;
1596
1691
  await makeEveryAppliedMutationDurable(input.indexState, input.failPartition);
1597
1692
  if (aTargetFailed(input.indexState, targets)) return 0;
@@ -1615,8 +1710,15 @@ async function writeOneBoundedCheckpoint(input) {
1615
1710
  input.hooks.onManifestCommitted(input.indexName, metadata.analysisGeneration ?? 0);
1616
1711
  recordWrittenVectors(vectorIndexes, liveVectors.written);
1617
1712
  await adoptVectorLayouts(store, vectorIndexes, liveVectors.layouts);
1713
+ const changes = changesSinceCapture();
1714
+ const droppedThisCheckpoint = changes !== null && capture2.recordsLeftInLog === 0 ? new Set(await adoptWrittenSegments(input, manager, segments.partitions, changes)) : /* @__PURE__ */ new Set();
1618
1715
  if (!input.holdsCheckpointFiles()) {
1619
- await removeCheckpointGarbage(store, await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage));
1716
+ const removable = await garbageThatNoFieldStillReads(store, vectorIndexes, written.garbage);
1717
+ const stillRead = metadata.textStorage === "disk" ? (await import('./module-BUFNZMEU.mjs')).keysThatSegmentsStillRead(manager) : /* @__PURE__ */ new Set();
1718
+ await removeCheckpointGarbage(
1719
+ store,
1720
+ removable.filter((key) => !droppedThisCheckpoint.has(key) && !stillRead.has(key))
1721
+ );
1620
1722
  }
1621
1723
  const checkpointDocumentCount = documentCount;
1622
1724
  await input.queueMetadataWrite(input.indexName, async () => {
@@ -2129,6 +2231,8 @@ function parseSegmentStartSeqNo2(key) {
2129
2231
  function createWalWriter(directory, config) {
2130
2232
  const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
2131
2233
  const commitKey = `${config.indexName}/wal/${config.partitionId}/commit`;
2234
+ const preparedKey = nextSegmentKey(config.indexName, config.partitionId);
2235
+ let nextSegmentPrepared = false;
2132
2236
  let handle = null;
2133
2237
  let activeKey = null;
2134
2238
  let activeStartSeqNo = 0;
@@ -2207,16 +2311,49 @@ function createWalWriter(directory, config) {
2207
2311
  await withDurabilityLock(flushActiveSegment);
2208
2312
  }
2209
2313
  }
2210
- async function rollToNewSegment(startSeqNo) {
2211
- await withDurabilityLock(async () => {
2212
- if (handle !== null) {
2213
- await flushActiveSegment();
2214
- await handle.close();
2215
- handle = null;
2216
- }
2217
- await openSegmentByKey(segmentKey(config.indexName, config.partitionId, startSeqNo));
2314
+ async function rollUnderLock(startSeqNo) {
2315
+ if (handle !== null) {
2218
2316
  await flushActiveSegment();
2317
+ await handle.close();
2318
+ handle = null;
2319
+ }
2320
+ await openSegmentByKey(segmentKey(config.indexName, config.partitionId, startSeqNo));
2321
+ await flushActiveSegment();
2322
+ }
2323
+ async function rollToNewSegment(startSeqNo) {
2324
+ await withDurabilityLock(() => rollUnderLock(startSeqNo));
2325
+ }
2326
+ function activeSegmentHoldsRecords() {
2327
+ return handle !== null && activeBytes > SEGMENT_HEADER_SIZE && highestAppendedSeqNo >= activeStartSeqNo;
2328
+ }
2329
+ async function prepareNextSegment() {
2330
+ const prepared = await directory.appendHandle(preparedKey);
2331
+ try {
2332
+ if (await prepared.size() > 0) await prepared.truncate(0);
2333
+ await prepared.append(writeSegmentHeader());
2334
+ await prepared.sync();
2335
+ nextSegmentPrepared = true;
2336
+ } finally {
2337
+ await prepared.close();
2338
+ }
2339
+ }
2340
+ async function swapToPreparedSegment(sealed) {
2341
+ const writer = await getMarkerWriter();
2342
+ const startSeqNo = highestAppendedSeqNo + 1;
2343
+ const durableSeqNo = Math.max(highestAppendedSeqNo, highestDurableSeqNo);
2344
+ const key = segmentKey(config.indexName, config.partitionId, startSeqNo);
2345
+ await Promise.all([sealed.sync(), directory.move(preparedKey, key)]);
2346
+ handle = await directory.appendHandle(key);
2347
+ activeKey = key;
2348
+ activeStartSeqNo = startSeqNo;
2349
+ activeBytes = SEGMENT_HEADER_SIZE;
2350
+ await writer.commit({
2351
+ activeSegmentSeqNo: startSeqNo,
2352
+ durableByteLength: SEGMENT_HEADER_SIZE,
2353
+ highestDurableSeqNo: durableSeqNo
2219
2354
  });
2355
+ highestDurableSeqNo = durableSeqNo;
2356
+ await sealed.close();
2220
2357
  }
2221
2358
  async function maybeRoll(nextSeqNo, appendedBytes) {
2222
2359
  if (handle === null || activeBytes <= SEGMENT_HEADER_SIZE) {
@@ -2302,6 +2439,19 @@ function createWalWriter(directory, config) {
2302
2439
  await coordinator.commit();
2303
2440
  },
2304
2441
  rollToNewSegment,
2442
+ async prepareCheckpointRoll() {
2443
+ if (activeSegmentHoldsRecords()) await prepareNextSegment();
2444
+ },
2445
+ async rollForCheckpoint() {
2446
+ await withDurabilityLock(async () => {
2447
+ const sealed = handle;
2448
+ const prepared = nextSegmentPrepared;
2449
+ nextSegmentPrepared = false;
2450
+ if (sealed === null || !activeSegmentHoldsRecords()) return;
2451
+ if (prepared) await swapToPreparedSegment(sealed);
2452
+ else await rollUnderLock(highestAppendedSeqNo + 1);
2453
+ });
2454
+ },
2305
2455
  async close() {
2306
2456
  await withDurabilityLock(async () => {
2307
2457
  if (handle !== null) {
@@ -2776,9 +2926,33 @@ async function readEarlierSegmentDocuments(store, key) {
2776
2926
 
2777
2927
  // src/persistence/durability/checkpoint-count.ts
2778
2928
  var decoder = new TextDecoder();
2929
+ async function countPagedPartition(store, partition) {
2930
+ const { directory } = store;
2931
+ if (directory === void 0 || partition.segments.length === 0) return null;
2932
+ const { countsAsLatestHolder, isPagedSegmentFile, openPagedSegmentScan } = await import('./module-BUFNZMEU.mjs');
2933
+ for (const ref of partition.segments) if (!await isPagedSegmentFile(directory, ref.key)) return null;
2934
+ const scans = [];
2935
+ try {
2936
+ for (const ref of partition.segments) scans.push(await openPagedSegmentScan(directory, ref));
2937
+ let live = 0;
2938
+ for (let at = 0; at < scans.length; at++) {
2939
+ for (let ordinal = 0; ordinal < scans[at].count; ordinal++) {
2940
+ if (countsAsLatestHolder(scans, at, scans[at].idAt(ordinal))) live += 1;
2941
+ }
2942
+ }
2943
+ return live;
2944
+ } finally {
2945
+ for (const scan of scans) scan.close();
2946
+ }
2947
+ }
2779
2948
  async function countCurrentLayout(store, manifest) {
2780
2949
  let total = 0;
2781
2950
  for (const partition of manifest.partitions) {
2951
+ const paged = await countPagedPartition(store, partition);
2952
+ if (paged !== null) {
2953
+ total += paged;
2954
+ continue;
2955
+ }
2782
2956
  const live = /* @__PURE__ */ new Set();
2783
2957
  for (const ref of partition.segments) {
2784
2958
  const contents = await readSegmentFile(store, ref);
@@ -3186,7 +3360,7 @@ async function recoverPersistedIndex(deps, indexName, metadataOnly) {
3186
3360
 
3187
3361
  // src/persistence/durability/manager.ts
3188
3362
  function createDurabilityManager(config, hooks, directoryOverride) {
3189
- const directory = createDurableDirectory(config.directory ?? "");
3363
+ const directory = directoryOverride ?? createDurableDirectory(config.directory ?? "");
3190
3364
  const canOffloadCheckpoint = directoryOverride === void 0;
3191
3365
  const segmentMaxBytes = config.segmentMaxBytes ?? DEFAULT_SEGMENT_MAX_BYTES;
3192
3366
  const checkpointIntervalMs = config.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
@@ -3369,6 +3543,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3369
3543
  await directoryWatch.claim();
3370
3544
  await recoverDirectoryRestores(directory);
3371
3545
  const names = await listPersistedIndexes(directory);
3546
+ await removeLeftoverRewriteFiles(directory, names);
3372
3547
  for (const indexName of names) {
3373
3548
  await recoverIndex(indexName, metadataOnly, false);
3374
3549
  }
@@ -3392,6 +3567,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
3392
3567
  keepWriteRecordsWith(keeper) {
3393
3568
  hooks.writeRecords = keeper;
3394
3569
  },
3570
+ rewriteStorage: () => ({ directory, writeRecords: hooks.writeRecords ?? null }),
3395
3571
  heldCheckpoints: () => heldCheckpoints.report(indexes),
3396
3572
  async commitPartition(indexName, partitionId) {
3397
3573
  const partition = indexes.get(indexName)?.partitions.get(partitionId);
@@ -3930,6 +4106,8 @@ function createDurabilityIntegration(tier, hooks) {
3930
4106
  },
3931
4107
  mustRewriteTerms: hooks.mustRewriteTerms,
3932
4108
  onManifestCommitted: hooks.onManifestCommitted,
4109
+ ...hooks.queueDiskAdoption === void 0 ? {} : { queueDiskAdoption: hooks.queueDiskAdoption },
4110
+ ...hooks.awaitCopiesOfIndex === void 0 ? {} : { awaitCopiesOfIndex: hooks.awaitCopiesOfIndex },
3933
4111
  createIndexFromMetadata: hooks.createIndexFromMetadata,
3934
4112
  recordCheckpoint: hooks.recordCheckpoint,
3935
4113
  onLayoutUpgrade: hooks.onLayoutUpgrade,
@@ -4012,6 +4190,8 @@ function createDurabilityFromTier(tier, wiring) {
4012
4190
  const entry = wiring.indexRegistry.get(indexName);
4013
4191
  if (entry !== void 0 && entry.analysisGeneration === analysisGeneration) entry.durableAnalysis = null;
4014
4192
  },
4193
+ queueDiskAdoption: wiring.queueDiskAdoption,
4194
+ awaitCopiesOfIndex: wiring.awaitCopiesOfIndex,
4015
4195
  createIndexFromMetadata: wiring.createIndexFromMetadata,
4016
4196
  recordCheckpoint: wiring.recordCheckpoint,
4017
4197
  onLayoutUpgrade: wiring.emitLayoutUpgrade,
@@ -1,8 +1,9 @@
1
- export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport } from './chunk-XVMOQQ34.mjs';
2
- import './chunk-QHNNULPW.mjs';
3
- import './chunk-ZAIZQOTM.mjs';
4
- import './chunk-26A5JFT2.mjs';
5
- import './chunk-4MFH226Q.mjs';
1
+ export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport } from './chunk-VFYMPCOV.mjs';
2
+ import './chunk-ZEV26AQ6.mjs';
3
+ import './chunk-WCFCHIBB.mjs';
4
+ import './chunk-5DB2VDGI.mjs';
6
5
  import './chunk-JPTQZWJQ.mjs';
6
+ import './chunk-26A5JFT2.mjs';
7
+ import './chunk-D33A5VMU.mjs';
7
8
  import './chunk-MZI3TEOW.mjs';
8
9
  import './chunk-35B42UFM.mjs';
@@ -0,0 +1,12 @@
1
+ export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-MWUK3DA5.mjs';
2
+ import './chunk-MVM33ZUL.mjs';
3
+ import './chunk-CCOAOO73.mjs';
4
+ import './chunk-TCLEOZ4P.mjs';
5
+ import './chunk-WCFCHIBB.mjs';
6
+ import './chunk-3ZBJYAS4.mjs';
7
+ import './chunk-5DB2VDGI.mjs';
8
+ import './chunk-JPTQZWJQ.mjs';
9
+ import './chunk-26A5JFT2.mjs';
10
+ import './chunk-D33A5VMU.mjs';
11
+ import './chunk-MZI3TEOW.mjs';
12
+ import './chunk-35B42UFM.mjs';