@delali/narsil 0.3.1-next.131 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
  4. package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
  5. package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
  6. package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  9. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
  10. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  11. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  12. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
  13. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
  14. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
  15. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
  16. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  17. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  18. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  19. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  20. package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
  21. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
  22. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  23. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  24. package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
  25. package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
  26. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  27. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
  28. package/dist/browser/import-TLB2A5LJ.mjs +9 -0
  29. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  30. package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
  31. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  32. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
  33. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  34. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
  35. package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
  36. package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
  37. package/dist/chunk-46NZOPHS.mjs +21 -0
  38. package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
  39. package/dist/chunk-6SNBQESJ.mjs +214 -0
  40. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  41. package/dist/chunk-7AUASRE7.mjs +653 -0
  42. package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
  43. package/dist/chunk-BOJZGUS6.mjs +177 -0
  44. package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
  45. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  46. package/dist/chunk-CZQ33DLW.mjs +78 -0
  47. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
  49. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  50. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  51. package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
  52. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  53. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  54. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  55. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  56. package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
  57. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  58. package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
  59. package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
  60. package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
  61. package/dist/chunk-L76W3IZJ.mjs +193 -0
  62. package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
  63. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  64. package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
  65. package/dist/chunk-NN33CZRE.mjs +86 -0
  66. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  67. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  68. package/dist/chunk-NRUXBYPH.mjs +121 -0
  69. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  70. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  71. package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
  72. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  73. package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
  74. package/dist/chunk-SRJBFG2T.mjs +881 -0
  75. package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
  76. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  77. package/dist/chunk-UUZCQQG2.mjs +209 -0
  78. package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
  79. package/dist/chunk-VJEGOELZ.mjs +177 -0
  80. package/dist/chunk-W2COJ5AH.mjs +148 -0
  81. package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
  82. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  83. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  84. package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
  85. package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
  86. package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
  87. package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
  88. package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  89. package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
  90. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  91. package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
  92. package/dist/client.d.ts +7 -7
  93. package/dist/client.mjs +11 -1
  94. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  95. package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
  96. package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
  97. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
  98. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  99. package/dist/distribution/coordinator/etcd.mjs +1 -1
  100. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/grpc.mjs +1 -1
  102. package/dist/distribution.d.ts +24 -8
  103. package/dist/distribution.mjs +982 -211
  104. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
  105. package/dist/embeddings/openai.mjs +2 -2
  106. package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
  107. package/dist/import-OOBUW3RR.mjs +44 -0
  108. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  109. package/dist/index.browser.mjs +5 -5
  110. package/dist/index.d.ts +6 -6
  111. package/dist/index.mjs +22 -22
  112. package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
  113. package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
  114. package/dist/module-IYPLWHGC.mjs +48 -0
  115. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  116. package/dist/partial-words.mjs +5 -5
  117. package/dist/pattern.mjs +3 -3
  118. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  119. package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
  120. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  121. package/dist/react.d.ts +9 -9
  122. package/dist/react.mjs +4 -0
  123. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  124. package/dist/rerankers/http.mjs +2 -2
  125. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
  126. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  127. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  128. package/dist/server.d.ts +12 -10
  129. package/dist/server.mjs +54 -21
  130. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
  131. package/dist/synonyms.mjs +6 -4
  132. package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
  133. package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
  134. package/dist/vector/search-worker.d.ts +1 -1
  135. package/dist/vector/search-worker.mjs +6 -5
  136. package/dist/workers/entry.mjs +8 -8
  137. package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
  138. package/openapi.json +114 -6
  139. package/package.json +12 -8
@@ -1,3 +1,5 @@
1
+ import { abandonChangeOf } from './chunk-XNWPP5NS.mjs';
2
+
1
3
  // src/narsil/restore-queue.ts
2
4
  function restoreDepsOf(core, host) {
3
5
  return {
@@ -9,7 +11,11 @@ function restoreDepsOf(core, host) {
9
11
  embeddingAdapters: core.embeddingAdapters,
10
12
  markAnalysisStale: core.analysisRebuild.markStale,
11
13
  clearAnalysisStale: core.analysisRebuild.clearStale,
12
- abandonAnalysisRebuild: core.analysisRebuild.abandon,
14
+ abandonAnalysisRebuild: async (indexName) => {
15
+ await core.analysisRebuild.abandon(indexName);
16
+ await abandonChangeOf(core, indexName);
17
+ },
18
+ abandonAnalysisChange: (indexName) => abandonChangeOf(core, indexName),
13
19
  indexState: core.indexState,
14
20
  copies: core.orchestrator,
15
21
  forgetWatermark: core.watermarkNotifier.forget
@@ -18,7 +24,7 @@ function restoreDepsOf(core, host) {
18
24
  function queueRestore(core, host, restoresInFlight, indexName, data) {
19
25
  const previous = restoresInFlight.get(indexName) ?? Promise.resolve();
20
26
  const restored = previous.then(async () => {
21
- const { restoreFromSnapshot } = await import('./snapshot-DOQLGGEX.mjs');
27
+ const { restoreFromSnapshot } = await import('./snapshot-QXBKHJAG.mjs');
22
28
  await restoreFromSnapshot(indexName, data, restoreDepsOf(core, host));
23
29
  });
24
30
  const settled = restored.then(
@@ -1,6 +1,6 @@
1
1
  import { MIN_RERANK_TIMEOUT_MS, MAX_RERANK_TIMEOUT_MS, MIN_RERANK_WINDOW, MAX_RERANK_WINDOW, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS, DEFAULT_RERANK_WINDOW } from './chunk-JPTQZWJQ.mjs';
2
- import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-VEGHFP2P.mjs';
3
- import { NarsilError, ErrorCodes, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, RESULT_WINDOW } from './chunk-UDGB2MWT.mjs';
2
+ import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-MZI3TEOW.mjs';
3
+ import { NarsilError, ErrorCodes, MIN_HYBRID_DEPTH, MAX_HYBRID_DEPTH, HYBRID_RESCORE_WINDOW, RESULT_WINDOW } from './chunk-35B42UFM.mjs';
4
4
 
5
5
  // src/search/hybrid-depth.ts
6
6
  function fixedHybridDepthOf(hybrid) {
@@ -1,13 +1,18 @@
1
- import { collectAndReturnPages, vectorsPerFile } from './chunk-VBBLTD7O.mjs';
2
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-NSCSR6MA.mjs';
3
- import './chunk-M5UAAXGU.mjs';
4
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-2Y3CO4EQ.mjs';
5
- import './chunk-26A5JFT2.mjs';
6
- import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-D7WQF6DV.mjs';
7
- import { MAX_FILE_BYTES } from './chunk-AIIJDWMG.mjs';
1
+ import { indexMetadataOf, writeCheckpointSegments, writeLiveVectors, commitCheckpointManifest, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-KTOK275N.mjs';
2
+ import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-ZEV26AQ6.mjs';
3
+ import { DEFAULT_COMPACTION_THRESHOLD, manifestKey, manifestFileRefs, encodeSegmentManifest, readSegmentFile } from './chunk-MVM33ZUL.mjs';
4
+ import './chunk-TCLEOZ4P.mjs';
5
+ import { vectorsPerFile } from './chunk-6RHTAVZX.mjs';
6
+ import './chunk-HKX2G6WH.mjs';
7
+ import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-LVY2LI35.mjs';
8
+ import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint, storedBytesOfSegments } from './chunk-R4WUDZQV.mjs';
9
+ import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateTextStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, isCompositePartition, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-5DB2VDGI.mjs';
8
10
  import './chunk-JPTQZWJQ.mjs';
9
- import './chunk-VEGHFP2P.mjs';
10
- import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
11
+ import './chunk-26A5JFT2.mjs';
12
+ import './chunk-OIZS5KIX.mjs';
13
+ import { MAX_FILE_BYTES } from './chunk-D33A5VMU.mjs';
14
+ import './chunk-MZI3TEOW.mjs';
15
+ import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
11
16
  import { decode } from '@msgpack/msgpack';
12
17
 
13
18
  // src/persistence/archive/memory-store.ts
@@ -67,10 +72,11 @@ function partitionVectorPlan(vectorIndex, docIds) {
67
72
  codes: null,
68
73
  vectors: vectorsToBytes(packed)
69
74
  };
70
- files.push({ payload, ordinals: new Int32Array(slice.length), dead: null });
75
+ files.push({ payload, ordinals: new Int32Array(slice.length), dead: null, magnitudes: null });
71
76
  }
72
77
  return {
73
78
  unchanged: false,
79
+ storesHeads: false,
74
80
  kept: [],
75
81
  keptFiles: [],
76
82
  newFiles: files.length,
@@ -162,6 +168,7 @@ async function buildArchiveFromMemory(input) {
162
168
 
163
169
  // src/persistence/archive/constants.ts
164
170
  var MAX_PAX_HEADER_BYTES = 65536;
171
+ var ARCHIVE_FILE_CHUNK_BYTES = 4194304;
165
172
 
166
173
  // src/persistence/archive/tar.ts
167
174
  var TAR_BLOCK_BYTES = 512;
@@ -346,17 +353,32 @@ async function readBlock(reader) {
346
353
  if (block.byteLength < TAR_BLOCK_BYTES) invalidArchive("the archive ends inside an entry header");
347
354
  return block;
348
355
  }
356
+ async function skipPadding(reader, size) {
357
+ const remainder = size % TAR_BLOCK_BYTES;
358
+ if (remainder === 0) return;
359
+ const padding = await reader.readExactly(TAR_BLOCK_BYTES - remainder);
360
+ if (padding === null || padding.byteLength < TAR_BLOCK_BYTES - remainder) {
361
+ invalidArchive("the archive ends inside the padding of an entry");
362
+ }
363
+ }
349
364
  async function readPadded(reader, size) {
350
365
  const body = await reader.readExactly(size) ?? new Uint8Array(0);
351
366
  if (body.byteLength < size) invalidArchive("the archive ends inside an entry");
352
- const remainder = size % TAR_BLOCK_BYTES;
353
- if (remainder !== 0) {
354
- const padding = await reader.readExactly(TAR_BLOCK_BYTES - remainder);
355
- if (padding === null || padding.byteLength < TAR_BLOCK_BYTES - remainder) {
356
- invalidArchive("the archive ends inside the padding of an entry");
367
+ await skipPadding(reader, size);
368
+ return body;
369
+ }
370
+ function entryBody(reader, size) {
371
+ let remaining = size;
372
+ async function* chunks() {
373
+ while (remaining > 0) {
374
+ const length = Math.min(ARCHIVE_FILE_CHUNK_BYTES, remaining);
375
+ const chunk = await reader.readExactly(length);
376
+ if (chunk === null || chunk.byteLength < length) invalidArchive("the archive ends inside an entry");
377
+ remaining -= length;
378
+ yield chunk;
357
379
  }
358
380
  }
359
- return body;
381
+ return chunks();
360
382
  }
361
383
  async function nextEntryHeader(reader) {
362
384
  const block = await readBlock(reader);
@@ -416,12 +438,16 @@ async function readSnapshotArchive(reader, handlers) {
416
438
  if (header.size !== ref.bytes) {
417
439
  invalidArchive(`the entry "${expected}" holds ${header.size} bytes where the manifest lists ${ref.bytes}`);
418
440
  }
419
- const bytes = await readPadded(reader, header.size);
420
- if (headerChecksumOf(bytes) !== ref.checksum) {
441
+ const start = await reader.peek(Math.min(HEADER_SIZE, header.size));
442
+ if (headerChecksumOf(start) !== ref.checksum) {
421
443
  invalidArchive(`the entry "${expected}" carries a checksum other than the manifest lists`, { entry: expected });
422
444
  }
423
445
  seen.add(expected);
424
- await handlers.file(expected, ref, bytes);
446
+ const entry = entryBody(reader, header.size);
447
+ await handlers.file(expected, ref, entry);
448
+ let unread = await entry.next();
449
+ while (unread.done !== true) unread = await entry.next();
450
+ await skipPadding(reader, header.size);
425
451
  }
426
452
  const trailing = await nextEntryHeader(reader);
427
453
  if (trailing !== null) {
@@ -437,12 +463,20 @@ function entryChunks(name, bytes) {
437
463
  (chunk) => chunk.byteLength > 0
438
464
  );
439
465
  }
466
+ async function* fileEntryChunks(source, name, ref) {
467
+ yield* tarEntryHeaders(name, ref.bytes).filter((chunk) => chunk.byteLength > 0);
468
+ for (let offset = 0; offset < ref.bytes; offset += ARCHIVE_FILE_CHUNK_BYTES) {
469
+ yield await source.readRange(ref, offset, Math.min(ARCHIVE_FILE_CHUNK_BYTES, ref.bytes - offset));
470
+ }
471
+ const padding = tarPadding(ref.bytes);
472
+ if (padding.byteLength > 0) yield padding;
473
+ }
440
474
  async function* archiveChunks(source) {
441
475
  yield* entryChunks(ARCHIVE_CONFIG_ENTRY, source.config);
442
476
  yield* entryChunks(ARCHIVE_MANIFEST_ENTRY, source.manifestBytes);
443
477
  const prefix = `${source.sourceIndexName}/`;
444
478
  for (const ref of manifestFileRefs(source.manifest)) {
445
- yield* entryChunks(ref.key.slice(prefix.length), await source.readFile(ref));
479
+ yield* fileEntryChunks(source, ref.key.slice(prefix.length), ref);
446
480
  }
447
481
  yield tarEnd();
448
482
  }
@@ -476,6 +510,27 @@ function archiveStream(source) {
476
510
  });
477
511
  }
478
512
 
513
+ // src/persistence/archive/file-entry.ts
514
+ async function writeArchiveFile(store, key, chunks) {
515
+ const { directory } = store;
516
+ if (directory === void 0) {
517
+ const held = [];
518
+ for await (const chunk of chunks) held.push(chunk);
519
+ await store.write(key, held);
520
+ return;
521
+ }
522
+ const receiving = `${key}.receiving`;
523
+ await directory.remove(receiving);
524
+ const handle = await directory.appendHandle(receiving);
525
+ try {
526
+ for await (const chunk of chunks) await handle.append(chunk);
527
+ await handle.sync();
528
+ } finally {
529
+ await handle.close();
530
+ }
531
+ await directory.move(receiving, key);
532
+ }
533
+
479
534
  // src/persistence/archive/validate.ts
480
535
  function requireDimension(key, found, expected) {
481
536
  if (expected === found) return;
@@ -499,6 +554,14 @@ function requireSchemaOfManifest(key, schema, manifest) {
499
554
  async function validateStagedCheckpoint(store, manifest) {
500
555
  for (const partition of manifest.partitions) {
501
556
  for (const ref of partition.segments) {
557
+ const { directory } = store;
558
+ if (directory !== void 0) {
559
+ const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-BUFNZMEU.mjs');
560
+ if (await isPagedSegmentFile(directory, ref.key)) {
561
+ requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
562
+ continue;
563
+ }
564
+ }
502
565
  const contents = await readSegmentFile(store, ref);
503
566
  requireSchemaOfManifest(ref.key, contents.schema, manifest);
504
567
  decodeEveryStoredDocument(contents.arrays.documentTable);
@@ -515,161 +578,6 @@ async function validateStagedCheckpoint(store, manifest) {
515
578
  }
516
579
  }
517
580
 
518
- // src/engine/snapshot/constants.ts
519
- var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
520
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
521
- var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
522
- var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
523
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
524
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
525
- var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
526
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
527
- var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
528
- var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
529
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
530
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
531
- var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
532
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
533
- var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
534
- var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
535
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
536
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
537
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
538
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
539
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
540
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
541
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
542
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
543
- var RESTORE_ESTIMATE_MARGIN = 1.25;
544
-
545
- // src/engine/snapshot/memory-budget.ts
546
- var MEGABYTE_BYTES = 1048576;
547
- var ARCHIVE_RATES = {
548
- worst: {
549
- heap: {
550
- perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
551
- perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
552
- perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
553
- },
554
- external: {
555
- perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
556
- perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
557
- perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
558
- }
559
- },
560
- floor: {
561
- heap: {
562
- perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
563
- perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
564
- perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
565
- },
566
- external: {
567
- perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
568
- perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
569
- perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
570
- }
571
- }
572
- };
573
- var EARLIER_SNAPSHOT_RATES = {
574
- worst: {
575
- heap: {
576
- perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
577
- perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
578
- perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
579
- },
580
- external: {
581
- perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
582
- perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
583
- perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
584
- }
585
- },
586
- floor: {
587
- heap: {
588
- perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
589
- perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
590
- perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
591
- },
592
- external: {
593
- perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
594
- perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
595
- perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
596
- }
597
- }
598
- };
599
- function bytesAt(rate, storedBytes, vectors) {
600
- let bytes = storedBytes * rate.perStoredByte;
601
- for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
602
- return bytes;
603
- }
604
- function footprintAt(rates, replacesIndex, storedBytes, vectors) {
605
- const chosen = replacesIndex ? rates.worst : rates.floor;
606
- const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
607
- return {
608
- heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
609
- externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
610
- };
611
- }
612
- function deadCountOf(dead) {
613
- if (dead === null) return 0;
614
- let count = 0;
615
- for (const byte of dead) {
616
- for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
617
- }
618
- return count;
619
- }
620
- function archiveFootprint(prepared, manifest, replacesIndex) {
621
- let segmentBytes = 0;
622
- for (const partition of manifest.partitions) {
623
- for (const segment of partition.segments) segmentBytes += segment.bytes;
624
- }
625
- const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
626
- const vectors = [];
627
- for (const field of manifest.vectors) {
628
- const dimension = dimensions.get(field.fieldPath) ?? 0;
629
- for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
630
- }
631
- return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
632
- }
633
- function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
634
- const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
635
- const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
636
- return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
637
- }
638
- function inMegabytes(bytes) {
639
- return Math.ceil(bytes / MEGABYTE_BYTES);
640
- }
641
- function shortfallOf(heapBytes, processBytes, countInUse) {
642
- const heap = readHeapStatistics();
643
- if (heap !== null) {
644
- const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
645
- const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
646
- if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
647
- }
648
- const resident = readResidentMemory();
649
- if (resident !== null) {
650
- const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
651
- if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
652
- }
653
- return null;
654
- }
655
- function refuse(indexName, replacesIndex, shortfall) {
656
- const { budget, neededBytes, availableBytes } = shortfall;
657
- const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
658
- const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
659
- throw new NarsilError(
660
- ErrorCodes.RESTORE_MEMORY_EXCEEDED,
661
- `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
662
- { indexName, budget, neededBytes, availableBytes }
663
- );
664
- }
665
- function requireRestoreMemory(indexName, footprint, replacesIndex) {
666
- const processBytes = footprint.heapBytes + footprint.externalBytes;
667
- if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
668
- const collected = collectAndReturnPages();
669
- const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
670
- if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
671
- }
672
-
673
581
  // src/workers/detached-index.ts
674
582
  function attachablePolicy(shared) {
675
583
  let attached = false;
@@ -709,6 +617,7 @@ function prepareRestoredIndex(configuration, deps, reject) {
709
617
  requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
710
618
  validateVectorPromotion(configFields.vectorPromotion);
711
619
  validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
620
+ validateTextStorage(configFields.textStorage, deps.filesystemDurability);
712
621
  if (configFields.required !== void 0 && configFields.required.length > 0) {
713
622
  validateRequiredFieldsInSchema(configFields.required, schema);
714
623
  }
@@ -735,12 +644,14 @@ function prepareRestoredIndex(configuration, deps, reject) {
735
644
  }
736
645
  const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
737
646
  const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
647
+ const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
738
648
  return {
739
649
  config,
740
650
  language,
741
651
  embeddingAdapter,
742
652
  adapterName,
743
653
  storedRevision,
654
+ analysisGeneration,
744
655
  storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
745
656
  synonyms
746
657
  };
@@ -775,7 +686,10 @@ function restoredRegistryEntry(prepared, manager, deps) {
775
686
  partitionCount: manager.partitionCount,
776
687
  analysisRevision: prepared.storedRevision,
777
688
  arrayPositionGap: prepared.storedArrayPositionGap,
778
- synonyms: prepared.synonyms?.encoded ?? null
689
+ synonyms: prepared.synonyms?.encoded ?? null,
690
+ analysisGeneration: prepared.analysisGeneration,
691
+ pendingAnalysis: null,
692
+ durableAnalysis: null
779
693
  };
780
694
  }
781
695
 
@@ -849,7 +763,14 @@ async function commitToStorage(indexName, restored, previousPartitions, deps) {
849
763
  }
850
764
  await staging.finish();
851
765
  const relocate = await staging.relocation().catch(() => null);
852
- if (relocate !== null) for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
766
+ if (relocate === null) return;
767
+ for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
768
+ const { manager } = restored.detached;
769
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
770
+ const partition = manager.getPartition(partitionId);
771
+ if (!isCompositePartition(partition)) continue;
772
+ for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
773
+ }
853
774
  }
854
775
  function activate(indexName, restored, entry, deps) {
855
776
  deps.executor.installDetachedIndex(restored.detached);
@@ -871,6 +792,7 @@ function activate(indexName, restored, entry, deps) {
871
792
  async function replaceExisting(indexName, restored, entry, deps) {
872
793
  const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
873
794
  try {
795
+ await deps.abandonAnalysisChange(indexName);
874
796
  await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
875
797
  await deps.indexState.replace(indexName, async (wasOpen) => {
876
798
  await deps.abandonAnalysisRebuild(indexName);
@@ -971,6 +893,9 @@ function partitionCountOf(manifest) {
971
893
  function missingPart(what) {
972
894
  throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
973
895
  }
896
+ function keepsTextOnDisk(prepared, staging) {
897
+ return prepared.config.textStorage === "disk" && staging.directory !== void 0;
898
+ }
974
899
  async function stageArchive(indexName, reader, staging, deps) {
975
900
  const staged = { prepared: null, manifest: null };
976
901
  await readSnapshotArchive(reader, {
@@ -981,17 +906,29 @@ async function stageArchive(indexName, reader, staging, deps) {
981
906
  if (staged.prepared === null) missingPart("configuration");
982
907
  requireSameSchema(staged.prepared, manifest2);
983
908
  const replacesIndex = deps.indexRegistry.has(indexName);
984
- requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2, replacesIndex), replacesIndex);
909
+ const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
910
+ requireRestoreMemory(
911
+ indexName,
912
+ archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
913
+ replacesIndex
914
+ );
985
915
  staged.manifest = renamed(manifest2, sourceName, indexName);
986
916
  },
987
- file: async (entryName, _ref, bytes) => {
917
+ file: async (entryName, _ref, chunks) => {
988
918
  if (staged.manifest === null) missingPart("manifest");
989
- await staging.write(`${indexName}/${entryName}`, [bytes]);
919
+ await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
990
920
  }
991
921
  });
992
922
  const { prepared, manifest } = staged;
993
923
  if (prepared === null) missingPart("configuration");
994
924
  if (manifest === null) missingPart("manifest");
925
+ const { directory } = staging;
926
+ if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
927
+ const { bytesThatSegmentsHoldInMemory } = await import('./module-BUFNZMEU.mjs');
928
+ const replacesIndex = deps.indexRegistry.has(indexName);
929
+ const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
930
+ requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
931
+ }
995
932
  await validateStagedCheckpoint(staging, manifest);
996
933
  return { prepared, manifest };
997
934
  }
@@ -1174,9 +1111,24 @@ async function checkpointForArchive(deps) {
1174
1111
  });
1175
1112
  }
1176
1113
  async function createSnapshot(deps) {
1177
- const config = await encodeIndexConfiguration(deps.entry);
1178
- const checkpoint = await checkpointForArchive(deps);
1179
- return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
1114
+ for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
1115
+ const checkpoint = await checkpointForArchive(deps);
1116
+ try {
1117
+ if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
1118
+ const config = await encodeIndexConfiguration(deps.entry);
1119
+ return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
1120
+ }
1121
+ } catch (error) {
1122
+ checkpoint.release();
1123
+ throw error;
1124
+ }
1125
+ checkpoint.release();
1126
+ }
1127
+ throw new NarsilError(
1128
+ ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
1129
+ `The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
1130
+ { indexName: deps.indexName }
1131
+ );
1180
1132
  }
1181
1133
  function requireSnapshotSource(data) {
1182
1134
  if (data instanceof Uint8Array) return data;
@@ -1,98 +1,20 @@
1
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-M5UAAXGU.mjs';
2
- import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-2Y3CO4EQ.mjs';
3
- import './chunk-26A5JFT2.mjs';
4
- import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-D7WQF6DV.mjs';
5
- import './chunk-AIIJDWMG.mjs';
1
+ import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-VFYMPCOV.mjs';
2
+ import './chunk-ZEV26AQ6.mjs';
3
+ import { encodeExactJson, encodeDocumentJson } from './chunk-CCOAOO73.mjs';
4
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-TCLEOZ4P.mjs';
5
+ import './chunk-WCFCHIBB.mjs';
6
+ import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-LVY2LI35.mjs';
7
+ import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-5DB2VDGI.mjs';
6
8
  import './chunk-JPTQZWJQ.mjs';
7
- import { EXPORT_CHUNK_BYTES } from './chunk-VEGHFP2P.mjs';
8
- import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
9
+ import './chunk-26A5JFT2.mjs';
10
+ import './chunk-D33A5VMU.mjs';
11
+ import { EXPORT_CHUNK_BYTES } from './chunk-MZI3TEOW.mjs';
12
+ import './chunk-35B42UFM.mjs';
9
13
  import { decode } from '@msgpack/msgpack';
10
14
 
11
- // src/engine/export/json-line.ts
12
- function refuse(path, kind, docId) {
13
- throw new NarsilError(
14
- ErrorCodes.DOC_VALIDATION_FAILED,
15
- `Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
16
- { docId, field: path }
17
- );
18
- }
19
- function encodeNumber(value, path, target) {
20
- if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
21
- if (!target.tagged) refuse(path, String(value), target.docId);
22
- return `{"${NUMBER_TAG}":"${String(value)}"}`;
23
- }
24
- function encodeDate(value, path, target) {
25
- const milliseconds = value.getTime();
26
- if (!target.tagged || Number.isNaN(milliseconds)) refuse(path, "a date", target.docId);
27
- const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
28
- const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
29
- return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
30
- }
31
- function encodeBytes(value, path, target) {
32
- if (!target.tagged) refuse(path, "bytes", target.docId);
33
- return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
34
- }
35
- function encodeList(value, path, target) {
36
- target.parts.push("[");
37
- for (let at = 0; at < value.length; at++) {
38
- if (at > 0) target.parts.push(",");
39
- encodeValue(value[at], `${path}[${at}]`, target);
40
- }
41
- target.parts.push("]");
42
- }
43
- function encodeMap(value, path, target) {
44
- const proto = Object.getPrototypeOf(value);
45
- if (proto !== Object.prototype && proto !== null) refuse(path, "a value other than a plain object", target.docId);
46
- const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
47
- const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
48
- target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
49
- for (let at = 0; at < entries.length; at++) {
50
- const [key, entry] = entries[at];
51
- if (at > 0) target.parts.push(",");
52
- target.parts.push(JSON.stringify(key), ":");
53
- encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
54
- }
55
- target.parts.push(wrapped ? "}}" : "}");
56
- }
57
- function encodeValue(value, path, target) {
58
- if (value === null) {
59
- target.parts.push("null");
60
- return;
61
- }
62
- switch (typeof value) {
63
- case "string":
64
- target.parts.push(JSON.stringify(value));
65
- return;
66
- case "boolean":
67
- target.parts.push(value ? "true" : "false");
68
- return;
69
- case "number":
70
- target.parts.push(encodeNumber(value, path, target));
71
- return;
72
- case "object":
73
- break;
74
- default:
75
- refuse(path, `a ${typeof value}`, target.docId);
76
- }
77
- if (Array.isArray(value)) encodeList(value, path, target);
78
- else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
79
- else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
80
- else encodeMap(value, path, target);
81
- }
82
- function encodeExactJson(value, docId) {
83
- const target = { docId, tagged: false, parts: [] };
84
- encodeValue(value, "", target);
85
- return target.parts.join("");
86
- }
87
- function encodeDocumentJson(document, docId) {
88
- const target = { docId, tagged: true, parts: [] };
89
- encodeValue(document, "", target);
90
- return target.parts.join("");
91
- }
92
-
93
- // src/engine/export/write.ts
94
15
  var VECTOR_FILE_ENVELOPE_VERSION = 2;
95
16
  var FLOAT32_BYTES = 4;
17
+ var ENVELOPE_VERSION_OFFSET = 4;
96
18
  async function locateVectors(checkpoint) {
97
19
  const located = /* @__PURE__ */ new Map();
98
20
  for (const field of checkpoint.manifest.vectors) {
@@ -140,7 +62,59 @@ async function winnersOf(checkpoint, partition) {
140
62
  }
141
63
  return winners;
142
64
  }
65
+ async function wholeSegmentScan(checkpoint, ref) {
66
+ const contents = await readSegment(checkpoint, ref);
67
+ const table = contents.arrays.documentTable;
68
+ const ids = /* @__PURE__ */ new Set();
69
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
70
+ return {
71
+ count: contents.arrays.documentCount,
72
+ schema: contents.schema,
73
+ tombstones: new Set(contents.tombstones),
74
+ idAt: (ordinal) => idAt(contents, ordinal),
75
+ holds: (docId) => ids.has(docId),
76
+ storedDocumentAt: (ordinal) => decodeStoredDocument(
77
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
78
+ table.discardedFields
79
+ ),
80
+ close: () => void 0
81
+ };
82
+ }
83
+ async function wholeFileVectorReads(checkpoint) {
84
+ const located = await locateVectors(checkpoint);
85
+ return {
86
+ async vectorOf(fieldPath, docId) {
87
+ const location = located.get(fieldPath)?.get(docId);
88
+ return location === void 0 ? null : readVector(checkpoint, location);
89
+ },
90
+ close: () => located.clear()
91
+ };
92
+ }
93
+ async function* pagedDocumentLines(checkpoint) {
94
+ const { openFileVectorReads, pagedPartitionLines } = await import('./module-BUFNZMEU.mjs');
95
+ const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
96
+ try {
97
+ for (const partition of checkpoint.manifest.partitions) {
98
+ yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
99
+ }
100
+ } finally {
101
+ vectors.close();
102
+ }
103
+ }
104
+ async function anyPartitionIsPaged(checkpoint) {
105
+ for (const partition of checkpoint.manifest.partitions) {
106
+ for (const ref of partition.segments) {
107
+ const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
108
+ if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
109
+ }
110
+ }
111
+ return false;
112
+ }
143
113
  async function* documentLines(checkpoint) {
114
+ if (await anyPartitionIsPaged(checkpoint)) {
115
+ yield* pagedDocumentLines(checkpoint);
116
+ return;
117
+ }
144
118
  const vectors = await locateVectors(checkpoint);
145
119
  for (const partition of checkpoint.manifest.partitions) {
146
120
  const winners = await winnersOf(checkpoint, partition);