@delali/narsil 0.3.1-next.140 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,14 +1,16 @@
1
- import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-QHNNULPW.mjs';
2
- import { vectorsPerFile } from './chunk-3EYDURFD.mjs';
3
- import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint } from './chunk-67KSBUYR.mjs';
4
- import './chunk-OIZS5KIX.mjs';
5
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-CGVBGHBX.mjs';
6
- import './chunk-6BSB4N6B.mjs';
7
- import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-5LYWZJE2.mjs';
8
- import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-ZAIZQOTM.mjs';
9
- import './chunk-26A5JFT2.mjs';
10
- import { MAX_FILE_BYTES } from './chunk-4MFH226Q.mjs';
1
+ import { indexMetadataOf, writeCheckpointSegments, writeLiveVectors, commitCheckpointManifest, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-KTOK275N.mjs';
2
+ import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-ZEV26AQ6.mjs';
3
+ import { DEFAULT_COMPACTION_THRESHOLD, manifestKey, manifestFileRefs, encodeSegmentManifest, readSegmentFile } from './chunk-MVM33ZUL.mjs';
4
+ import './chunk-TCLEOZ4P.mjs';
5
+ import { vectorsPerFile } from './chunk-6RHTAVZX.mjs';
6
+ import './chunk-HKX2G6WH.mjs';
7
+ import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-LVY2LI35.mjs';
8
+ import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint, storedBytesOfSegments } from './chunk-R4WUDZQV.mjs';
9
+ import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateTextStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, isCompositePartition, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-5DB2VDGI.mjs';
11
10
  import './chunk-JPTQZWJQ.mjs';
11
+ import './chunk-26A5JFT2.mjs';
12
+ import './chunk-OIZS5KIX.mjs';
13
+ import { MAX_FILE_BYTES } from './chunk-D33A5VMU.mjs';
12
14
  import './chunk-MZI3TEOW.mjs';
13
15
  import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
14
16
  import { decode } from '@msgpack/msgpack';
@@ -70,10 +72,11 @@ function partitionVectorPlan(vectorIndex, docIds) {
70
72
  codes: null,
71
73
  vectors: vectorsToBytes(packed)
72
74
  };
73
- files.push({ payload, ordinals: new Int32Array(slice.length), dead: null });
75
+ files.push({ payload, ordinals: new Int32Array(slice.length), dead: null, magnitudes: null });
74
76
  }
75
77
  return {
76
78
  unchanged: false,
79
+ storesHeads: false,
77
80
  kept: [],
78
81
  keptFiles: [],
79
82
  newFiles: files.length,
@@ -165,6 +168,7 @@ async function buildArchiveFromMemory(input) {
165
168
 
166
169
  // src/persistence/archive/constants.ts
167
170
  var MAX_PAX_HEADER_BYTES = 65536;
171
+ var ARCHIVE_FILE_CHUNK_BYTES = 4194304;
168
172
 
169
173
  // src/persistence/archive/tar.ts
170
174
  var TAR_BLOCK_BYTES = 512;
@@ -349,17 +353,32 @@ async function readBlock(reader) {
349
353
  if (block.byteLength < TAR_BLOCK_BYTES) invalidArchive("the archive ends inside an entry header");
350
354
  return block;
351
355
  }
356
+ async function skipPadding(reader, size) {
357
+ const remainder = size % TAR_BLOCK_BYTES;
358
+ if (remainder === 0) return;
359
+ const padding = await reader.readExactly(TAR_BLOCK_BYTES - remainder);
360
+ if (padding === null || padding.byteLength < TAR_BLOCK_BYTES - remainder) {
361
+ invalidArchive("the archive ends inside the padding of an entry");
362
+ }
363
+ }
352
364
  async function readPadded(reader, size) {
353
365
  const body = await reader.readExactly(size) ?? new Uint8Array(0);
354
366
  if (body.byteLength < size) invalidArchive("the archive ends inside an entry");
355
- const remainder = size % TAR_BLOCK_BYTES;
356
- if (remainder !== 0) {
357
- const padding = await reader.readExactly(TAR_BLOCK_BYTES - remainder);
358
- if (padding === null || padding.byteLength < TAR_BLOCK_BYTES - remainder) {
359
- invalidArchive("the archive ends inside the padding of an entry");
367
+ await skipPadding(reader, size);
368
+ return body;
369
+ }
370
+ function entryBody(reader, size) {
371
+ let remaining = size;
372
+ async function* chunks() {
373
+ while (remaining > 0) {
374
+ const length = Math.min(ARCHIVE_FILE_CHUNK_BYTES, remaining);
375
+ const chunk = await reader.readExactly(length);
376
+ if (chunk === null || chunk.byteLength < length) invalidArchive("the archive ends inside an entry");
377
+ remaining -= length;
378
+ yield chunk;
360
379
  }
361
380
  }
362
- return body;
381
+ return chunks();
363
382
  }
364
383
  async function nextEntryHeader(reader) {
365
384
  const block = await readBlock(reader);
@@ -419,12 +438,16 @@ async function readSnapshotArchive(reader, handlers) {
419
438
  if (header.size !== ref.bytes) {
420
439
  invalidArchive(`the entry "${expected}" holds ${header.size} bytes where the manifest lists ${ref.bytes}`);
421
440
  }
422
- const bytes = await readPadded(reader, header.size);
423
- if (headerChecksumOf(bytes) !== ref.checksum) {
441
+ const start = await reader.peek(Math.min(HEADER_SIZE, header.size));
442
+ if (headerChecksumOf(start) !== ref.checksum) {
424
443
  invalidArchive(`the entry "${expected}" carries a checksum other than the manifest lists`, { entry: expected });
425
444
  }
426
445
  seen.add(expected);
427
- await handlers.file(expected, ref, bytes);
446
+ const entry = entryBody(reader, header.size);
447
+ await handlers.file(expected, ref, entry);
448
+ let unread = await entry.next();
449
+ while (unread.done !== true) unread = await entry.next();
450
+ await skipPadding(reader, header.size);
428
451
  }
429
452
  const trailing = await nextEntryHeader(reader);
430
453
  if (trailing !== null) {
@@ -440,12 +463,20 @@ function entryChunks(name, bytes) {
440
463
  (chunk) => chunk.byteLength > 0
441
464
  );
442
465
  }
466
+ async function* fileEntryChunks(source, name, ref) {
467
+ yield* tarEntryHeaders(name, ref.bytes).filter((chunk) => chunk.byteLength > 0);
468
+ for (let offset = 0; offset < ref.bytes; offset += ARCHIVE_FILE_CHUNK_BYTES) {
469
+ yield await source.readRange(ref, offset, Math.min(ARCHIVE_FILE_CHUNK_BYTES, ref.bytes - offset));
470
+ }
471
+ const padding = tarPadding(ref.bytes);
472
+ if (padding.byteLength > 0) yield padding;
473
+ }
443
474
  async function* archiveChunks(source) {
444
475
  yield* entryChunks(ARCHIVE_CONFIG_ENTRY, source.config);
445
476
  yield* entryChunks(ARCHIVE_MANIFEST_ENTRY, source.manifestBytes);
446
477
  const prefix = `${source.sourceIndexName}/`;
447
478
  for (const ref of manifestFileRefs(source.manifest)) {
448
- yield* entryChunks(ref.key.slice(prefix.length), await source.readFile(ref));
479
+ yield* fileEntryChunks(source, ref.key.slice(prefix.length), ref);
449
480
  }
450
481
  yield tarEnd();
451
482
  }
@@ -479,6 +510,27 @@ function archiveStream(source) {
479
510
  });
480
511
  }
481
512
 
513
+ // src/persistence/archive/file-entry.ts
514
+ async function writeArchiveFile(store, key, chunks) {
515
+ const { directory } = store;
516
+ if (directory === void 0) {
517
+ const held = [];
518
+ for await (const chunk of chunks) held.push(chunk);
519
+ await store.write(key, held);
520
+ return;
521
+ }
522
+ const receiving = `${key}.receiving`;
523
+ await directory.remove(receiving);
524
+ const handle = await directory.appendHandle(receiving);
525
+ try {
526
+ for await (const chunk of chunks) await handle.append(chunk);
527
+ await handle.sync();
528
+ } finally {
529
+ await handle.close();
530
+ }
531
+ await directory.move(receiving, key);
532
+ }
533
+
482
534
  // src/persistence/archive/validate.ts
483
535
  function requireDimension(key, found, expected) {
484
536
  if (expected === found) return;
@@ -502,6 +554,14 @@ function requireSchemaOfManifest(key, schema, manifest) {
502
554
  async function validateStagedCheckpoint(store, manifest) {
503
555
  for (const partition of manifest.partitions) {
504
556
  for (const ref of partition.segments) {
557
+ const { directory } = store;
558
+ if (directory !== void 0) {
559
+ const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-BUFNZMEU.mjs');
560
+ if (await isPagedSegmentFile(directory, ref.key)) {
561
+ requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
562
+ continue;
563
+ }
564
+ }
505
565
  const contents = await readSegmentFile(store, ref);
506
566
  requireSchemaOfManifest(ref.key, contents.schema, manifest);
507
567
  decodeEveryStoredDocument(contents.arrays.documentTable);
@@ -557,6 +617,7 @@ function prepareRestoredIndex(configuration, deps, reject) {
557
617
  requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
558
618
  validateVectorPromotion(configFields.vectorPromotion);
559
619
  validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
620
+ validateTextStorage(configFields.textStorage, deps.filesystemDurability);
560
621
  if (configFields.required !== void 0 && configFields.required.length > 0) {
561
622
  validateRequiredFieldsInSchema(configFields.required, schema);
562
623
  }
@@ -702,7 +763,14 @@ async function commitToStorage(indexName, restored, previousPartitions, deps) {
702
763
  }
703
764
  await staging.finish();
704
765
  const relocate = await staging.relocation().catch(() => null);
705
- if (relocate !== null) for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
766
+ if (relocate === null) return;
767
+ for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
768
+ const { manager } = restored.detached;
769
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
770
+ const partition = manager.getPartition(partitionId);
771
+ if (!isCompositePartition(partition)) continue;
772
+ for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
773
+ }
706
774
  }
707
775
  function activate(indexName, restored, entry, deps) {
708
776
  deps.executor.installDetachedIndex(restored.detached);
@@ -825,6 +893,9 @@ function partitionCountOf(manifest) {
825
893
  function missingPart(what) {
826
894
  throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
827
895
  }
896
+ function keepsTextOnDisk(prepared, staging) {
897
+ return prepared.config.textStorage === "disk" && staging.directory !== void 0;
898
+ }
828
899
  async function stageArchive(indexName, reader, staging, deps) {
829
900
  const staged = { prepared: null, manifest: null };
830
901
  await readSnapshotArchive(reader, {
@@ -835,17 +906,29 @@ async function stageArchive(indexName, reader, staging, deps) {
835
906
  if (staged.prepared === null) missingPart("configuration");
836
907
  requireSameSchema(staged.prepared, manifest2);
837
908
  const replacesIndex = deps.indexRegistry.has(indexName);
838
- requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2, replacesIndex), replacesIndex);
909
+ const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
910
+ requireRestoreMemory(
911
+ indexName,
912
+ archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
913
+ replacesIndex
914
+ );
839
915
  staged.manifest = renamed(manifest2, sourceName, indexName);
840
916
  },
841
- file: async (entryName, _ref, bytes) => {
917
+ file: async (entryName, _ref, chunks) => {
842
918
  if (staged.manifest === null) missingPart("manifest");
843
- await staging.write(`${indexName}/${entryName}`, [bytes]);
919
+ await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
844
920
  }
845
921
  });
846
922
  const { prepared, manifest } = staged;
847
923
  if (prepared === null) missingPart("configuration");
848
924
  if (manifest === null) missingPart("manifest");
925
+ const { directory } = staging;
926
+ if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
927
+ const { bytesThatSegmentsHoldInMemory } = await import('./module-BUFNZMEU.mjs');
928
+ const replacesIndex = deps.indexRegistry.has(indexName);
929
+ const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
930
+ requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
931
+ }
849
932
  await validateStagedCheckpoint(staging, manifest);
850
933
  return { prepared, manifest };
851
934
  }
@@ -1,100 +1,20 @@
1
- import { NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-XVMOQQ34.mjs';
2
- import './chunk-QHNNULPW.mjs';
3
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-6BSB4N6B.mjs';
4
- import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-5LYWZJE2.mjs';
5
- import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-ZAIZQOTM.mjs';
6
- import './chunk-26A5JFT2.mjs';
7
- import './chunk-4MFH226Q.mjs';
1
+ import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-VFYMPCOV.mjs';
2
+ import './chunk-ZEV26AQ6.mjs';
3
+ import { encodeExactJson, encodeDocumentJson } from './chunk-CCOAOO73.mjs';
4
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-TCLEOZ4P.mjs';
5
+ import './chunk-WCFCHIBB.mjs';
6
+ import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-LVY2LI35.mjs';
7
+ import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-5DB2VDGI.mjs';
8
8
  import './chunk-JPTQZWJQ.mjs';
9
+ import './chunk-26A5JFT2.mjs';
10
+ import './chunk-D33A5VMU.mjs';
9
11
  import { EXPORT_CHUNK_BYTES } from './chunk-MZI3TEOW.mjs';
10
- import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
12
+ import './chunk-35B42UFM.mjs';
11
13
  import { decode } from '@msgpack/msgpack';
12
14
 
13
- // src/engine/export/json-line.ts
14
- function refuse(path, kind, docId) {
15
- throw new NarsilError(
16
- ErrorCodes.DOC_VALIDATION_FAILED,
17
- `Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
18
- { docId, field: path }
19
- );
20
- }
21
- function encodeNumber(value, path, target) {
22
- if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
23
- if (!target.tagged) refuse(path, String(value), target.docId);
24
- return `{"${NUMBER_TAG}":"${String(value)}"}`;
25
- }
26
- function encodeDate(value, path, target) {
27
- const milliseconds = value.getTime();
28
- if (!target.tagged || Number.isNaN(milliseconds)) refuse(path, "a date", target.docId);
29
- const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
30
- const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
31
- return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
32
- }
33
- function encodeBytes(value, path, target) {
34
- if (!target.tagged) refuse(path, "bytes", target.docId);
35
- return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
36
- }
37
- function encodeList(value, path, target) {
38
- target.parts.push("[");
39
- for (let at = 0; at < value.length; at++) {
40
- if (at > 0) target.parts.push(",");
41
- encodeValue(value[at], `${path}[${at}]`, target);
42
- }
43
- target.parts.push("]");
44
- }
45
- function encodeMap(value, path, target) {
46
- const proto = Object.getPrototypeOf(value);
47
- if (proto !== Object.prototype && proto !== null) refuse(path, "a value other than a plain object", target.docId);
48
- const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
49
- const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
50
- target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
51
- for (let at = 0; at < entries.length; at++) {
52
- const [key, entry] = entries[at];
53
- if (at > 0) target.parts.push(",");
54
- target.parts.push(JSON.stringify(key), ":");
55
- encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
56
- }
57
- target.parts.push(wrapped ? "}}" : "}");
58
- }
59
- function encodeValue(value, path, target) {
60
- if (value === null) {
61
- target.parts.push("null");
62
- return;
63
- }
64
- switch (typeof value) {
65
- case "string":
66
- target.parts.push(JSON.stringify(value));
67
- return;
68
- case "boolean":
69
- target.parts.push(value ? "true" : "false");
70
- return;
71
- case "number":
72
- target.parts.push(encodeNumber(value, path, target));
73
- return;
74
- case "object":
75
- break;
76
- default:
77
- refuse(path, `a ${typeof value}`, target.docId);
78
- }
79
- if (Array.isArray(value)) encodeList(value, path, target);
80
- else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
81
- else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
82
- else encodeMap(value, path, target);
83
- }
84
- function encodeExactJson(value, docId) {
85
- const target = { docId, tagged: false, parts: [] };
86
- encodeValue(value, "", target);
87
- return target.parts.join("");
88
- }
89
- function encodeDocumentJson(document, docId) {
90
- const target = { docId, tagged: true, parts: [] };
91
- encodeValue(document, "", target);
92
- return target.parts.join("");
93
- }
94
-
95
- // src/engine/export/write.ts
96
15
  var VECTOR_FILE_ENVELOPE_VERSION = 2;
97
16
  var FLOAT32_BYTES = 4;
17
+ var ENVELOPE_VERSION_OFFSET = 4;
98
18
  async function locateVectors(checkpoint) {
99
19
  const located = /* @__PURE__ */ new Map();
100
20
  for (const field of checkpoint.manifest.vectors) {
@@ -142,7 +62,59 @@ async function winnersOf(checkpoint, partition) {
142
62
  }
143
63
  return winners;
144
64
  }
65
+ async function wholeSegmentScan(checkpoint, ref) {
66
+ const contents = await readSegment(checkpoint, ref);
67
+ const table = contents.arrays.documentTable;
68
+ const ids = /* @__PURE__ */ new Set();
69
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
70
+ return {
71
+ count: contents.arrays.documentCount,
72
+ schema: contents.schema,
73
+ tombstones: new Set(contents.tombstones),
74
+ idAt: (ordinal) => idAt(contents, ordinal),
75
+ holds: (docId) => ids.has(docId),
76
+ storedDocumentAt: (ordinal) => decodeStoredDocument(
77
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
78
+ table.discardedFields
79
+ ),
80
+ close: () => void 0
81
+ };
82
+ }
83
+ async function wholeFileVectorReads(checkpoint) {
84
+ const located = await locateVectors(checkpoint);
85
+ return {
86
+ async vectorOf(fieldPath, docId) {
87
+ const location = located.get(fieldPath)?.get(docId);
88
+ return location === void 0 ? null : readVector(checkpoint, location);
89
+ },
90
+ close: () => located.clear()
91
+ };
92
+ }
93
+ async function* pagedDocumentLines(checkpoint) {
94
+ const { openFileVectorReads, pagedPartitionLines } = await import('./module-BUFNZMEU.mjs');
95
+ const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
96
+ try {
97
+ for (const partition of checkpoint.manifest.partitions) {
98
+ yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
99
+ }
100
+ } finally {
101
+ vectors.close();
102
+ }
103
+ }
104
+ async function anyPartitionIsPaged(checkpoint) {
105
+ for (const partition of checkpoint.manifest.partitions) {
106
+ for (const ref of partition.segments) {
107
+ const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
108
+ if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
109
+ }
110
+ }
111
+ return false;
112
+ }
145
113
  async function* documentLines(checkpoint) {
114
+ if (await anyPartitionIsPaged(checkpoint)) {
115
+ yield* pagedDocumentLines(checkpoint);
116
+ return;
117
+ }
146
118
  const vectors = await locateVectors(checkpoint);
147
119
  for (const partition of checkpoint.manifest.partitions) {
148
120
  const winners = await winnersOf(checkpoint, partition);
@@ -1,6 +1,6 @@
1
1
  import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
2
- import { C as ClusterNamespace } from './config-Brysl9NO.js';
3
- import { E as ErrorDetail, i as TaskStore } from './tasks-DjGjK70Y.js';
2
+ import { C as ClusterNamespace } from './config-CIG5yDyg.js';
3
+ import { E as ErrorDetail, i as TaskStore } from './tasks-CjRJRGTb.js';
4
4
 
5
5
  /**
6
6
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -4,14 +4,14 @@ function holdsChangeWork(entry) {
4
4
  }
5
5
  async function abandonChangeOf(core, indexName) {
6
6
  if (!holdsChangeWork(core.indexRegistry.get(indexName))) return;
7
- const { abandonAnalysisChange } = await import('./analysis-change-JN23AZQP.mjs');
7
+ const { abandonAnalysisChange } = await import('./analysis-change-HQUT2Z3I.mjs');
8
8
  await abandonAnalysisChange(core, indexName);
9
9
  }
10
10
  async function resumeChangeOf(core, indexName) {
11
11
  if ((core.indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
12
12
  if (core.mutationCtx.capacityScope === "partition") return;
13
13
  if (core.executor.getManager(indexName) === void 0) return;
14
- const { resumeAnalysisChange } = await import('./analysis-change-JN23AZQP.mjs');
14
+ const { resumeAnalysisChange } = await import('./analysis-change-HQUT2Z3I.mjs');
15
15
  void resumeAnalysisChange(core, indexName).catch(() => void 0);
16
16
  }
17
17
  async function resumeEveryChange(core) {
@@ -1,4 +1,4 @@
1
- import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-UBXJQJII.mjs';
1
+ import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-KGOXHLLZ.mjs';
2
2
  import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
3
3
 
4
4
  // src/narsil/internals.ts
@@ -0,0 +1,214 @@
1
+ import { untagDocumentLine } from './chunk-BOJZGUS6.mjs';
2
+ import { chunksOf } from './chunk-CZQ33DLW.mjs';
3
+ import { restoredConfigFields, restoredEmbedding } from './chunk-L76W3IZJ.mjs';
4
+ import { requireConfigurationShape } from './chunk-IO5F7ZBE.mjs';
5
+ import { MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_IMPORT_REFUSALS_REPORTED } from './chunk-EEBCDP36.mjs';
6
+ import { flattenSchema } from './chunk-KGOXHLLZ.mjs';
7
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
8
+
9
+ // src/engine/export/lines.ts
10
+ var NEWLINE = 10;
11
+ function joined(parts, total) {
12
+ if (parts.length === 1) return parts[0];
13
+ const bytes = new Uint8Array(total);
14
+ let offset = 0;
15
+ for (const part of parts) {
16
+ bytes.set(part, offset);
17
+ offset += part.byteLength;
18
+ }
19
+ return bytes;
20
+ }
21
+ async function* exportLines(source, maxLineBytes) {
22
+ const decoder = new TextDecoder("utf-8", { fatal: true });
23
+ let parts = [];
24
+ let pending = 0;
25
+ let overflowing = false;
26
+ let line = 0;
27
+ const finishLine = () => {
28
+ line += 1;
29
+ if (overflowing) {
30
+ overflowing = false;
31
+ parts = [];
32
+ pending = 0;
33
+ return { line, refused: "too-long" };
34
+ }
35
+ const bytes = joined(parts, pending);
36
+ parts = [];
37
+ pending = 0;
38
+ try {
39
+ return { line, text: decoder.decode(bytes) };
40
+ } catch {
41
+ return { line, refused: "not-utf8" };
42
+ }
43
+ };
44
+ for await (const chunk of chunksOf(source)) {
45
+ let start = 0;
46
+ for (; ; ) {
47
+ const newline = chunk.indexOf(NEWLINE, start);
48
+ const end = newline < 0 ? chunk.byteLength : newline;
49
+ const piece = chunk.subarray(start, end);
50
+ if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
51
+ if (piece.byteLength > 0) parts.push(piece.slice());
52
+ pending += piece.byteLength;
53
+ } else {
54
+ overflowing = true;
55
+ parts = [];
56
+ }
57
+ if (newline < 0) break;
58
+ yield finishLine();
59
+ start = newline + 1;
60
+ }
61
+ }
62
+ if (pending > 0 || overflowing) yield finishLine();
63
+ }
64
+
65
+ // src/engine/export/import.ts
66
+ var DOCUMENT_EXPORT_FORMAT = "narsil-export";
67
+ var DOCUMENT_EXPORT_VERSION = 1;
68
+ function invalidHeader(message) {
69
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
70
+ }
71
+ function isRecord(value) {
72
+ return typeof value === "object" && value !== null && !Array.isArray(value);
73
+ }
74
+ function configOfHeader(line) {
75
+ if (line === void 0) invalidHeader("is missing");
76
+ if (!("text" in line)) {
77
+ invalidHeader(
78
+ line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
79
+ );
80
+ }
81
+ let raw;
82
+ try {
83
+ raw = JSON.parse(line.text);
84
+ } catch {
85
+ invalidHeader("is not JSON");
86
+ }
87
+ if (!isRecord(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
88
+ invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
89
+ }
90
+ const configuration = requireConfigurationShape(raw.index, "document export header");
91
+ const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
92
+ const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
93
+ return {
94
+ schema: configuration.schema,
95
+ language: configuration.language,
96
+ ...fields,
97
+ ...embedding === void 0 ? {} : {
98
+ embedding: {
99
+ fields: embedding.fields,
100
+ ...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
101
+ }
102
+ }
103
+ };
104
+ }
105
+ async function prepareIndex(engine, indexName, config) {
106
+ if (!engine.listIndexes().some((index) => index.name === indexName)) {
107
+ await engine.createIndex(indexName, config);
108
+ return;
109
+ }
110
+ const existing = flattenSchema(engine.getStats(indexName).schema);
111
+ const exported = flattenSchema(config.schema);
112
+ const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
113
+ for (const field of fields) {
114
+ if (existing[field] === exported[field]) continue;
115
+ throw new NarsilError(
116
+ ErrorCodes.CONFIG_INVALID,
117
+ `Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
118
+ { indexName, field }
119
+ );
120
+ }
121
+ }
122
+ function highestReportedAt(refused) {
123
+ let highest = 0;
124
+ for (let at = 1; at < refused.length; at++) {
125
+ if (refused[at].line > refused[highest].line) highest = at;
126
+ }
127
+ return highest;
128
+ }
129
+ function refuse(state, line, code, message) {
130
+ state.result.refusedCount += 1;
131
+ const refused = state.result.refused;
132
+ if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
133
+ refused.push({ line, code, message });
134
+ if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
135
+ return;
136
+ }
137
+ if (line >= refused[state.highestReported].line) return;
138
+ refused[state.highestReported] = { line, code, message };
139
+ state.highestReported = highestReportedAt(refused);
140
+ }
141
+ async function flush(state) {
142
+ if (state.batch.length === 0) return;
143
+ const batch = state.batch;
144
+ state.batch = [];
145
+ state.batchIds = /* @__PURE__ */ new Set();
146
+ const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
147
+ const inserts = [];
148
+ const updates = [];
149
+ for (const entry of batch) {
150
+ if (await state.engine.has(state.indexName, entry.docId))
151
+ updates.push({ docId: entry.docId, document: entry.document });
152
+ else inserts.push(entry.document);
153
+ }
154
+ for (const outcome of [
155
+ inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
156
+ updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
157
+ ]) {
158
+ if (outcome === null) continue;
159
+ state.result.imported += outcome.succeeded.length;
160
+ for (const failure of outcome.failed) {
161
+ const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
162
+ refuse(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
163
+ }
164
+ }
165
+ }
166
+ async function takeLine(state, entry) {
167
+ if (!("text" in entry)) {
168
+ const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
169
+ refuse(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
170
+ return;
171
+ }
172
+ if (entry.text.length === 0) return;
173
+ let document;
174
+ try {
175
+ document = JSON.parse(entry.text);
176
+ } catch {
177
+ refuse(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
178
+ return;
179
+ }
180
+ if (!isRecord(document) || typeof document.id !== "string" || document.id.length === 0) {
181
+ refuse(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
182
+ return;
183
+ }
184
+ try {
185
+ untagDocumentLine(entry.text, document);
186
+ } catch (error) {
187
+ if (!(error instanceof NarsilError)) throw error;
188
+ refuse(state, entry.line, error.code, error.message);
189
+ return;
190
+ }
191
+ if (state.batchIds.has(document.id)) await flush(state);
192
+ state.batch.push({ line: entry.line, docId: document.id, document });
193
+ state.batchIds.add(document.id);
194
+ if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
195
+ }
196
+ async function readDocumentExport(engine, indexName, source) {
197
+ const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
198
+ const header = await lines.next();
199
+ await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
200
+ const state = {
201
+ engine,
202
+ indexName,
203
+ batch: [],
204
+ batchIds: /* @__PURE__ */ new Set(),
205
+ result: { imported: 0, refusedCount: 0, refused: [] },
206
+ highestReported: 0
207
+ };
208
+ for await (const entry of lines) await takeLine(state, entry);
209
+ await flush(state);
210
+ state.result.refused.sort((a, b) => a.line - b.line);
211
+ return state.result;
212
+ }
213
+
214
+ export { DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, readDocumentExport };