@delali/narsil 0.3.1-next.90 → 0.3.1-next.94

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-4JTBL63P.mjs} +13 -5
  4. package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
  5. package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
  6. package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-HC6DMR4T.mjs} +2445 -2107
  7. package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-PP6N5CCS.mjs} +2 -2
  8. package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
  9. package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
  10. package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
  11. package/dist/browser/release-76MXRPGV.mjs +33 -0
  12. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  13. package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
  14. package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-PPTZ565G.mjs} +368 -103
  15. package/dist/browser/{write-6SBYTS62.mjs → write-2ZAA7UDH.mjs} +4 -4
  16. package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-xsJfdFoQ.d.ts} +11 -3
  17. package/dist/chunk-2GZGBOQI.mjs +426 -0
  18. package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
  19. package/dist/chunk-7KC56574.mjs +507 -0
  20. package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
  21. package/dist/chunk-7X5JST5L.mjs +114 -0
  22. package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
  23. package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
  24. package/dist/{chunk-R6BMMQXH.mjs → chunk-C45FUYPI.mjs} +193 -79
  25. package/dist/{chunk-EMXAYPTQ.mjs → chunk-CHOGQSHK.mjs} +2326 -3503
  26. package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
  27. package/dist/{chunk-VY4DXZ5P.mjs → chunk-EWBXO45L.mjs} +3 -3
  28. package/dist/{chunk-QAMUC2BJ.mjs → chunk-EX62BQT6.mjs} +10 -9
  29. package/dist/{chunk-MWJIAREY.mjs → chunk-EZD4MWOU.mjs} +4 -4
  30. package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
  31. package/dist/{chunk-URGLSNKR.mjs → chunk-GJZHU4FC.mjs} +202 -16
  32. package/dist/{chunk-S4LQDSUL.mjs → chunk-J53IHLG7.mjs} +7 -7
  33. package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
  34. package/dist/chunk-KOKM4PQN.mjs +52 -0
  35. package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
  36. package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
  37. package/dist/{chunk-V7YUSKKZ.mjs → chunk-QWWCLRXX.mjs} +10 -6
  38. package/dist/chunk-QY7GC2MW.mjs +33 -0
  39. package/dist/{chunk-DCK4P5BI.mjs → chunk-R36MIGBU.mjs} +100 -59
  40. package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
  41. package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
  42. package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
  43. package/dist/{chunk-XSZVS4S3.mjs → chunk-RUIXODYU.mjs} +9 -4
  44. package/dist/{chunk-A3UB2MXL.mjs → chunk-SBCEHIPL.mjs} +4 -507
  45. package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
  46. package/dist/chunk-UAUZGXJP.mjs +3591 -0
  47. package/dist/{chunk-CL4OM7GB.mjs → chunk-WBS3EYC4.mjs} +12 -8
  48. package/dist/{chunk-J2BPXTD7.mjs → chunk-WLOF6HSP.mjs} +3 -3
  49. package/dist/chunk-X5CV2E2B.mjs +1 -0
  50. package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
  51. package/dist/chunk-ZHS44YKF.mjs +35 -0
  52. package/dist/{client-BMK53IR2.d.ts → client-CgayPw2R.d.ts} +27 -3
  53. package/dist/client.d.ts +9 -8
  54. package/dist/client.mjs +16 -2
  55. package/dist/{config-UJx1dyX1.d.ts → config-3kBsQ0Pi.d.ts} +2 -2
  56. package/dist/{config-BUR-7eVQ.d.ts → config-B-RnO6dr.d.ts} +2 -2
  57. package/dist/direct-executor-IJTRU3YN.mjs +22 -0
  58. package/dist/distribution/coordinator/etcd.d.ts +3 -2
  59. package/dist/distribution/coordinator/etcd.mjs +5 -1
  60. package/dist/distribution/coordinator/in-memory.d.ts +3 -2
  61. package/dist/distribution/coordinator/in-memory.mjs +4 -0
  62. package/dist/distribution/transport/grpc.mjs +3 -3
  63. package/dist/distribution/transport/in-memory.mjs +1 -1
  64. package/dist/distribution/transport/tcp.mjs +2 -2
  65. package/dist/distribution.d.ts +43 -9
  66. package/dist/distribution.mjs +4620 -4293
  67. package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-Y55SCPFC.mjs} +524 -93
  68. package/dist/embeddings/openai.mjs +1 -1
  69. package/dist/{engine-9X6W5naj.d.ts → engine-DSUfY2F0.d.ts} +53 -17
  70. package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
  71. package/dist/index.browser.mjs +2 -2
  72. package/dist/index.d.ts +44 -25
  73. package/dist/index.mjs +22 -18
  74. package/dist/{memory-CV9NAxLY.d.ts → memory-wuXcYOD1.d.ts} +99 -79
  75. package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
  76. package/dist/partial-words.mjs +5 -4
  77. package/dist/pattern.mjs +3 -3
  78. package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
  79. package/dist/persistence/durability/checkpoint-worker.mjs +27 -23
  80. package/dist/react.d.ts +8 -7
  81. package/dist/react.mjs +1 -0
  82. package/dist/release-BH3DKW3T.mjs +33 -0
  83. package/dist/replace-CEO5KJJ3.mjs +57 -0
  84. package/dist/restore-queue-4SG45BLF.mjs +35 -0
  85. package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
  86. package/dist/server.d.ts +11 -10
  87. package/dist/server.mjs +38 -11
  88. package/dist/snapshot-XBKE7PUG.mjs +738 -0
  89. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  90. package/dist/synonyms.d.ts +22 -0
  91. package/dist/synonyms.mjs +12 -0
  92. package/dist/{tasks-BGaZJyoY.d.ts → tasks-Bj9WBlFG.d.ts} +4 -1
  93. package/dist/{thread-ZP3UMIZT.mjs → thread-U7XYJWGG.mjs} +17 -12
  94. package/dist/vector/search-worker.d.ts +2 -1
  95. package/dist/vector/search-worker.mjs +6 -6
  96. package/dist/workers/entry.mjs +6 -6
  97. package/dist/{write-PXDKPG2R.mjs → write-BMNV7P2H.mjs} +24 -20
  98. package/package.json +15 -7
  99. package/dist/chunk-PR4O7CAP.mjs +0 -18
  100. package/dist/chunk-X4SEXL4U.mjs +0 -2310
  101. package/dist/chunk-Y2F66IQJ.mjs +0 -74
  102. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  103. package/dist/direct-executor-EGESD26S.mjs +0 -18
  104. package/dist/snapshot-22CUEYYW.mjs +0 -469
@@ -1,9 +1,9 @@
1
- import { vectorsPerFile } from './chunk-3ZRMZIH4.mjs';
2
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts } from './chunk-3GZQKGVQ.mjs';
3
- import './chunk-JACCR3BJ.mjs';
4
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, setNestedValue, decodeIndexConfiguration, flattenSchema } from './chunk-RAHUQOJL.mjs';
5
- import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, decodeVectorIndexParts, bytesToVectors } from './chunk-NQW2YXEX.mjs';
6
- import { NarsilError, ErrorCodes, MAX_FILE_BYTES, BATCH_CHUNK_SIZE } from './chunk-FHFFWLEK.mjs';
1
+ import { vectorsPerFile } from './chunk-5S74HQRT.mjs';
2
+ import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-4JTBL63P.mjs';
3
+ import './chunk-PP6N5CCS.mjs';
4
+ import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, readHeapStatistics, readResidentMemory, prepareStoredSynonyms, rejectSnapshot, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-HC6DMR4T.mjs';
5
+ import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-WKPJ4E73.mjs';
6
+ import { NarsilError, ErrorCodes, MAX_FILE_BYTES } from './chunk-HARBSUWP.mjs';
7
7
  import { decode } from '@msgpack/msgpack';
8
8
 
9
9
  // src/persistence/archive/memory-store.ts
@@ -498,6 +498,136 @@ async function validateStagedCheckpoint(store, manifest) {
498
498
  }
499
499
  }
500
500
 
501
+ // src/engine/snapshot/constants.ts
502
+ var RESTORE_HEAP_BYTES_PER_STORED_BYTE = 0.25;
503
+ var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 9.82;
504
+ var RESTORE_HEAP_BYTES_PER_VECTOR = 330;
505
+ var RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.3;
506
+ var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 923;
507
+ var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.25;
508
+ var RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE = 5.07;
509
+ var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 18.12;
510
+ var RESTORE_EXTERNAL_BYTES_PER_VECTOR = 1853;
511
+ var RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 11.62;
512
+ var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2676;
513
+ var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 30.19;
514
+
515
+ // src/engine/snapshot/memory-budget.ts
516
+ var MEGABYTE_BYTES = 1048576;
517
+ var ARCHIVE_RATES = {
518
+ heap: {
519
+ perStoredByte: RESTORE_HEAP_BYTES_PER_STORED_BYTE,
520
+ perVector: RESTORE_HEAP_BYTES_PER_VECTOR,
521
+ perVectorDimension: RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION
522
+ },
523
+ external: {
524
+ perStoredByte: RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE,
525
+ perVector: RESTORE_EXTERNAL_BYTES_PER_VECTOR,
526
+ perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
527
+ }
528
+ };
529
+ var EARLIER_SNAPSHOT_RATES = {
530
+ heap: {
531
+ perStoredByte: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
532
+ perVector: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
533
+ perVectorDimension: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
534
+ },
535
+ external: {
536
+ perStoredByte: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
537
+ perVector: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
538
+ perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
539
+ }
540
+ };
541
+ function bytesAt(rate, storedBytes, vectors) {
542
+ let bytes = storedBytes * rate.perStoredByte;
543
+ for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
544
+ return bytes;
545
+ }
546
+ function footprintAt(rates, storedBytes, vectors) {
547
+ return {
548
+ heapBytes: bytesAt(rates.heap, storedBytes, vectors),
549
+ externalBytes: bytesAt(rates.external, storedBytes, vectors)
550
+ };
551
+ }
552
+ function deadCountOf(dead) {
553
+ if (dead === null) return 0;
554
+ let count = 0;
555
+ for (const byte of dead) {
556
+ for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
557
+ }
558
+ return count;
559
+ }
560
+ function archiveFootprint(prepared, manifest) {
561
+ let segmentBytes = 0;
562
+ for (const partition of manifest.partitions) {
563
+ for (const segment of partition.segments) segmentBytes += segment.bytes;
564
+ }
565
+ const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
566
+ const vectors = [];
567
+ for (const field of manifest.vectors) {
568
+ const dimension = dimensions.get(field.fieldPath) ?? 0;
569
+ for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
570
+ }
571
+ return footprintAt(ARCHIVE_RATES, segmentBytes, vectors);
572
+ }
573
+ function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts) {
574
+ const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
575
+ const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
576
+ return footprintAt(EARLIER_SNAPSHOT_RATES, partitionBytes, vectors);
577
+ }
578
+ function inMegabytes(bytes) {
579
+ return Math.ceil(bytes / MEGABYTE_BYTES);
580
+ }
581
+ function refuse(indexName, budget, neededBytes, availableBytes) {
582
+ const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
583
+ throw new NarsilError(
584
+ ErrorCodes.RESTORE_MEMORY_EXCEEDED,
585
+ `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again`,
586
+ { indexName, budget, neededBytes, availableBytes }
587
+ );
588
+ }
589
+ function requireRestoreMemory(indexName, footprint) {
590
+ const heap = readHeapStatistics();
591
+ if (heap !== null) {
592
+ const availableBytes = (heap.configuredLimitBytes ?? heap.limitBytes) - heap.usedBytes;
593
+ if (footprint.heapBytes > availableBytes) refuse(indexName, "heap", footprint.heapBytes, availableBytes);
594
+ }
595
+ const resident = readResidentMemory();
596
+ if (resident !== null) {
597
+ const neededBytes = footprint.heapBytes + footprint.externalBytes;
598
+ const availableBytes = resident.limitBytes - resident.residentBytes;
599
+ if (neededBytes > availableBytes) refuse(indexName, "process", neededBytes, availableBytes);
600
+ }
601
+ }
602
+
603
+ // src/workers/detached-index.ts
604
+ function attachablePolicy(shared) {
605
+ let attached = false;
606
+ const policy = {
607
+ get enabled() {
608
+ return attached && shared.enabled;
609
+ },
610
+ get count() {
611
+ return shared.count;
612
+ },
613
+ get host() {
614
+ return shared.host;
615
+ }
616
+ };
617
+ return {
618
+ policy,
619
+ attach() {
620
+ attached = true;
621
+ }
622
+ };
623
+ }
624
+ function createDetachedIndex(indexName, config, language, synonyms, executor) {
625
+ const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
626
+ const entry = executor.createDetachedEntry(indexName, config, language, policy);
627
+ entry.manager.synonyms = synonyms;
628
+ return { indexName, manager: entry.manager, entry, attach };
629
+ }
630
+
501
631
  // src/engine/snapshot/restore-setup.ts
502
632
  function prepareRestoredIndex(configuration, deps, reject) {
503
633
  const configFields = restoredConfigFields(configuration, reject);
@@ -534,15 +664,120 @@ function prepareRestoredIndex(configuration, deps, reject) {
534
664
  }
535
665
  }
536
666
  const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
537
- return { config, language, embeddingAdapter, adapterName, storedRevision };
667
+ const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
668
+ return { config, language, embeddingAdapter, adapterName, storedRevision, synonyms };
538
669
  }
539
- async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
670
+ function restoredSynonyms(raw, config, language, reject) {
671
+ if (raw === void 0 || raw === null) return null;
672
+ return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
673
+ }
674
+ async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
540
675
  const loadingVectorCode = loadVectorCodeFor(prepared.config.schema);
541
676
  if (loadingVectorCode !== null) await loadingVectorCode;
542
- if (deps.indexRegistry.has(indexName)) {
543
- await deps.dropIndex(indexName);
677
+ const detached = createDetachedIndex(
678
+ indexName,
679
+ prepared.config,
680
+ prepared.language,
681
+ prepared.synonyms?.lookup ?? null,
682
+ deps.executor
683
+ );
684
+ while (detached.manager.partitionCount < partitionCount) detached.manager.addPartition();
685
+ return detached;
686
+ }
687
+ function restoredRegistryEntry(prepared, manager, deps) {
688
+ return {
689
+ config: prepared.config,
690
+ language: prepared.language,
691
+ embeddingAdapter: prepared.embeddingAdapter,
692
+ embeddingAdapterName: prepared.adapterName,
693
+ vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
694
+ indexUuid: null,
695
+ heldPartitions: null,
696
+ documentCount: manager.countDocuments(),
697
+ partitionCount: manager.partitionCount,
698
+ analysisRevision: prepared.storedRevision,
699
+ synonyms: prepared.synonyms?.encoded ?? null
700
+ };
701
+ }
702
+
703
+ // src/engine/snapshot/replace.ts
704
+ function releaseTimedOut(indexName, timeoutMs) {
705
+ return new NarsilError(
706
+ ErrorCodes.WORKER_TIMEOUT,
707
+ `The restore of index "${indexName}" stopped because a worker held the index past ${timeoutMs} ms, and the index keeps its earlier contents`,
708
+ { indexName, timeoutMs }
709
+ );
710
+ }
711
+ async function withinLimit(indexName, release, work, deps) {
712
+ release.settled = work.catch(() => {
713
+ release.reloads = true;
714
+ });
715
+ let timer;
716
+ const limit = new Promise((_, reject) => {
717
+ timer = setTimeout(() => {
718
+ release.timedOut = true;
719
+ reject(releaseTimedOut(indexName, deps.releaseTimeoutMs));
720
+ }, deps.releaseTimeoutMs);
721
+ });
722
+ try {
723
+ await Promise.race([work, limit]);
724
+ } finally {
725
+ clearTimeout(timer);
726
+ }
727
+ }
728
+ function vectorIndexesOf(indexName, deps) {
729
+ const manager = deps.executor.getManager(indexName);
730
+ return manager === void 0 ? [] : [...manager.getVectorIndexes().values()];
731
+ }
732
+ async function holdVectorCopies(vectorIndexes, release) {
733
+ for (const vectorIndex of vectorIndexes) {
734
+ if (release.held.includes(vectorIndex)) continue;
735
+ release.held.push(vectorIndex);
736
+ await vectorIndex.holdWorkerCopies();
737
+ }
738
+ }
739
+ async function releaseCopies(indexName, release, deps) {
740
+ release.reloads = await deps.copies.releaseIndexCopies(indexName);
741
+ await holdVectorCopies(vectorIndexesOf(indexName, deps), release);
742
+ }
743
+ async function endRelease(indexName, release, deps) {
744
+ await release.settled;
745
+ for (const vectorIndex of release.held) vectorIndex.resumeWorkerCopies();
746
+ await deps.copies.endCopyRelease(indexName, release.reloads);
747
+ }
748
+ function persistedCheckpoint(indexName, partitionCount, deps) {
749
+ const durable = deps.durability?.manager;
750
+ if (durable === void 0) return [];
751
+ return Array.from({ length: partitionCount }, (_, partitionId) => ({
752
+ partitionId,
753
+ lastSeqNo: durable.highestPersistedSeqNo(indexName, partitionId),
754
+ primaryTerm: SINGLE_NODE_PRIMARY_TERM
755
+ }));
756
+ }
757
+ async function commitToStorage(indexName, restored, previousPartitions, deps) {
758
+ const durable = deps.durability?.manager;
759
+ const staging = restored.staging;
760
+ if (durable === void 0 || staging === null) return;
761
+ const previous = persistedCheckpoint(indexName, previousPartitions, deps);
762
+ if (previousPartitions > 0) await durable.unloadIndex(indexName);
763
+ const vectorIndexes = [...restored.detached.manager.getVectorIndexes().values()];
764
+ try {
765
+ for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
766
+ await staging.commit();
767
+ } catch (error) {
768
+ if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previous);
769
+ throw error;
544
770
  }
771
+ await staging.finish();
772
+ const relocate = await staging.relocation().catch(() => null);
773
+ if (relocate !== null) for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
774
+ }
775
+ function activate(indexName, restored, entry, deps) {
776
+ deps.executor.installDetachedIndex(restored.detached);
777
+ deps.indexRegistry.set(indexName, entry);
778
+ restored.detached.attach();
545
779
  deps.clearAnalysisStale(indexName);
780
+ const { prepared } = restored;
546
781
  if (prepared.storedRevision !== prepared.language.revision) {
547
782
  deps.markAnalysisStale({
548
783
  indexName,
@@ -551,42 +786,68 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
551
786
  currentRevision: prepared.language.revision
552
787
  });
553
788
  }
554
- deps.executor.createIndex(indexName, prepared.config, prepared.language);
555
- deps.indexRegistry.set(indexName, {
556
- config: prepared.config,
557
- language: prepared.language,
558
- embeddingAdapter: prepared.embeddingAdapter,
559
- embeddingAdapterName: prepared.adapterName,
560
- vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
561
- indexUuid: null,
562
- heldPartitions: null,
563
- documentCount: 0,
564
- partitionCount,
565
- analysisRevision: prepared.storedRevision
566
- });
567
- await deps.indexState.registerOpen(indexName);
568
- const release = await deps.indexState.acquire(indexName, false);
569
- const manager = deps.requireManager(indexName);
570
- while (manager.partitionCount < partitionCount) manager.addPartition();
571
- return { manager, release };
789
+ if (restored.staging !== null) deps.durability?.manager.adoptRestoredCheckpoint?.(indexName, restored.checkpoint);
790
+ deps.forgetWatermark(indexName);
572
791
  }
573
- async function abandonRestoredIndex(indexName, deps) {
792
+ async function replaceExisting(indexName, restored, entry, deps) {
793
+ const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
574
794
  try {
575
- deps.executor.dropIndex(indexName);
576
- } catch (_) {
795
+ await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
796
+ await deps.indexState.replace(indexName, async (wasOpen) => {
797
+ await deps.abandonAnalysisRebuild(indexName);
798
+ const previous = wasOpen ? deps.executor.getManager(indexName) : void 0;
799
+ const vectorIndexes = previous === void 0 ? [] : vectorIndexesOf(indexName, deps);
800
+ await withinLimit(indexName, release, holdVectorCopies(vectorIndexes, release), deps);
801
+ for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
802
+ await commitToStorage(indexName, restored, previous?.partitionCount ?? 0, deps);
803
+ activate(indexName, restored, entry, deps);
804
+ });
805
+ } finally {
806
+ const ending = endRelease(indexName, release, deps).catch(() => void 0);
807
+ if (!release.timedOut) await ending;
577
808
  }
578
- deps.indexRegistry.delete(indexName);
579
- deps.indexState.forget(indexName);
580
- if (deps.durability) {
581
- await deps.durability.manager.removeIndex(indexName).catch(() => void 0);
809
+ }
810
+ async function createFresh(indexName, restored, entry, deps) {
811
+ deps.copies.desyncIndex(indexName);
812
+ deps.executor.installDetachedIndex(restored.detached);
813
+ deps.indexRegistry.set(indexName, entry);
814
+ try {
815
+ await deps.indexState.registerReplacing(indexName, async () => {
816
+ await commitToStorage(indexName, restored, 0, deps);
817
+ activate(indexName, restored, entry, deps);
818
+ });
819
+ } catch (error) {
820
+ if (deps.indexRegistry.get(indexName) === entry) deps.indexRegistry.delete(indexName);
821
+ throw error;
822
+ } finally {
823
+ await deps.copies.resyncIndex(indexName, false).catch(() => void 0);
582
824
  }
583
825
  }
584
- function recordRestoredCounts(indexName, manager, deps) {
585
- const entry = deps.indexRegistry.get(indexName);
586
- if (entry !== void 0) {
587
- entry.documentCount = manager.countDocuments();
588
- entry.partitionCount = manager.partitionCount;
826
+ async function stageMetadata(indexName, staging, entry) {
827
+ const metadata = indexMetadataOf(indexName, entry, entry.partitionCount);
828
+ await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
829
+ }
830
+ async function installRestoredIndex(indexName, restored, deps) {
831
+ const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
832
+ let replaced;
833
+ try {
834
+ if (restored.staging !== null) await stageMetadata(indexName, restored.staging, entry);
835
+ replaced = deps.indexRegistry.get(indexName);
836
+ if (replaced !== void 0) await replaceExisting(indexName, restored, entry, deps);
837
+ else await createFresh(indexName, restored, entry, deps);
838
+ } catch (error) {
839
+ if (deps.executor.getManager(indexName) === restored.detached.manager) deps.executor.dropIndex(indexName);
840
+ else deps.executor.discardDetachedIndex(restored.detached);
841
+ throw error;
842
+ }
843
+ await restored.staging?.removeRetired().catch(() => void 0);
844
+ const durable = deps.durability?.manager;
845
+ if (restored.staging === null && durable !== void 0) {
846
+ await durable.persistMetadata(indexName);
847
+ await durable.checkpoint(indexName);
589
848
  }
849
+ if (replaced !== void 0) await deps.announceIndexDropped(indexName, replaced.config);
850
+ await deps.announceIndexCreated(indexName, restored.prepared.config);
590
851
  }
591
852
 
592
853
  // src/engine/snapshot/archive-restore.ts
@@ -637,6 +898,9 @@ async function stageArchive(indexName, reader, staging, deps) {
637
898
  manifest: async (manifest2, sourceName) => {
638
899
  if (staged.prepared === null) missingPart("configuration");
639
900
  requireSameSchema(staged.prepared, manifest2);
901
+ if (deps.indexRegistry.has(indexName)) {
902
+ requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2));
903
+ }
640
904
  staged.manifest = renamed(manifest2, sourceName, indexName);
641
905
  },
642
906
  file: async (entryName, _ref, bytes) => {
@@ -650,44 +914,30 @@ async function stageArchive(indexName, reader, staging, deps) {
650
914
  await validateStagedCheckpoint(staging, manifest);
651
915
  return { prepared, manifest };
652
916
  }
653
- async function replaceWithStaged(indexName, staged, staging, memory, deps) {
917
+ async function loadStagedIndex(indexName, staged, staging, store, deps) {
654
918
  const { prepared, manifest } = staged;
655
- const durable = deps.durability?.manager;
656
- const created = await createRestoredIndex(indexName, prepared, partitionCountOf(manifest), deps);
919
+ await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
920
+ const detached = await prepareDetachedIndex(indexName, prepared, partitionCountOf(manifest), deps);
657
921
  try {
658
- if (staging !== null && durable !== void 0) await durable.persistMetadata(indexName);
659
- const store = staging === null ? memory : await staging.commit();
660
- await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
661
922
  await loadSegmentedCheckpoint(
662
923
  store,
663
924
  indexName,
664
925
  manifest,
665
- { manager: created.manager, vectorIndexes: created.manager.getVectorIndexes() },
926
+ { manager: detached.manager, vectorIndexes: detached.manager.getVectorIndexes() },
666
927
  true
667
928
  );
668
- recordRestoredCounts(indexName, created.manager, deps);
669
- if (durable?.adoptRestoredCheckpoint !== void 0 && staging !== null) {
670
- durable.adoptRestoredCheckpoint(indexName, manifest.checkpoint);
671
- await durable.persistMetadata(indexName);
672
- } else if (durable !== void 0) {
673
- await durable.persistMetadata(indexName);
674
- await durable.checkpoint(indexName);
675
- }
676
- await deps.announceIndexCreated(indexName, prepared.config);
677
- return created;
678
929
  } catch (error) {
679
- created.release();
680
- await abandonRestoredIndex(indexName, deps);
930
+ deps.executor.discardDetachedIndex(detached);
681
931
  throw error;
682
932
  }
933
+ return { prepared, detached, staging, checkpoint: manifest.checkpoint };
683
934
  }
684
935
  async function restoreArchive(indexName, reader, deps) {
685
936
  const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
686
- const memory = memoryCheckpointStore();
937
+ const store = staging?.store ?? memoryCheckpointStore();
687
938
  try {
688
- const staged = await stageArchive(indexName, reader, staging?.store ?? memory, deps);
689
- const created = await replaceWithStaged(indexName, staged, staging, memory, deps);
690
- created.release();
939
+ const staged = await stageArchive(indexName, reader, store, deps);
940
+ await installRestoredIndex(indexName, await loadStagedIndex(indexName, staged, staging, store, deps), deps);
691
941
  } finally {
692
942
  await staging?.discard().catch(() => void 0);
693
943
  await reader.close();
@@ -740,16 +990,20 @@ function vectorsOfSnapshot(envelope) {
740
990
  }
741
991
  return byDocument;
742
992
  }
993
+ function footprintOf(envelope, prepared) {
994
+ let partitionBytes = 0;
995
+ for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
996
+ const vectorCounts = /* @__PURE__ */ new Map();
997
+ for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
998
+ let count = 0;
999
+ for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
1000
+ vectorCounts.set(fieldPath, count);
1001
+ }
1002
+ return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts);
1003
+ }
743
1004
  function documentsOfPartition(envelope, payload) {
744
1005
  return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
745
1006
  }
746
- function withVectors(fields, vectors) {
747
- if (vectors === void 0 || vectors.size === 0) return fields;
748
- const nested = [...vectors.keys()].some((fieldPath) => fieldPath.includes("."));
749
- const document = nested ? structuredClone(fields) : { ...fields };
750
- for (const [fieldPath, vector] of vectors) setNestedValue(document, fieldPath, vector);
751
- return document;
752
- }
753
1007
  function insertRestoredDocument(manager, docId, fields, vectors) {
754
1008
  manager.insert(docId, fields, { skipClone: true });
755
1009
  if (vectors === void 0) return;
@@ -760,51 +1014,62 @@ function insertRestoredDocument(manager, docId, fields, vectors) {
760
1014
  throw err;
761
1015
  }
762
1016
  }
763
- async function insertRestoredChunk(indexName, manager, chunk, vectors, deps) {
764
- if (deps.durability === null) {
765
- for (const [docId, fields] of chunk) insertRestoredDocument(manager, docId, fields, vectors.get(docId));
766
- return;
767
- }
768
- const mutations = chunk.map(([docId, fields]) => ({
769
- docId,
770
- document: withVectors(fields, vectors.get(docId)),
771
- apply: () => insertRestoredDocument(manager, docId, fields, vectors.get(docId))
772
- }));
773
- for (const outcome of await deps.durability.recordMutationBatch(indexName, mutations)) {
774
- if (!outcome.ok) throw outcome.error;
775
- }
776
- }
777
- async function insertSnapshotDocuments(indexName, envelope, manager, deps) {
1017
+ function insertSnapshotDocuments(envelope, manager) {
778
1018
  const vectors = vectorsOfSnapshot(envelope);
779
1019
  for (const payload of envelope.partitions ?? []) {
780
- const documents = Object.entries(documentsOfPartition(envelope, payload)).map(
781
- ([docId, stored]) => [docId, stored.fields]
782
- );
783
- for (let start = 0; start < documents.length; start += BATCH_CHUNK_SIZE) {
784
- await insertRestoredChunk(indexName, manager, documents.slice(start, start + BATCH_CHUNK_SIZE), vectors, deps);
1020
+ for (const [docId, stored] of Object.entries(documentsOfPartition(envelope, payload))) {
1021
+ insertRestoredDocument(manager, docId, stored.fields, vectors.get(docId));
785
1022
  }
786
1023
  }
787
1024
  for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
788
1025
  }
1026
+ async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
1027
+ const entry = restoredRegistryEntry(prepared, built.manager, deps);
1028
+ const archive = await buildArchiveFromMemory({
1029
+ indexName,
1030
+ metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
1031
+ manager: built.manager,
1032
+ vectorIndexes: built.manager.getVectorIndexes(),
1033
+ appliedSeqNo: () => 0,
1034
+ partitionId: null
1035
+ });
1036
+ try {
1037
+ for (const ref of manifestFileRefs(archive.manifest))
1038
+ await staging.store.write(ref.key, [await archive.readFile(ref)]);
1039
+ } finally {
1040
+ archive.release();
1041
+ }
1042
+ await validateStagedCheckpoint(staging.store, archive.manifest);
1043
+ return archive.manifest;
1044
+ }
1045
+ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps) {
1046
+ const built = await prepareDetachedIndex(indexName, prepared, envelope.partitions?.length ?? 0, deps);
1047
+ let manifest;
1048
+ try {
1049
+ insertSnapshotDocuments(envelope, built.manager);
1050
+ if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
1051
+ manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
1052
+ } catch (err) {
1053
+ deps.executor.discardDetachedIndex(built);
1054
+ throw err;
1055
+ }
1056
+ deps.executor.discardDetachedIndex(built);
1057
+ return loadStagedIndex(indexName, { prepared, manifest }, staging, staging.store, deps);
1058
+ }
789
1059
  async function restoreEarlierSnapshot(indexName, data, deps) {
790
1060
  const envelope = await decodeEarlierSnapshot(data);
791
1061
  const prepared = prepareRestoredIndex(envelope, deps);
1062
+ if (deps.indexRegistry.has(indexName)) requireRestoreMemory(indexName, footprintOf(envelope, prepared));
792
1063
  const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
793
- const partitionCount = envelope.partitions?.length ?? 0;
794
- const { manager, release } = await createRestoredIndex(indexName, analysedNow, partitionCount, deps);
1064
+ const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
795
1065
  try {
796
- await insertSnapshotDocuments(indexName, envelope, manager, deps);
797
- recordRestoredCounts(indexName, manager, deps);
798
- if (deps.durability) {
799
- await deps.durability.manager.persistMetadata(indexName);
800
- await deps.durability.manager.checkpoint(indexName);
801
- }
802
- await deps.announceIndexCreated(indexName, prepared.config);
803
- } catch (err) {
804
- await abandonRestoredIndex(indexName, deps);
805
- throw err;
1066
+ await installRestoredIndex(
1067
+ indexName,
1068
+ await buildRestoredIndex(indexName, envelope, analysedNow, staging, deps),
1069
+ deps
1070
+ );
806
1071
  } finally {
807
- release();
1072
+ await staging?.discard().catch(() => void 0);
808
1073
  }
809
1074
  }
810
1075
 
@@ -1,7 +1,7 @@
1
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-JACCR3BJ.mjs';
2
- import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-RAHUQOJL.mjs';
3
- import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-NQW2YXEX.mjs';
4
- import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-FHFFWLEK.mjs';
1
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-PP6N5CCS.mjs';
2
+ import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-HC6DMR4T.mjs';
3
+ import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-WKPJ4E73.mjs';
4
+ import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-HARBSUWP.mjs';
5
5
  import { decode } from '@msgpack/msgpack';
6
6
 
7
7
  // src/engine/export/json-line.ts
@@ -1,6 +1,6 @@
1
1
  import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
2
- import { C as ClusterNamespace } from './config-UJx1dyX1.js';
3
- import { h as TaskStore } from './tasks-BGaZJyoY.js';
2
+ import { C as ClusterNamespace } from './config-3kBsQ0Pi.js';
3
+ import { h as TaskStore } from './tasks-Bj9WBlFG.js';
4
4
 
5
5
  /**
6
6
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -284,6 +284,14 @@ declare const INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
284
284
  * @public
285
285
  */
286
286
  declare const DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
287
+ /**
288
+ * Marks a server that reads the synonym set of an index at
289
+ * `GET /indexes/{name}/synonyms` and replaces it at
290
+ * `POST /indexes/{name}/synonyms`.
291
+ *
292
+ * @public
293
+ */
294
+ declare const SYNONYMS_CAPABILITY = "indexes.synonyms";
287
295
  /**
288
296
  * Every capability this server announces at `/capabilities`.
289
297
  *
@@ -295,4 +303,4 @@ declare const DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
295
303
  */
296
304
  declare const SERVER_CAPABILITIES: readonly string[];
297
305
 
298
- export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, TASK_FILTER_CAPABILITY as a, type ServerOptions as b, type RequestContext as c, type RequestDenial as d, type ServerLimits as e };
306
+ export { ASYNC_IMPORT_CAPABILITY as A, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, SYNONYMS_CAPABILITY as a, TASK_FILTER_CAPABILITY as b, type ServerOptions as c, type RequestContext as d, type RequestDenial as e, type ServerLimits as f };