@delali/narsil 0.3.1-next.91 → 0.3.1-next.95

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-66KYAOIK.mjs → chunk-3U636ADD.mjs} +2 -2
  4. package/dist/browser/{chunk-NOUZ6RH3.mjs → chunk-D3CC2NMZ.mjs} +238 -114
  5. package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-FYFQJKFW.mjs} +208 -78
  6. package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
  7. package/dist/browser/{chunk-B6XE7KXY.mjs → chunk-LULHQXCK.mjs} +4 -4
  8. package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
  9. package/dist/browser/{durability-wiring-F4OGQUSU.mjs → durability-wiring-3ZRKSPJD.mjs} +498 -71
  10. package/dist/browser/{module-E25UDADM.mjs → module-IEOQI5GI.mjs} +102 -54
  11. package/dist/browser/release-76MXRPGV.mjs +33 -0
  12. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  13. package/dist/browser/restore-queue-RBULSOVX.mjs +35 -0
  14. package/dist/browser/{snapshot-5EJ7I7QY.mjs → snapshot-Y3NGUJHX.mjs} +362 -103
  15. package/dist/browser/{write-K5V4XQ72.mjs → write-ZUBDNLCH.mjs} +4 -4
  16. package/dist/{capabilities-CvExBThq.d.ts → capabilities-BvxvFzVA.d.ts} +2 -2
  17. package/dist/{chunk-NVL4ZR3K.mjs → chunk-2GZGBOQI.mjs} +2 -2
  18. package/dist/{chunk-XLJ2S2BH.mjs → chunk-3LYY62M6.mjs} +40 -21
  19. package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
  20. package/dist/{chunk-T4DB5YFB.mjs → chunk-54HYC5FT.mjs} +6 -6
  21. package/dist/{chunk-ZIJ63TBT.mjs → chunk-7KC56574.mjs} +1 -1
  22. package/dist/{chunk-67Z2V6TC.mjs → chunk-7QONURHW.mjs} +3 -3
  23. package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
  24. package/dist/{chunk-6VZ3Q73N.mjs → chunk-CHOGQSHK.mjs} +44 -25
  25. package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
  26. package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
  27. package/dist/{chunk-43ORORN6.mjs → chunk-GJZHU4FC.mjs} +3 -3
  28. package/dist/{chunk-R7OSDAFK.mjs → chunk-HBVLF3HA.mjs} +1 -1
  29. package/dist/{chunk-ULRPNYAG.mjs → chunk-ICL2AELD.mjs} +120 -62
  30. package/dist/{chunk-3RDS4UHC.mjs → chunk-J7SDAOTP.mjs} +4 -4
  31. package/dist/{chunk-A4G7JSYO.mjs → chunk-JFUE6F2Y.mjs} +98 -49
  32. package/dist/chunk-KOKM4PQN.mjs +52 -0
  33. package/dist/{chunk-7OQHSDK6.mjs → chunk-L26LEDHO.mjs} +7 -7
  34. package/dist/{chunk-EONIHG3V.mjs → chunk-MASE2LTJ.mjs} +7 -7
  35. package/dist/{chunk-MXS5HWIH.mjs → chunk-MWT5WCJ5.mjs} +89 -17
  36. package/dist/{chunk-3TTYK5A7.mjs → chunk-Q4UTU5ZE.mjs} +9 -9
  37. package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
  38. package/dist/chunk-QY7GC2MW.mjs +33 -0
  39. package/dist/{chunk-BJ2YKRNX.mjs → chunk-R36MIGBU.mjs} +86 -54
  40. package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
  41. package/dist/{chunk-UXQE2ROB.mjs → chunk-RUIXODYU.mjs} +3 -2
  42. package/dist/{chunk-5ER245FI.mjs → chunk-SBCEHIPL.mjs} +3 -3
  43. package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
  44. package/dist/{chunk-UKSUENHK.mjs → chunk-T2QN2FA4.mjs} +40 -26
  45. package/dist/{chunk-CO4MRKST.mjs → chunk-WLOF6HSP.mjs} +2 -2
  46. package/dist/{chunk-MKWUXWQH.mjs → chunk-ZHS44YKF.mjs} +1 -1
  47. package/dist/{client-Cc42b42j.d.ts → client-DKwterMb.d.ts} +2 -2
  48. package/dist/client.d.ts +6 -6
  49. package/dist/client.mjs +1 -0
  50. package/dist/{config-DyRowvVP.d.ts → config-6YBtxekG.d.ts} +1 -1
  51. package/dist/{config-BYne-yCI.d.ts → config-DPphZfuM.d.ts} +1 -1
  52. package/dist/direct-executor-5KJGVQHM.mjs +22 -0
  53. package/dist/distribution/coordinator/etcd.mjs +1 -1
  54. package/dist/distribution/transport/grpc.mjs +1 -1
  55. package/dist/distribution.d.ts +5 -5
  56. package/dist/distribution.mjs +26 -26
  57. package/dist/{durability-wiring-6AN4QTT3.mjs → durability-wiring-FRPV6JLZ.mjs} +518 -91
  58. package/dist/embeddings/openai.mjs +1 -1
  59. package/dist/{engine-96_kLtSY.d.ts → engine-BGt9lI1V.d.ts} +11 -11
  60. package/dist/index.browser.mjs +2 -2
  61. package/dist/index.d.ts +5 -5
  62. package/dist/index.mjs +18 -18
  63. package/dist/{memory-dd3GESdc.d.ts → memory-COdoGthn.d.ts} +96 -76
  64. package/dist/{module-WIDYIJLY.mjs → module-IUTNXWMV.mjs} +72 -38
  65. package/dist/partial-words.mjs +5 -5
  66. package/dist/pattern.mjs +3 -3
  67. package/dist/persistence/durability/checkpoint-worker.mjs +21 -21
  68. package/dist/react.d.ts +6 -6
  69. package/dist/react.mjs +1 -0
  70. package/dist/release-BH3DKW3T.mjs +33 -0
  71. package/dist/replace-CEO5KJJ3.mjs +57 -0
  72. package/dist/restore-queue-I5SI6AUQ.mjs +35 -0
  73. package/dist/server.d.ts +8 -8
  74. package/dist/server.mjs +10 -10
  75. package/dist/snapshot-YC6TS4OP.mjs +738 -0
  76. package/dist/synonyms.mjs +3 -3
  77. package/dist/{tasks-DA1Ff0G4.d.ts → tasks-BJmamiC4.d.ts} +1 -1
  78. package/dist/{thread-VTDLLYXC.mjs → thread-U7XYJWGG.mjs} +15 -12
  79. package/dist/vector/search-worker.mjs +6 -6
  80. package/dist/workers/entry.mjs +5 -5
  81. package/dist/{write-OJR3H4BI.mjs → write-M2OALSA4.mjs} +20 -20
  82. package/package.json +7 -7
  83. package/dist/chunk-Y2F66IQJ.mjs +0 -74
  84. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  85. package/dist/direct-executor-CHXVCBN7.mjs +0 -22
  86. package/dist/snapshot-JFZVOGZV.mjs +0 -479
@@ -1,9 +1,9 @@
1
- import { vectorsPerFile } from './chunk-3ZRMZIH4.mjs';
2
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, documentsOfPayloadV2, deserializePayloadV1, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts } from './chunk-B6XE7KXY.mjs';
3
- import './chunk-66KYAOIK.mjs';
4
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, loadVectorCodeFor, prepareStoredSynonyms, rejectSnapshot, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, setNestedValue, decodeIndexConfiguration, flattenSchema } from './chunk-NOUZ6RH3.mjs';
5
- import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, decodeVectorIndexParts, bytesToVectors } from './chunk-NQW2YXEX.mjs';
6
- import { NarsilError, ErrorCodes, MAX_FILE_BYTES, BATCH_CHUNK_SIZE } from './chunk-FHFFWLEK.mjs';
1
+ import { vectorsPerFile } from './chunk-FYFQJKFW.mjs';
2
+ import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-LULHQXCK.mjs';
3
+ import './chunk-3U636ADD.mjs';
4
+ import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, readHeapStatistics, readResidentMemory, prepareStoredSynonyms, rejectSnapshot, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-D3CC2NMZ.mjs';
5
+ import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-WKPJ4E73.mjs';
6
+ import { NarsilError, ErrorCodes, MAX_FILE_BYTES } from './chunk-HARBSUWP.mjs';
7
7
  import { decode } from '@msgpack/msgpack';
8
8
 
9
9
  // src/persistence/archive/memory-store.ts
@@ -498,6 +498,136 @@ async function validateStagedCheckpoint(store, manifest) {
498
498
  }
499
499
  }
500
500
 
501
+ // src/engine/snapshot/constants.ts
502
+ var RESTORE_HEAP_BYTES_PER_STORED_BYTE = 0.25;
503
+ var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 9.82;
504
+ var RESTORE_HEAP_BYTES_PER_VECTOR = 330;
505
+ var RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.3;
506
+ var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 923;
507
+ var RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.25;
508
+ var RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE = 5.07;
509
+ var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 18.12;
510
+ var RESTORE_EXTERNAL_BYTES_PER_VECTOR = 1853;
511
+ var RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 11.62;
512
+ var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2676;
513
+ var RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 30.19;
514
+
515
+ // src/engine/snapshot/memory-budget.ts
516
+ var MEGABYTE_BYTES = 1048576;
517
+ var ARCHIVE_RATES = {
518
+ heap: {
519
+ perStoredByte: RESTORE_HEAP_BYTES_PER_STORED_BYTE,
520
+ perVector: RESTORE_HEAP_BYTES_PER_VECTOR,
521
+ perVectorDimension: RESTORE_HEAP_BYTES_PER_VECTOR_DIMENSION
522
+ },
523
+ external: {
524
+ perStoredByte: RESTORE_EXTERNAL_BYTES_PER_STORED_BYTE,
525
+ perVector: RESTORE_EXTERNAL_BYTES_PER_VECTOR,
526
+ perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
527
+ }
528
+ };
529
+ var EARLIER_SNAPSHOT_RATES = {
530
+ heap: {
531
+ perStoredByte: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
532
+ perVector: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
533
+ perVectorDimension: RESTORE_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
534
+ },
535
+ external: {
536
+ perStoredByte: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
537
+ perVector: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
538
+ perVectorDimension: RESTORE_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
539
+ }
540
+ };
541
+ function bytesAt(rate, storedBytes, vectors) {
542
+ let bytes = storedBytes * rate.perStoredByte;
543
+ for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
544
+ return bytes;
545
+ }
546
+ function footprintAt(rates, storedBytes, vectors) {
547
+ return {
548
+ heapBytes: bytesAt(rates.heap, storedBytes, vectors),
549
+ externalBytes: bytesAt(rates.external, storedBytes, vectors)
550
+ };
551
+ }
552
+ function deadCountOf(dead) {
553
+ if (dead === null) return 0;
554
+ let count = 0;
555
+ for (const byte of dead) {
556
+ for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
557
+ }
558
+ return count;
559
+ }
560
+ function archiveFootprint(prepared, manifest) {
561
+ let segmentBytes = 0;
562
+ for (const partition of manifest.partitions) {
563
+ for (const segment of partition.segments) segmentBytes += segment.bytes;
564
+ }
565
+ const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
566
+ const vectors = [];
567
+ for (const field of manifest.vectors) {
568
+ const dimension = dimensions.get(field.fieldPath) ?? 0;
569
+ for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
570
+ }
571
+ return footprintAt(ARCHIVE_RATES, segmentBytes, vectors);
572
+ }
573
+ function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts) {
574
+ const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
575
+ const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
576
+ return footprintAt(EARLIER_SNAPSHOT_RATES, partitionBytes, vectors);
577
+ }
578
+ function inMegabytes(bytes) {
579
+ return Math.ceil(bytes / MEGABYTE_BYTES);
580
+ }
581
+ function refuse(indexName, budget, neededBytes, availableBytes) {
582
+ const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
583
+ throw new NarsilError(
584
+ ErrorCodes.RESTORE_MEMORY_EXCEEDED,
585
+ `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again`,
586
+ { indexName, budget, neededBytes, availableBytes }
587
+ );
588
+ }
589
+ function requireRestoreMemory(indexName, footprint) {
590
+ const heap = readHeapStatistics();
591
+ if (heap !== null) {
592
+ const availableBytes = (heap.configuredLimitBytes ?? heap.limitBytes) - heap.usedBytes;
593
+ if (footprint.heapBytes > availableBytes) refuse(indexName, "heap", footprint.heapBytes, availableBytes);
594
+ }
595
+ const resident = readResidentMemory();
596
+ if (resident !== null) {
597
+ const neededBytes = footprint.heapBytes + footprint.externalBytes;
598
+ const availableBytes = resident.limitBytes - resident.residentBytes;
599
+ if (neededBytes > availableBytes) refuse(indexName, "process", neededBytes, availableBytes);
600
+ }
601
+ }
602
+
603
+ // src/workers/detached-index.ts
604
+ function attachablePolicy(shared) {
605
+ let attached = false;
606
+ const policy = {
607
+ get enabled() {
608
+ return attached && shared.enabled;
609
+ },
610
+ get count() {
611
+ return shared.count;
612
+ },
613
+ get host() {
614
+ return shared.host;
615
+ }
616
+ };
617
+ return {
618
+ policy,
619
+ attach() {
620
+ attached = true;
621
+ }
622
+ };
623
+ }
624
+ function createDetachedIndex(indexName, config, language, synonyms, executor) {
625
+ const { policy, attach } = attachablePolicy(executor.vectorWorkerCopies);
626
+ const entry = executor.createDetachedEntry(indexName, config, language, policy);
627
+ entry.manager.synonyms = synonyms;
628
+ return { indexName, manager: entry.manager, entry, attach };
629
+ }
630
+
501
631
  // src/engine/snapshot/restore-setup.ts
502
632
  function prepareRestoredIndex(configuration, deps, reject) {
503
633
  const configFields = restoredConfigFields(configuration, reject);
@@ -541,23 +671,21 @@ function restoredSynonyms(raw, config, language, reject) {
541
671
  if (raw === void 0 || raw === null) return null;
542
672
  return prepareStoredSynonyms(raw, config, language, (message) => reject(`synonyms: ${message}`));
543
673
  }
544
- async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
674
+ async function prepareDetachedIndex(indexName, prepared, partitionCount, deps) {
545
675
  const loadingVectorCode = loadVectorCodeFor(prepared.config.schema);
546
676
  if (loadingVectorCode !== null) await loadingVectorCode;
547
- if (deps.indexRegistry.has(indexName)) {
548
- await deps.dropIndex(indexName);
549
- }
550
- deps.clearAnalysisStale(indexName);
551
- if (prepared.storedRevision !== prepared.language.revision) {
552
- deps.markAnalysisStale({
553
- indexName,
554
- language: prepared.language.name,
555
- storedRevision: prepared.storedRevision,
556
- currentRevision: prepared.language.revision
557
- });
558
- }
559
- deps.executor.createIndex(indexName, prepared.config, prepared.language, false, prepared.synonyms?.lookup ?? null);
560
- deps.indexRegistry.set(indexName, {
677
+ const detached = createDetachedIndex(
678
+ indexName,
679
+ prepared.config,
680
+ prepared.language,
681
+ prepared.synonyms?.lookup ?? null,
682
+ deps.executor
683
+ );
684
+ while (detached.manager.partitionCount < partitionCount) detached.manager.addPartition();
685
+ return detached;
686
+ }
687
+ function restoredRegistryEntry(prepared, manager, deps) {
688
+ return {
561
689
  config: prepared.config,
562
690
  language: prepared.language,
563
691
  embeddingAdapter: prepared.embeddingAdapter,
@@ -565,34 +693,161 @@ async function createRestoredIndex(indexName, prepared, partitionCount, deps) {
565
693
  vectorFieldPaths: deps.getVectorFieldPaths(prepared.config.schema),
566
694
  indexUuid: null,
567
695
  heldPartitions: null,
568
- documentCount: 0,
569
- partitionCount,
696
+ documentCount: manager.countDocuments(),
697
+ partitionCount: manager.partitionCount,
570
698
  analysisRevision: prepared.storedRevision,
571
699
  synonyms: prepared.synonyms?.encoded ?? null
700
+ };
701
+ }
702
+
703
+ // src/engine/snapshot/replace.ts
704
+ function releaseTimedOut(indexName, timeoutMs) {
705
+ return new NarsilError(
706
+ ErrorCodes.WORKER_TIMEOUT,
707
+ `The restore of index "${indexName}" stopped because a worker held the index past ${timeoutMs} ms, and the index keeps its earlier contents`,
708
+ { indexName, timeoutMs }
709
+ );
710
+ }
711
+ async function withinLimit(indexName, release, work, deps) {
712
+ release.settled = work.catch(() => {
713
+ release.reloads = true;
714
+ });
715
+ let timer;
716
+ const limit = new Promise((_, reject) => {
717
+ timer = setTimeout(() => {
718
+ release.timedOut = true;
719
+ reject(releaseTimedOut(indexName, deps.releaseTimeoutMs));
720
+ }, deps.releaseTimeoutMs);
572
721
  });
573
- await deps.indexState.registerOpen(indexName);
574
- const release = await deps.indexState.acquire(indexName, false);
575
- const manager = deps.requireManager(indexName);
576
- while (manager.partitionCount < partitionCount) manager.addPartition();
577
- return { manager, release };
722
+ try {
723
+ await Promise.race([work, limit]);
724
+ } finally {
725
+ clearTimeout(timer);
726
+ }
727
+ }
728
+ function vectorIndexesOf(indexName, deps) {
729
+ const manager = deps.executor.getManager(indexName);
730
+ return manager === void 0 ? [] : [...manager.getVectorIndexes().values()];
731
+ }
732
+ async function holdVectorCopies(vectorIndexes, release) {
733
+ for (const vectorIndex of vectorIndexes) {
734
+ if (release.held.includes(vectorIndex)) continue;
735
+ release.held.push(vectorIndex);
736
+ await vectorIndex.holdWorkerCopies();
737
+ }
578
738
  }
579
- async function abandonRestoredIndex(indexName, deps) {
739
+ async function releaseCopies(indexName, release, deps) {
740
+ release.reloads = await deps.copies.releaseIndexCopies(indexName);
741
+ await holdVectorCopies(vectorIndexesOf(indexName, deps), release);
742
+ }
743
+ async function endRelease(indexName, release, deps) {
744
+ await release.settled;
745
+ for (const vectorIndex of release.held) vectorIndex.resumeWorkerCopies();
746
+ await deps.copies.endCopyRelease(indexName, release.reloads);
747
+ }
748
+ function persistedCheckpoint(indexName, partitionCount, deps) {
749
+ const durable = deps.durability?.manager;
750
+ if (durable === void 0) return [];
751
+ return Array.from({ length: partitionCount }, (_, partitionId) => ({
752
+ partitionId,
753
+ lastSeqNo: durable.highestPersistedSeqNo(indexName, partitionId),
754
+ primaryTerm: SINGLE_NODE_PRIMARY_TERM
755
+ }));
756
+ }
757
+ async function commitToStorage(indexName, restored, previousPartitions, deps) {
758
+ const durable = deps.durability?.manager;
759
+ const staging = restored.staging;
760
+ if (durable === void 0 || staging === null) return;
761
+ const previous = persistedCheckpoint(indexName, previousPartitions, deps);
762
+ if (previousPartitions > 0) await durable.unloadIndex(indexName);
763
+ const vectorIndexes = [...restored.detached.manager.getVectorIndexes().values()];
764
+ try {
765
+ for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
766
+ await staging.commit();
767
+ } catch (error) {
768
+ if (previousPartitions > 0) durable.adoptRestoredCheckpoint?.(indexName, previous);
769
+ throw error;
770
+ }
771
+ await staging.finish();
772
+ const relocate = await staging.relocation().catch(() => null);
773
+ if (relocate !== null) for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
774
+ }
775
+ function activate(indexName, restored, entry, deps) {
776
+ deps.executor.installDetachedIndex(restored.detached);
777
+ deps.indexRegistry.set(indexName, entry);
778
+ restored.detached.attach();
779
+ deps.clearAnalysisStale(indexName);
780
+ const { prepared } = restored;
781
+ if (prepared.storedRevision !== prepared.language.revision) {
782
+ deps.markAnalysisStale({
783
+ indexName,
784
+ language: prepared.language.name,
785
+ storedRevision: prepared.storedRevision,
786
+ currentRevision: prepared.language.revision
787
+ });
788
+ }
789
+ if (restored.staging !== null) deps.durability?.manager.adoptRestoredCheckpoint?.(indexName, restored.checkpoint);
790
+ deps.forgetWatermark(indexName);
791
+ }
792
+ async function replaceExisting(indexName, restored, entry, deps) {
793
+ const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
580
794
  try {
581
- deps.executor.dropIndex(indexName);
582
- } catch (_) {
795
+ await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
796
+ await deps.indexState.replace(indexName, async (wasOpen) => {
797
+ await deps.abandonAnalysisRebuild(indexName);
798
+ const previous = wasOpen ? deps.executor.getManager(indexName) : void 0;
799
+ const vectorIndexes = previous === void 0 ? [] : vectorIndexesOf(indexName, deps);
800
+ await withinLimit(indexName, release, holdVectorCopies(vectorIndexes, release), deps);
801
+ for (const vectorIndex of vectorIndexes) await vectorIndex.releaseVectorFiles();
802
+ await commitToStorage(indexName, restored, previous?.partitionCount ?? 0, deps);
803
+ activate(indexName, restored, entry, deps);
804
+ });
805
+ } finally {
806
+ const ending = endRelease(indexName, release, deps).catch(() => void 0);
807
+ if (!release.timedOut) await ending;
583
808
  }
584
- deps.indexRegistry.delete(indexName);
585
- deps.indexState.forget(indexName);
586
- if (deps.durability) {
587
- await deps.durability.manager.removeIndex(indexName).catch(() => void 0);
809
+ }
810
+ async function createFresh(indexName, restored, entry, deps) {
811
+ deps.copies.desyncIndex(indexName);
812
+ deps.executor.installDetachedIndex(restored.detached);
813
+ deps.indexRegistry.set(indexName, entry);
814
+ try {
815
+ await deps.indexState.registerReplacing(indexName, async () => {
816
+ await commitToStorage(indexName, restored, 0, deps);
817
+ activate(indexName, restored, entry, deps);
818
+ });
819
+ } catch (error) {
820
+ if (deps.indexRegistry.get(indexName) === entry) deps.indexRegistry.delete(indexName);
821
+ throw error;
822
+ } finally {
823
+ await deps.copies.resyncIndex(indexName, false).catch(() => void 0);
588
824
  }
589
825
  }
590
- function recordRestoredCounts(indexName, manager, deps) {
591
- const entry = deps.indexRegistry.get(indexName);
592
- if (entry !== void 0) {
593
- entry.documentCount = manager.countDocuments();
594
- entry.partitionCount = manager.partitionCount;
826
+ async function stageMetadata(indexName, staging, entry) {
827
+ const metadata = indexMetadataOf(indexName, entry, entry.partitionCount);
828
+ await staging.store.write(`${indexName}/meta`, [await writeMetadataEnvelope(metadata, { checksum: true })]);
829
+ }
830
+ async function installRestoredIndex(indexName, restored, deps) {
831
+ const entry = restoredRegistryEntry(restored.prepared, restored.detached.manager, deps);
832
+ let replaced;
833
+ try {
834
+ if (restored.staging !== null) await stageMetadata(indexName, restored.staging, entry);
835
+ replaced = deps.indexRegistry.get(indexName);
836
+ if (replaced !== void 0) await replaceExisting(indexName, restored, entry, deps);
837
+ else await createFresh(indexName, restored, entry, deps);
838
+ } catch (error) {
839
+ if (deps.executor.getManager(indexName) === restored.detached.manager) deps.executor.dropIndex(indexName);
840
+ else deps.executor.discardDetachedIndex(restored.detached);
841
+ throw error;
842
+ }
843
+ await restored.staging?.removeRetired().catch(() => void 0);
844
+ const durable = deps.durability?.manager;
845
+ if (restored.staging === null && durable !== void 0) {
846
+ await durable.persistMetadata(indexName);
847
+ await durable.checkpoint(indexName);
595
848
  }
849
+ if (replaced !== void 0) await deps.announceIndexDropped(indexName, replaced.config);
850
+ await deps.announceIndexCreated(indexName, restored.prepared.config);
596
851
  }
597
852
 
598
853
  // src/engine/snapshot/archive-restore.ts
@@ -643,6 +898,9 @@ async function stageArchive(indexName, reader, staging, deps) {
643
898
  manifest: async (manifest2, sourceName) => {
644
899
  if (staged.prepared === null) missingPart("configuration");
645
900
  requireSameSchema(staged.prepared, manifest2);
901
+ if (deps.indexRegistry.has(indexName)) {
902
+ requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2));
903
+ }
646
904
  staged.manifest = renamed(manifest2, sourceName, indexName);
647
905
  },
648
906
  file: async (entryName, _ref, bytes) => {
@@ -656,44 +914,30 @@ async function stageArchive(indexName, reader, staging, deps) {
656
914
  await validateStagedCheckpoint(staging, manifest);
657
915
  return { prepared, manifest };
658
916
  }
659
- async function replaceWithStaged(indexName, staged, staging, memory, deps) {
917
+ async function loadStagedIndex(indexName, staged, staging, store, deps) {
660
918
  const { prepared, manifest } = staged;
661
- const durable = deps.durability?.manager;
662
- const created = await createRestoredIndex(indexName, prepared, partitionCountOf(manifest), deps);
919
+ await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
920
+ const detached = await prepareDetachedIndex(indexName, prepared, partitionCountOf(manifest), deps);
663
921
  try {
664
- if (staging !== null && durable !== void 0) await durable.persistMetadata(indexName);
665
- const store = staging === null ? memory : await staging.commit();
666
- await store.write(manifestKey(indexName), (await encodeSegmentManifest(manifest)).chunks);
667
922
  await loadSegmentedCheckpoint(
668
923
  store,
669
924
  indexName,
670
925
  manifest,
671
- { manager: created.manager, vectorIndexes: created.manager.getVectorIndexes() },
926
+ { manager: detached.manager, vectorIndexes: detached.manager.getVectorIndexes() },
672
927
  true
673
928
  );
674
- recordRestoredCounts(indexName, created.manager, deps);
675
- if (durable?.adoptRestoredCheckpoint !== void 0 && staging !== null) {
676
- durable.adoptRestoredCheckpoint(indexName, manifest.checkpoint);
677
- await durable.persistMetadata(indexName);
678
- } else if (durable !== void 0) {
679
- await durable.persistMetadata(indexName);
680
- await durable.checkpoint(indexName);
681
- }
682
- await deps.announceIndexCreated(indexName, prepared.config);
683
- return created;
684
929
  } catch (error) {
685
- created.release();
686
- await abandonRestoredIndex(indexName, deps);
930
+ deps.executor.discardDetachedIndex(detached);
687
931
  throw error;
688
932
  }
933
+ return { prepared, detached, staging, checkpoint: manifest.checkpoint };
689
934
  }
690
935
  async function restoreArchive(indexName, reader, deps) {
691
936
  const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
692
- const memory = memoryCheckpointStore();
937
+ const store = staging?.store ?? memoryCheckpointStore();
693
938
  try {
694
- const staged = await stageArchive(indexName, reader, staging?.store ?? memory, deps);
695
- const created = await replaceWithStaged(indexName, staged, staging, memory, deps);
696
- created.release();
939
+ const staged = await stageArchive(indexName, reader, store, deps);
940
+ await installRestoredIndex(indexName, await loadStagedIndex(indexName, staged, staging, store, deps), deps);
697
941
  } finally {
698
942
  await staging?.discard().catch(() => void 0);
699
943
  await reader.close();
@@ -746,16 +990,20 @@ function vectorsOfSnapshot(envelope) {
746
990
  }
747
991
  return byDocument;
748
992
  }
993
+ function footprintOf(envelope, prepared) {
994
+ let partitionBytes = 0;
995
+ for (const payload of envelope.partitions ?? []) partitionBytes += payload.byteLength;
996
+ const vectorCounts = /* @__PURE__ */ new Map();
997
+ for (const [fieldPath, rawParts] of Object.entries(envelope.vectorIndexes ?? {})) {
998
+ let count = 0;
999
+ for (const part of decodeVectorIndexParts(rawParts)) count += part.docIds.length;
1000
+ vectorCounts.set(fieldPath, count);
1001
+ }
1002
+ return earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts);
1003
+ }
749
1004
  function documentsOfPartition(envelope, payload) {
750
1005
  return envelope.version === 2 ? documentsOfPayloadV2(payload).documents : deserializePayloadV1(payload).documents;
751
1006
  }
752
- function withVectors(fields, vectors) {
753
- if (vectors === void 0 || vectors.size === 0) return fields;
754
- const nested = [...vectors.keys()].some((fieldPath) => fieldPath.includes("."));
755
- const document = nested ? structuredClone(fields) : { ...fields };
756
- for (const [fieldPath, vector] of vectors) setNestedValue(document, fieldPath, vector);
757
- return document;
758
- }
759
1007
  function insertRestoredDocument(manager, docId, fields, vectors) {
760
1008
  manager.insert(docId, fields, { skipClone: true });
761
1009
  if (vectors === void 0) return;
@@ -766,51 +1014,62 @@ function insertRestoredDocument(manager, docId, fields, vectors) {
766
1014
  throw err;
767
1015
  }
768
1016
  }
769
- async function insertRestoredChunk(indexName, manager, chunk, vectors, deps) {
770
- if (deps.durability === null) {
771
- for (const [docId, fields] of chunk) insertRestoredDocument(manager, docId, fields, vectors.get(docId));
772
- return;
773
- }
774
- const mutations = chunk.map(([docId, fields]) => ({
775
- docId,
776
- document: withVectors(fields, vectors.get(docId)),
777
- apply: () => insertRestoredDocument(manager, docId, fields, vectors.get(docId))
778
- }));
779
- for (const outcome of await deps.durability.recordMutationBatch(indexName, mutations)) {
780
- if (!outcome.ok) throw outcome.error;
781
- }
782
- }
783
- async function insertSnapshotDocuments(indexName, envelope, manager, deps) {
1017
+ function insertSnapshotDocuments(envelope, manager) {
784
1018
  const vectors = vectorsOfSnapshot(envelope);
785
1019
  for (const payload of envelope.partitions ?? []) {
786
- const documents = Object.entries(documentsOfPartition(envelope, payload)).map(
787
- ([docId, stored]) => [docId, stored.fields]
788
- );
789
- for (let start = 0; start < documents.length; start += BATCH_CHUNK_SIZE) {
790
- await insertRestoredChunk(indexName, manager, documents.slice(start, start + BATCH_CHUNK_SIZE), vectors, deps);
1020
+ for (const [docId, stored] of Object.entries(documentsOfPartition(envelope, payload))) {
1021
+ insertRestoredDocument(manager, docId, stored.fields, vectors.get(docId));
791
1022
  }
792
1023
  }
793
1024
  for (const vectorIndex of manager.getVectorIndexes().values()) vectorIndex.scheduleBuild();
794
1025
  }
1026
+ async function stageBuiltIndex(indexName, prepared, built, staging, deps) {
1027
+ const entry = restoredRegistryEntry(prepared, built.manager, deps);
1028
+ const archive = await buildArchiveFromMemory({
1029
+ indexName,
1030
+ metadata: indexMetadataOf(indexName, entry, built.manager.partitionCount),
1031
+ manager: built.manager,
1032
+ vectorIndexes: built.manager.getVectorIndexes(),
1033
+ appliedSeqNo: () => 0,
1034
+ partitionId: null
1035
+ });
1036
+ try {
1037
+ for (const ref of manifestFileRefs(archive.manifest))
1038
+ await staging.store.write(ref.key, [await archive.readFile(ref)]);
1039
+ } finally {
1040
+ archive.release();
1041
+ }
1042
+ await validateStagedCheckpoint(staging.store, archive.manifest);
1043
+ return archive.manifest;
1044
+ }
1045
+ async function buildRestoredIndex(indexName, envelope, prepared, staging, deps) {
1046
+ const built = await prepareDetachedIndex(indexName, prepared, envelope.partitions?.length ?? 0, deps);
1047
+ let manifest;
1048
+ try {
1049
+ insertSnapshotDocuments(envelope, built.manager);
1050
+ if (staging === null) return { prepared, detached: built, staging: null, checkpoint: [] };
1051
+ manifest = await stageBuiltIndex(indexName, prepared, built, staging, deps);
1052
+ } catch (err) {
1053
+ deps.executor.discardDetachedIndex(built);
1054
+ throw err;
1055
+ }
1056
+ deps.executor.discardDetachedIndex(built);
1057
+ return loadStagedIndex(indexName, { prepared, manifest }, staging, staging.store, deps);
1058
+ }
795
1059
  async function restoreEarlierSnapshot(indexName, data, deps) {
796
1060
  const envelope = await decodeEarlierSnapshot(data);
797
1061
  const prepared = prepareRestoredIndex(envelope, deps);
1062
+ if (deps.indexRegistry.has(indexName)) requireRestoreMemory(indexName, footprintOf(envelope, prepared));
798
1063
  const analysedNow = { ...prepared, storedRevision: prepared.language.revision };
799
- const partitionCount = envelope.partitions?.length ?? 0;
800
- const { manager, release } = await createRestoredIndex(indexName, analysedNow, partitionCount, deps);
1064
+ const staging = deps.durability?.manager.stageRestore?.(indexName) ?? null;
801
1065
  try {
802
- await insertSnapshotDocuments(indexName, envelope, manager, deps);
803
- recordRestoredCounts(indexName, manager, deps);
804
- if (deps.durability) {
805
- await deps.durability.manager.persistMetadata(indexName);
806
- await deps.durability.manager.checkpoint(indexName);
807
- }
808
- await deps.announceIndexCreated(indexName, prepared.config);
809
- } catch (err) {
810
- await abandonRestoredIndex(indexName, deps);
811
- throw err;
1066
+ await installRestoredIndex(
1067
+ indexName,
1068
+ await buildRestoredIndex(indexName, envelope, analysedNow, staging, deps),
1069
+ deps
1070
+ );
812
1071
  } finally {
813
- release();
1072
+ await staging?.discard().catch(() => void 0);
814
1073
  }
815
1074
  }
816
1075
 
@@ -1,7 +1,7 @@
1
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-66KYAOIK.mjs';
2
- import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-NOUZ6RH3.mjs';
3
- import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-NQW2YXEX.mjs';
4
- import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-FHFFWLEK.mjs';
1
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-3U636ADD.mjs';
2
+ import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-D3CC2NMZ.mjs';
3
+ import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-WKPJ4E73.mjs';
4
+ import { EXPORT_CHUNK_BYTES, NarsilError, ErrorCodes } from './chunk-HARBSUWP.mjs';
5
5
  import { decode } from '@msgpack/msgpack';
6
6
 
7
7
  // src/engine/export/json-line.ts
@@ -1,6 +1,6 @@
1
1
  import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
2
- import { C as ClusterNamespace } from './config-DyRowvVP.js';
3
- import { h as TaskStore } from './tasks-DA1Ff0G4.js';
2
+ import { C as ClusterNamespace } from './config-6YBtxekG.js';
3
+ import { h as TaskStore } from './tasks-BJmamiC4.js';
4
4
 
5
5
  /**
6
6
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -1,7 +1,7 @@
1
1
  import { parseSynonymRules, encodeSynonymRules, decodeSynonymRules, SYNONYM_EXPANSION_CACHE_ENTRIES, SYNONYMS_ADDED_PER_QUERY } from './chunk-7X5JST5L.mjs';
2
- import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-ZIJ63TBT.mjs';
2
+ import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-7KC56574.mjs';
3
3
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
4
- import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
4
+ import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
5
5
 
6
6
  // src/analysis/registry.ts
7
7
  var tokenizers = /* @__PURE__ */ new Map();