@delali/narsil 0.3.1-next.130 → 0.3.1-next.140

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-JN23AZQP.mjs +43 -0
  4. package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
  5. package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
  6. package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
  9. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
  10. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
  11. package/dist/browser/chunk-67KSBUYR.mjs +173 -0
  12. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
  13. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
  14. package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
  15. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  16. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  17. package/dist/browser/chunk-QHNNULPW.mjs +265 -0
  18. package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
  19. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
  20. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
  21. package/dist/browser/import-Q7MXD6BS.mjs +8 -0
  22. package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
  23. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  24. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
  25. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  26. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
  27. package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
  28. package/dist/{capabilities-Bl8G9taj.d.ts → capabilities-DXAmysqJ.d.ts} +11 -11
  29. package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
  30. package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
  31. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  32. package/dist/{chunk-QQTXT65X.mjs → chunk-B5I3IL4H.mjs} +12 -7
  33. package/dist/chunk-BDIDUW2B.mjs +387 -0
  34. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  35. package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
  36. package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
  37. package/dist/chunk-CZQ33DLW.mjs +78 -0
  38. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  39. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  40. package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
  41. package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
  42. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  43. package/dist/chunk-HMDT3I63.mjs +21 -0
  44. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  45. package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
  46. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  47. package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
  49. package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
  50. package/dist/chunk-LK3MOTXF.mjs +174 -0
  51. package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
  52. package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
  53. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  54. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  55. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  56. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  57. package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
  58. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  59. package/dist/{chunk-XABMCBHA.mjs → chunk-SBMAWTRG.mjs} +684 -99
  60. package/dist/chunk-TCWQJL7O.mjs +188 -0
  61. package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
  62. package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
  63. package/dist/chunk-UEV53BBO.mjs +480 -0
  64. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  65. package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
  66. package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
  67. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  68. package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
  69. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  70. package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
  71. package/dist/{chunk-YBWSV5DV.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  72. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  73. package/dist/{client-Cx-TiqPE.d.ts → client-DApBeLbk.d.ts} +18 -3
  74. package/dist/client.d.ts +7 -8
  75. package/dist/client.mjs +11 -1
  76. package/dist/{events-DErop1LM.d.ts → config-B0H-vp4U.d.ts} +424 -2
  77. package/dist/{config-CzB9YvF_.d.ts → config-Brysl9NO.d.ts} +3 -3
  78. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
  79. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  80. package/dist/distribution/coordinator/etcd.mjs +1 -1
  81. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  82. package/dist/distribution/transport/grpc.mjs +1 -1
  83. package/dist/distribution.d.ts +24 -9
  84. package/dist/distribution.mjs +882 -192
  85. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
  86. package/dist/embeddings/openai.mjs +2 -2
  87. package/dist/{engine-B_W9sdZK.d.ts → engine-CuLMbH1-.d.ts} +64 -7
  88. package/dist/import-QRZAQ5RH.mjs +43 -0
  89. package/dist/{index-CKI1fDEl.d.ts → index-HYnqS1SE.d.ts} +2 -2
  90. package/dist/index.browser.mjs +4 -4
  91. package/dist/index.d.ts +6 -7
  92. package/dist/index.mjs +20 -20
  93. package/dist/{long-query-thread-WVSMIBYP.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
  94. package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
  95. package/dist/partial-words.mjs +5 -5
  96. package/dist/pattern.mjs +3 -3
  97. package/dist/persistence/durability/checkpoint-worker.d.ts +12 -1
  98. package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
  99. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  100. package/dist/react.d.ts +11 -8
  101. package/dist/react.mjs +4 -0
  102. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  103. package/dist/rerankers/http.mjs +2 -2
  104. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
  105. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  106. package/dist/{schema-CS-clLun.d.ts → schema-BTXbHCqA.d.ts} +4 -1
  107. package/dist/server.d.ts +200 -11
  108. package/dist/server.mjs +66 -23
  109. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
  110. package/dist/synonyms.mjs +6 -4
  111. package/dist/tasks-DjGjK70Y.d.ts +720 -0
  112. package/dist/{thread-DNK5XWXG.mjs → thread-USB3GOPY.mjs} +13 -13
  113. package/dist/vector/search-worker.d.ts +1 -1
  114. package/dist/vector/search-worker.mjs +5 -5
  115. package/dist/workers/entry.mjs +7 -7
  116. package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
  117. package/openapi.json +5310 -0
  118. package/package.json +16 -8
  119. package/dist/config-hOWGBeC6.d.ts +0 -414
  120. package/dist/tasks-BfF-VRhm.d.ts +0 -302
@@ -1,13 +1,16 @@
1
- import { collectAndReturnPages, vectorsPerFile } from './chunk-VBBLTD7O.mjs';
2
- import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-NSCSR6MA.mjs';
3
- import './chunk-M5UAAXGU.mjs';
4
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, restoredConfigFields, restoredEmbedding, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, rejectSnapshot, readHeapStatistics, readResidentMemory, extractVectorFieldsFromSchema, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-2Y3CO4EQ.mjs';
1
+ import { createByteReader, restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-QHNNULPW.mjs';
2
+ import { vectorsPerFile } from './chunk-3EYDURFD.mjs';
3
+ import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint } from './chunk-67KSBUYR.mjs';
4
+ import './chunk-OIZS5KIX.mjs';
5
+ import { indexMetadataOf, writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, writeLiveVectors, commitCheckpointManifest, manifestKey, decodeSegmentManifest, SINGLE_NODE_PRIMARY_TERM, partitionContentsOf, manifestFileRefs, unpackIndexSnapshotEnvelope, encodeSegmentManifest, loadSegmentedCheckpoint, writeMetadataEnvelope, openManifestPayload, listedKeysOf, decodeManifestPayload, readSegmentFile, readVectorFile, readGraphParts, documentsOfPayloadV2, deserializePayloadV1 } from './chunk-CGVBGHBX.mjs';
6
+ import './chunk-6BSB4N6B.mjs';
7
+ import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-5LYWZJE2.mjs';
8
+ import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, ARRAY_POSITION_GAP, raiseEarlierReleaseGraphSetting, getLanguage, validateSchema, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateVectorPromotion, validateVectorStorage, validateRequiredFieldsInSchema, resolveIndexAnalysis, validateEmbeddingConfig, storedArrayPositionGap, prepareStoredSynonyms, loadVectorCodeFor, decodeEveryStoredDocument, parseVectorDimension, arrayPositionsAreStale, insertDocumentVectors, decodeIndexConfiguration, flattenSchema } from './chunk-ZAIZQOTM.mjs';
5
9
  import './chunk-26A5JFT2.mjs';
6
- import { decodeVectorIndexParts, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, bytesToVectors } from './chunk-D7WQF6DV.mjs';
7
- import { MAX_FILE_BYTES } from './chunk-AIIJDWMG.mjs';
10
+ import { MAX_FILE_BYTES } from './chunk-4MFH226Q.mjs';
8
11
  import './chunk-JPTQZWJQ.mjs';
9
- import './chunk-VEGHFP2P.mjs';
10
- import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
12
+ import './chunk-MZI3TEOW.mjs';
13
+ import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
11
14
  import { decode } from '@msgpack/msgpack';
12
15
 
13
16
  // src/persistence/archive/memory-store.ts
@@ -515,161 +518,6 @@ async function validateStagedCheckpoint(store, manifest) {
515
518
  }
516
519
  }
517
520
 
518
- // src/engine/snapshot/constants.ts
519
- var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
520
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
521
- var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
522
- var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
523
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
524
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
525
- var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
526
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
527
- var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
528
- var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
529
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
530
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
531
- var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
532
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
533
- var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
534
- var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
535
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
536
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
537
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
538
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
539
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
540
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
541
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
542
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
543
- var RESTORE_ESTIMATE_MARGIN = 1.25;
544
-
545
- // src/engine/snapshot/memory-budget.ts
546
- var MEGABYTE_BYTES = 1048576;
547
- var ARCHIVE_RATES = {
548
- worst: {
549
- heap: {
550
- perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
551
- perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
552
- perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
553
- },
554
- external: {
555
- perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
556
- perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
557
- perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
558
- }
559
- },
560
- floor: {
561
- heap: {
562
- perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
563
- perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
564
- perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
565
- },
566
- external: {
567
- perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
568
- perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
569
- perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
570
- }
571
- }
572
- };
573
- var EARLIER_SNAPSHOT_RATES = {
574
- worst: {
575
- heap: {
576
- perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
577
- perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
578
- perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
579
- },
580
- external: {
581
- perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
582
- perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
583
- perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
584
- }
585
- },
586
- floor: {
587
- heap: {
588
- perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
589
- perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
590
- perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
591
- },
592
- external: {
593
- perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
594
- perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
595
- perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
596
- }
597
- }
598
- };
599
- function bytesAt(rate, storedBytes, vectors) {
600
- let bytes = storedBytes * rate.perStoredByte;
601
- for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
602
- return bytes;
603
- }
604
- function footprintAt(rates, replacesIndex, storedBytes, vectors) {
605
- const chosen = replacesIndex ? rates.worst : rates.floor;
606
- const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
607
- return {
608
- heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
609
- externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
610
- };
611
- }
612
- function deadCountOf(dead) {
613
- if (dead === null) return 0;
614
- let count = 0;
615
- for (const byte of dead) {
616
- for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
617
- }
618
- return count;
619
- }
620
- function archiveFootprint(prepared, manifest, replacesIndex) {
621
- let segmentBytes = 0;
622
- for (const partition of manifest.partitions) {
623
- for (const segment of partition.segments) segmentBytes += segment.bytes;
624
- }
625
- const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
626
- const vectors = [];
627
- for (const field of manifest.vectors) {
628
- const dimension = dimensions.get(field.fieldPath) ?? 0;
629
- for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
630
- }
631
- return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
632
- }
633
- function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
634
- const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
635
- const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
636
- return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
637
- }
638
- function inMegabytes(bytes) {
639
- return Math.ceil(bytes / MEGABYTE_BYTES);
640
- }
641
- function shortfallOf(heapBytes, processBytes, countInUse) {
642
- const heap = readHeapStatistics();
643
- if (heap !== null) {
644
- const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
645
- const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
646
- if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
647
- }
648
- const resident = readResidentMemory();
649
- if (resident !== null) {
650
- const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
651
- if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
652
- }
653
- return null;
654
- }
655
- function refuse(indexName, replacesIndex, shortfall) {
656
- const { budget, neededBytes, availableBytes } = shortfall;
657
- const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
658
- const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
659
- throw new NarsilError(
660
- ErrorCodes.RESTORE_MEMORY_EXCEEDED,
661
- `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
662
- { indexName, budget, neededBytes, availableBytes }
663
- );
664
- }
665
- function requireRestoreMemory(indexName, footprint, replacesIndex) {
666
- const processBytes = footprint.heapBytes + footprint.externalBytes;
667
- if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
668
- const collected = collectAndReturnPages();
669
- const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
670
- if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
671
- }
672
-
673
521
  // src/workers/detached-index.ts
674
522
  function attachablePolicy(shared) {
675
523
  let attached = false;
@@ -735,12 +583,14 @@ function prepareRestoredIndex(configuration, deps, reject) {
735
583
  }
736
584
  const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
737
585
  const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
586
+ const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
738
587
  return {
739
588
  config,
740
589
  language,
741
590
  embeddingAdapter,
742
591
  adapterName,
743
592
  storedRevision,
593
+ analysisGeneration,
744
594
  storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
745
595
  synonyms
746
596
  };
@@ -775,7 +625,10 @@ function restoredRegistryEntry(prepared, manager, deps) {
775
625
  partitionCount: manager.partitionCount,
776
626
  analysisRevision: prepared.storedRevision,
777
627
  arrayPositionGap: prepared.storedArrayPositionGap,
778
- synonyms: prepared.synonyms?.encoded ?? null
628
+ synonyms: prepared.synonyms?.encoded ?? null,
629
+ analysisGeneration: prepared.analysisGeneration,
630
+ pendingAnalysis: null,
631
+ durableAnalysis: null
779
632
  };
780
633
  }
781
634
 
@@ -871,6 +724,7 @@ function activate(indexName, restored, entry, deps) {
871
724
  async function replaceExisting(indexName, restored, entry, deps) {
872
725
  const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
873
726
  try {
727
+ await deps.abandonAnalysisChange(indexName);
874
728
  await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
875
729
  await deps.indexState.replace(indexName, async (wasOpen) => {
876
730
  await deps.abandonAnalysisRebuild(indexName);
@@ -1174,9 +1028,24 @@ async function checkpointForArchive(deps) {
1174
1028
  });
1175
1029
  }
1176
1030
  async function createSnapshot(deps) {
1177
- const config = await encodeIndexConfiguration(deps.entry);
1178
- const checkpoint = await checkpointForArchive(deps);
1179
- return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
1031
+ for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
1032
+ const checkpoint = await checkpointForArchive(deps);
1033
+ try {
1034
+ if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
1035
+ const config = await encodeIndexConfiguration(deps.entry);
1036
+ return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
1037
+ }
1038
+ } catch (error) {
1039
+ checkpoint.release();
1040
+ throw error;
1041
+ }
1042
+ checkpoint.release();
1043
+ }
1044
+ throw new NarsilError(
1045
+ ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
1046
+ `The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
1047
+ { indexName: deps.indexName }
1048
+ );
1180
1049
  }
1181
1050
  function requireSnapshotSource(data) {
1182
1051
  if (data instanceof Uint8Array) return data;
@@ -1,11 +1,13 @@
1
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-M5UAAXGU.mjs';
2
- import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-2Y3CO4EQ.mjs';
1
+ import { NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-XVMOQQ34.mjs';
2
+ import './chunk-QHNNULPW.mjs';
3
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-6BSB4N6B.mjs';
4
+ import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-5LYWZJE2.mjs';
5
+ import { decodeStoredDocument, openChecksummedEnvelope } from './chunk-ZAIZQOTM.mjs';
3
6
  import './chunk-26A5JFT2.mjs';
4
- import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-D7WQF6DV.mjs';
5
- import './chunk-AIIJDWMG.mjs';
7
+ import './chunk-4MFH226Q.mjs';
6
8
  import './chunk-JPTQZWJQ.mjs';
7
- import { EXPORT_CHUNK_BYTES } from './chunk-VEGHFP2P.mjs';
8
- import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
9
+ import { EXPORT_CHUNK_BYTES } from './chunk-MZI3TEOW.mjs';
10
+ import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
9
11
  import { decode } from '@msgpack/msgpack';
10
12
 
11
13
  // src/engine/export/json-line.ts
@@ -1,6 +1,6 @@
1
1
  import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
2
- import { C as ClusterNamespace } from './config-CzB9YvF_.js';
3
- import { h as TaskStore } from './tasks-BfF-VRhm.js';
2
+ import { C as ClusterNamespace } from './config-Brysl9NO.js';
3
+ import { E as ErrorDetail, i as TaskStore } from './tasks-DjGjK70Y.js';
4
4
 
5
5
  /**
6
6
  * Context handed to {@link OnRequestHook} for every inbound request, captured
@@ -256,14 +256,7 @@ interface NarsilServer {
256
256
  */
257
257
  interface ErrorEnvelope {
258
258
  /** This describes the failure. */
259
- error: {
260
- /** This code comes from either {@link ServerErrorCodes} or {@link ErrorCodes}. */
261
- code: string;
262
- /** This describes the failure, for a log or a person. */
263
- message: string;
264
- /** These values are behind the failure, such as the field or limit involved. */
265
- details?: Record<string, unknown>;
266
- };
259
+ error: ErrorDetail;
267
260
  }
268
261
 
269
262
  /**
@@ -293,6 +286,13 @@ declare const TASK_FILTER_CAPABILITY = "tasks.filter";
293
286
  * @public
294
287
  */
295
288
  declare const REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
289
+ /**
290
+ * Marks that `POST /indexes/{name}/_change-analysis` changes the language, the
291
+ * tokeniser, or the stop words of a live index.
292
+ *
293
+ * @public
294
+ */
295
+ declare const CHANGE_ANALYSIS_CAPABILITY = "indexes.changeAnalysis";
296
296
  /**
297
297
  * Marks that `POST /indexes/{name}/_open` and `POST /indexes/{name}/_close`
298
298
  * load and release an index's server memory.
@@ -334,4 +334,4 @@ declare const BM25_TUNING_CAPABILITY = "indexes.bm25";
334
334
  */
335
335
  declare const SERVER_CAPABILITIES: readonly string[];
336
336
 
337
- export { ASYNC_IMPORT_CAPABILITY as A, BM25_TUNING_CAPABILITY as B, type CorsOptions as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, SYNONYMS_CAPABILITY as a, TASK_FILTER_CAPABILITY as b, type ServerOptions as c, type RequestContext as d, type RequestDenial as e, type ServerLimits as f };
337
+ export { ASYNC_IMPORT_CAPABILITY as A, BM25_TUNING_CAPABILITY as B, CHANGE_ANALYSIS_CAPABILITY as C, DOCUMENT_EXPORT_CAPABILITY as D, type ErrorEnvelope as E, INDEX_LIFECYCLE_CAPABILITY as I, type NarsilServer as N, type OnRequestHook as O, REBUILD_ANALYSIS_CAPABILITY as R, SERVER_CAPABILITIES as S, TASK_CANCEL_CAPABILITY as T, SYNONYMS_CAPABILITY as a, TASK_FILTER_CAPABILITY as b, type ServerOptions as c, type CorsOptions as d, type RequestContext as e, type RequestDenial as f, type ServerLimits as g };
@@ -1,8 +1,8 @@
1
1
  import { nodeThreadId } from './chunk-W55X2BYN.mjs';
2
2
  import { getFs, getPath } from './chunk-HLPRHOSO.mjs';
3
- import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-2JWNJSNE.mjs';
3
+ import { DIRECTORY_LOCK_ATTEMPTS, DIRECTORY_LOCK_SETTLE_MS, DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS } from './chunk-CPCGEIOM.mjs';
4
4
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
5
- import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
5
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
6
6
  import { openSync, close, readFileSync } from 'fs';
7
7
  import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
8
8
  import { endianness } from 'os';
@@ -1,9 +1,9 @@
1
- import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-Y2U5VVWX.mjs';
2
- import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-X5CNUKD5.mjs';
1
+ import { VECTOR_FILE_MAX_VECTORS, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors, VECTOR_GRAPH_PART_VECTORS, VECTOR_GRAPH_PART_PAYLOAD_VERSION, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, vectorsToBytes } from './chunk-ICW45TTU.mjs';
2
+ import { createOrdinalFilter, addToOrdinalFilter, osqBitsOf, STORE_ORDINAL_LAYOUT_GENERATION, removeFromOrdinalFilter, createHNSWIndex, STORE_CALIBRATION_GENERATION, graphWithoutNodes, createOsqScratch, osqQuantize } from './chunk-K3AQOZVM.mjs';
3
3
  import { osqRecordBytes, writeRecord } from './chunk-37CD3OKS.mjs';
4
- import { MAX_DOC_ID_LENGTH } from './chunk-RNSC4NFI.mjs';
4
+ import { MAX_DOC_ID_LENGTH } from './chunk-EEBCDP36.mjs';
5
5
  import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
6
- import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
6
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
7
7
 
8
8
  // src/vector/filter-ordinals.ts
9
9
  var UNRESOLVED_ORDINAL = -1;
@@ -1,4 +1,4 @@
1
- import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
1
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
2
2
  import { existsSync } from 'fs';
3
3
  import { createRequire } from 'module';
4
4
  import { join, dirname } from 'path';
@@ -1,9 +1,9 @@
1
- import { sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, respondJson, withServerRerankWindow, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-XABMCBHA.mjs';
1
+ import { sendRelayed, parseJson, validateQuery, rejectInvalid, refuseRegex, respondJson, withServerRerankWindow, respondError, loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedBodyBudget, createSharedRequestGate, registerServerRoutes, closeWhenTheProcessExits, statusLine, sendError, REQUEST_THREAD_MOVE_AFTER_MS, MAX_LONG_QUERIES_IN_FLIGHT, createDocumentReadHandlers, SERVER_CAPABILITIES, versionReport, serverRoutes, validateSuggest, SEARCH_CLOCK_SAMPLE_UNITS, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-SBMAWTRG.mjs';
2
2
  import { withRerankQueryText } from './chunk-KZ3NX2P7.mjs';
3
- import { executeListDocuments, executeSuggest, executeQuery, lazyQueryBinding, executePreflight } from './chunk-6M6SDH3S.mjs';
4
- import { holdsQuotedText, resolveProjection, projectionKeepsField, setNestedValue } from './chunk-2MKX2SCO.mjs';
5
- import { extractVectorFieldsFromSchema } from './chunk-PU6SKH7O.mjs';
6
- import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-CZ7ISLXA.mjs';
3
+ import { executeListDocuments, executeSuggest, executeQuery, lazyQueryBinding, executePreflight } from './chunk-XMQNOD5V.mjs';
4
+ import { holdsQuotedText, resolveProjection, projectionKeepsField, setNestedValue } from './chunk-U4X4AWKL.mjs';
5
+ import { extractVectorFieldsFromSchema } from './chunk-UBXJQJII.mjs';
6
+ import { ServerErrorCodes, ErrorCodes, NarsilError, isNarsilError } from './chunk-HNHCNTVG.mjs';
7
7
 
8
8
  // src/server/request-threads/messages.ts
9
9
  function isLongQueryPortNote(message) {
@@ -273,7 +273,8 @@ var SearchMoved = class extends Error {
273
273
  }
274
274
  };
275
275
  function moveClock(moveAfterMs) {
276
- const moveAt = performance.now() + moveAfterMs;
276
+ let moveAt = performance.now() + moveAfterMs;
277
+ let overranOnce = false;
277
278
  let pending = 0;
278
279
  const clock = {
279
280
  tick(units) {
@@ -282,7 +283,11 @@ function moveClock(moveAfterMs) {
282
283
  },
283
284
  check() {
284
285
  pending = 0;
285
- if (performance.now() > moveAt) throw new SearchMoved();
286
+ const now = performance.now();
287
+ if (now <= moveAt) return;
288
+ if (overranOnce) throw new SearchMoved();
289
+ overranOnce = true;
290
+ moveAt = now + moveAfterMs;
286
291
  }
287
292
  };
288
293
  return clock;