@delali/narsil 0.3.1-next.130 → 0.3.1-next.140

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-JN23AZQP.mjs +43 -0
  4. package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
  5. package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
  6. package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
  9. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
  10. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
  11. package/dist/browser/chunk-67KSBUYR.mjs +173 -0
  12. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
  13. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
  14. package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
  15. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  16. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  17. package/dist/browser/chunk-QHNNULPW.mjs +265 -0
  18. package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
  19. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
  20. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
  21. package/dist/browser/import-Q7MXD6BS.mjs +8 -0
  22. package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
  23. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  24. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
  25. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  26. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
  27. package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
  28. package/dist/{capabilities-Bl8G9taj.d.ts → capabilities-DXAmysqJ.d.ts} +11 -11
  29. package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
  30. package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
  31. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  32. package/dist/{chunk-QQTXT65X.mjs → chunk-B5I3IL4H.mjs} +12 -7
  33. package/dist/chunk-BDIDUW2B.mjs +387 -0
  34. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  35. package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
  36. package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
  37. package/dist/chunk-CZQ33DLW.mjs +78 -0
  38. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  39. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  40. package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
  41. package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
  42. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  43. package/dist/chunk-HMDT3I63.mjs +21 -0
  44. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  45. package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
  46. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  47. package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
  49. package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
  50. package/dist/chunk-LK3MOTXF.mjs +174 -0
  51. package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
  52. package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
  53. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  54. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  55. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  56. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  57. package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
  58. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  59. package/dist/{chunk-XABMCBHA.mjs → chunk-SBMAWTRG.mjs} +684 -99
  60. package/dist/chunk-TCWQJL7O.mjs +188 -0
  61. package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
  62. package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
  63. package/dist/chunk-UEV53BBO.mjs +480 -0
  64. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  65. package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
  66. package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
  67. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  68. package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
  69. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  70. package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
  71. package/dist/{chunk-YBWSV5DV.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  72. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  73. package/dist/{client-Cx-TiqPE.d.ts → client-DApBeLbk.d.ts} +18 -3
  74. package/dist/client.d.ts +7 -8
  75. package/dist/client.mjs +11 -1
  76. package/dist/{events-DErop1LM.d.ts → config-B0H-vp4U.d.ts} +424 -2
  77. package/dist/{config-CzB9YvF_.d.ts → config-Brysl9NO.d.ts} +3 -3
  78. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
  79. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  80. package/dist/distribution/coordinator/etcd.mjs +1 -1
  81. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  82. package/dist/distribution/transport/grpc.mjs +1 -1
  83. package/dist/distribution.d.ts +24 -9
  84. package/dist/distribution.mjs +882 -192
  85. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
  86. package/dist/embeddings/openai.mjs +2 -2
  87. package/dist/{engine-B_W9sdZK.d.ts → engine-CuLMbH1-.d.ts} +64 -7
  88. package/dist/import-QRZAQ5RH.mjs +43 -0
  89. package/dist/{index-CKI1fDEl.d.ts → index-HYnqS1SE.d.ts} +2 -2
  90. package/dist/index.browser.mjs +4 -4
  91. package/dist/index.d.ts +6 -7
  92. package/dist/index.mjs +20 -20
  93. package/dist/{long-query-thread-WVSMIBYP.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
  94. package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
  95. package/dist/partial-words.mjs +5 -5
  96. package/dist/pattern.mjs +3 -3
  97. package/dist/persistence/durability/checkpoint-worker.d.ts +12 -1
  98. package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
  99. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  100. package/dist/react.d.ts +11 -8
  101. package/dist/react.mjs +4 -0
  102. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  103. package/dist/rerankers/http.mjs +2 -2
  104. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
  105. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  106. package/dist/{schema-CS-clLun.d.ts → schema-BTXbHCqA.d.ts} +4 -1
  107. package/dist/server.d.ts +200 -11
  108. package/dist/server.mjs +66 -23
  109. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
  110. package/dist/synonyms.mjs +6 -4
  111. package/dist/tasks-DjGjK70Y.d.ts +720 -0
  112. package/dist/{thread-DNK5XWXG.mjs → thread-USB3GOPY.mjs} +13 -13
  113. package/dist/vector/search-worker.d.ts +1 -1
  114. package/dist/vector/search-worker.mjs +5 -5
  115. package/dist/workers/entry.mjs +7 -7
  116. package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
  117. package/openapi.json +5310 -0
  118. package/package.json +16 -8
  119. package/dist/config-hOWGBeC6.d.ts +0 -414
  120. package/dist/tasks-BfF-VRhm.d.ts +0 -302
@@ -1,56 +1,59 @@
1
- import { documentsOfPayloadV2 } from './chunk-JJ6EUMXB.mjs';
2
- import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-WVRDRLIC.mjs';
3
- import './chunk-SOHMMNCH.mjs';
4
- import { indexMetadataOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, readVectorFile, readGraphParts, deserializePayloadV1, SINGLE_NODE_PRIMARY_TERM } from './chunk-K77FJXKS.mjs';
5
- import { decodeVectorIndexParts, bytesToVectors } from './chunk-Y2U5VVWX.mjs';
6
- import './chunk-X5CNUKD5.mjs';
7
- import './chunk-DJHHF2L5.mjs';
1
+ import { documentsOfPayloadV2 } from './chunk-PNNVTKMT.mjs';
2
+ import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-FKARMUWH.mjs';
3
+ import { indexMetadataOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, readVectorFile, readGraphParts, deserializePayloadV1, SINGLE_NODE_PRIMARY_TERM } from './chunk-MTQTX4BP.mjs';
4
+ import { createByteReader } from './chunk-CZQ33DLW.mjs';
5
+ import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-TCWQJL7O.mjs';
6
+ import './chunk-6VHWXAOI.mjs';
7
+ import { decodeVectorIndexParts, bytesToVectors } from './chunk-ICW45TTU.mjs';
8
+ import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, archiveFootprint } from './chunk-LK3MOTXF.mjs';
9
+ import './chunk-K3AQOZVM.mjs';
8
10
  import './chunk-37CD3OKS.mjs';
9
- import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-2JWNJSNE.mjs';
10
- import './chunk-7OIQ2SIO.mjs';
11
+ import './chunk-DJHHF2L5.mjs';
12
+ import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-CPCGEIOM.mjs';
13
+ import './chunk-LLDFIKGP.mjs';
11
14
  import './chunk-P36MUFEJ.mjs';
12
15
  import './chunk-YH5OIL7B.mjs';
13
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, decodeIndexConfiguration, prepareStoredSynonyms, arrayPositionsAreStale, insertDocumentVectors } from './chunk-TGHNXN7W.mjs';
14
- import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-6M6SDH3S.mjs';
15
- import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-JT3YZGCE.mjs';
16
- import './chunk-NNO7CO3A.mjs';
16
+ import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, decodeIndexConfiguration, prepareStoredSynonyms, arrayPositionsAreStale, insertDocumentVectors } from './chunk-W2XDK66Z.mjs';
17
+ import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-XMQNOD5V.mjs';
18
+ import './chunk-XXKWEG53.mjs';
19
+ import './chunk-UIKN2OTQ.mjs';
17
20
  import './chunk-AV4IJKD2.mjs';
18
- import { decodeEveryStoredDocument } from './chunk-RWESFSJI.mjs';
19
- import { ARRAY_POSITION_GAP } from './chunk-2MKX2SCO.mjs';
20
- import { getLanguage } from './chunk-ZR4VM4A6.mjs';
21
- import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
21
+ import { decodeEveryStoredDocument } from './chunk-IBATGS3J.mjs';
22
+ import { ARRAY_POSITION_GAP } from './chunk-U4X4AWKL.mjs';
23
+ import './chunk-N37VMC62.mjs';
24
+ import './chunk-ZATRDAYU.mjs';
22
25
  import './chunk-EBIHRL7B.mjs';
23
26
  import './chunk-HJSN7UYG.mjs';
24
- import './chunk-RNSC4NFI.mjs';
25
- import './chunk-TGP5M5R7.mjs';
26
- import './chunk-U2BKYEC7.mjs';
27
+ import './chunk-EEBCDP36.mjs';
27
28
  import './chunk-KOKM4PQN.mjs';
28
29
  import './chunk-QSK35UC2.mjs';
29
- import './chunk-2C3WXBQS.mjs';
30
- import './chunk-2SEL6SBS.mjs';
30
+ import './chunk-74XB2SXI.mjs';
31
+ import './chunk-RKAVHNRW.mjs';
31
32
  import './chunk-WQBCBKY3.mjs';
32
33
  import './chunk-JUKYYXSP.mjs';
33
- import { parseVectorDimension } from './chunk-EUCX5K74.mjs';
34
- import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, flattenSchema } from './chunk-PU6SKH7O.mjs';
34
+ import { parseVectorDimension } from './chunk-J3EHATEZ.mjs';
35
+ import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, flattenSchema } from './chunk-UBXJQJII.mjs';
35
36
  import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
36
37
  import './chunk-FQBFZAWG.mjs';
37
38
  import './chunk-VJS4JFBQ.mjs';
38
39
  import './chunk-GES7U2PM.mjs';
39
- import './chunk-7XY6VD7B.mjs';
40
- import { resolveIndexAnalysis } from './chunk-6SSVIW7Y.mjs';
40
+ import './chunk-ZSWHPKGM.mjs';
41
+ import { resolveIndexAnalysis } from './chunk-O4LJQI4G.mjs';
41
42
  import './chunk-X5CV2E2B.mjs';
42
- import './chunk-7X5JST5L.mjs';
43
- import './chunk-D5TFRB5G.mjs';
43
+ import { getLanguage } from './chunk-WNBRW4Q2.mjs';
44
+ import './chunk-TGP5M5R7.mjs';
45
+ import './chunk-U2BKYEC7.mjs';
46
+ import './chunk-NPNK57CA.mjs';
44
47
  import './chunk-4UMZ7KQL.mjs';
45
48
  import './chunk-R6A773WK.mjs';
46
49
  import './chunk-YTJCGKM4.mjs';
47
- import { collectAndReturnPages } from './chunk-RIIUZJQ3.mjs';
50
+ import './chunk-RIIUZJQ3.mjs';
48
51
  import './chunk-UJWGQIXU.mjs';
49
- import { loadVectorCodeFor } from './chunk-HCGJKDZV.mjs';
52
+ import { loadVectorCodeFor } from './chunk-FFVLPCLH.mjs';
50
53
  import './chunk-22YPZVMG.mjs';
51
54
  import './chunk-RFBLQLLZ.mjs';
52
55
  import './chunk-L5ORED7V.mjs';
53
- import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
56
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
54
57
  import { decode } from '@msgpack/msgpack';
55
58
 
56
59
  // src/persistence/archive/validate.ts
@@ -92,161 +95,6 @@ async function validateStagedCheckpoint(store, manifest) {
92
95
  }
93
96
  }
94
97
 
95
- // src/engine/snapshot/constants.ts
96
- var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
97
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
98
- var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
99
- var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
100
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
101
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
102
- var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
103
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
104
- var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
105
- var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
106
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
107
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
108
- var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
109
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
110
- var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
111
- var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
112
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
113
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
114
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
115
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
116
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
117
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
118
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
119
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
120
- var RESTORE_ESTIMATE_MARGIN = 1.25;
121
-
122
- // src/engine/snapshot/memory-budget.ts
123
- var MEGABYTE_BYTES = 1048576;
124
- var ARCHIVE_RATES = {
125
- worst: {
126
- heap: {
127
- perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
128
- perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
129
- perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
130
- },
131
- external: {
132
- perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
133
- perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
134
- perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
135
- }
136
- },
137
- floor: {
138
- heap: {
139
- perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
140
- perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
141
- perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
142
- },
143
- external: {
144
- perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
145
- perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
146
- perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
147
- }
148
- }
149
- };
150
- var EARLIER_SNAPSHOT_RATES = {
151
- worst: {
152
- heap: {
153
- perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
154
- perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
155
- perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
156
- },
157
- external: {
158
- perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
159
- perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
160
- perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
161
- }
162
- },
163
- floor: {
164
- heap: {
165
- perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
166
- perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
167
- perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
168
- },
169
- external: {
170
- perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
171
- perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
172
- perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
173
- }
174
- }
175
- };
176
- function bytesAt(rate, storedBytes, vectors) {
177
- let bytes = storedBytes * rate.perStoredByte;
178
- for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
179
- return bytes;
180
- }
181
- function footprintAt(rates, replacesIndex, storedBytes, vectors) {
182
- const chosen = replacesIndex ? rates.worst : rates.floor;
183
- const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
184
- return {
185
- heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
186
- externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
187
- };
188
- }
189
- function deadCountOf(dead) {
190
- if (dead === null) return 0;
191
- let count = 0;
192
- for (const byte of dead) {
193
- for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
194
- }
195
- return count;
196
- }
197
- function archiveFootprint(prepared, manifest, replacesIndex) {
198
- let segmentBytes = 0;
199
- for (const partition of manifest.partitions) {
200
- for (const segment of partition.segments) segmentBytes += segment.bytes;
201
- }
202
- const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
203
- const vectors = [];
204
- for (const field of manifest.vectors) {
205
- const dimension = dimensions.get(field.fieldPath) ?? 0;
206
- for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
207
- }
208
- return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
209
- }
210
- function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
211
- const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
212
- const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
213
- return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
214
- }
215
- function inMegabytes(bytes) {
216
- return Math.ceil(bytes / MEGABYTE_BYTES);
217
- }
218
- function shortfallOf(heapBytes, processBytes, countInUse) {
219
- const heap = readHeapStatistics();
220
- if (heap !== null) {
221
- const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
222
- const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
223
- if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
224
- }
225
- const resident = readResidentMemory();
226
- if (resident !== null) {
227
- const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
228
- if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
229
- }
230
- return null;
231
- }
232
- function refuse(indexName, replacesIndex, shortfall) {
233
- const { budget, neededBytes, availableBytes } = shortfall;
234
- const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
235
- const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
236
- throw new NarsilError(
237
- ErrorCodes.RESTORE_MEMORY_EXCEEDED,
238
- `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
239
- { indexName, budget, neededBytes, availableBytes }
240
- );
241
- }
242
- function requireRestoreMemory(indexName, footprint, replacesIndex) {
243
- const processBytes = footprint.heapBytes + footprint.externalBytes;
244
- if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
245
- const collected = collectAndReturnPages();
246
- const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
247
- if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
248
- }
249
-
250
98
  // src/workers/detached-index.ts
251
99
  function attachablePolicy(shared) {
252
100
  let attached = false;
@@ -312,12 +160,14 @@ function prepareRestoredIndex(configuration, deps, reject) {
312
160
  }
313
161
  const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
314
162
  const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
163
+ const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
315
164
  return {
316
165
  config,
317
166
  language,
318
167
  embeddingAdapter,
319
168
  adapterName,
320
169
  storedRevision,
170
+ analysisGeneration,
321
171
  storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
322
172
  synonyms
323
173
  };
@@ -352,7 +202,10 @@ function restoredRegistryEntry(prepared, manager, deps) {
352
202
  partitionCount: manager.partitionCount,
353
203
  analysisRevision: prepared.storedRevision,
354
204
  arrayPositionGap: prepared.storedArrayPositionGap,
355
- synonyms: prepared.synonyms?.encoded ?? null
205
+ synonyms: prepared.synonyms?.encoded ?? null,
206
+ analysisGeneration: prepared.analysisGeneration,
207
+ pendingAnalysis: null,
208
+ durableAnalysis: null
356
209
  };
357
210
  }
358
211
 
@@ -448,6 +301,7 @@ function activate(indexName, restored, entry, deps) {
448
301
  async function replaceExisting(indexName, restored, entry, deps) {
449
302
  const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
450
303
  try {
304
+ await deps.abandonAnalysisChange(indexName);
451
305
  await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
452
306
  await deps.indexState.replace(indexName, async (wasOpen) => {
453
307
  await deps.abandonAnalysisRebuild(indexName);
@@ -751,9 +605,24 @@ async function checkpointForArchive(deps) {
751
605
  });
752
606
  }
753
607
  async function createSnapshot(deps) {
754
- const config = await encodeIndexConfiguration(deps.entry);
755
- const checkpoint = await checkpointForArchive(deps);
756
- return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
608
+ for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
609
+ const checkpoint = await checkpointForArchive(deps);
610
+ try {
611
+ if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
612
+ const config = await encodeIndexConfiguration(deps.entry);
613
+ return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
614
+ }
615
+ } catch (error) {
616
+ checkpoint.release();
617
+ throw error;
618
+ }
619
+ checkpoint.release();
620
+ }
621
+ throw new NarsilError(
622
+ ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
623
+ `The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
624
+ { indexName: deps.indexName }
625
+ );
757
626
  }
758
627
  function requireSnapshotSource(data) {
759
628
  if (data instanceof Uint8Array) return data;
package/dist/synonyms.mjs CHANGED
@@ -1,11 +1,13 @@
1
- import { synonymSearchModule } from './chunk-6SSVIW7Y.mjs';
1
+ import { synonymSearchModule } from './chunk-O4LJQI4G.mjs';
2
2
  import './chunk-X5CV2E2B.mjs';
3
- import './chunk-7X5JST5L.mjs';
4
- import './chunk-D5TFRB5G.mjs';
3
+ import './chunk-WNBRW4Q2.mjs';
4
+ import './chunk-TGP5M5R7.mjs';
5
+ import './chunk-U2BKYEC7.mjs';
6
+ import './chunk-NPNK57CA.mjs';
5
7
  import './chunk-4UMZ7KQL.mjs';
6
8
  import './chunk-R6A773WK.mjs';
7
9
  import './chunk-YTJCGKM4.mjs';
8
- import './chunk-CZ7ISLXA.mjs';
10
+ import './chunk-HNHCNTVG.mjs';
9
11
 
10
12
  // src/synonyms/index.ts
11
13
  var synonyms = synonymSearchModule;