@delali/narsil 0.3.1-next.131 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
  4. package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
  5. package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
  6. package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  9. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
  10. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  11. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  12. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
  13. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
  14. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
  15. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
  16. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  17. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  18. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  19. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  20. package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
  21. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
  22. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  23. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  24. package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
  25. package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
  26. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  27. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
  28. package/dist/browser/import-TLB2A5LJ.mjs +9 -0
  29. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  30. package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
  31. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  32. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
  33. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  34. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
  35. package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
  36. package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
  37. package/dist/chunk-46NZOPHS.mjs +21 -0
  38. package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
  39. package/dist/chunk-6SNBQESJ.mjs +214 -0
  40. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  41. package/dist/chunk-7AUASRE7.mjs +653 -0
  42. package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
  43. package/dist/chunk-BOJZGUS6.mjs +177 -0
  44. package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
  45. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  46. package/dist/chunk-CZQ33DLW.mjs +78 -0
  47. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
  49. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  50. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  51. package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
  52. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  53. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  54. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  55. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  56. package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
  57. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  58. package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
  59. package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
  60. package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
  61. package/dist/chunk-L76W3IZJ.mjs +193 -0
  62. package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
  63. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  64. package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
  65. package/dist/chunk-NN33CZRE.mjs +86 -0
  66. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  67. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  68. package/dist/chunk-NRUXBYPH.mjs +121 -0
  69. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  70. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  71. package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
  72. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  73. package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
  74. package/dist/chunk-SRJBFG2T.mjs +881 -0
  75. package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
  76. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  77. package/dist/chunk-UUZCQQG2.mjs +209 -0
  78. package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
  79. package/dist/chunk-VJEGOELZ.mjs +177 -0
  80. package/dist/chunk-W2COJ5AH.mjs +148 -0
  81. package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
  82. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  83. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  84. package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
  85. package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
  86. package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
  87. package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
  88. package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  89. package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
  90. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  91. package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
  92. package/dist/client.d.ts +7 -7
  93. package/dist/client.mjs +11 -1
  94. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  95. package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
  96. package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
  97. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
  98. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  99. package/dist/distribution/coordinator/etcd.mjs +1 -1
  100. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/grpc.mjs +1 -1
  102. package/dist/distribution.d.ts +24 -8
  103. package/dist/distribution.mjs +982 -211
  104. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
  105. package/dist/embeddings/openai.mjs +2 -2
  106. package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
  107. package/dist/import-OOBUW3RR.mjs +44 -0
  108. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  109. package/dist/index.browser.mjs +5 -5
  110. package/dist/index.d.ts +6 -6
  111. package/dist/index.mjs +22 -22
  112. package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
  113. package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
  114. package/dist/module-IYPLWHGC.mjs +48 -0
  115. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  116. package/dist/partial-words.mjs +5 -5
  117. package/dist/pattern.mjs +3 -3
  118. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  119. package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
  120. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  121. package/dist/react.d.ts +9 -9
  122. package/dist/react.mjs +4 -0
  123. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  124. package/dist/rerankers/http.mjs +2 -2
  125. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
  126. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  127. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  128. package/dist/server.d.ts +12 -10
  129. package/dist/server.mjs +54 -21
  130. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
  131. package/dist/synonyms.mjs +6 -4
  132. package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
  133. package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
  134. package/dist/vector/search-worker.d.ts +1 -1
  135. package/dist/vector/search-worker.mjs +6 -5
  136. package/dist/workers/entry.mjs +8 -8
  137. package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
  138. package/openapi.json +114 -6
  139. package/package.json +12 -8
package/dist/server.mjs CHANGED
@@ -1,36 +1,37 @@
1
- import { answerRequest } from './chunk-5W7AEVTH.mjs';
2
- import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, respondAnswer, parseTaskListQuery, parseJsonOptional, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-RCMNL7RI.mjs';
3
- export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-RCMNL7RI.mjs';
4
- import { rerankOnHost } from './chunk-PATMDIMN.mjs';
5
- import { writeBlockLookupOf, indexReadsOf, isRecord, validateIndexName } from './chunk-T4OFG4IQ.mjs';
6
- import { readMemoryFigures } from './chunk-4GDLFDPZ.mjs';
1
+ import { answerRequest } from './chunk-ZMBFWQXN.mjs';
2
+ import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createBodyBudgetBuffer, createRouteRunner, corsHeaderReader, createSharedBodyBudget, MAIN_THREAD_GATE_SLOT, createSharedRequestGate, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_RERANK_WINDOW, DEFAULT_MAX_BODY_BYTES_IN_FLIGHT_FRACTION, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, MAX_BODY_BYTES_IN_FLIGHT_FRACTION, createDocumentReadHandlers, sendError, respondError, respondJson, sendText, PROMETHEUS_TEXT_CONTENT_TYPE, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, closeEveryThreadAtExit, REQUEST_THREAD_CLOSE_AT_EXIT_MS, clearGateSlot, clearBodyBudgetSlot, createCloseAtExitWords, createDepartureWord, departureConfirmed, parseJson, validateCreateIndex, rejectInvalid, isRemoveOnlyBatch, validateBatch, serializeBatchResult, parseWriteJson, validateDocumentBody, validateSuggest, validateQuery, refuseRegex, withServerRerankWindow, respondAnswer, parseTaskListQuery, parseJsonOptional, validateChangeAnalysis, validateRebalance, badRequest, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, parseRefusalOf, IMPORT_FLUSHES_IN_FLIGHT, confirmDeparture, SERVER_CAPABILITIES, toHttpError, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-SBMAWTRG.mjs';
3
+ export { ASYNC_IMPORT_CAPABILITY, BM25_TUNING_CAPABILITY, CHANGE_ANALYSIS_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, SYNONYMS_CAPABILITY, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-SBMAWTRG.mjs';
4
+ import { rerankOnHost } from './chunk-HJJWY6PT.mjs';
5
+ import { writeBlockLookupOf, indexReadsOf, isRecord, validateIndexName } from './chunk-BRUUBXCB.mjs';
6
+ import { readMemoryFigures } from './chunk-DD2673TX.mjs';
7
+ import './chunk-L76W3IZJ.mjs';
7
8
  import './chunk-IWVEG5CX.mjs';
8
9
  import './chunk-DJHHF2L5.mjs';
9
- import './chunk-P36MUFEJ.mjs';
10
- import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-JT3YZGCE.mjs';
10
+ import './chunk-JDDXV6UF.mjs';
11
+ import { rerankWindowCapOf, engineCoreOf, providedDocId, resolveVectorText } from './chunk-5WYVUPGD.mjs';
12
+ import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
11
13
  import { detectRuntime } from './chunk-AV4IJKD2.mjs';
12
- import './chunk-ZR4VM4A6.mjs';
14
+ import './chunk-EEBCDP36.mjs';
15
+ import './chunk-N37VMC62.mjs';
13
16
  import './chunk-ZATRDAYU.mjs';
14
17
  import './chunk-EBIHRL7B.mjs';
15
- import { MAX_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
16
- import './chunk-RNSC4NFI.mjs';
17
- import './chunk-TGP5M5R7.mjs';
18
- import './chunk-U2BKYEC7.mjs';
19
- import './chunk-2C3WXBQS.mjs';
18
+ import './chunk-74XB2SXI.mjs';
20
19
  import './chunk-WQBCBKY3.mjs';
21
- import './chunk-PU6SKH7O.mjs';
20
+ import './chunk-KGOXHLLZ.mjs';
22
21
  import './chunk-U476IXKQ.mjs';
23
22
  import './chunk-FQBFZAWG.mjs';
24
23
  import './chunk-VJS4JFBQ.mjs';
25
- import './chunk-7XY6VD7B.mjs';
26
- import './chunk-7X5JST5L.mjs';
24
+ import './chunk-ZSWHPKGM.mjs';
25
+ import './chunk-WNBRW4Q2.mjs';
26
+ import './chunk-TGP5M5R7.mjs';
27
+ import './chunk-U2BKYEC7.mjs';
27
28
  import './chunk-R6A773WK.mjs';
28
29
  import './chunk-YTJCGKM4.mjs';
29
30
  import { startIdleHeapCleanup } from './chunk-RIIUZJQ3.mjs';
30
31
  import './chunk-22YPZVMG.mjs';
31
32
  import './chunk-L5ORED7V.mjs';
32
- import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-CZ7ISLXA.mjs';
33
- export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-CZ7ISLXA.mjs';
33
+ import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-HNHCNTVG.mjs';
34
+ export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-HNHCNTVG.mjs';
34
35
  import { randomUUID } from 'crypto';
35
36
  import { createReadStream } from 'fs';
36
37
  import { MessageChannel } from 'worker_threads';
@@ -151,6 +152,24 @@ function createAdminHandlers(deps) {
151
152
  const record = await tasks.start("rebuildAnalysis", name, () => engine.rebuildAnalysis(name));
152
153
  respondJson(ctx, record, 202);
153
154
  }
155
+ async function changeAnalysis(ctx) {
156
+ const body = parseJsonOptional(ctx);
157
+ if (!body) return;
158
+ const failure = validateChangeAnalysis(body);
159
+ if (failure) {
160
+ rejectInvalid(ctx, failure);
161
+ return;
162
+ }
163
+ const name = ctx.params[0];
164
+ try {
165
+ await requireIndexExists(name);
166
+ } catch (err) {
167
+ respondError(ctx, err);
168
+ return;
169
+ }
170
+ const record = await tasks.start("changeAnalysis", name, () => engine.changeAnalysis(name, body));
171
+ respondJson(ctx, record, 202);
172
+ }
154
173
  async function rebalance(ctx) {
155
174
  const body = parseJsonOptional(ctx);
156
175
  if (!body) return;
@@ -250,6 +269,7 @@ function createAdminHandlers(deps) {
250
269
  optimize,
251
270
  rebalance,
252
271
  rebuildAnalysis,
272
+ changeAnalysis,
253
273
  restore,
254
274
  getTask,
255
275
  listTasks,
@@ -767,15 +787,27 @@ function mapHttpIndexConfig(http) {
767
787
  if (http.defaultScoring !== void 0) config.defaultScoring = http.defaultScoring;
768
788
  if (http.bm25 !== void 0) config.bm25 = http.bm25;
769
789
  if (http.trackPositions !== void 0) config.trackPositions = http.trackPositions;
790
+ if (http.textStorage !== void 0) config.textStorage = http.textStorage;
770
791
  if (http.surfaceForms !== void 0) config.surfaceForms = http.surfaceForms === true;
771
792
  if (http.vectorPromotion !== void 0) config.vectorPromotion = http.vectorPromotion;
772
793
  if (http.strict !== void 0) config.strict = http.strict;
773
794
  if (http.required !== void 0) config.required = http.required;
774
795
  if (http.patternValueLimit !== void 0) config.patternValueLimit = http.patternValueLimit;
775
796
  if (http.synonyms !== void 0) config.synonyms = http.synonyms;
776
- if (http.stopWords !== void 0) {
797
+ if (http.tokenizer !== void 0) {
798
+ if (typeof http.tokenizer !== "string") {
799
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Field "tokenizer" must be a registered name');
800
+ }
801
+ config.tokenizer = http.tokenizer;
802
+ }
803
+ if (typeof http.stopWords === "string") {
804
+ config.stopWords = http.stopWords;
805
+ } else if (http.stopWords !== void 0) {
777
806
  if (!Array.isArray(http.stopWords) || http.stopWords.some((w) => typeof w !== "string")) {
778
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'Field "stopWords" must be an array of strings');
807
+ throw new NarsilError(
808
+ ErrorCodes.CONFIG_INVALID,
809
+ 'Field "stopWords" must be a registered name or an array of strings'
810
+ );
779
811
  }
780
812
  config.stopWords = new Set(http.stopWords);
781
813
  }
@@ -1718,6 +1750,7 @@ function mainThreadHandlers(deps) {
1718
1750
  multiGet: doc.multiGet,
1719
1751
  checkpoint: admin.checkpoint,
1720
1752
  rebuildAnalysis: admin.rebuildAnalysis,
1753
+ changeAnalysis: admin.changeAnalysis,
1721
1754
  snapshot: admin.snapshot,
1722
1755
  vectorMaintenance: admin.vectorMaintenance,
1723
1756
  compact: admin.compact,
@@ -1,56 +1,63 @@
1
- import { documentsOfPayloadV2 } from './chunk-JJ6EUMXB.mjs';
2
- import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive } from './chunk-WVRDRLIC.mjs';
3
- import './chunk-SOHMMNCH.mjs';
4
- import { indexMetadataOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, readVectorFile, readGraphParts, deserializePayloadV1, SINGLE_NODE_PRIMARY_TERM } from './chunk-K77FJXKS.mjs';
5
- import { decodeVectorIndexParts, bytesToVectors } from './chunk-Y2U5VVWX.mjs';
6
- import './chunk-X5CNUKD5.mjs';
7
- import './chunk-DJHHF2L5.mjs';
1
+ import { documentsOfPayloadV2 } from './chunk-NJJCPFXN.mjs';
2
+ import { buildArchiveFromMemory, archiveStream, memoryCheckpointStore, readSnapshotArchive, writeArchiveFile } from './chunk-KZL72VTC.mjs';
3
+ import { indexMetadataOf, unpackIndexSnapshotEnvelope, loadSegmentedCheckpoint, writeMetadataEnvelope, deserializePayloadV1, SINGLE_NODE_PRIMARY_TERM } from './chunk-YHUTF6EA.mjs';
4
+ import { readVectorFile, readGraphParts } from './chunk-NRUXBYPH.mjs';
5
+ import { createByteReader } from './chunk-CZQ33DLW.mjs';
6
+ import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-L76W3IZJ.mjs';
7
+ import './chunk-W2COJ5AH.mjs';
8
+ import './chunk-Z4HFMYNX.mjs';
9
+ import { decodeVectorIndexParts, bytesToVectors } from './chunk-ICW45TTU.mjs';
10
+ import { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, requireRestoreMemory, earlierSnapshotFootprint, storedBytesOfSegments, archiveFootprint } from './chunk-VJEGOELZ.mjs';
11
+ import './chunk-TNBIK2LQ.mjs';
8
12
  import './chunk-37CD3OKS.mjs';
9
- import { manifestKey, encodeSegmentManifest, manifestFileRefs, readSegmentFile } from './chunk-2JWNJSNE.mjs';
10
- import './chunk-7OIQ2SIO.mjs';
11
- import './chunk-P36MUFEJ.mjs';
13
+ import './chunk-DJHHF2L5.mjs';
14
+ import './chunk-LA4WNDHL.mjs';
15
+ import './chunk-JDDXV6UF.mjs';
16
+ import { readSegmentFile } from './chunk-7AUASRE7.mjs';
17
+ import { manifestKey, encodeSegmentManifest, manifestFileRefs } from './chunk-UUZCQQG2.mjs';
18
+ import './chunk-KEIYVZUX.mjs';
12
19
  import './chunk-YH5OIL7B.mjs';
13
- import { encodeIndexConfiguration, createByteReader, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, decodeIndexConfiguration, prepareStoredSynonyms, arrayPositionsAreStale, insertDocumentVectors } from './chunk-TGHNXN7W.mjs';
14
- import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-6M6SDH3S.mjs';
15
- import { restoredConfigFields, restoredEmbedding, rejectSnapshot } from './chunk-JT3YZGCE.mjs';
16
- import './chunk-NNO7CO3A.mjs';
20
+ import { encodeIndexConfiguration, hasNrslMagic, HEADER_SIZE, readHeader, validateRequiredFieldsInSchema, validateEmbeddingConfig, storedArrayPositionGap, decodeIndexConfiguration, prepareStoredSynonyms, arrayPositionsAreStale, insertDocumentVectors } from './chunk-IO5F7ZBE.mjs';
21
+ import { validatePatternValueLimit, requireSurfaceFormsForPartialFields } from './chunk-VEFN6X6Q.mjs';
22
+ import './chunk-5WYVUPGD.mjs';
23
+ import './chunk-HJSN7UYG.mjs';
24
+ import './chunk-UIKN2OTQ.mjs';
17
25
  import './chunk-AV4IJKD2.mjs';
18
- import { decodeEveryStoredDocument } from './chunk-RWESFSJI.mjs';
19
- import { ARRAY_POSITION_GAP } from './chunk-2MKX2SCO.mjs';
20
- import { getLanguage } from './chunk-ZR4VM4A6.mjs';
21
- import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
26
+ import './chunk-EEBCDP36.mjs';
27
+ import { decodeEveryStoredDocument } from './chunk-WJGO5PGV.mjs';
28
+ import { ARRAY_POSITION_GAP, isCompositePartition } from './chunk-ZPWS2SKJ.mjs';
29
+ import './chunk-N37VMC62.mjs';
30
+ import './chunk-ZATRDAYU.mjs';
22
31
  import './chunk-EBIHRL7B.mjs';
23
- import './chunk-HJSN7UYG.mjs';
24
- import './chunk-RNSC4NFI.mjs';
25
- import './chunk-TGP5M5R7.mjs';
26
- import './chunk-U2BKYEC7.mjs';
27
32
  import './chunk-KOKM4PQN.mjs';
28
33
  import './chunk-QSK35UC2.mjs';
29
- import './chunk-2C3WXBQS.mjs';
30
- import './chunk-2SEL6SBS.mjs';
34
+ import './chunk-74XB2SXI.mjs';
35
+ import './chunk-RKAVHNRW.mjs';
31
36
  import './chunk-WQBCBKY3.mjs';
32
37
  import './chunk-JUKYYXSP.mjs';
33
- import { parseVectorDimension } from './chunk-EUCX5K74.mjs';
34
- import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, extractVectorFieldsFromSchema, flattenSchema } from './chunk-PU6SKH7O.mjs';
38
+ import { parseVectorDimension } from './chunk-DQNKVOJ2.mjs';
39
+ import { raiseEarlierReleaseGraphSetting, validateSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, flattenSchema } from './chunk-KGOXHLLZ.mjs';
35
40
  import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
36
41
  import './chunk-FQBFZAWG.mjs';
37
42
  import './chunk-VJS4JFBQ.mjs';
38
43
  import './chunk-GES7U2PM.mjs';
39
- import './chunk-7XY6VD7B.mjs';
40
- import { resolveIndexAnalysis } from './chunk-6SSVIW7Y.mjs';
44
+ import './chunk-ZSWHPKGM.mjs';
45
+ import { resolveIndexAnalysis } from './chunk-O4LJQI4G.mjs';
41
46
  import './chunk-X5CV2E2B.mjs';
42
- import './chunk-7X5JST5L.mjs';
43
- import './chunk-D5TFRB5G.mjs';
47
+ import { getLanguage } from './chunk-WNBRW4Q2.mjs';
48
+ import './chunk-TGP5M5R7.mjs';
49
+ import './chunk-U2BKYEC7.mjs';
50
+ import './chunk-NPNK57CA.mjs';
44
51
  import './chunk-4UMZ7KQL.mjs';
45
52
  import './chunk-R6A773WK.mjs';
46
53
  import './chunk-YTJCGKM4.mjs';
47
- import { collectAndReturnPages } from './chunk-RIIUZJQ3.mjs';
48
- import './chunk-UJWGQIXU.mjs';
49
- import { loadVectorCodeFor } from './chunk-HCGJKDZV.mjs';
54
+ import './chunk-RIIUZJQ3.mjs';
55
+ import './chunk-QHPYS427.mjs';
56
+ import { loadVectorCodeFor } from './chunk-GLZHNSGB.mjs';
50
57
  import './chunk-22YPZVMG.mjs';
51
58
  import './chunk-RFBLQLLZ.mjs';
52
59
  import './chunk-L5ORED7V.mjs';
53
- import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
60
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
54
61
  import { decode } from '@msgpack/msgpack';
55
62
 
56
63
  // src/persistence/archive/validate.ts
@@ -76,6 +83,14 @@ function requireSchemaOfManifest(key, schema, manifest) {
76
83
  async function validateStagedCheckpoint(store, manifest) {
77
84
  for (const partition of manifest.partitions) {
78
85
  for (const ref of partition.segments) {
86
+ const { directory } = store;
87
+ if (directory !== void 0) {
88
+ const { checkPagedSegmentFully, isPagedSegmentFile } = await import('./module-IYPLWHGC.mjs');
89
+ if (await isPagedSegmentFile(directory, ref.key)) {
90
+ requireSchemaOfManifest(ref.key, await checkPagedSegmentFully(directory, ref), manifest);
91
+ continue;
92
+ }
93
+ }
79
94
  const contents = await readSegmentFile(store, ref);
80
95
  requireSchemaOfManifest(ref.key, contents.schema, manifest);
81
96
  decodeEveryStoredDocument(contents.arrays.documentTable);
@@ -92,161 +107,6 @@ async function validateStagedCheckpoint(store, manifest) {
92
107
  }
93
108
  }
94
109
 
95
- // src/engine/snapshot/constants.ts
96
- var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
97
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
98
- var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
99
- var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
100
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
101
- var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
102
- var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
103
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
104
- var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
105
- var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
106
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
107
- var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
108
- var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
109
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
110
- var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
111
- var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
112
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
113
- var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
114
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
115
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
116
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
117
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
118
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
119
- var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
120
- var RESTORE_ESTIMATE_MARGIN = 1.25;
121
-
122
- // src/engine/snapshot/memory-budget.ts
123
- var MEGABYTE_BYTES = 1048576;
124
- var ARCHIVE_RATES = {
125
- worst: {
126
- heap: {
127
- perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
128
- perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
129
- perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
130
- },
131
- external: {
132
- perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
133
- perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
134
- perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
135
- }
136
- },
137
- floor: {
138
- heap: {
139
- perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
140
- perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
141
- perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
142
- },
143
- external: {
144
- perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
145
- perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
146
- perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
147
- }
148
- }
149
- };
150
- var EARLIER_SNAPSHOT_RATES = {
151
- worst: {
152
- heap: {
153
- perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
154
- perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
155
- perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
156
- },
157
- external: {
158
- perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
159
- perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
160
- perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
161
- }
162
- },
163
- floor: {
164
- heap: {
165
- perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
166
- perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
167
- perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
168
- },
169
- external: {
170
- perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
171
- perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
172
- perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
173
- }
174
- }
175
- };
176
- function bytesAt(rate, storedBytes, vectors) {
177
- let bytes = storedBytes * rate.perStoredByte;
178
- for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
179
- return bytes;
180
- }
181
- function footprintAt(rates, replacesIndex, storedBytes, vectors) {
182
- const chosen = replacesIndex ? rates.worst : rates.floor;
183
- const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
184
- return {
185
- heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
186
- externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
187
- };
188
- }
189
- function deadCountOf(dead) {
190
- if (dead === null) return 0;
191
- let count = 0;
192
- for (const byte of dead) {
193
- for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
194
- }
195
- return count;
196
- }
197
- function archiveFootprint(prepared, manifest, replacesIndex) {
198
- let segmentBytes = 0;
199
- for (const partition of manifest.partitions) {
200
- for (const segment of partition.segments) segmentBytes += segment.bytes;
201
- }
202
- const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
203
- const vectors = [];
204
- for (const field of manifest.vectors) {
205
- const dimension = dimensions.get(field.fieldPath) ?? 0;
206
- for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
207
- }
208
- return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
209
- }
210
- function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
211
- const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
212
- const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
213
- return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
214
- }
215
- function inMegabytes(bytes) {
216
- return Math.ceil(bytes / MEGABYTE_BYTES);
217
- }
218
- function shortfallOf(heapBytes, processBytes, countInUse) {
219
- const heap = readHeapStatistics();
220
- if (heap !== null) {
221
- const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
222
- const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
223
- if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
224
- }
225
- const resident = readResidentMemory();
226
- if (resident !== null) {
227
- const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
228
- if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
229
- }
230
- return null;
231
- }
232
- function refuse(indexName, replacesIndex, shortfall) {
233
- const { budget, neededBytes, availableBytes } = shortfall;
234
- const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
235
- const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
236
- throw new NarsilError(
237
- ErrorCodes.RESTORE_MEMORY_EXCEEDED,
238
- `The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
239
- { indexName, budget, neededBytes, availableBytes }
240
- );
241
- }
242
- function requireRestoreMemory(indexName, footprint, replacesIndex) {
243
- const processBytes = footprint.heapBytes + footprint.externalBytes;
244
- if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
245
- const collected = collectAndReturnPages();
246
- const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
247
- if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
248
- }
249
-
250
110
  // src/workers/detached-index.ts
251
111
  function attachablePolicy(shared) {
252
112
  let attached = false;
@@ -286,6 +146,7 @@ function prepareRestoredIndex(configuration, deps, reject) {
286
146
  requireSurfaceFormsForPartialFields(schema, configFields.surfaceForms);
287
147
  validateVectorPromotion(configFields.vectorPromotion);
288
148
  validateVectorStorage(configFields.vectorPromotion, deps.filesystemDurability);
149
+ validateTextStorage(configFields.textStorage, deps.filesystemDurability);
289
150
  if (configFields.required !== void 0 && configFields.required.length > 0) {
290
151
  validateRequiredFieldsInSchema(configFields.required, schema);
291
152
  }
@@ -312,12 +173,14 @@ function prepareRestoredIndex(configuration, deps, reject) {
312
173
  }
313
174
  const storedRevision = typeof configuration.analysisRevision === "string" ? configuration.analysisRevision : null;
314
175
  const synonyms = restoredSynonyms(configuration.synonyms, config, language, reject ?? rejectSnapshot);
176
+ const analysisGeneration = Number.isSafeInteger(configuration.analysisGeneration) ? Math.max(0, configuration.analysisGeneration) : 0;
315
177
  return {
316
178
  config,
317
179
  language,
318
180
  embeddingAdapter,
319
181
  adapterName,
320
182
  storedRevision,
183
+ analysisGeneration,
321
184
  storedArrayPositionGap: storedArrayPositionGap(configuration.arrayPositionGap),
322
185
  synonyms
323
186
  };
@@ -352,7 +215,10 @@ function restoredRegistryEntry(prepared, manager, deps) {
352
215
  partitionCount: manager.partitionCount,
353
216
  analysisRevision: prepared.storedRevision,
354
217
  arrayPositionGap: prepared.storedArrayPositionGap,
355
- synonyms: prepared.synonyms?.encoded ?? null
218
+ synonyms: prepared.synonyms?.encoded ?? null,
219
+ analysisGeneration: prepared.analysisGeneration,
220
+ pendingAnalysis: null,
221
+ durableAnalysis: null
356
222
  };
357
223
  }
358
224
 
@@ -426,7 +292,14 @@ async function commitToStorage(indexName, restored, previousPartitions, deps) {
426
292
  }
427
293
  await staging.finish();
428
294
  const relocate = await staging.relocation().catch(() => null);
429
- if (relocate !== null) for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
295
+ if (relocate === null) return;
296
+ for (const vectorIndex of vectorIndexes) vectorIndex.relocateVectorFiles(relocate);
297
+ const { manager } = restored.detached;
298
+ for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) {
299
+ const partition = manager.getPartition(partitionId);
300
+ if (!isCompositePartition(partition)) continue;
301
+ for (const segment of partition.frozenSegments()) segment.paged?.relocate(relocate);
302
+ }
430
303
  }
431
304
  function activate(indexName, restored, entry, deps) {
432
305
  deps.executor.installDetachedIndex(restored.detached);
@@ -448,6 +321,7 @@ function activate(indexName, restored, entry, deps) {
448
321
  async function replaceExisting(indexName, restored, entry, deps) {
449
322
  const release = { reloads: false, held: [], settled: Promise.resolve(), timedOut: false };
450
323
  try {
324
+ await deps.abandonAnalysisChange(indexName);
451
325
  await withinLimit(indexName, release, releaseCopies(indexName, release, deps), deps);
452
326
  await deps.indexState.replace(indexName, async (wasOpen) => {
453
327
  await deps.abandonAnalysisRebuild(indexName);
@@ -548,6 +422,9 @@ function partitionCountOf(manifest) {
548
422
  function missingPart(what) {
549
423
  throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The archive holds no ${what}`);
550
424
  }
425
+ function keepsTextOnDisk(prepared, staging) {
426
+ return prepared.config.textStorage === "disk" && staging.directory !== void 0;
427
+ }
551
428
  async function stageArchive(indexName, reader, staging, deps) {
552
429
  const staged = { prepared: null, manifest: null };
553
430
  await readSnapshotArchive(reader, {
@@ -558,17 +435,29 @@ async function stageArchive(indexName, reader, staging, deps) {
558
435
  if (staged.prepared === null) missingPart("configuration");
559
436
  requireSameSchema(staged.prepared, manifest2);
560
437
  const replacesIndex = deps.indexRegistry.has(indexName);
561
- requireRestoreMemory(indexName, archiveFootprint(staged.prepared, manifest2, replacesIndex), replacesIndex);
438
+ const segmentBytes = keepsTextOnDisk(staged.prepared, staging) ? 0 : storedBytesOfSegments(manifest2);
439
+ requireRestoreMemory(
440
+ indexName,
441
+ archiveFootprint(staged.prepared, manifest2, replacesIndex, segmentBytes),
442
+ replacesIndex
443
+ );
562
444
  staged.manifest = renamed(manifest2, sourceName, indexName);
563
445
  },
564
- file: async (entryName, _ref, bytes) => {
446
+ file: async (entryName, _ref, chunks) => {
565
447
  if (staged.manifest === null) missingPart("manifest");
566
- await staging.write(`${indexName}/${entryName}`, [bytes]);
448
+ await writeArchiveFile(staging, `${indexName}/${entryName}`, chunks);
567
449
  }
568
450
  });
569
451
  const { prepared, manifest } = staged;
570
452
  if (prepared === null) missingPart("configuration");
571
453
  if (manifest === null) missingPart("manifest");
454
+ const { directory } = staging;
455
+ if (keepsTextOnDisk(prepared, staging) && directory !== void 0) {
456
+ const { bytesThatSegmentsHoldInMemory } = await import('./module-IYPLWHGC.mjs');
457
+ const replacesIndex = deps.indexRegistry.has(indexName);
458
+ const segmentBytes = await bytesThatSegmentsHoldInMemory(directory, manifest);
459
+ requireRestoreMemory(indexName, archiveFootprint(prepared, manifest, replacesIndex, segmentBytes), replacesIndex);
460
+ }
572
461
  await validateStagedCheckpoint(staging, manifest);
573
462
  return { prepared, manifest };
574
463
  }
@@ -751,9 +640,24 @@ async function checkpointForArchive(deps) {
751
640
  });
752
641
  }
753
642
  async function createSnapshot(deps) {
754
- const config = await encodeIndexConfiguration(deps.entry);
755
- const checkpoint = await checkpointForArchive(deps);
756
- return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
643
+ for (let attempt = 0; attempt < MAX_SNAPSHOT_ANALYSIS_ATTEMPTS; attempt += 1) {
644
+ const checkpoint = await checkpointForArchive(deps);
645
+ try {
646
+ if ((checkpoint.manifest.analysisGeneration ?? 0) === deps.entry.analysisGeneration) {
647
+ const config = await encodeIndexConfiguration(deps.entry);
648
+ return archiveStream({ ...checkpoint, config, sourceIndexName: deps.indexName });
649
+ }
650
+ } catch (error) {
651
+ checkpoint.release();
652
+ throw error;
653
+ }
654
+ checkpoint.release();
655
+ }
656
+ throw new NarsilError(
657
+ ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
658
+ `The analysis of index "${deps.indexName}" changed while the engine wrote its snapshot, so try the snapshot again once the change has a durable checkpoint`,
659
+ { indexName: deps.indexName }
660
+ );
757
661
  }
758
662
  function requireSnapshotSource(data) {
759
663
  if (data instanceof Uint8Array) return data;
package/dist/synonyms.mjs CHANGED
@@ -1,11 +1,13 @@
1
- import { synonymSearchModule } from './chunk-6SSVIW7Y.mjs';
1
+ import { synonymSearchModule } from './chunk-O4LJQI4G.mjs';
2
2
  import './chunk-X5CV2E2B.mjs';
3
- import './chunk-7X5JST5L.mjs';
4
- import './chunk-D5TFRB5G.mjs';
3
+ import './chunk-WNBRW4Q2.mjs';
4
+ import './chunk-TGP5M5R7.mjs';
5
+ import './chunk-U2BKYEC7.mjs';
6
+ import './chunk-NPNK57CA.mjs';
5
7
  import './chunk-4UMZ7KQL.mjs';
6
8
  import './chunk-R6A773WK.mjs';
7
9
  import './chunk-YTJCGKM4.mjs';
8
- import './chunk-CZ7ISLXA.mjs';
10
+ import './chunk-HNHCNTVG.mjs';
9
11
 
10
12
  // src/synonyms/index.ts
11
13
  var synonyms = synonymSearchModule;
@@ -1,8 +1,8 @@
1
- import { I as IndexInfo, P as PartitionStatsResult, V as VectorMaintenanceResult, M as MemoryStats } from './config-D1cDv3sE.js';
2
- import { A as AnyDocument, I as InsertOptions, B as BM25Params, W as WriteOptions } from './schema-BTXbHCqA.js';
1
+ import { I as IndexInfo, P as PartitionStatsResult, V as VectorMaintenanceResult, M as MemoryStats } from './config-BhZxIeb1.js';
2
+ import { A as AnyDocument, I as InsertOptions, B as BM25Params, W as WriteOptions } from './schema-B-Runv0n.js';
3
3
  import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
4
- import { N as NodeRole, c as NodeMemoryReport } from './index-HYnqS1SE.js';
5
- import { a as ClusterNodeInfo, N as NodeReadiness } from './config-CmYJO34X.js';
4
+ import { N as NodeRole, c as NodeMemoryReport } from './index-DsEurhSl.js';
5
+ import { a as ClusterNodeInfo, N as NodeReadiness } from './config-CIG5yDyg.js';
6
6
 
7
7
  /**
8
8
  * The body of `POST /indexes`, which the server sends once the index exists.
@@ -425,6 +425,20 @@ interface BatchBody {
425
425
  /** These settings apply to the whole batch. */
426
426
  options?: InsertOptions;
427
427
  }
428
+ /**
429
+ * The body of `POST /indexes/{name}/_change-analysis`, where a key that you
430
+ * leave out keeps its current value.
431
+ *
432
+ * @public
433
+ */
434
+ interface ChangeAnalysisBody {
435
+ /** This is the name of a language registered on the server. */
436
+ language?: string;
437
+ /** This is a name registered on the server, or `null` for the language default. */
438
+ tokenizer?: string | null;
439
+ /** This is a name registered on the server, a list of words, or `null` for the language default. */
440
+ stopWords?: string | string[] | null;
441
+ }
428
442
  /**
429
443
  * The body of `POST /indexes/{name}/_rebalance`.
430
444
  *
@@ -459,10 +473,14 @@ interface HttpIndexConfig {
459
473
  defaultScoring?: 'local' | 'dfs' | 'broadcast';
460
474
  /** These parameters tune BM25 for this index and for each of its text fields. */
461
475
  bm25?: BM25Params;
462
- /** This list replaces the language's stop words. */
463
- stopWords?: string[];
476
+ /** This is the name of a tokeniser registered on the server, which then splits text in place of the language's own rules. */
477
+ tokenizer?: string;
478
+ /** This name of a stop word set registered on the server, or this list, replaces the language's stop words. */
479
+ stopWords?: string | string[];
464
480
  /** Setting this records term positions in each posting, which the `.nrsl` format carries for readers that match phrases. Highlighting works without it. */
465
481
  trackPositions?: boolean;
482
+ /** Setting this to `disk` lets the index answer from segment files that the server reads by position, which needs filesystem durability. The default is `memory`. */
483
+ textStorage?: 'memory' | 'disk';
466
484
  /** Setting this to false returns index stems from suggestions and prefix expansion, instead of the words users typed. */
467
485
  surfaceForms?: boolean;
468
486
  /** These settings control when vector fields move to an HNSW graph, and how that graph is built. */
@@ -551,7 +569,7 @@ interface MemoryStatsResponse extends MemoryStats {
551
569
  *
552
570
  * @public
553
571
  */
554
- type TaskType = 'optimizeVectors' | 'rebalance' | 'restore' | 'import' | 'importExport' | 'rebuildAnalysis';
572
+ type TaskType = 'optimizeVectors' | 'rebalance' | 'restore' | 'import' | 'importExport' | 'rebuildAnalysis' | 'changeAnalysis';
555
573
  /**
556
574
  * Where a long-running operation stands.
557
575
  *
@@ -701,4 +719,4 @@ interface TaskStore {
701
719
  shutdown?(): Promise<void>;
702
720
  }
703
721
 
704
- export type { IndexStateResponse as A, BatchBody as B, CapabilitiesResponse as C, DocumentBody as D, ErrorDetail as E, MultiGetResponse as F, ReadinessResponse as G, HttpIndexConfig as H, ImportError as I, ReplicationFactorResponse as J, VectorMaintenanceResponse as K, LivenessResponse as L, MemoryStatsResponse as M, VersionResponse as N, OkResponse as O, BatchFailure as P, ClusterMemberReport as Q, RebalanceBody as R, SynonymSetBody as S, TaskListPage as T, ClusterNodeReport as U, VectorFieldBody as V, PartitionAllocationReport as W, CreateIndexEmbedding as a, ImportResult as b, TaskListQuery as c, TaskProgress as d, TaskRecord as e, TaskStatus as f, TaskType as g, TaskStore as h, CreateIndexRequest as i, InsertBody as j, MultiGetBody as k, ReplicationFactorBody as l, BatchResponse as m, ClusterAllocationResponse as n, ClusterTopologyResponse as o, DocumentCountResponse as p, DocumentExistsResponse as q, DocumentIdResponse as r, DocumentPutResponse as s, DocumentRemovedResponse as t, DocumentResponse as u, IndexClearedResponse as v, IndexCreatedResponse as w, IndexDroppedResponse as x, IndexListResponse as y, IndexPartitionsResponse as z };
722
+ export type { IndexPartitionsResponse as A, BatchBody as B, CapabilitiesResponse as C, DocumentBody as D, ErrorDetail as E, IndexStateResponse as F, MultiGetResponse as G, HttpIndexConfig as H, ImportError as I, ReadinessResponse as J, ReplicationFactorResponse as K, LivenessResponse as L, MemoryStatsResponse as M, VectorMaintenanceResponse as N, OkResponse as O, VersionResponse as P, BatchFailure as Q, RebalanceBody as R, SynonymSetBody as S, TaskListPage as T, ClusterMemberReport as U, VectorFieldBody as V, ClusterNodeReport as W, PartitionAllocationReport as X, ChangeAnalysisBody as a, CreateIndexEmbedding as b, ImportResult as c, TaskListQuery as d, TaskProgress as e, TaskRecord as f, TaskStatus as g, TaskType as h, TaskStore as i, CreateIndexRequest as j, InsertBody as k, MultiGetBody as l, ReplicationFactorBody as m, BatchResponse as n, ClusterAllocationResponse as o, ClusterTopologyResponse as p, DocumentCountResponse as q, DocumentExistsResponse as r, DocumentIdResponse as s, DocumentPutResponse as t, DocumentRemovedResponse as u, DocumentResponse as v, IndexClearedResponse as w, IndexCreatedResponse as x, IndexDroppedResponse as y, IndexListResponse as z };