@delali/narsil 0.3.1-next.131 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
  4. package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
  5. package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
  6. package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
  7. package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
  8. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  9. package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
  10. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  11. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  12. package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
  13. package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
  14. package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
  15. package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
  16. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  17. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  18. package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
  19. package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
  20. package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
  21. package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
  22. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  23. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  24. package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
  25. package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
  26. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  27. package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
  28. package/dist/browser/import-TLB2A5LJ.mjs +9 -0
  29. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  30. package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
  31. package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
  32. package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
  33. package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
  34. package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
  35. package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
  36. package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
  37. package/dist/chunk-46NZOPHS.mjs +21 -0
  38. package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
  39. package/dist/chunk-6SNBQESJ.mjs +214 -0
  40. package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
  41. package/dist/chunk-7AUASRE7.mjs +653 -0
  42. package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
  43. package/dist/chunk-BOJZGUS6.mjs +177 -0
  44. package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
  45. package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
  46. package/dist/chunk-CZQ33DLW.mjs +78 -0
  47. package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
  48. package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
  49. package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
  50. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  51. package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
  52. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  53. package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
  54. package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
  55. package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
  56. package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
  57. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  58. package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
  59. package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
  60. package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
  61. package/dist/chunk-L76W3IZJ.mjs +193 -0
  62. package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
  63. package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
  64. package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
  65. package/dist/chunk-NN33CZRE.mjs +86 -0
  66. package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
  67. package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
  68. package/dist/chunk-NRUXBYPH.mjs +121 -0
  69. package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
  70. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  71. package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
  72. package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
  73. package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
  74. package/dist/chunk-SRJBFG2T.mjs +881 -0
  75. package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
  76. package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
  77. package/dist/chunk-UUZCQQG2.mjs +209 -0
  78. package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
  79. package/dist/chunk-VJEGOELZ.mjs +177 -0
  80. package/dist/chunk-W2COJ5AH.mjs +148 -0
  81. package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
  82. package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
  83. package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
  84. package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
  85. package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
  86. package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
  87. package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
  88. package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
  89. package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
  90. package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
  91. package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
  92. package/dist/client.d.ts +7 -7
  93. package/dist/client.mjs +11 -1
  94. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  95. package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
  96. package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
  97. package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
  98. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  99. package/dist/distribution/coordinator/etcd.mjs +1 -1
  100. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/grpc.mjs +1 -1
  102. package/dist/distribution.d.ts +24 -8
  103. package/dist/distribution.mjs +982 -211
  104. package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
  105. package/dist/embeddings/openai.mjs +2 -2
  106. package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
  107. package/dist/import-OOBUW3RR.mjs +44 -0
  108. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  109. package/dist/index.browser.mjs +5 -5
  110. package/dist/index.d.ts +6 -6
  111. package/dist/index.mjs +22 -22
  112. package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
  113. package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
  114. package/dist/module-IYPLWHGC.mjs +48 -0
  115. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  116. package/dist/partial-words.mjs +5 -5
  117. package/dist/pattern.mjs +3 -3
  118. package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
  119. package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
  120. package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
  121. package/dist/react.d.ts +9 -9
  122. package/dist/react.mjs +4 -0
  123. package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
  124. package/dist/rerankers/http.mjs +2 -2
  125. package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
  126. package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
  127. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  128. package/dist/server.d.ts +12 -10
  129. package/dist/server.mjs +54 -21
  130. package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
  131. package/dist/synonyms.mjs +6 -4
  132. package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
  133. package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
  134. package/dist/vector/search-worker.d.ts +1 -1
  135. package/dist/vector/search-worker.mjs +6 -5
  136. package/dist/workers/entry.mjs +8 -8
  137. package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
  138. package/openapi.json +114 -6
  139. package/package.json +12 -8
@@ -1,27 +1,27 @@
1
- import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, lazyQueryBinding, executeQuery, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-6M6SDH3S.mjs';
2
- import { resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-JT3YZGCE.mjs';
3
- import { splitWorkerBudget, resolveWorkerCount, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-NNO7CO3A.mjs';
1
+ import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-VEFN6X6Q.mjs';
2
+ import { resolveVectorText, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-5WYVUPGD.mjs';
3
+ import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
4
+ import { splitWorkerBudget, resolveWorkerCount, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-UIKN2OTQ.mjs';
4
5
  import { detectRuntime } from './chunk-AV4IJKD2.mjs';
5
- import { isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, resolvePartitionInsertOptions, createFrozenSegment } from './chunk-RWESFSJI.mjs';
6
- import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, mergePartitionStats, createPartitionIndex, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-2MKX2SCO.mjs';
7
- import { generateId, getLanguage } from './chunk-ZR4VM4A6.mjs';
6
+ import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-EEBCDP36.mjs';
7
+ import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-WJGO5PGV.mjs';
8
+ import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, createPartitionIndex, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-ZPWS2SKJ.mjs';
9
+ import { generateId } from './chunk-N37VMC62.mjs';
8
10
  import { readHeapStatistics } from './chunk-ZATRDAYU.mjs';
9
11
  import { countEngineThread } from './chunk-EBIHRL7B.mjs';
10
- import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
11
- import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-RNSC4NFI.mjs';
12
- import { requireSearchBackendSettings } from './chunk-2C3WXBQS.mjs';
13
- import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-2SEL6SBS.mjs';
14
- import { getNestedValue } from './chunk-EUCX5K74.mjs';
15
- import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-PU6SKH7O.mjs';
12
+ import { requireSearchBackendSettings } from './chunk-74XB2SXI.mjs';
13
+ import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-RKAVHNRW.mjs';
14
+ import { getNestedValue } from './chunk-DQNKVOJ2.mjs';
15
+ import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KGOXHLLZ.mjs';
16
16
  import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-U476IXKQ.mjs';
17
- import { isSingleStringFieldType } from './chunk-7XY6VD7B.mjs';
18
- import { withOpenIndex, oneAtATime } from './chunk-6SSVIW7Y.mjs';
17
+ import { isSingleStringFieldType } from './chunk-ZSWHPKGM.mjs';
18
+ import { withOpenIndex, oneAtATime } from './chunk-O4LJQI4G.mjs';
19
+ import { getLanguage } from './chunk-WNBRW4Q2.mjs';
19
20
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
20
- import { createRequestId, isValidWorkerResponse } from './chunk-UJWGQIXU.mjs';
21
- import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-HCGJKDZV.mjs';
21
+ import { createRequestId, isValidWorkerResponse } from './chunk-QHPYS427.mjs';
22
+ import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-GLZHNSGB.mjs';
22
23
  import { noteThreadSlotLive, threadSlotOfWorker, noteThreadSlotGone } from './chunk-RFBLQLLZ.mjs';
23
- import { MAX_DOCUMENT_NESTING_DEPTH } from './chunk-L5ORED7V.mjs';
24
- import { ErrorCodes, NarsilError } from './chunk-CZ7ISLXA.mjs';
24
+ import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
25
25
  import { encode, decode } from '@msgpack/msgpack';
26
26
  import * as zlib from 'zlib';
27
27
 
@@ -152,130 +152,6 @@ function requireValidPatternWorkCap(cap) {
152
152
  );
153
153
  }
154
154
 
155
- // src/engine/orchestration/turn.ts
156
- function afterCurrentTurn(callback) {
157
- if (typeof setImmediate === "function") {
158
- setImmediate(callback);
159
- return;
160
- }
161
- setTimeout(callback, 0);
162
- }
163
- function yieldToEventLoop() {
164
- return new Promise((resolve) => afterCurrentTurn(resolve));
165
- }
166
-
167
- // src/partitioning/constants.ts
168
- var REBALANCE_CHUNK_SIZE = 1e3;
169
- var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
170
- var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
171
- var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
172
-
173
- // src/partitioning/rebalancer.ts
174
- function createRebalancer() {
175
- const activeManagers = /* @__PURE__ */ new WeakSet();
176
- let activeCount = 0;
177
- async function rebalance(manager, newPartitionCount, router, onProgress) {
178
- if (activeManagers.has(manager)) {
179
- throw new NarsilError(
180
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
181
- "A rebalance operation is already in progress"
182
- );
183
- }
184
- if (newPartitionCount <= 0) {
185
- throw new NarsilError(
186
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
187
- `New partition count must be greater than 0, received ${newPartitionCount}`,
188
- { newPartitionCount }
189
- );
190
- }
191
- if (newPartitionCount === manager.partitionCount) {
192
- throw new NarsilError(
193
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
194
- `New partition count (${newPartitionCount}) is the same as the current count`,
195
- { newPartitionCount, currentPartitionCount: manager.partitionCount }
196
- );
197
- }
198
- activeManagers.add(manager);
199
- activeCount++;
200
- try {
201
- const currentPartitions = manager.getAllPartitions();
202
- let documentsTotal = 0;
203
- for (const partition of currentPartitions) {
204
- documentsTotal += partition.count();
205
- }
206
- onProgress?.({
207
- phase: "scanning",
208
- documentsProcessed: 0,
209
- documentsTotal
210
- });
211
- const newPartitions = [];
212
- for (let i = 0; i < newPartitionCount; i++) {
213
- newPartitions.push(createPartitionIndex(i, manager.config.trackPositions ?? true));
214
- }
215
- let documentsProcessed = 0;
216
- let chunkFill = 0;
217
- const insertOptions = {
218
- validate: false,
219
- skipClone: true,
220
- stopWordOverride: manager.analysis.stopWords,
221
- customTokenizer: manager.analysis.customTokenizer,
222
- collectSurfaces: manager.config.surfaceForms !== false
223
- };
224
- for (const partition of currentPartitions) {
225
- const docIds = [...partition.docIds()];
226
- for (const docId of docIds) {
227
- const document = partition.getRef(docId);
228
- if (!document) continue;
229
- const targetPartitionId = router.route(docId, newPartitionCount);
230
- const targetPartition = newPartitions[targetPartitionId];
231
- if (!targetPartition.has(docId)) {
232
- targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
233
- documentsProcessed++;
234
- }
235
- chunkFill++;
236
- if (chunkFill >= REBALANCE_CHUNK_SIZE) {
237
- chunkFill = 0;
238
- onProgress?.({
239
- phase: "moving",
240
- documentsProcessed,
241
- documentsTotal
242
- });
243
- await yieldToEventLoop();
244
- }
245
- }
246
- }
247
- if (chunkFill > 0) {
248
- onProgress?.({
249
- phase: "moving",
250
- documentsProcessed,
251
- documentsTotal
252
- });
253
- await yieldToEventLoop();
254
- }
255
- onProgress?.({
256
- phase: "swapping",
257
- documentsProcessed,
258
- documentsTotal
259
- });
260
- manager.setPartitions(newPartitions);
261
- onProgress?.({
262
- phase: "complete",
263
- documentsProcessed,
264
- documentsTotal
265
- });
266
- } finally {
267
- activeManagers.delete(manager);
268
- activeCount--;
269
- }
270
- }
271
- return {
272
- rebalance,
273
- isRebalancing() {
274
- return activeCount > 0;
275
- }
276
- };
277
- }
278
-
279
155
  // src/plugins/registry.ts
280
156
  var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
281
157
  function isThenable(value) {
@@ -479,40 +355,73 @@ function validateRequiredFieldsInSchema(required, schema) {
479
355
  }
480
356
  }
481
357
 
358
+ // src/engine/orchestration/turn.ts
359
+ function afterCurrentTurn(callback) {
360
+ if (typeof setImmediate === "function") {
361
+ setImmediate(callback);
362
+ return;
363
+ }
364
+ setTimeout(callback, 0);
365
+ }
366
+ function yieldToEventLoop() {
367
+ return new Promise((resolve) => afterCurrentTurn(resolve));
368
+ }
369
+
370
+ // src/partitioning/constants.ts
371
+ var REBALANCE_CHUNK_SIZE = 1e3;
372
+ var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
373
+ var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
374
+ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
375
+
376
+ // src/partitioning/rewrite-spill-of.ts
377
+ async function rewriteSpillOf(storage, manager, target) {
378
+ if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
379
+ const { openRewriteSpill } = await import('./module-IYPLWHGC.mjs');
380
+ return openRewriteSpill(storage, manager, target);
381
+ }
382
+
482
383
  // src/partitioning/text-rebuild.ts
384
+ function textRebuildTargetOf(config, language, analysis) {
385
+ return { config, language, analysis, partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis) };
386
+ }
483
387
  function interrupted(pass) {
484
388
  return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
485
389
  }
486
390
  async function buildPartition(pass, source, partitionId) {
487
- const { manager, options } = pass;
488
- const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
391
+ const { manager, language, options } = pass;
392
+ const target = pass.spill?.newPartition(partitionId) ?? createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
489
393
  const docIds = [...source.docIds()];
490
394
  target.beginBatch();
491
395
  for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
492
396
  const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
493
397
  for (let i = start; i < end; i++) {
494
398
  const document = manager.getRef(docIds[i]);
495
- if (document !== void 0) target.insert(docIds[i], document, manager.schema, manager.language, options);
399
+ if (document !== void 0) target.insert(docIds[i], document, manager.schema, language, options);
400
+ }
401
+ if (pass.spill !== null) {
402
+ target.endBatch();
403
+ await pass.spill.spillIfFull(target, partitionId);
404
+ target.beginBatch();
496
405
  }
497
406
  await yieldToEventLoop();
498
407
  if (interrupted(pass)) return null;
499
408
  }
500
409
  target.endBatch();
501
- if (manager.partialWordAnalysis !== null) target.preparePartialWords(manager.partialWordAnalysis);
410
+ if (pass.partialWordAnalysis !== null) target.preparePartialWords(pass.partialWordAnalysis);
502
411
  return target;
503
412
  }
504
413
  function applyWrittenDocuments(pass, built) {
505
- const { manager, options } = pass;
414
+ const { manager, language, options } = pass;
506
415
  for (const docId of pass.log.takeWrittenDocIds()) {
507
416
  for (const partition of built) {
508
417
  if (!partition.has(docId)) continue;
509
- partition.remove(docId, manager.schema, manager.language, options);
418
+ partition.remove(docId, manager.schema, language, options);
510
419
  break;
511
420
  }
512
421
  const partitionId = manager.partitionIdOf(docId);
513
422
  const document = manager.getRef(docId);
514
423
  if (partitionId === void 0 || document === void 0) continue;
515
- built[partitionId].insert(docId, document, manager.schema, manager.language, options);
424
+ built[partitionId].insert(docId, document, manager.schema, language, options);
516
425
  }
517
426
  }
518
427
  async function rebuildOnce(pass) {
@@ -529,19 +438,45 @@ async function rebuildOnce(pass) {
529
438
  await yieldToEventLoop();
530
439
  if (interrupted(pass)) return false;
531
440
  }
532
- applyWrittenDocuments(pass, built);
533
- pass.manager.adoptRebuiltPartitions(built);
534
- pass.hooks.onSwitch();
535
- return true;
441
+ const keepCurrent = () => {
442
+ if (interrupted(pass)) return false;
443
+ applyWrittenDocuments(pass, built);
444
+ return true;
445
+ };
446
+ if (pass.hooks.holdBeforeSwitch !== void 0 && !await pass.hooks.holdBeforeSwitch(keepCurrent)) return false;
447
+ const swap = () => {
448
+ if (interrupted(pass)) return false;
449
+ applyWrittenDocuments(pass, built);
450
+ pass.manager.adoptRebuiltPartitions(built);
451
+ pass.hooks.onSwitch();
452
+ return true;
453
+ };
454
+ return pass.hooks.runSwitch === void 0 ? swap() : pass.hooks.runSwitch(swap);
536
455
  }
537
- async function rebuildTextIndexes(manager, hooks) {
538
- const options = resolvePartitionInsertOptions(manager.config, manager.analysis, { validate: false, skipClone: true });
456
+ async function rebuildTextIndexes(manager, hooks, target) {
457
+ const options = resolvePartitionInsertOptions(
458
+ target?.config ?? manager.config,
459
+ target?.analysis ?? manager.analysis,
460
+ {
461
+ validate: false,
462
+ skipClone: true
463
+ }
464
+ );
465
+ const language = target?.language ?? manager.language;
466
+ const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
467
+ const spill = await rewriteSpillOf(hooks.rewriteStorage?.(), manager, target);
539
468
  while (hooks.isCurrent()) {
540
469
  const log = manager.openWriteLog();
470
+ let swapped = false;
541
471
  try {
542
- if (await rebuildOnce({ manager, log, hooks, options })) return true;
472
+ swapped = await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options, spill });
473
+ if (swapped) {
474
+ hooks.onRewriteSpill?.(spill);
475
+ return true;
476
+ }
543
477
  } finally {
544
478
  log.close();
479
+ if (!swapped) await spill?.abandon();
545
480
  }
546
481
  }
547
482
  return false;
@@ -575,8 +510,10 @@ function toError(value) {
575
510
  function wireAnalysisRebuild(wiring) {
576
511
  return createAnalysisRebuildCoordinator(wiring.config, {
577
512
  getManager: wiring.getManager,
513
+ rewriteStorage: () => wiring.durabilityManager?.rewriteStorage?.(),
578
514
  desyncIndex: wiring.desyncIndex,
579
515
  resyncIndex: wiring.resyncIndex,
516
+ hasPendingChange: wiring.hasPendingChange,
580
517
  async persistAnalysisRevision(indexName) {
581
518
  const durability = wiring.durabilityManager;
582
519
  if (durability === null) {
@@ -594,6 +531,7 @@ function wireAnalysisRebuild(wiring) {
594
531
  }
595
532
  function createAnalysisRebuildCoordinator(config, deps) {
596
533
  const stale = /* @__PURE__ */ new Map();
534
+ const exclusiveRuns = /* @__PURE__ */ new Map();
597
535
  const rebuildMode = config?.rebuild ?? "auto";
598
536
  let queue = Promise.resolve();
599
537
  let active = null;
@@ -606,8 +544,13 @@ function createAnalysisRebuildCoordinator(config, deps) {
606
544
  const partitionCount = manager.partitionCount;
607
545
  deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
608
546
  const wasPromoted = deps.desyncIndex(indexName);
547
+ let spill = null;
609
548
  try {
610
549
  const rebuilt = await rebuildTextIndexes(manager, {
550
+ rewriteStorage: () => deps.rewriteStorage?.(),
551
+ onRewriteSpill(kept) {
552
+ spill = kept;
553
+ },
611
554
  isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
612
555
  onPartitionBuilt(partitionsBuilt) {
613
556
  entry.partitionsRebuilt = partitionsBuilt;
@@ -622,6 +565,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
622
565
  return;
623
566
  }
624
567
  await deps.persistAnalysisRevision(indexName);
568
+ await spill?.removeUnread(manager);
625
569
  await deps.resyncIndex(indexName, wasPromoted);
626
570
  deps.emit({
627
571
  indexName,
@@ -669,7 +613,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
669
613
  await config?.onStaleAnalysis?.({ ...entry.index, documentCount }, () => enqueue(indexName, entry));
670
614
  }
671
615
  async function review(indexName, entry) {
672
- if (entry.run !== null || deps.getManager(indexName) === void 0) return;
616
+ if (entry.run !== null || deps.getManager(indexName) === void 0 || deps.hasPendingChange(indexName)) return;
673
617
  try {
674
618
  await announce(indexName, entry);
675
619
  } catch (err) {
@@ -701,8 +645,22 @@ function createAnalysisRebuildCoordinator(config, deps) {
701
645
  return stale.has(indexName);
702
646
  },
703
647
  isRunning(indexName) {
648
+ if ((exclusiveRuns.get(indexName) ?? 0) > 0) return true;
704
649
  return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
705
650
  },
651
+ runExclusive(indexName, work, turnEnds) {
652
+ exclusiveRuns.set(indexName, (exclusiveRuns.get(indexName) ?? 0) + 1);
653
+ const run = queue.then(work).finally(() => {
654
+ const remaining = (exclusiveRuns.get(indexName) ?? 1) - 1;
655
+ if (remaining === 0) exclusiveRuns.delete(indexName);
656
+ else exclusiveRuns.set(indexName, remaining);
657
+ });
658
+ queue = Promise.race([run, turnEnds ?? run]).then(
659
+ () => void 0,
660
+ () => void 0
661
+ );
662
+ return run;
663
+ },
706
664
  async abandon(indexName) {
707
665
  const entry = stale.get(indexName);
708
666
  if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
@@ -881,6 +839,9 @@ function reconstructSchemaFromMetadata(metadata) {
881
839
  if (metadata.trackPositions !== void 0) {
882
840
  config.trackPositions = metadata.trackPositions;
883
841
  }
842
+ if (metadata.textStorage !== void 0) {
843
+ config.textStorage = metadata.textStorage;
844
+ }
884
845
  if (metadata.strict !== void 0) {
885
846
  config.strict = metadata.strict;
886
847
  }
@@ -958,6 +919,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
958
919
  if (loadingVectorCode !== null) await loadingVectorCode;
959
920
  const existing = deps.indexRegistry.get(metadata.indexName);
960
921
  if (existing !== void 0) {
922
+ if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
923
+ const { adoptSavedAnalysis } = await import('./analysis-change-HQUT2Z3I.mjs');
924
+ adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
925
+ }
961
926
  if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
962
927
  deps.executor.createIndex(
963
928
  metadata.indexName,
@@ -988,6 +953,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
988
953
  const synonyms = prepared?.encoded ?? null;
989
954
  try {
990
955
  validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
956
+ validateTextStorage(indexConfig.textStorage, deps.filesystemDurability);
991
957
  if (loadData) {
992
958
  deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
993
959
  }
@@ -1006,7 +972,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1006
972
  partitionCount: metadata.partitionCount,
1007
973
  analysisRevision: metadata.analysisRevision ?? null,
1008
974
  arrayPositionGap: metadata.arrayPositionGap ?? null,
1009
- synonyms
975
+ synonyms,
976
+ analysisGeneration: metadata.analysisGeneration ?? 0,
977
+ pendingAnalysis: metadata.pendingAnalysis ?? null,
978
+ durableAnalysis: null
1010
979
  });
1011
980
  if (loadData) await deps.indexState().registerOpen(metadata.indexName);
1012
981
  else deps.indexState().registerClosed(metadata.indexName);
@@ -1406,6 +1375,7 @@ function wireIndexState(wiring) {
1406
1375
  }
1407
1376
  try {
1408
1377
  await durability.manager.recoverIndex(indexName);
1378
+ await wiring.resumeChange(indexName);
1409
1379
  await analysisRebuild.review(indexName);
1410
1380
  await orchestrator.openIndex(indexName);
1411
1381
  await wiring.onOpen?.(indexName);
@@ -1735,6 +1705,9 @@ function checkHeapAfterRecovery(core) {
1735
1705
 
1736
1706
  // src/core/partition/frozen/share.ts
1737
1707
  function shareFrozenSegment(segment, analysis) {
1708
+ if (segment.paged !== null) {
1709
+ return { snapshot: segment.paged.snapshot, tombstonedDocIds: segment.tombstonedDocIds() };
1710
+ }
1738
1711
  if (segment.sharedSnapshot !== null) {
1739
1712
  const tombstonedDocIds = segment.tombstonedDocIds();
1740
1713
  if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
@@ -2106,14 +2079,18 @@ function assertConfigReachesWorker(indexName, config, bootstrapModule) {
2106
2079
  { indexName }
2107
2080
  );
2108
2081
  }
2082
+ if (bootstrapModule !== void 0) return;
2109
2083
  const language = config.language ?? "english";
2110
- if (language !== "english" && bootstrapModule === void 0) {
2084
+ const requireRegistered = (setting, name) => {
2111
2085
  throw new NarsilError(
2112
2086
  ErrorCodes.CONFIG_INVALID,
2113
- `Index "${indexName}" uses language "${language}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
2114
- { indexName, language }
2087
+ `Index "${indexName}" uses ${setting} "${name}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
2088
+ { indexName, [setting]: name }
2115
2089
  );
2116
- }
2090
+ };
2091
+ if (language !== "english") requireRegistered("language", language);
2092
+ if (typeof config.tokenizer === "string") requireRegistered("tokenizer", config.tokenizer);
2093
+ if (typeof config.stopWords === "string") requireRegistered("stopWords", config.stopWords);
2117
2094
  }
2118
2095
  function workerIneligibility(indexName, config, bootstrapModule) {
2119
2096
  try {
@@ -2517,7 +2494,7 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
2517
2494
  }
2518
2495
  async function compactIndexSegments(state, indexName, policy) {
2519
2496
  const manager = state.executor.getManager(indexName);
2520
- if (!manager) return;
2497
+ if (!manager || manager.config.textStorage === "disk") return;
2521
2498
  const partitionIds = /* @__PURE__ */ new Set();
2522
2499
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) partitionIds.add(partitionId);
2523
2500
  for (const partitionId of state.segmentLedger.get(indexName)?.keys() ?? []) partitionIds.add(partitionId);
@@ -3991,6 +3968,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
3991
3968
  scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
3992
3969
  replicateToWorkers: replicate,
3993
3970
  awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
3971
+ queueDiskAdoption: (action) => queueForCopies(state, action),
3994
3972
  awaitWrites: (indexName) => awaitWritesApplied(state, indexName),
3995
3973
  awaitCompactions: () => awaitCompactions(state),
3996
3974
  openIndex,
@@ -4064,7 +4042,7 @@ async function emitWriteEventAfter(source, indexName, write) {
4064
4042
  // src/engine/core.ts
4065
4043
  async function createEngineCore(config, hooks, capacityScope = "index") {
4066
4044
  validateWorkerConfig(config?.workers, config?.lifecycle);
4067
- const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-7MLQ7J64.mjs')).resolveRerankSettings(config);
4045
+ const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-DYV75DCB.mjs')).resolveRerankSettings(config);
4068
4046
  const eventHandlers = eventHandlersOf(config?.events);
4069
4047
  const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
4070
4048
  const vectorCopyPolicy = {
@@ -4075,7 +4053,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4075
4053
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
4076
4054
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
4077
4055
  }
4078
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-S7OX2MAI.mjs');
4056
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-324D2DDD.mjs');
4079
4057
  const filesystemDurability = durabilityTier?.kind === "wal";
4080
4058
  const executor = createDirectExecutor({
4081
4059
  vectorWorkerCopies: vectorCopyPolicy,
@@ -4135,7 +4113,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4135
4113
  },
4136
4114
  vectorCopyPolicy
4137
4115
  );
4138
- const rebalancer = createRebalancer();
4139
4116
  const rebalanceRouter = createPartitionRouter();
4140
4117
  const waqMap = /* @__PURE__ */ new Map();
4141
4118
  const rebalanceTargets = /* @__PURE__ */ new Map();
@@ -4195,6 +4172,8 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4195
4172
  emitCheckpointHeld(report) {
4196
4173
  emitEngineEvent(eventHandlers, "checkpointHeld", report);
4197
4174
  },
4175
+ queueDiskAdoption: orchestrator.queueDiskAdoption,
4176
+ awaitCopiesOfIndex: (indexName) => orchestrator.awaitReplication(indexName),
4198
4177
  publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve(),
4199
4178
  recordCheckpoint(indexName, documentCount, partitionCount) {
4200
4179
  const entry = indexRegistry.get(indexName);
@@ -4245,6 +4224,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4245
4224
  entry.analysisRevision = entry.language.revision;
4246
4225
  entry.arrayPositionGap = ARRAY_POSITION_GAP;
4247
4226
  },
4227
+ hasPendingChange: (indexName) => (indexRegistry.get(indexName)?.pendingAnalysis ?? null) !== null,
4248
4228
  durabilityManager: durability?.manager ?? null
4249
4229
  });
4250
4230
  const indexState = wireIndexState({
@@ -4256,6 +4236,10 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4256
4236
  indexRegistry,
4257
4237
  rebalancingIndexes,
4258
4238
  requireManager,
4239
+ resumeChange: async (indexName) => {
4240
+ if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
4241
+ await (await import('./analysis-change-entry-TZTG4AZB.mjs')).resumeChangeOf(engineCore, indexName);
4242
+ },
4259
4243
  onClose: hooks?.onIndexClose,
4260
4244
  onOpen: (indexName) => {
4261
4245
  heapPressureNotifier.check(indexName);
@@ -4284,7 +4268,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4284
4268
  checkHeapPressure: heapPressureNotifier.check
4285
4269
  };
4286
4270
  const rebalanceCtx = {
4287
- rebalancer,
4288
4271
  router: rebalanceRouter,
4289
4272
  waqMap,
4290
4273
  rebalancingIndexes,
@@ -4296,7 +4279,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4296
4279
  checkWatermark: watermarkNotifier.check,
4297
4280
  requireIndex
4298
4281
  };
4299
- return {
4282
+ const engineCore = {
4300
4283
  executor,
4301
4284
  pluginRegistry,
4302
4285
  durability,
@@ -4311,7 +4294,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4311
4294
  shutdownState,
4312
4295
  abortController,
4313
4296
  orchestrator,
4314
- rebalancer,
4315
4297
  rebalanceRouter,
4316
4298
  rebalancingIndexes,
4317
4299
  waqMap,
@@ -4327,6 +4309,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4327
4309
  rebalanceCtx,
4328
4310
  patternWorkCap: config?.patternWorkCap ?? DEFAULT_PATTERN_WORK_CAP
4329
4311
  };
4312
+ return engineCore;
4330
4313
  }
4331
4314
 
4332
4315
  // src/engine/lifecycle.ts
@@ -4337,6 +4320,7 @@ async function startEngineCore(core, config) {
4337
4320
  checkHeapAfterRecovery(core);
4338
4321
  }
4339
4322
  if (core.invalidation) await core.invalidation.start();
4323
+ if (core.durability) await (await import('./analysis-change-entry-TZTG4AZB.mjs')).resumeEveryChange(core);
4340
4324
  if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
4341
4325
  } catch (err) {
4342
4326
  await core.durability?.manager.shutdown().catch(() => void 0);
@@ -4374,7 +4358,7 @@ async function shutdownEngine(core) {
4374
4358
  }
4375
4359
  }
4376
4360
  if (core.rerankAdapters.size > 0) {
4377
- await (await import('./runtime-7MLQ7J64.mjs')).shutdownRerankAdapters(core.rerankAdapters);
4361
+ await (await import('./runtime-DYV75DCB.mjs')).shutdownRerankAdapters(core.rerankAdapters);
4378
4362
  }
4379
4363
  await orchestrator.shutdown();
4380
4364
  await executor.shutdown();
@@ -4479,6 +4463,7 @@ async function checksumOfChunks(chunks) {
4479
4463
  // src/serialization/header.ts
4480
4464
  var MAGIC_BYTES = new Uint8Array([78, 82, 83, 76]);
4481
4465
  var HEADER_SIZE = 32;
4466
+ var RESERVED_SIZE = 14;
4482
4467
  var MAX_PAYLOAD_LENGTH = 4294967295;
4483
4468
  var COMPRESSION_ALGORITHMS = {
4484
4469
  0: "none",
@@ -4591,9 +4576,12 @@ function requireWithinFileLimit(byteLength, what) {
4591
4576
  );
4592
4577
  }
4593
4578
  async function packChecksummedEnvelope(payloadChunks, envelopeFormatVersion, what) {
4579
+ requireWithinFileLimit(HEADER_SIZE + payloadByteLength(payloadChunks), what);
4580
+ return packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, await checksumOfChunks(payloadChunks));
4581
+ }
4582
+ function packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, checksum) {
4594
4583
  const payloadLength = payloadByteLength(payloadChunks);
4595
4584
  requireWithinFileLimit(HEADER_SIZE + payloadLength, what);
4596
- const checksum = await checksumOfChunks(payloadChunks);
4597
4585
  const [major, minor, patch] = engineVersionParts();
4598
4586
  const header = writeHeader({
4599
4587
  envelopeFormatVersion,
@@ -4711,7 +4699,8 @@ function indexConfigurationOf(entry) {
4711
4699
  entry.embeddingAdapterName,
4712
4700
  entry.analysisRevision,
4713
4701
  entry.arrayPositionGap,
4714
- synonyms
4702
+ synonyms,
4703
+ entry.analysisGeneration
4715
4704
  );
4716
4705
  }
4717
4706
  function requireConfigurationWithinSizeLimit(config, language) {
@@ -4732,7 +4721,7 @@ function requireConfigurationWithinSizeLimit(config, language) {
4732
4721
  { bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
4733
4722
  );
4734
4723
  }
4735
- function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms) {
4724
+ function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms, analysisGeneration = 0) {
4736
4725
  const bm25 = config.bm25;
4737
4726
  return {
4738
4727
  schema: config.schema,
@@ -4742,6 +4731,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
4742
4731
  ...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
4743
4732
  ...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
4744
4733
  ...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
4734
+ ...analysisGeneration > 0 ? { analysisGeneration } : {},
4745
4735
  ...bm25 !== void 0 ? {
4746
4736
  bm25: {
4747
4737
  ...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
@@ -4753,6 +4743,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
4753
4743
  ...config.partitions !== void 0 ? { partitionConfig: config.partitions } : {},
4754
4744
  ...config.defaultScoring !== void 0 ? { defaultScoring: config.defaultScoring } : {},
4755
4745
  ...config.trackPositions !== void 0 ? { trackPositions: config.trackPositions } : {},
4746
+ ...config.textStorage !== void 0 ? { textStorage: config.textStorage } : {},
4756
4747
  ...config.strict !== void 0 ? { strict: config.strict } : {},
4757
4748
  ...config.required !== void 0 ? { required: config.required } : {},
4758
4749
  ...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
@@ -4841,6 +4832,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
4841
4832
  requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
4842
4833
  validateVectorPromotion(indexConfig.vectorPromotion);
4843
4834
  validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
4835
+ validateTextStorage(indexConfig.textStorage, core.filesystemDurability);
4844
4836
  if (indexConfig.bm25 !== void 0) {
4845
4837
  validateBM25Params(
4846
4838
  indexConfig.bm25,
@@ -4916,7 +4908,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
4916
4908
  partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
4917
4909
  analysisRevision: language.revision,
4918
4910
  arrayPositionGap: ARRAY_POSITION_GAP,
4919
- synonyms: synonyms?.encoded ?? null
4911
+ synonyms: synonyms?.encoded ?? null,
4912
+ analysisGeneration: 0,
4913
+ pendingAnalysis: null,
4914
+ durableAnalysis: null
4920
4915
  });
4921
4916
  await core.indexState.registerOpen(name);
4922
4917
  if (core.durability) {
@@ -4927,8 +4922,14 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
4927
4922
  async function dropEngineIndex(core, name) {
4928
4923
  core.guardShutdown();
4929
4924
  const entry = core.requireIndex(name);
4925
+ const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
4926
+ const abandonChange = async () => {
4927
+ await (await import('./analysis-change-entry-TZTG4AZB.mjs')).abandonChangeOf(core, name);
4928
+ };
4929
+ if (changeRecorded()) await abandonChange();
4930
4930
  await core.indexState.drop(name, async () => {
4931
4931
  await core.analysisRebuild.abandon(name);
4932
+ if (changeRecorded()) await abandonChange();
4932
4933
  await core.orchestrator.closeIndex(name);
4933
4934
  if (core.durability) {
4934
4935
  try {
@@ -6178,276 +6179,6 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
6178
6179
  return { succeeded, failed };
6179
6180
  }
6180
6181
 
6181
- // src/partitioning/write-ahead-queue.ts
6182
- function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
6183
- const entries = [];
6184
- const docStates = /* @__PURE__ */ new Map();
6185
- let nextSequence = 1;
6186
- let resolveReplayed = () => void 0;
6187
- const replayed = new Promise((resolve) => {
6188
- resolveReplayed = resolve;
6189
- });
6190
- return {
6191
- push(entry) {
6192
- if (entries.length >= maxSize) {
6193
- throw new NarsilError(
6194
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
6195
- `Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
6196
- { maxSize, currentSize: entries.length }
6197
- );
6198
- }
6199
- const seq = nextSequence++;
6200
- entries.push({ ...entry, sequenceNumber: seq });
6201
- docStates.set(entry.docId, entry.action === "remove" ? "absent" : "present");
6202
- return seq;
6203
- },
6204
- drain() {
6205
- const result = entries.splice(0);
6206
- result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
6207
- return result;
6208
- },
6209
- clear() {
6210
- entries.length = 0;
6211
- docStates.clear();
6212
- },
6213
- bufferedDocState(docId) {
6214
- return docStates.get(docId);
6215
- },
6216
- whenReplayed() {
6217
- return replayed;
6218
- },
6219
- markReplayed() {
6220
- resolveReplayed();
6221
- },
6222
- get size() {
6223
- return entries.length;
6224
- },
6225
- get isFull() {
6226
- return entries.length >= maxSize;
6227
- }
6228
- };
6229
- }
6230
-
6231
- // src/engine/rebalance-executor.ts
6232
- function warnHookError(hook, err) {
6233
- console.warn(`${hook} plugin hook error:`, err instanceof Error ? err.message : String(err));
6234
- }
6235
- function applyReplayUpdateVectors(entry, vectorFieldPaths) {
6236
- const updateVectors = /* @__PURE__ */ new Map();
6237
- for (const fieldPath of vectorFieldPaths) {
6238
- updateVectors.set(fieldPath, extractVectorFromDoc(entry.document, fieldPath));
6239
- }
6240
- let updateDoc = entry.document;
6241
- if (updateVectors.size > 0) {
6242
- updateDoc = structuredClone(entry.document);
6243
- for (const fieldPath of updateVectors.keys()) {
6244
- deleteNestedValue(updateDoc, fieldPath);
6245
- }
6246
- }
6247
- return { updateDoc, updateVectors };
6248
- }
6249
- async function replayEntry(manager, indexName, entry, ctx) {
6250
- const vecIndexes = manager.getVectorIndexes();
6251
- const vectorFieldPaths = ctx.requireIndex(indexName).vectorFieldPaths;
6252
- if (entry.action === "insert" && entry.document) {
6253
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(
6254
- entry.document,
6255
- vectorFieldPaths
6256
- );
6257
- const overwrote = manager.has(entry.docId);
6258
- const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
6259
- if (overwrote) {
6260
- removeDocumentVectors(entry.docId, vecIndexes);
6261
- manager.remove(entry.docId, removedSpellings);
6262
- }
6263
- manager.insert(entry.docId, partitionDoc);
6264
- if (extractedVectors.size > 0) {
6265
- insertDocumentVectors(entry.docId, extractedVectors, vecIndexes, manager.partitionIdOf(entry.docId));
6266
- for (const fieldPath of extractedVectors.keys()) {
6267
- vecIndexes.get(fieldPath)?.scheduleBuild();
6268
- }
6269
- }
6270
- try {
6271
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: entry.docId, document: entry.document });
6272
- } catch (err) {
6273
- warnHookError("afterInsert", err);
6274
- }
6275
- const replicated = {
6276
- indexName,
6277
- docId: entry.docId,
6278
- document: entry.document,
6279
- requestId: `replicate-insert-${entry.docId}`
6280
- };
6281
- await ctx.orchestrator.replicateToWorkers(
6282
- overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
6283
- );
6284
- return;
6285
- }
6286
- if (entry.action === "remove") {
6287
- const removedSpellings = manager.removedSpellingsOf(entry.docId);
6288
- manager.remove(entry.docId, removedSpellings);
6289
- removeDocumentVectors(entry.docId, vecIndexes);
6290
- try {
6291
- await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
6292
- } catch (err) {
6293
- warnHookError("afterRemove", err);
6294
- }
6295
- await ctx.orchestrator.replicateToWorkers({
6296
- type: "remove",
6297
- indexName,
6298
- docId: entry.docId,
6299
- requestId: `replicate-remove-${entry.docId}`,
6300
- removedSpellings
6301
- });
6302
- return;
6303
- }
6304
- if (entry.action === "update" && entry.document) {
6305
- const oldDocument = manager.get(entry.docId);
6306
- const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths);
6307
- const removedSpellings = manager.removedSpellingsOf(entry.docId);
6308
- if (manager.has(entry.docId)) {
6309
- manager.update(entry.docId, updateDoc, void 0, removedSpellings);
6310
- } else {
6311
- manager.insert(entry.docId, updateDoc);
6312
- }
6313
- for (const [fieldPath, newVec] of updateVectors) {
6314
- const vecIndex = vecIndexes.get(fieldPath);
6315
- if (!vecIndex) continue;
6316
- if (newVec === null) {
6317
- if (vecIndex.has(entry.docId)) {
6318
- vecIndex.remove(entry.docId);
6319
- }
6320
- } else {
6321
- const oldVec = vecIndex.getVector(entry.docId);
6322
- if (!vectorsEqual(oldVec, newVec)) {
6323
- if (vecIndex.has(entry.docId)) {
6324
- vecIndex.remove(entry.docId);
6325
- }
6326
- vecIndex.insert(entry.docId, newVec);
6327
- vecIndex.scheduleBuild();
6328
- }
6329
- }
6330
- }
6331
- try {
6332
- await ctx.pluginRegistry.runHook("afterUpdate", {
6333
- indexName,
6334
- docId: entry.docId,
6335
- oldDocument: oldDocument ?? {},
6336
- newDocument: entry.document
6337
- });
6338
- } catch (err) {
6339
- warnHookError("afterUpdate", err);
6340
- }
6341
- await ctx.orchestrator.replicateToWorkers({
6342
- type: "update",
6343
- indexName,
6344
- docId: entry.docId,
6345
- document: entry.document,
6346
- requestId: `replicate-update-${entry.docId}`,
6347
- removedSpellings
6348
- });
6349
- }
6350
- }
6351
- async function replayQueued(manager, indexName, waq, ctx) {
6352
- while (true) {
6353
- const entries = waq.drain();
6354
- if (entries.length === 0) break;
6355
- for (const entry of entries) {
6356
- try {
6357
- await replayEntry(manager, indexName, entry, ctx);
6358
- } catch (replayErr) {
6359
- const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
6360
- if (!isMissing) {
6361
- console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
6362
- }
6363
- }
6364
- }
6365
- }
6366
- }
6367
- async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
6368
- if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
6369
- throw new NarsilError(
6370
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
6371
- `Target partition count must be a positive integer, got ${targetPartitionCount}`,
6372
- { targetPartitionCount }
6373
- );
6374
- }
6375
- const entry = ctx.requireIndex(indexName);
6376
- const maxPartitions = entry.config.partitions?.maxPartitions;
6377
- if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
6378
- throw new NarsilError(
6379
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
6380
- `Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
6381
- { targetPartitionCount, maxPartitions }
6382
- );
6383
- }
6384
- if (ctx.rebalancingIndexes.has(indexName)) {
6385
- throw new NarsilError(
6386
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
6387
- `Index "${indexName}" is already being rebalanced`
6388
- );
6389
- }
6390
- if (targetPartitionCount === manager.partitionCount) {
6391
- return;
6392
- }
6393
- const waq = createWriteAheadQueue();
6394
- ctx.waqMap.set(indexName, waq);
6395
- ctx.rebalancingIndexes.add(indexName);
6396
- ctx.rebalanceTargets.set(indexName, targetPartitionCount);
6397
- const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
6398
- try {
6399
- const oldCount = manager.partitionCount;
6400
- await ctx.rebalancer.rebalance(manager, targetPartitionCount, ctx.router, (progress) => {
6401
- if (progress.phase === "complete") {
6402
- emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
6403
- indexName,
6404
- oldCount,
6405
- newCount: targetPartitionCount
6406
- });
6407
- }
6408
- });
6409
- try {
6410
- await ctx.pluginRegistry.runHook("onPartitionSplit", {
6411
- indexName,
6412
- oldPartitionCount: oldCount,
6413
- newPartitionCount: targetPartitionCount
6414
- });
6415
- } catch (hookErr) {
6416
- console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
6417
- }
6418
- await replayQueued(manager, indexName, waq, ctx);
6419
- if (ctx.durabilityManager) {
6420
- await ctx.durabilityManager.persistMetadata(indexName);
6421
- await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
6422
- await replayQueued(manager, indexName, waq, ctx);
6423
- }
6424
- await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
6425
- do {
6426
- await replayQueued(manager, indexName, waq, ctx);
6427
- } while (waq.size > 0);
6428
- } catch (err) {
6429
- try {
6430
- await replayQueued(manager, indexName, waq, ctx);
6431
- await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
6432
- do {
6433
- await replayQueued(manager, indexName, waq, ctx);
6434
- } while (waq.size > 0);
6435
- } catch (recoveryErr) {
6436
- console.warn(
6437
- `Rebalance failure recovery for index "${indexName}" did not complete:`,
6438
- recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)
6439
- );
6440
- }
6441
- throw err;
6442
- } finally {
6443
- ctx.rebalancingIndexes.delete(indexName);
6444
- ctx.rebalanceTargets.delete(indexName);
6445
- ctx.waqMap.delete(indexName);
6446
- waq.markReplayed();
6447
- }
6448
- ctx.checkWatermark(indexName);
6449
- }
6450
-
6451
6182
  // src/engine/vector-maintenance.ts
6452
6183
  function resolveVectorIndexes(manager, indexName, fieldName) {
6453
6184
  const all = manager.getVectorIndexes();
@@ -6487,6 +6218,7 @@ function getVectorMaintenanceStatus(manager) {
6487
6218
  bufferSize: status.bufferSize,
6488
6219
  building: status.building,
6489
6220
  waitingForMemory: status.waitingForMemory,
6221
+ loadingFromFiles: status.loadingFromFiles === true,
6490
6222
  estimatedCompactMs: status.estimatedCompactMs,
6491
6223
  estimatedOptimizeMs: status.estimatedOptimizeMs
6492
6224
  });
@@ -6570,461 +6302,6 @@ function bm25Operations(core) {
6570
6302
  };
6571
6303
  }
6572
6304
 
6573
- // src/persistence/archive/byte-reader.ts
6574
- async function* chunksOf(source) {
6575
- if (source instanceof Uint8Array) {
6576
- yield source;
6577
- return;
6578
- }
6579
- if (typeof source.getReader === "function") {
6580
- const reader = source.getReader();
6581
- try {
6582
- for (; ; ) {
6583
- const step = await reader.read();
6584
- if (step.done) return;
6585
- yield step.value;
6586
- }
6587
- } finally {
6588
- reader.releaseLock();
6589
- }
6590
- }
6591
- yield* source;
6592
- }
6593
- function createByteReader(source) {
6594
- const iterator = chunksOf(source);
6595
- const pending = [];
6596
- let pendingBytes = 0;
6597
- let ended = false;
6598
- async function fill(length) {
6599
- while (pendingBytes < length && !ended) {
6600
- const step = await iterator.next();
6601
- if (step.done) {
6602
- ended = true;
6603
- break;
6604
- }
6605
- if (step.value.byteLength === 0) continue;
6606
- pending.push(step.value);
6607
- pendingBytes += step.value.byteLength;
6608
- }
6609
- }
6610
- function take(length) {
6611
- const out = new Uint8Array(length);
6612
- let filled = 0;
6613
- while (filled < length) {
6614
- const head = pending[0];
6615
- const needed = length - filled;
6616
- if (head.byteLength <= needed) {
6617
- out.set(head, filled);
6618
- filled += head.byteLength;
6619
- pending.shift();
6620
- } else {
6621
- out.set(head.subarray(0, needed), filled);
6622
- pending[0] = head.subarray(needed);
6623
- filled += needed;
6624
- }
6625
- }
6626
- pendingBytes -= length;
6627
- return out;
6628
- }
6629
- return {
6630
- async readExactly(length) {
6631
- await fill(length);
6632
- if (pendingBytes === 0 && length > 0) return null;
6633
- if (pendingBytes < length) return take(pendingBytes);
6634
- return take(length);
6635
- },
6636
- async peek(length) {
6637
- await fill(length);
6638
- const available = Math.min(length, pendingBytes);
6639
- const copy = take(available);
6640
- pending.unshift(copy);
6641
- pendingBytes += copy.byteLength;
6642
- return copy;
6643
- },
6644
- async close() {
6645
- await iterator.return(void 0);
6646
- }
6647
- };
6648
- }
6649
-
6650
- // src/engine/export/lines.ts
6651
- var NEWLINE = 10;
6652
- function joined(parts, total) {
6653
- if (parts.length === 1) return parts[0];
6654
- const bytes = new Uint8Array(total);
6655
- let offset = 0;
6656
- for (const part of parts) {
6657
- bytes.set(part, offset);
6658
- offset += part.byteLength;
6659
- }
6660
- return bytes;
6661
- }
6662
- async function* exportLines(source, maxLineBytes) {
6663
- const decoder = new TextDecoder("utf-8", { fatal: true });
6664
- let parts = [];
6665
- let pending = 0;
6666
- let overflowing = false;
6667
- let line = 0;
6668
- const finishLine = () => {
6669
- line += 1;
6670
- if (overflowing) {
6671
- overflowing = false;
6672
- parts = [];
6673
- pending = 0;
6674
- return { line, refused: "too-long" };
6675
- }
6676
- const bytes = joined(parts, pending);
6677
- parts = [];
6678
- pending = 0;
6679
- try {
6680
- return { line, text: decoder.decode(bytes) };
6681
- } catch {
6682
- return { line, refused: "not-utf8" };
6683
- }
6684
- };
6685
- for await (const chunk of chunksOf(source)) {
6686
- let start = 0;
6687
- for (; ; ) {
6688
- const newline = chunk.indexOf(NEWLINE, start);
6689
- const end = newline < 0 ? chunk.byteLength : newline;
6690
- const piece = chunk.subarray(start, end);
6691
- if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
6692
- if (piece.byteLength > 0) parts.push(piece.slice());
6693
- pending += piece.byteLength;
6694
- } else {
6695
- overflowing = true;
6696
- parts = [];
6697
- }
6698
- if (newline < 0) break;
6699
- yield finishLine();
6700
- start = newline + 1;
6701
- }
6702
- }
6703
- if (pending > 0 || overflowing) yield finishLine();
6704
- }
6705
-
6706
- // src/serialization/base64.ts
6707
- var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
6708
- var PAD = 61;
6709
- var BYTES_PER_GROUP = 3;
6710
- var CHARACTERS_PER_GROUP = 4;
6711
- var ASCII_LIMIT = 128;
6712
- var ENCODE = new Uint8Array(ALPHABET.length);
6713
- var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
6714
- for (let at = 0; at < ALPHABET.length; at++) {
6715
- ENCODE[at] = ALPHABET.charCodeAt(at);
6716
- DECODE[ALPHABET.charCodeAt(at)] = at;
6717
- }
6718
- var asciiDecoder = new TextDecoder();
6719
- function encodeBase64(bytes) {
6720
- const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
6721
- let write = 0;
6722
- let read = 0;
6723
- for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
6724
- const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
6725
- out[write++] = ENCODE[group >>> 18];
6726
- out[write++] = ENCODE[group >>> 12 & 63];
6727
- out[write++] = ENCODE[group >>> 6 & 63];
6728
- out[write++] = ENCODE[group & 63];
6729
- }
6730
- const left = bytes.byteLength - read;
6731
- if (left > 0) {
6732
- const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
6733
- out[write++] = ENCODE[group >>> 18];
6734
- out[write++] = ENCODE[group >>> 12 & 63];
6735
- out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
6736
- out[write++] = PAD;
6737
- }
6738
- return asciiDecoder.decode(out);
6739
- }
6740
- function sextetAt(text, at) {
6741
- const code = text.charCodeAt(at);
6742
- return code < ASCII_LIMIT ? DECODE[code] : -1;
6743
- }
6744
- function decodeExactBase64(text) {
6745
- if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
6746
- const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
6747
- const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
6748
- const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
6749
- let write = 0;
6750
- for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
6751
- const a2 = sextetAt(text, at);
6752
- const b2 = sextetAt(text, at + 1);
6753
- const c2 = sextetAt(text, at + 2);
6754
- const d = sextetAt(text, at + 3);
6755
- if ((a2 | b2 | c2 | d) < 0) return null;
6756
- const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
6757
- out[write++] = group >>> 16;
6758
- out[write++] = group >>> 8 & 255;
6759
- out[write++] = group & 255;
6760
- }
6761
- if (padding === 0) return out;
6762
- const a = sextetAt(text, fullEnd);
6763
- const b = sextetAt(text, fullEnd + 1);
6764
- const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
6765
- if ((a | b | c) < 0) return null;
6766
- if (padding === 2 && (b & 15) !== 0) return null;
6767
- if (padding === 1 && (c & 3) !== 0) return null;
6768
- out[write++] = a << 2 | b >>> 4;
6769
- if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
6770
- return out;
6771
- }
6772
-
6773
- // src/engine/export/tags.ts
6774
- var TAG_PREFIX = "$";
6775
- var DATE_TAG = "$date";
6776
- var BYTES_TAG = "$bytes";
6777
- var NUMBER_TAG = "$number";
6778
- var OBJECT_TAG = "$object";
6779
- var MILLISECONDS_PER_SECOND = 1e3;
6780
- var NANOSECONDS_PER_MILLISECOND = 1e6;
6781
- var MAX_NANOSECONDS = 999999999;
6782
- var MAX_DATE_MILLISECONDS = 864e13;
6783
- var ESCAPED_TAG_PREFIX = "\\u0024";
6784
- var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
6785
- ["NaN", Number.NaN],
6786
- ["Infinity", Number.POSITIVE_INFINITY],
6787
- ["-Infinity", Number.NEGATIVE_INFINITY]
6788
- ]);
6789
- function refuse(path, reason) {
6790
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
6791
- field: path
6792
- });
6793
- }
6794
- function isRecord(value) {
6795
- return typeof value === "object" && value !== null && !Array.isArray(value);
6796
- }
6797
- function startsWithTagPrefix(key) {
6798
- return key.startsWith(TAG_PREFIX);
6799
- }
6800
- function soleTagKey(record) {
6801
- let sole = null;
6802
- for (const key of Object.keys(record)) {
6803
- if (sole !== null) return null;
6804
- sole = key;
6805
- }
6806
- return sole !== null && startsWithTagPrefix(sole) ? sole : null;
6807
- }
6808
- function dateOf(value, path) {
6809
- if (!Array.isArray(value) || value.length !== 2)
6810
- refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
6811
- const seconds = value[0];
6812
- const nanoseconds = value[1];
6813
- if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
6814
- refuse(
6815
- path,
6816
- `holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
6817
- );
6818
- }
6819
- if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
6820
- refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
6821
- }
6822
- const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
6823
- if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
6824
- refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
6825
- return new Date(milliseconds);
6826
- }
6827
- function bytesOf(value, path) {
6828
- const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
6829
- if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
6830
- return bytes;
6831
- }
6832
- function numberOf(value, path) {
6833
- const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
6834
- if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
6835
- return number;
6836
- }
6837
- function requireDepth(path, depth) {
6838
- if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
6839
- }
6840
- function untagMap(record, path, depth) {
6841
- requireDepth(path, depth);
6842
- for (const key of Object.keys(record)) {
6843
- record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
6844
- }
6845
- return record;
6846
- }
6847
- function untagList(list, path, depth) {
6848
- requireDepth(path, depth);
6849
- for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
6850
- return list;
6851
- }
6852
- function untagTag(tag, value, path, depth) {
6853
- switch (tag) {
6854
- case DATE_TAG:
6855
- return dateOf(value, path);
6856
- case BYTES_TAG:
6857
- return bytesOf(value, path);
6858
- case NUMBER_TAG:
6859
- return numberOf(value, path);
6860
- case OBJECT_TAG:
6861
- if (!isRecord(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
6862
- return untagMap(value, path, depth);
6863
- default:
6864
- return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
6865
- }
6866
- }
6867
- function untag(value, path, depth) {
6868
- if (typeof value !== "object" || value === null) return value;
6869
- if (Array.isArray(value)) return untagList(value, path, depth);
6870
- const record = value;
6871
- const tag = soleTagKey(record);
6872
- return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
6873
- }
6874
- function untagDocumentLine(text, parsed) {
6875
- if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
6876
- return untagMap(parsed, "", 0);
6877
- }
6878
-
6879
- // src/engine/export/import.ts
6880
- var DOCUMENT_EXPORT_FORMAT = "narsil-export";
6881
- var DOCUMENT_EXPORT_VERSION = 1;
6882
- function invalidHeader(message) {
6883
- throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
6884
- }
6885
- function isRecord2(value) {
6886
- return typeof value === "object" && value !== null && !Array.isArray(value);
6887
- }
6888
- function configOfHeader(line) {
6889
- if (line === void 0) invalidHeader("is missing");
6890
- if (!("text" in line)) {
6891
- invalidHeader(
6892
- line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
6893
- );
6894
- }
6895
- let raw;
6896
- try {
6897
- raw = JSON.parse(line.text);
6898
- } catch {
6899
- invalidHeader("is not JSON");
6900
- }
6901
- if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
6902
- invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
6903
- }
6904
- const configuration = requireConfigurationShape(raw.index, "document export header");
6905
- const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
6906
- const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
6907
- return {
6908
- schema: configuration.schema,
6909
- language: configuration.language,
6910
- ...fields,
6911
- ...embedding === void 0 ? {} : {
6912
- embedding: {
6913
- fields: embedding.fields,
6914
- ...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
6915
- }
6916
- }
6917
- };
6918
- }
6919
- async function prepareIndex(engine, indexName, config) {
6920
- if (!engine.listIndexes().some((index) => index.name === indexName)) {
6921
- await engine.createIndex(indexName, config);
6922
- return;
6923
- }
6924
- const existing = flattenSchema(engine.getStats(indexName).schema);
6925
- const exported = flattenSchema(config.schema);
6926
- const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
6927
- for (const field of fields) {
6928
- if (existing[field] === exported[field]) continue;
6929
- throw new NarsilError(
6930
- ErrorCodes.CONFIG_INVALID,
6931
- `Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
6932
- { indexName, field }
6933
- );
6934
- }
6935
- }
6936
- function highestReportedAt(refused) {
6937
- let highest = 0;
6938
- for (let at = 1; at < refused.length; at++) {
6939
- if (refused[at].line > refused[highest].line) highest = at;
6940
- }
6941
- return highest;
6942
- }
6943
- function refuse2(state, line, code, message) {
6944
- state.result.refusedCount += 1;
6945
- const refused = state.result.refused;
6946
- if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
6947
- refused.push({ line, code, message });
6948
- if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
6949
- return;
6950
- }
6951
- if (line >= refused[state.highestReported].line) return;
6952
- refused[state.highestReported] = { line, code, message };
6953
- state.highestReported = highestReportedAt(refused);
6954
- }
6955
- async function flush(state) {
6956
- if (state.batch.length === 0) return;
6957
- const batch = state.batch;
6958
- state.batch = [];
6959
- state.batchIds = /* @__PURE__ */ new Set();
6960
- const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
6961
- const inserts = [];
6962
- const updates = [];
6963
- for (const entry of batch) {
6964
- if (await state.engine.has(state.indexName, entry.docId))
6965
- updates.push({ docId: entry.docId, document: entry.document });
6966
- else inserts.push(entry.document);
6967
- }
6968
- for (const outcome of [
6969
- inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
6970
- updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
6971
- ]) {
6972
- if (outcome === null) continue;
6973
- state.result.imported += outcome.succeeded.length;
6974
- for (const failure of outcome.failed) {
6975
- const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
6976
- refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
6977
- }
6978
- }
6979
- }
6980
- async function takeLine(state, entry) {
6981
- if (!("text" in entry)) {
6982
- const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
6983
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
6984
- return;
6985
- }
6986
- if (entry.text.length === 0) return;
6987
- let document;
6988
- try {
6989
- document = JSON.parse(entry.text);
6990
- } catch {
6991
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
6992
- return;
6993
- }
6994
- if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
6995
- refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
6996
- return;
6997
- }
6998
- try {
6999
- untagDocumentLine(entry.text, document);
7000
- } catch (error) {
7001
- if (!(error instanceof NarsilError)) throw error;
7002
- refuse2(state, entry.line, error.code, error.message);
7003
- return;
7004
- }
7005
- if (state.batchIds.has(document.id)) await flush(state);
7006
- state.batch.push({ line: entry.line, docId: document.id, document });
7007
- state.batchIds.add(document.id);
7008
- if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
7009
- }
7010
- async function readDocumentExport(engine, indexName, source) {
7011
- const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
7012
- const header = await lines.next();
7013
- await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
7014
- const state = {
7015
- engine,
7016
- indexName,
7017
- batch: [],
7018
- batchIds: /* @__PURE__ */ new Set(),
7019
- result: { imported: 0, refusedCount: 0, refused: [] },
7020
- highestReported: 0
7021
- };
7022
- for await (const entry of lines) await takeLine(state, entry);
7023
- await flush(state);
7024
- state.result.refused.sort((a, b) => a.line - b.line);
7025
- return state.result;
7026
- }
7027
-
7028
6305
  // src/narsil/portability.ts
7029
6306
  function portabilityOperations(core, config, engine) {
7030
6307
  async function withOpenIndex2(indexName, action) {
@@ -7056,14 +6333,14 @@ function portabilityOperations(core, config, engine) {
7056
6333
  return {
7057
6334
  snapshot(indexName) {
7058
6335
  return withOpenIndex2(indexName, async () => {
7059
- const { createSnapshot } = await import('./snapshot-DOQLGGEX.mjs');
6336
+ const { createSnapshot } = await import('./snapshot-CGM65VCD.mjs');
7060
6337
  return createSnapshot(snapshotDepsOf(indexName));
7061
6338
  });
7062
6339
  },
7063
6340
  async restore(indexName, data) {
7064
6341
  core.guardShutdown();
7065
6342
  validateIndexName(indexName);
7066
- const restored = import('./restore-queue-JAMNRC6T.mjs').then(
6343
+ const restored = import('./restore-queue-FEYAC4SB.mjs').then(
7067
6344
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
7068
6345
  );
7069
6346
  await emitWriteEventAfter(core, indexName, restored);
@@ -7072,8 +6349,8 @@ function portabilityOperations(core, config, engine) {
7072
6349
  exportDocuments(indexName) {
7073
6350
  return withOpenIndex2(indexName, async () => {
7074
6351
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
7075
- import('./snapshot-DOQLGGEX.mjs'),
7076
- import('./write-ZKR7SNRT.mjs')
6352
+ import('./snapshot-CGM65VCD.mjs'),
6353
+ import('./write-2DZPEQL2.mjs')
7077
6354
  ]);
7078
6355
  const header = indexConfigurationOf(core.requireIndex(indexName));
7079
6356
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -7081,7 +6358,10 @@ function portabilityOperations(core, config, engine) {
7081
6358
  },
7082
6359
  importDocumentExport(indexName, data) {
7083
6360
  core.guardShutdown();
7084
- return emitWriteEventAfter(core, indexName, readDocumentExport(engine(), indexName, data));
6361
+ const imported = import('./import-OOBUW3RR.mjs').then(
6362
+ ({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
6363
+ );
6364
+ return emitWriteEventAfter(core, indexName, imported);
7085
6365
  }
7086
6366
  };
7087
6367
  }
@@ -7092,13 +6372,19 @@ async function enterRead(core, indexName) {
7092
6372
  if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop();
7093
6373
  }
7094
6374
  async function readUnderOneAnalysis(core, indexName, read) {
7095
- let analysisStale = core.analysisRebuild.isStale(indexName);
7096
- let result = await read();
7097
- while (core.analysisRebuild.isStale(indexName) !== analysisStale) {
7098
- analysisStale = core.analysisRebuild.isStale(indexName);
7099
- result = await read();
6375
+ const manager = core.requireManager(indexName);
6376
+ for (; ; ) {
6377
+ const analysis = manager.analysis;
6378
+ const analysisStale = core.analysisRebuild.isStale(indexName);
6379
+ try {
6380
+ const result = await read();
6381
+ if (manager.analysis === analysis && core.analysisRebuild.isStale(indexName) === analysisStale) {
6382
+ return { result, analysisStale };
6383
+ }
6384
+ } catch (error) {
6385
+ if (manager.analysis === analysis) throw error;
6386
+ }
7100
6387
  }
7101
- return { result, analysisStale };
7102
6388
  }
7103
6389
  function phraseReadinessOf(core, indexName, config) {
7104
6390
  return () => {
@@ -7131,7 +6417,7 @@ async function runEngineQuery(core, indexName, params, options) {
7131
6417
  const manager = core.requireManager(indexName);
7132
6418
  const scoped = scopedParams(params, options);
7133
6419
  await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
7134
- const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-7MLQ7J64.mjs');
6420
+ const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-DYV75DCB.mjs');
7135
6421
  const resolvedParams = await resolveVectorText(
7136
6422
  rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
7137
6423
  entry.embeddingAdapter,
@@ -7139,25 +6425,24 @@ async function runEngineQuery(core, indexName, params, options) {
7139
6425
  entry.embeddingAdapterName
7140
6426
  );
7141
6427
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
7142
- const context = {
7143
- manager,
7144
- language: entry.language,
7145
- config: entry.config,
7146
- workerSearch,
7147
- indexName,
7148
- broadcastStats: broadcastStatsFor(core, options),
7149
- partitionIds: options?.partitionIds,
7150
- rescoreWindow: options?.rescoreWindow,
7151
- indexBm25: options?.indexBm25,
7152
- cursorBinding: lazyQueryBinding(params),
7153
- patternWorkCap: core.patternWorkCap,
7154
- rerank: rerank?.rerankRunnerFor(core),
7155
- requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
7156
- };
7157
6428
  const { result, analysisStale } = await readUnderOneAnalysis(
7158
6429
  core,
7159
6430
  indexName,
7160
- () => executeQuery(resolvedParams, context)
6431
+ () => executeQuery(resolvedParams, {
6432
+ manager,
6433
+ language: manager.language,
6434
+ config: manager.config,
6435
+ workerSearch,
6436
+ indexName,
6437
+ broadcastStats: broadcastStatsFor(core, options),
6438
+ partitionIds: options?.partitionIds,
6439
+ rescoreWindow: options?.rescoreWindow,
6440
+ indexBm25: options?.indexBm25,
6441
+ cursorBinding: lazyQueryBinding(params),
6442
+ patternWorkCap: core.patternWorkCap,
6443
+ rerank: rerank?.rerankRunnerFor(core),
6444
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
6445
+ })
7161
6446
  );
7162
6447
  if (analysisStale) {
7163
6448
  result.analysisStale = true;
@@ -7191,23 +6476,22 @@ async function runEnginePreflight(core, indexName, params, options) {
7191
6476
  entry.embeddingAdapterName
7192
6477
  );
7193
6478
  const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
7194
- const context = {
7195
- manager,
7196
- language: entry.language,
7197
- config: entry.config,
7198
- workerSearch,
7199
- indexName,
7200
- broadcastStats: broadcastStatsFor(core, options),
7201
- partitionIds: options?.partitionIds,
7202
- indexBm25: options?.indexBm25,
7203
- cursorBinding: lazyQueryBinding(params),
7204
- patternWorkCap: core.patternWorkCap,
7205
- requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
7206
- };
7207
6479
  const { result, analysisStale } = await readUnderOneAnalysis(
7208
6480
  core,
7209
6481
  indexName,
7210
- () => executePreflight(resolvedParams, context)
6482
+ () => executePreflight(resolvedParams, {
6483
+ manager,
6484
+ language: manager.language,
6485
+ config: manager.config,
6486
+ workerSearch,
6487
+ indexName,
6488
+ broadcastStats: broadcastStatsFor(core, options),
6489
+ partitionIds: options?.partitionIds,
6490
+ indexBm25: options?.indexBm25,
6491
+ cursorBinding: lazyQueryBinding(params),
6492
+ patternWorkCap: core.patternWorkCap,
6493
+ requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
6494
+ })
7211
6495
  );
7212
6496
  if (analysisStale) {
7213
6497
  result.analysisStale = true;
@@ -7323,8 +6607,11 @@ function createNarsilFromCore(core, config) {
7323
6607
  release();
7324
6608
  }
7325
6609
  }
6610
+ const rebalancesStarting = /* @__PURE__ */ new Map();
7326
6611
  function withOpenIndexWrite(indexName, action) {
7327
- return emitWriteEventAfter(core, indexName, withOpenIndex2(indexName, action));
6612
+ const starting = rebalancesStarting.get(indexName);
6613
+ const written = starting === void 0 ? withOpenIndex2(indexName, action) : starting.then(() => withOpenIndex2(indexName, action));
6614
+ return emitWriteEventAfter(core, indexName, written);
7328
6615
  }
7329
6616
  const narsil = {
7330
6617
  createIndex(name, indexConfig) {
@@ -7448,9 +6735,21 @@ function createNarsilFromCore(core, config) {
7448
6735
  return runEngineSuggest(core, indexName, params);
7449
6736
  },
7450
6737
  async rebuildAnalysis(indexName) {
6738
+ guardShutdown();
6739
+ const entry = requireIndex(indexName);
6740
+ await withOpenIndexWrite(indexName, async () => {
6741
+ if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
6742
+ return (await import('./analysis-change-HQUT2Z3I.mjs')).resumeAnalysisChange(core, indexName);
6743
+ });
6744
+ },
6745
+ async changeAnalysis(indexName, change) {
7451
6746
  guardShutdown();
7452
6747
  requireIndex(indexName);
7453
- await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
6748
+ const changed = withOpenIndex2(
6749
+ indexName,
6750
+ async () => (await import('./analysis-change-HQUT2Z3I.mjs')).changeIndexAnalysis(core, indexName, change)
6751
+ );
6752
+ await emitWriteEventAfter(core, indexName, changed);
7454
6753
  },
7455
6754
  ...portabilityOperations(core, config, () => narsil),
7456
6755
  ...synonymOperations(core),
@@ -7469,15 +6768,29 @@ function createNarsilFromCore(core, config) {
7469
6768
  await executor.execute({ type: "clear", indexName, requestId: indexName });
7470
6769
  await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
7471
6770
  core.watermarkNotifier.forget(indexName);
6771
+ const durable = core.durability?.manager;
6772
+ await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
7472
6773
  });
7473
6774
  },
7474
6775
  async rebalance(indexName, targetPartitionCount) {
7475
6776
  guardShutdown();
7476
6777
  requireIndex(indexName);
7477
- return withOpenIndexWrite(
7478
- indexName,
7479
- () => executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx)
7480
- );
6778
+ let writesMayFollow = () => void 0;
6779
+ const started = new Promise((resolve) => {
6780
+ writesMayFollow = resolve;
6781
+ });
6782
+ function releaseWrites() {
6783
+ writesMayFollow();
6784
+ if (rebalancesStarting.get(indexName) === started) rebalancesStarting.delete(indexName);
6785
+ }
6786
+ rebalancesStarting.set(indexName, started);
6787
+ const rebalanced = withOpenIndex2(indexName, async () => {
6788
+ const { executeRebalance } = await import('./analysis-change-HQUT2Z3I.mjs');
6789
+ const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
6790
+ releaseWrites();
6791
+ return running;
6792
+ });
6793
+ return emitWriteEventAfter(core, indexName, rebalanced).finally(releaseWrites);
7481
6794
  },
7482
6795
  async updatePartitionConfig(indexName, partitionConfig) {
7483
6796
  guardShutdown();
@@ -7596,4 +6909,4 @@ async function createNarsil(config) {
7596
6909
  // src/index.ts
7597
6910
  var VERSION = "0.1.0";
7598
6911
 
7599
- export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, crc322 as crc32, crc32Final, crc32Init, crc32Update, createByteReader, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, emptySerializablePartition, encodeBase64, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, startsWithTagPrefix, storedArrayPositionGap, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader, yieldToEventLoop };
6912
+ export { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, REBALANCE_CHUNK_SIZE, RESERVED_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, checksumOfChunks, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, deleteNestedValue, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineVersionParts, extractVectorFromDoc, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, requireWithinFileLimit, resolveRunnableConfig, rewriteSpillOf, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, validateEmbeddingConfig, validateRequiredFieldsInSchema, vectorsEqual, writeHeader, yieldToEventLoop };