@delali/narsil 0.3.1-next.140 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,25 +1,25 @@
1
- import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-XMQNOD5V.mjs';
2
- import { resolveVectorText, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-XXKWEG53.mjs';
1
+ import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-VEFN6X6Q.mjs';
2
+ import { resolveVectorText, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-5WYVUPGD.mjs';
3
+ import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
3
4
  import { splitWorkerBudget, resolveWorkerCount, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-UIKN2OTQ.mjs';
4
5
  import { detectRuntime } from './chunk-AV4IJKD2.mjs';
5
- import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-IBATGS3J.mjs';
6
- import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, createPartitionIndex, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-U4X4AWKL.mjs';
6
+ import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-EEBCDP36.mjs';
7
+ import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-WJGO5PGV.mjs';
8
+ import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, createPartitionIndex, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-ZPWS2SKJ.mjs';
7
9
  import { generateId } from './chunk-N37VMC62.mjs';
8
10
  import { readHeapStatistics } from './chunk-ZATRDAYU.mjs';
9
11
  import { countEngineThread } from './chunk-EBIHRL7B.mjs';
10
- import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
11
- import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-EEBCDP36.mjs';
12
12
  import { requireSearchBackendSettings } from './chunk-74XB2SXI.mjs';
13
13
  import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-RKAVHNRW.mjs';
14
- import { getNestedValue } from './chunk-J3EHATEZ.mjs';
15
- import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-UBXJQJII.mjs';
14
+ import { getNestedValue } from './chunk-DQNKVOJ2.mjs';
15
+ import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KGOXHLLZ.mjs';
16
16
  import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-U476IXKQ.mjs';
17
17
  import { isSingleStringFieldType } from './chunk-ZSWHPKGM.mjs';
18
18
  import { withOpenIndex, oneAtATime } from './chunk-O4LJQI4G.mjs';
19
19
  import { getLanguage } from './chunk-WNBRW4Q2.mjs';
20
20
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
21
- import { createRequestId, isValidWorkerResponse } from './chunk-UJWGQIXU.mjs';
22
- import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-FFVLPCLH.mjs';
21
+ import { createRequestId, isValidWorkerResponse } from './chunk-QHPYS427.mjs';
22
+ import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-GLZHNSGB.mjs';
23
23
  import { noteThreadSlotLive, threadSlotOfWorker, noteThreadSlotGone } from './chunk-RFBLQLLZ.mjs';
24
24
  import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
25
25
  import { encode, decode } from '@msgpack/msgpack';
@@ -152,130 +152,6 @@ function requireValidPatternWorkCap(cap) {
152
152
  );
153
153
  }
154
154
 
155
- // src/engine/orchestration/turn.ts
156
- function afterCurrentTurn(callback) {
157
- if (typeof setImmediate === "function") {
158
- setImmediate(callback);
159
- return;
160
- }
161
- setTimeout(callback, 0);
162
- }
163
- function yieldToEventLoop() {
164
- return new Promise((resolve) => afterCurrentTurn(resolve));
165
- }
166
-
167
- // src/partitioning/constants.ts
168
- var REBALANCE_CHUNK_SIZE = 1e3;
169
- var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
170
- var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
171
- var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
172
-
173
- // src/partitioning/rebalancer.ts
174
- function createRebalancer() {
175
- const activeManagers = /* @__PURE__ */ new WeakSet();
176
- let activeCount = 0;
177
- async function rebalance(manager, newPartitionCount, router, onProgress) {
178
- if (activeManagers.has(manager)) {
179
- throw new NarsilError(
180
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
181
- "A rebalance operation is already in progress"
182
- );
183
- }
184
- if (newPartitionCount <= 0) {
185
- throw new NarsilError(
186
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
187
- `New partition count must be greater than 0, received ${newPartitionCount}`,
188
- { newPartitionCount }
189
- );
190
- }
191
- if (newPartitionCount === manager.partitionCount) {
192
- throw new NarsilError(
193
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
194
- `New partition count (${newPartitionCount}) is the same as the current count`,
195
- { newPartitionCount, currentPartitionCount: manager.partitionCount }
196
- );
197
- }
198
- activeManagers.add(manager);
199
- activeCount++;
200
- try {
201
- const currentPartitions = manager.getAllPartitions();
202
- let documentsTotal = 0;
203
- for (const partition of currentPartitions) {
204
- documentsTotal += partition.count();
205
- }
206
- onProgress?.({
207
- phase: "scanning",
208
- documentsProcessed: 0,
209
- documentsTotal
210
- });
211
- const newPartitions = [];
212
- for (let i = 0; i < newPartitionCount; i++) {
213
- newPartitions.push(createPartitionIndex(i, manager.config.trackPositions ?? true));
214
- }
215
- let documentsProcessed = 0;
216
- let chunkFill = 0;
217
- const insertOptions = {
218
- validate: false,
219
- skipClone: true,
220
- stopWordOverride: manager.analysis.stopWords,
221
- customTokenizer: manager.analysis.customTokenizer,
222
- collectSurfaces: manager.config.surfaceForms !== false
223
- };
224
- for (const partition of currentPartitions) {
225
- const docIds = [...partition.docIds()];
226
- for (const docId of docIds) {
227
- const document = partition.getRef(docId);
228
- if (!document) continue;
229
- const targetPartitionId = router.route(docId, newPartitionCount);
230
- const targetPartition = newPartitions[targetPartitionId];
231
- if (!targetPartition.has(docId)) {
232
- targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
233
- documentsProcessed++;
234
- }
235
- chunkFill++;
236
- if (chunkFill >= REBALANCE_CHUNK_SIZE) {
237
- chunkFill = 0;
238
- onProgress?.({
239
- phase: "moving",
240
- documentsProcessed,
241
- documentsTotal
242
- });
243
- await yieldToEventLoop();
244
- }
245
- }
246
- }
247
- if (chunkFill > 0) {
248
- onProgress?.({
249
- phase: "moving",
250
- documentsProcessed,
251
- documentsTotal
252
- });
253
- await yieldToEventLoop();
254
- }
255
- onProgress?.({
256
- phase: "swapping",
257
- documentsProcessed,
258
- documentsTotal
259
- });
260
- manager.setPartitions(newPartitions);
261
- onProgress?.({
262
- phase: "complete",
263
- documentsProcessed,
264
- documentsTotal
265
- });
266
- } finally {
267
- activeManagers.delete(manager);
268
- activeCount--;
269
- }
270
- }
271
- return {
272
- rebalance,
273
- isRebalancing() {
274
- return activeCount > 0;
275
- }
276
- };
277
- }
278
-
279
155
  // src/plugins/registry.ts
280
156
  var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
281
157
  function isThenable(value) {
@@ -479,6 +355,31 @@ function validateRequiredFieldsInSchema(required, schema) {
479
355
  }
480
356
  }
481
357
 
358
+ // src/engine/orchestration/turn.ts
359
+ function afterCurrentTurn(callback) {
360
+ if (typeof setImmediate === "function") {
361
+ setImmediate(callback);
362
+ return;
363
+ }
364
+ setTimeout(callback, 0);
365
+ }
366
+ function yieldToEventLoop() {
367
+ return new Promise((resolve) => afterCurrentTurn(resolve));
368
+ }
369
+
370
+ // src/partitioning/constants.ts
371
+ var REBALANCE_CHUNK_SIZE = 1e3;
372
+ var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
373
+ var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
374
+ var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
375
+
376
+ // src/partitioning/rewrite-spill-of.ts
377
+ async function rewriteSpillOf(storage, manager, target) {
378
+ if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
379
+ const { openRewriteSpill } = await import('./module-IYPLWHGC.mjs');
380
+ return openRewriteSpill(storage, manager, target);
381
+ }
382
+
482
383
  // src/partitioning/text-rebuild.ts
483
384
  function textRebuildTargetOf(config, language, analysis) {
484
385
  return { config, language, analysis, partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis) };
@@ -488,7 +389,7 @@ function interrupted(pass) {
488
389
  }
489
390
  async function buildPartition(pass, source, partitionId) {
490
391
  const { manager, language, options } = pass;
491
- const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
392
+ const target = pass.spill?.newPartition(partitionId) ?? createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
492
393
  const docIds = [...source.docIds()];
493
394
  target.beginBatch();
494
395
  for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
@@ -497,6 +398,11 @@ async function buildPartition(pass, source, partitionId) {
497
398
  const document = manager.getRef(docIds[i]);
498
399
  if (document !== void 0) target.insert(docIds[i], document, manager.schema, language, options);
499
400
  }
401
+ if (pass.spill !== null) {
402
+ target.endBatch();
403
+ await pass.spill.spillIfFull(target, partitionId);
404
+ target.beginBatch();
405
+ }
500
406
  await yieldToEventLoop();
501
407
  if (interrupted(pass)) return null;
502
408
  }
@@ -558,12 +464,19 @@ async function rebuildTextIndexes(manager, hooks, target) {
558
464
  );
559
465
  const language = target?.language ?? manager.language;
560
466
  const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
467
+ const spill = await rewriteSpillOf(hooks.rewriteStorage?.(), manager, target);
561
468
  while (hooks.isCurrent()) {
562
469
  const log = manager.openWriteLog();
470
+ let swapped = false;
563
471
  try {
564
- if (await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options })) return true;
472
+ swapped = await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options, spill });
473
+ if (swapped) {
474
+ hooks.onRewriteSpill?.(spill);
475
+ return true;
476
+ }
565
477
  } finally {
566
478
  log.close();
479
+ if (!swapped) await spill?.abandon();
567
480
  }
568
481
  }
569
482
  return false;
@@ -597,6 +510,7 @@ function toError(value) {
597
510
  function wireAnalysisRebuild(wiring) {
598
511
  return createAnalysisRebuildCoordinator(wiring.config, {
599
512
  getManager: wiring.getManager,
513
+ rewriteStorage: () => wiring.durabilityManager?.rewriteStorage?.(),
600
514
  desyncIndex: wiring.desyncIndex,
601
515
  resyncIndex: wiring.resyncIndex,
602
516
  hasPendingChange: wiring.hasPendingChange,
@@ -630,8 +544,13 @@ function createAnalysisRebuildCoordinator(config, deps) {
630
544
  const partitionCount = manager.partitionCount;
631
545
  deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
632
546
  const wasPromoted = deps.desyncIndex(indexName);
547
+ let spill = null;
633
548
  try {
634
549
  const rebuilt = await rebuildTextIndexes(manager, {
550
+ rewriteStorage: () => deps.rewriteStorage?.(),
551
+ onRewriteSpill(kept) {
552
+ spill = kept;
553
+ },
635
554
  isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
636
555
  onPartitionBuilt(partitionsBuilt) {
637
556
  entry.partitionsRebuilt = partitionsBuilt;
@@ -646,6 +565,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
646
565
  return;
647
566
  }
648
567
  await deps.persistAnalysisRevision(indexName);
568
+ await spill?.removeUnread(manager);
649
569
  await deps.resyncIndex(indexName, wasPromoted);
650
570
  deps.emit({
651
571
  indexName,
@@ -919,6 +839,9 @@ function reconstructSchemaFromMetadata(metadata) {
919
839
  if (metadata.trackPositions !== void 0) {
920
840
  config.trackPositions = metadata.trackPositions;
921
841
  }
842
+ if (metadata.textStorage !== void 0) {
843
+ config.textStorage = metadata.textStorage;
844
+ }
922
845
  if (metadata.strict !== void 0) {
923
846
  config.strict = metadata.strict;
924
847
  }
@@ -997,7 +920,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
997
920
  const existing = deps.indexRegistry.get(metadata.indexName);
998
921
  if (existing !== void 0) {
999
922
  if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
1000
- const { adoptSavedAnalysis } = await import('./analysis-change-JN23AZQP.mjs');
923
+ const { adoptSavedAnalysis } = await import('./analysis-change-HQUT2Z3I.mjs');
1001
924
  adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
1002
925
  }
1003
926
  if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
@@ -1030,6 +953,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1030
953
  const synonyms = prepared?.encoded ?? null;
1031
954
  try {
1032
955
  validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
956
+ validateTextStorage(indexConfig.textStorage, deps.filesystemDurability);
1033
957
  if (loadData) {
1034
958
  deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
1035
959
  }
@@ -1781,6 +1705,9 @@ function checkHeapAfterRecovery(core) {
1781
1705
 
1782
1706
  // src/core/partition/frozen/share.ts
1783
1707
  function shareFrozenSegment(segment, analysis) {
1708
+ if (segment.paged !== null) {
1709
+ return { snapshot: segment.paged.snapshot, tombstonedDocIds: segment.tombstonedDocIds() };
1710
+ }
1784
1711
  if (segment.sharedSnapshot !== null) {
1785
1712
  const tombstonedDocIds = segment.tombstonedDocIds();
1786
1713
  if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
@@ -2567,7 +2494,7 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
2567
2494
  }
2568
2495
  async function compactIndexSegments(state, indexName, policy) {
2569
2496
  const manager = state.executor.getManager(indexName);
2570
- if (!manager) return;
2497
+ if (!manager || manager.config.textStorage === "disk") return;
2571
2498
  const partitionIds = /* @__PURE__ */ new Set();
2572
2499
  for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) partitionIds.add(partitionId);
2573
2500
  for (const partitionId of state.segmentLedger.get(indexName)?.keys() ?? []) partitionIds.add(partitionId);
@@ -4041,6 +3968,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
4041
3968
  scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
4042
3969
  replicateToWorkers: replicate,
4043
3970
  awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
3971
+ queueDiskAdoption: (action) => queueForCopies(state, action),
4044
3972
  awaitWrites: (indexName) => awaitWritesApplied(state, indexName),
4045
3973
  awaitCompactions: () => awaitCompactions(state),
4046
3974
  openIndex,
@@ -4125,7 +4053,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4125
4053
  if (config?.lifecycle !== void 0 && durabilityTier === null) {
4126
4054
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
4127
4055
  }
4128
- const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-AYOTAPNQ.mjs');
4056
+ const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-324D2DDD.mjs');
4129
4057
  const filesystemDurability = durabilityTier?.kind === "wal";
4130
4058
  const executor = createDirectExecutor({
4131
4059
  vectorWorkerCopies: vectorCopyPolicy,
@@ -4185,7 +4113,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4185
4113
  },
4186
4114
  vectorCopyPolicy
4187
4115
  );
4188
- const rebalancer = createRebalancer();
4189
4116
  const rebalanceRouter = createPartitionRouter();
4190
4117
  const waqMap = /* @__PURE__ */ new Map();
4191
4118
  const rebalanceTargets = /* @__PURE__ */ new Map();
@@ -4245,6 +4172,8 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4245
4172
  emitCheckpointHeld(report) {
4246
4173
  emitEngineEvent(eventHandlers, "checkpointHeld", report);
4247
4174
  },
4175
+ queueDiskAdoption: orchestrator.queueDiskAdoption,
4176
+ awaitCopiesOfIndex: (indexName) => orchestrator.awaitReplication(indexName),
4248
4177
  publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve(),
4249
4178
  recordCheckpoint(indexName, documentCount, partitionCount) {
4250
4179
  const entry = indexRegistry.get(indexName);
@@ -4309,7 +4238,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4309
4238
  requireManager,
4310
4239
  resumeChange: async (indexName) => {
4311
4240
  if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
4312
- await (await import('./analysis-change-entry-5JSIMEKG.mjs')).resumeChangeOf(engineCore, indexName);
4241
+ await (await import('./analysis-change-entry-TZTG4AZB.mjs')).resumeChangeOf(engineCore, indexName);
4313
4242
  },
4314
4243
  onClose: hooks?.onIndexClose,
4315
4244
  onOpen: (indexName) => {
@@ -4339,7 +4268,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4339
4268
  checkHeapPressure: heapPressureNotifier.check
4340
4269
  };
4341
4270
  const rebalanceCtx = {
4342
- rebalancer,
4343
4271
  router: rebalanceRouter,
4344
4272
  waqMap,
4345
4273
  rebalancingIndexes,
@@ -4366,7 +4294,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
4366
4294
  shutdownState,
4367
4295
  abortController,
4368
4296
  orchestrator,
4369
- rebalancer,
4370
4297
  rebalanceRouter,
4371
4298
  rebalancingIndexes,
4372
4299
  waqMap,
@@ -4393,7 +4320,7 @@ async function startEngineCore(core, config) {
4393
4320
  checkHeapAfterRecovery(core);
4394
4321
  }
4395
4322
  if (core.invalidation) await core.invalidation.start();
4396
- if (core.durability) await (await import('./analysis-change-entry-5JSIMEKG.mjs')).resumeEveryChange(core);
4323
+ if (core.durability) await (await import('./analysis-change-entry-TZTG4AZB.mjs')).resumeEveryChange(core);
4397
4324
  if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
4398
4325
  } catch (err) {
4399
4326
  await core.durability?.manager.shutdown().catch(() => void 0);
@@ -4536,6 +4463,7 @@ async function checksumOfChunks(chunks) {
4536
4463
  // src/serialization/header.ts
4537
4464
  var MAGIC_BYTES = new Uint8Array([78, 82, 83, 76]);
4538
4465
  var HEADER_SIZE = 32;
4466
+ var RESERVED_SIZE = 14;
4539
4467
  var MAX_PAYLOAD_LENGTH = 4294967295;
4540
4468
  var COMPRESSION_ALGORITHMS = {
4541
4469
  0: "none",
@@ -4648,9 +4576,12 @@ function requireWithinFileLimit(byteLength, what) {
4648
4576
  );
4649
4577
  }
4650
4578
  async function packChecksummedEnvelope(payloadChunks, envelopeFormatVersion, what) {
4579
+ requireWithinFileLimit(HEADER_SIZE + payloadByteLength(payloadChunks), what);
4580
+ return packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, await checksumOfChunks(payloadChunks));
4581
+ }
4582
+ function packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, checksum) {
4651
4583
  const payloadLength = payloadByteLength(payloadChunks);
4652
4584
  requireWithinFileLimit(HEADER_SIZE + payloadLength, what);
4653
- const checksum = await checksumOfChunks(payloadChunks);
4654
4585
  const [major, minor, patch] = engineVersionParts();
4655
4586
  const header = writeHeader({
4656
4587
  envelopeFormatVersion,
@@ -4812,6 +4743,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
4812
4743
  ...config.partitions !== void 0 ? { partitionConfig: config.partitions } : {},
4813
4744
  ...config.defaultScoring !== void 0 ? { defaultScoring: config.defaultScoring } : {},
4814
4745
  ...config.trackPositions !== void 0 ? { trackPositions: config.trackPositions } : {},
4746
+ ...config.textStorage !== void 0 ? { textStorage: config.textStorage } : {},
4815
4747
  ...config.strict !== void 0 ? { strict: config.strict } : {},
4816
4748
  ...config.required !== void 0 ? { required: config.required } : {},
4817
4749
  ...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
@@ -4900,6 +4832,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
4900
4832
  requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
4901
4833
  validateVectorPromotion(indexConfig.vectorPromotion);
4902
4834
  validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
4835
+ validateTextStorage(indexConfig.textStorage, core.filesystemDurability);
4903
4836
  if (indexConfig.bm25 !== void 0) {
4904
4837
  validateBM25Params(
4905
4838
  indexConfig.bm25,
@@ -4991,7 +4924,7 @@ async function dropEngineIndex(core, name) {
4991
4924
  const entry = core.requireIndex(name);
4992
4925
  const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
4993
4926
  const abandonChange = async () => {
4994
- await (await import('./analysis-change-entry-5JSIMEKG.mjs')).abandonChangeOf(core, name);
4927
+ await (await import('./analysis-change-entry-TZTG4AZB.mjs')).abandonChangeOf(core, name);
4995
4928
  };
4996
4929
  if (changeRecorded()) await abandonChange();
4997
4930
  await core.indexState.drop(name, async () => {
@@ -6246,284 +6179,6 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
6246
6179
  return { succeeded, failed };
6247
6180
  }
6248
6181
 
6249
- // src/partitioning/write-ahead-queue.ts
6250
- function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
6251
- const entries = [];
6252
- const docStates = /* @__PURE__ */ new Map();
6253
- let nextSequence = 1;
6254
- let resolveReplayed = () => void 0;
6255
- const replayed = new Promise((resolve) => {
6256
- resolveReplayed = resolve;
6257
- });
6258
- return {
6259
- push(entry) {
6260
- if (entries.length >= maxSize) {
6261
- throw new NarsilError(
6262
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
6263
- `Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
6264
- { maxSize, currentSize: entries.length }
6265
- );
6266
- }
6267
- const seq = nextSequence++;
6268
- entries.push({ ...entry, sequenceNumber: seq });
6269
- docStates.set(entry.docId, entry.action === "remove" ? "absent" : "present");
6270
- return seq;
6271
- },
6272
- drain() {
6273
- const result = entries.splice(0);
6274
- result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
6275
- return result;
6276
- },
6277
- clear() {
6278
- entries.length = 0;
6279
- docStates.clear();
6280
- },
6281
- bufferedDocState(docId) {
6282
- return docStates.get(docId);
6283
- },
6284
- whenReplayed() {
6285
- return replayed;
6286
- },
6287
- markReplayed() {
6288
- resolveReplayed();
6289
- },
6290
- get size() {
6291
- return entries.length;
6292
- },
6293
- get isFull() {
6294
- return entries.length >= maxSize;
6295
- }
6296
- };
6297
- }
6298
-
6299
- // src/engine/rebalance-executor.ts
6300
- function warnHookError(hook, err) {
6301
- console.warn(`${hook} plugin hook error:`, err instanceof Error ? err.message : String(err));
6302
- }
6303
- function applyReplayUpdateVectors(entry, vectorFieldPaths) {
6304
- const updateVectors = /* @__PURE__ */ new Map();
6305
- for (const fieldPath of vectorFieldPaths) {
6306
- updateVectors.set(fieldPath, extractVectorFromDoc(entry.document, fieldPath));
6307
- }
6308
- let updateDoc = entry.document;
6309
- if (updateVectors.size > 0) {
6310
- updateDoc = structuredClone(entry.document);
6311
- for (const fieldPath of updateVectors.keys()) {
6312
- deleteNestedValue(updateDoc, fieldPath);
6313
- }
6314
- }
6315
- return { updateDoc, updateVectors };
6316
- }
6317
- async function replayEntry(manager, indexName, entry, ctx) {
6318
- const vecIndexes = manager.getVectorIndexes();
6319
- const vectorFieldPaths = ctx.requireIndex(indexName).vectorFieldPaths;
6320
- if (entry.action === "insert" && entry.document) {
6321
- const { partitionDoc, extractedVectors } = prepareDocumentVectors(
6322
- entry.document,
6323
- vectorFieldPaths
6324
- );
6325
- const overwrote = manager.has(entry.docId);
6326
- const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
6327
- if (overwrote) {
6328
- removeDocumentVectors(entry.docId, vecIndexes);
6329
- manager.remove(entry.docId, removedSpellings);
6330
- }
6331
- manager.insert(entry.docId, partitionDoc);
6332
- if (extractedVectors.size > 0) {
6333
- insertDocumentVectors(entry.docId, extractedVectors, vecIndexes, manager.partitionIdOf(entry.docId));
6334
- for (const fieldPath of extractedVectors.keys()) {
6335
- vecIndexes.get(fieldPath)?.scheduleBuild();
6336
- }
6337
- }
6338
- try {
6339
- await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: entry.docId, document: entry.document });
6340
- } catch (err) {
6341
- warnHookError("afterInsert", err);
6342
- }
6343
- const replicated = {
6344
- indexName,
6345
- docId: entry.docId,
6346
- document: entry.document,
6347
- requestId: `replicate-insert-${entry.docId}`
6348
- };
6349
- await ctx.orchestrator.replicateToWorkers(
6350
- overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
6351
- );
6352
- return;
6353
- }
6354
- if (entry.action === "remove") {
6355
- const removedSpellings = manager.removedSpellingsOf(entry.docId);
6356
- manager.remove(entry.docId, removedSpellings);
6357
- removeDocumentVectors(entry.docId, vecIndexes);
6358
- try {
6359
- await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
6360
- } catch (err) {
6361
- warnHookError("afterRemove", err);
6362
- }
6363
- await ctx.orchestrator.replicateToWorkers({
6364
- type: "remove",
6365
- indexName,
6366
- docId: entry.docId,
6367
- requestId: `replicate-remove-${entry.docId}`,
6368
- removedSpellings
6369
- });
6370
- return;
6371
- }
6372
- if (entry.action === "update" && entry.document) {
6373
- const oldDocument = manager.get(entry.docId);
6374
- const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths);
6375
- const removedSpellings = manager.removedSpellingsOf(entry.docId);
6376
- if (manager.has(entry.docId)) {
6377
- manager.update(entry.docId, updateDoc, void 0, removedSpellings);
6378
- } else {
6379
- manager.insert(entry.docId, updateDoc);
6380
- }
6381
- for (const [fieldPath, newVec] of updateVectors) {
6382
- const vecIndex = vecIndexes.get(fieldPath);
6383
- if (!vecIndex) continue;
6384
- if (newVec === null) {
6385
- if (vecIndex.has(entry.docId)) {
6386
- vecIndex.remove(entry.docId);
6387
- }
6388
- } else {
6389
- const oldVec = vecIndex.getVector(entry.docId);
6390
- if (!vectorsEqual(oldVec, newVec)) {
6391
- if (vecIndex.has(entry.docId)) {
6392
- vecIndex.remove(entry.docId);
6393
- }
6394
- vecIndex.insert(entry.docId, newVec);
6395
- vecIndex.scheduleBuild();
6396
- }
6397
- }
6398
- }
6399
- try {
6400
- await ctx.pluginRegistry.runHook("afterUpdate", {
6401
- indexName,
6402
- docId: entry.docId,
6403
- oldDocument: oldDocument ?? {},
6404
- newDocument: entry.document
6405
- });
6406
- } catch (err) {
6407
- warnHookError("afterUpdate", err);
6408
- }
6409
- await ctx.orchestrator.replicateToWorkers({
6410
- type: "update",
6411
- indexName,
6412
- docId: entry.docId,
6413
- document: entry.document,
6414
- requestId: `replicate-update-${entry.docId}`,
6415
- removedSpellings
6416
- });
6417
- }
6418
- }
6419
- async function replayQueued(manager, indexName, waq, ctx) {
6420
- while (true) {
6421
- const entries = waq.drain();
6422
- if (entries.length === 0) break;
6423
- for (const entry of entries) {
6424
- try {
6425
- await replayEntry(manager, indexName, entry, ctx);
6426
- } catch (replayErr) {
6427
- const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
6428
- if (!isMissing) {
6429
- console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
6430
- }
6431
- }
6432
- }
6433
- }
6434
- }
6435
- async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
6436
- if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
6437
- throw new NarsilError(
6438
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
6439
- `Target partition count must be a positive integer, got ${targetPartitionCount}`,
6440
- { targetPartitionCount }
6441
- );
6442
- }
6443
- const entry = ctx.requireIndex(indexName);
6444
- const maxPartitions = entry.config.partitions?.maxPartitions;
6445
- if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
6446
- throw new NarsilError(
6447
- ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
6448
- `Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
6449
- { targetPartitionCount, maxPartitions }
6450
- );
6451
- }
6452
- if (ctx.rebalancingIndexes.has(indexName)) {
6453
- throw new NarsilError(
6454
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
6455
- `Index "${indexName}" is already being rebalanced`
6456
- );
6457
- }
6458
- const registered = ctx.requireIndex(indexName);
6459
- if (registered.pendingAnalysis !== null || registered.durableAnalysis !== null) {
6460
- throw new NarsilError(
6461
- ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
6462
- `Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
6463
- { indexName }
6464
- );
6465
- }
6466
- if (targetPartitionCount === manager.partitionCount) {
6467
- return;
6468
- }
6469
- const waq = createWriteAheadQueue();
6470
- ctx.waqMap.set(indexName, waq);
6471
- ctx.rebalancingIndexes.add(indexName);
6472
- ctx.rebalanceTargets.set(indexName, targetPartitionCount);
6473
- const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
6474
- try {
6475
- const oldCount = manager.partitionCount;
6476
- await ctx.rebalancer.rebalance(manager, targetPartitionCount, ctx.router, (progress) => {
6477
- if (progress.phase === "complete") {
6478
- emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
6479
- indexName,
6480
- oldCount,
6481
- newCount: targetPartitionCount
6482
- });
6483
- }
6484
- });
6485
- try {
6486
- await ctx.pluginRegistry.runHook("onPartitionSplit", {
6487
- indexName,
6488
- oldPartitionCount: oldCount,
6489
- newPartitionCount: targetPartitionCount
6490
- });
6491
- } catch (hookErr) {
6492
- console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
6493
- }
6494
- await replayQueued(manager, indexName, waq, ctx);
6495
- if (ctx.durabilityManager) {
6496
- await ctx.durabilityManager.persistMetadata(indexName);
6497
- await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
6498
- await replayQueued(manager, indexName, waq, ctx);
6499
- }
6500
- await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
6501
- do {
6502
- await replayQueued(manager, indexName, waq, ctx);
6503
- } while (waq.size > 0);
6504
- } catch (err) {
6505
- try {
6506
- await replayQueued(manager, indexName, waq, ctx);
6507
- await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
6508
- do {
6509
- await replayQueued(manager, indexName, waq, ctx);
6510
- } while (waq.size > 0);
6511
- } catch (recoveryErr) {
6512
- console.warn(
6513
- `Rebalance failure recovery for index "${indexName}" did not complete:`,
6514
- recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)
6515
- );
6516
- }
6517
- throw err;
6518
- } finally {
6519
- ctx.rebalancingIndexes.delete(indexName);
6520
- ctx.rebalanceTargets.delete(indexName);
6521
- ctx.waqMap.delete(indexName);
6522
- waq.markReplayed();
6523
- }
6524
- ctx.checkWatermark(indexName);
6525
- }
6526
-
6527
6182
  // src/engine/vector-maintenance.ts
6528
6183
  function resolveVectorIndexes(manager, indexName, fieldName) {
6529
6184
  const all = manager.getVectorIndexes();
@@ -6563,6 +6218,7 @@ function getVectorMaintenanceStatus(manager) {
6563
6218
  bufferSize: status.bufferSize,
6564
6219
  building: status.building,
6565
6220
  waitingForMemory: status.waitingForMemory,
6221
+ loadingFromFiles: status.loadingFromFiles === true,
6566
6222
  estimatedCompactMs: status.estimatedCompactMs,
6567
6223
  estimatedOptimizeMs: status.estimatedOptimizeMs
6568
6224
  });
@@ -6677,14 +6333,14 @@ function portabilityOperations(core, config, engine) {
6677
6333
  return {
6678
6334
  snapshot(indexName) {
6679
6335
  return withOpenIndex2(indexName, async () => {
6680
- const { createSnapshot } = await import('./snapshot-BTRZ3MZK.mjs');
6336
+ const { createSnapshot } = await import('./snapshot-CGM65VCD.mjs');
6681
6337
  return createSnapshot(snapshotDepsOf(indexName));
6682
6338
  });
6683
6339
  },
6684
6340
  async restore(indexName, data) {
6685
6341
  core.guardShutdown();
6686
6342
  validateIndexName(indexName);
6687
- const restored = import('./restore-queue-RYSRRFS4.mjs').then(
6343
+ const restored = import('./restore-queue-FEYAC4SB.mjs').then(
6688
6344
  ({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
6689
6345
  );
6690
6346
  await emitWriteEventAfter(core, indexName, restored);
@@ -6693,8 +6349,8 @@ function portabilityOperations(core, config, engine) {
6693
6349
  exportDocuments(indexName) {
6694
6350
  return withOpenIndex2(indexName, async () => {
6695
6351
  const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
6696
- import('./snapshot-BTRZ3MZK.mjs'),
6697
- import('./write-GHTA56OW.mjs')
6352
+ import('./snapshot-CGM65VCD.mjs'),
6353
+ import('./write-2DZPEQL2.mjs')
6698
6354
  ]);
6699
6355
  const header = indexConfigurationOf(core.requireIndex(indexName));
6700
6356
  return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
@@ -6702,7 +6358,7 @@ function portabilityOperations(core, config, engine) {
6702
6358
  },
6703
6359
  importDocumentExport(indexName, data) {
6704
6360
  core.guardShutdown();
6705
- const imported = import('./import-QRZAQ5RH.mjs').then(
6361
+ const imported = import('./import-OOBUW3RR.mjs').then(
6706
6362
  ({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
6707
6363
  );
6708
6364
  return emitWriteEventAfter(core, indexName, imported);
@@ -6951,8 +6607,11 @@ function createNarsilFromCore(core, config) {
6951
6607
  release();
6952
6608
  }
6953
6609
  }
6610
+ const rebalancesStarting = /* @__PURE__ */ new Map();
6954
6611
  function withOpenIndexWrite(indexName, action) {
6955
- return emitWriteEventAfter(core, indexName, withOpenIndex2(indexName, action));
6612
+ const starting = rebalancesStarting.get(indexName);
6613
+ const written = starting === void 0 ? withOpenIndex2(indexName, action) : starting.then(() => withOpenIndex2(indexName, action));
6614
+ return emitWriteEventAfter(core, indexName, written);
6956
6615
  }
6957
6616
  const narsil = {
6958
6617
  createIndex(name, indexConfig) {
@@ -7080,16 +6739,17 @@ function createNarsilFromCore(core, config) {
7080
6739
  const entry = requireIndex(indexName);
7081
6740
  await withOpenIndexWrite(indexName, async () => {
7082
6741
  if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
7083
- return (await import('./analysis-change-JN23AZQP.mjs')).resumeAnalysisChange(core, indexName);
6742
+ return (await import('./analysis-change-HQUT2Z3I.mjs')).resumeAnalysisChange(core, indexName);
7084
6743
  });
7085
6744
  },
7086
6745
  async changeAnalysis(indexName, change) {
7087
6746
  guardShutdown();
7088
6747
  requireIndex(indexName);
7089
- await withOpenIndexWrite(
6748
+ const changed = withOpenIndex2(
7090
6749
  indexName,
7091
- async () => (await import('./analysis-change-JN23AZQP.mjs')).changeIndexAnalysis(core, indexName, change)
6750
+ async () => (await import('./analysis-change-HQUT2Z3I.mjs')).changeIndexAnalysis(core, indexName, change)
7092
6751
  );
6752
+ await emitWriteEventAfter(core, indexName, changed);
7093
6753
  },
7094
6754
  ...portabilityOperations(core, config, () => narsil),
7095
6755
  ...synonymOperations(core),
@@ -7108,15 +6768,29 @@ function createNarsilFromCore(core, config) {
7108
6768
  await executor.execute({ type: "clear", indexName, requestId: indexName });
7109
6769
  await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
7110
6770
  core.watermarkNotifier.forget(indexName);
6771
+ const durable = core.durability?.manager;
6772
+ await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
7111
6773
  });
7112
6774
  },
7113
6775
  async rebalance(indexName, targetPartitionCount) {
7114
6776
  guardShutdown();
7115
6777
  requireIndex(indexName);
7116
- return withOpenIndexWrite(
7117
- indexName,
7118
- () => executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx)
7119
- );
6778
+ let writesMayFollow = () => void 0;
6779
+ const started = new Promise((resolve) => {
6780
+ writesMayFollow = resolve;
6781
+ });
6782
+ function releaseWrites() {
6783
+ writesMayFollow();
6784
+ if (rebalancesStarting.get(indexName) === started) rebalancesStarting.delete(indexName);
6785
+ }
6786
+ rebalancesStarting.set(indexName, started);
6787
+ const rebalanced = withOpenIndex2(indexName, async () => {
6788
+ const { executeRebalance } = await import('./analysis-change-HQUT2Z3I.mjs');
6789
+ const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
6790
+ releaseWrites();
6791
+ return running;
6792
+ });
6793
+ return emitWriteEventAfter(core, indexName, rebalanced).finally(releaseWrites);
7120
6794
  },
7121
6795
  async updatePartitionConfig(indexName, partitionConfig) {
7122
6796
  guardShutdown();
@@ -7235,4 +6909,4 @@ async function createNarsil(config) {
7235
6909
  // src/index.ts
7236
6910
  var VERSION = "0.1.0";
7237
6911
 
7238
- export { HEADER_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader, yieldToEventLoop };
6912
+ export { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, REBALANCE_CHUNK_SIZE, RESERVED_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, checksumOfChunks, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, deleteNestedValue, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineVersionParts, extractVectorFromDoc, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, requireWithinFileLimit, resolveRunnableConfig, rewriteSpillOf, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, validateEmbeddingConfig, validateRequiredFieldsInSchema, vectorsEqual, writeHeader, yieldToEventLoop };