@delali/narsil 0.3.1-next.140 → 0.3.1-next.143

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,8 +1,8 @@
1
- import { requireAnalysisRebuildMemory } from './chunk-LK3MOTXF.mjs';
2
- import { requireConfigurationWithinSizeLimit, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, emitEngineEvent, prepareConfiguredSynonyms, rebuildTextIndexes, textRebuildTargetOf } from './chunk-W2XDK66Z.mjs';
3
- import { resolvePartialWordAnalysis } from './chunk-IBATGS3J.mjs';
4
- import { ARRAY_POSITION_GAP } from './chunk-U4X4AWKL.mjs';
1
+ import { requireAnalysisRebuildMemory } from './chunk-VJEGOELZ.mjs';
2
+ import { requireConfigurationWithinSizeLimit, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, prepareConfiguredSynonyms, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue } from './chunk-IO5F7ZBE.mjs';
5
3
  import { ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-EEBCDP36.mjs';
4
+ import { resolvePartialWordAnalysis } from './chunk-WJGO5PGV.mjs';
5
+ import { createPartitionIndex, ARRAY_POSITION_GAP } from './chunk-ZPWS2SKJ.mjs';
6
6
  import { configUnderAnalysis, resolveIndexAnalysis, targetOfPending } from './chunk-O4LJQI4G.mjs';
7
7
  import { getLanguage } from './chunk-WNBRW4Q2.mjs';
8
8
  import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
@@ -168,6 +168,401 @@ function requireSynonymsUnderTarget(entry, target) {
168
168
  prepareConfiguredSynonyms({ ...target.config, synonyms: rulesOfStoredSynonyms(entry.synonyms) }, target.language);
169
169
  }
170
170
 
171
+ // src/partitioning/rebalancer.ts
172
+ function createRebalancer() {
173
+ const activeManagers = /* @__PURE__ */ new WeakSet();
174
+ let activeCount = 0;
175
+ async function rebalance(manager, newPartitionCount, router, onProgress, spill = null) {
176
+ if (activeManagers.has(manager)) {
177
+ throw new NarsilError(
178
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
179
+ "A rebalance operation is already in progress"
180
+ );
181
+ }
182
+ if (newPartitionCount <= 0) {
183
+ throw new NarsilError(
184
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
185
+ `New partition count must be greater than 0, received ${newPartitionCount}`,
186
+ { newPartitionCount }
187
+ );
188
+ }
189
+ if (newPartitionCount === manager.partitionCount) {
190
+ throw new NarsilError(
191
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
192
+ `New partition count (${newPartitionCount}) is the same as the current count`,
193
+ { newPartitionCount, currentPartitionCount: manager.partitionCount }
194
+ );
195
+ }
196
+ activeManagers.add(manager);
197
+ activeCount++;
198
+ try {
199
+ const currentPartitions = manager.getAllPartitions();
200
+ let documentsTotal = 0;
201
+ for (const partition of currentPartitions) {
202
+ documentsTotal += partition.count();
203
+ }
204
+ onProgress?.({
205
+ phase: "scanning",
206
+ documentsProcessed: 0,
207
+ documentsTotal
208
+ });
209
+ const newPartitions = [];
210
+ for (let i = 0; i < newPartitionCount; i++) {
211
+ newPartitions.push(spill?.newPartition(i) ?? createPartitionIndex(i, manager.config.trackPositions ?? true));
212
+ }
213
+ let documentsProcessed = 0;
214
+ let chunkFill = 0;
215
+ const insertOptions = {
216
+ validate: false,
217
+ skipClone: true,
218
+ stopWordOverride: manager.analysis.stopWords,
219
+ customTokenizer: manager.analysis.customTokenizer,
220
+ collectSurfaces: manager.config.surfaceForms !== false
221
+ };
222
+ for (const partition of currentPartitions) {
223
+ const docIds = [...partition.docIds()];
224
+ for (const docId of docIds) {
225
+ const document = partition.getRef(docId);
226
+ if (!document) continue;
227
+ const targetPartitionId = router.route(docId, newPartitionCount);
228
+ const targetPartition = newPartitions[targetPartitionId];
229
+ if (!targetPartition.has(docId)) {
230
+ targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
231
+ documentsProcessed++;
232
+ }
233
+ chunkFill++;
234
+ if (chunkFill >= REBALANCE_CHUNK_SIZE) {
235
+ chunkFill = 0;
236
+ onProgress?.({
237
+ phase: "moving",
238
+ documentsProcessed,
239
+ documentsTotal
240
+ });
241
+ for (let at = 0; at < newPartitions.length; at++) await spill?.spillIfFull(newPartitions[at], at);
242
+ await yieldToEventLoop();
243
+ }
244
+ }
245
+ }
246
+ if (chunkFill > 0) {
247
+ onProgress?.({
248
+ phase: "moving",
249
+ documentsProcessed,
250
+ documentsTotal
251
+ });
252
+ await yieldToEventLoop();
253
+ }
254
+ onProgress?.({
255
+ phase: "swapping",
256
+ documentsProcessed,
257
+ documentsTotal
258
+ });
259
+ manager.setPartitions(newPartitions);
260
+ onProgress?.({
261
+ phase: "complete",
262
+ documentsProcessed,
263
+ documentsTotal
264
+ });
265
+ } finally {
266
+ activeManagers.delete(manager);
267
+ activeCount--;
268
+ }
269
+ }
270
+ return {
271
+ rebalance,
272
+ isRebalancing() {
273
+ return activeCount > 0;
274
+ }
275
+ };
276
+ }
277
+
278
+ // src/partitioning/write-ahead-queue.ts
279
+ function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
280
+ const entries = [];
281
+ const docStates = /* @__PURE__ */ new Map();
282
+ let nextSequence = 1;
283
+ let resolveReplayed = () => void 0;
284
+ const replayed = new Promise((resolve) => {
285
+ resolveReplayed = resolve;
286
+ });
287
+ return {
288
+ push(entry) {
289
+ if (entries.length >= maxSize) {
290
+ throw new NarsilError(
291
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
292
+ `Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
293
+ { maxSize, currentSize: entries.length }
294
+ );
295
+ }
296
+ const seq = nextSequence++;
297
+ entries.push({ ...entry, sequenceNumber: seq });
298
+ docStates.set(entry.docId, entry.action === "remove" ? "absent" : "present");
299
+ return seq;
300
+ },
301
+ drain() {
302
+ const result = entries.splice(0);
303
+ result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
304
+ return result;
305
+ },
306
+ clear() {
307
+ entries.length = 0;
308
+ docStates.clear();
309
+ },
310
+ bufferedDocState(docId) {
311
+ return docStates.get(docId);
312
+ },
313
+ whenReplayed() {
314
+ return replayed;
315
+ },
316
+ markReplayed() {
317
+ resolveReplayed();
318
+ },
319
+ get size() {
320
+ return entries.length;
321
+ },
322
+ get isFull() {
323
+ return entries.length >= maxSize;
324
+ }
325
+ };
326
+ }
327
+
328
+ // src/engine/rebalance-executor.ts
329
+ function warnHookError(hook, err) {
330
+ console.warn(`${hook} plugin hook error:`, err instanceof Error ? err.message : String(err));
331
+ }
332
+ function applyReplayUpdateVectors(entry, vectorFieldPaths) {
333
+ const updateVectors = /* @__PURE__ */ new Map();
334
+ for (const fieldPath of vectorFieldPaths) {
335
+ updateVectors.set(fieldPath, extractVectorFromDoc(entry.document, fieldPath));
336
+ }
337
+ let updateDoc = entry.document;
338
+ if (updateVectors.size > 0) {
339
+ updateDoc = structuredClone(entry.document);
340
+ for (const fieldPath of updateVectors.keys()) {
341
+ deleteNestedValue(updateDoc, fieldPath);
342
+ }
343
+ }
344
+ return { updateDoc, updateVectors };
345
+ }
346
+ async function replayEntry(manager, indexName, entry, ctx) {
347
+ const vecIndexes = manager.getVectorIndexes();
348
+ const vectorFieldPaths = ctx.requireIndex(indexName).vectorFieldPaths;
349
+ if (entry.action === "insert" && entry.document) {
350
+ const { partitionDoc, extractedVectors } = prepareDocumentVectors(
351
+ entry.document,
352
+ vectorFieldPaths
353
+ );
354
+ const overwrote = manager.has(entry.docId);
355
+ const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
356
+ if (overwrote) {
357
+ removeDocumentVectors(entry.docId, vecIndexes);
358
+ manager.remove(entry.docId, removedSpellings);
359
+ }
360
+ manager.insert(entry.docId, partitionDoc);
361
+ if (extractedVectors.size > 0) {
362
+ insertDocumentVectors(entry.docId, extractedVectors, vecIndexes, manager.partitionIdOf(entry.docId));
363
+ for (const fieldPath of extractedVectors.keys()) {
364
+ vecIndexes.get(fieldPath)?.scheduleBuild();
365
+ }
366
+ }
367
+ try {
368
+ await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: entry.docId, document: entry.document });
369
+ } catch (err) {
370
+ warnHookError("afterInsert", err);
371
+ }
372
+ const replicated = {
373
+ indexName,
374
+ docId: entry.docId,
375
+ document: entry.document,
376
+ requestId: `replicate-insert-${entry.docId}`
377
+ };
378
+ await ctx.orchestrator.replicateToWorkers(
379
+ overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
380
+ );
381
+ return;
382
+ }
383
+ if (entry.action === "remove") {
384
+ const removedSpellings = manager.removedSpellingsOf(entry.docId);
385
+ manager.remove(entry.docId, removedSpellings);
386
+ removeDocumentVectors(entry.docId, vecIndexes);
387
+ try {
388
+ await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
389
+ } catch (err) {
390
+ warnHookError("afterRemove", err);
391
+ }
392
+ await ctx.orchestrator.replicateToWorkers({
393
+ type: "remove",
394
+ indexName,
395
+ docId: entry.docId,
396
+ requestId: `replicate-remove-${entry.docId}`,
397
+ removedSpellings
398
+ });
399
+ return;
400
+ }
401
+ if (entry.action === "update" && entry.document) {
402
+ const oldDocument = manager.get(entry.docId);
403
+ const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths);
404
+ const removedSpellings = manager.removedSpellingsOf(entry.docId);
405
+ if (manager.has(entry.docId)) {
406
+ manager.update(entry.docId, updateDoc, void 0, removedSpellings);
407
+ } else {
408
+ manager.insert(entry.docId, updateDoc);
409
+ }
410
+ for (const [fieldPath, newVec] of updateVectors) {
411
+ const vecIndex = vecIndexes.get(fieldPath);
412
+ if (!vecIndex) continue;
413
+ if (newVec === null) {
414
+ if (vecIndex.has(entry.docId)) {
415
+ vecIndex.remove(entry.docId);
416
+ }
417
+ } else {
418
+ const oldVec = vecIndex.getVector(entry.docId);
419
+ if (!vectorsEqual(oldVec, newVec)) {
420
+ if (vecIndex.has(entry.docId)) {
421
+ vecIndex.remove(entry.docId);
422
+ }
423
+ vecIndex.insert(entry.docId, newVec);
424
+ vecIndex.scheduleBuild();
425
+ }
426
+ }
427
+ }
428
+ try {
429
+ await ctx.pluginRegistry.runHook("afterUpdate", {
430
+ indexName,
431
+ docId: entry.docId,
432
+ oldDocument: oldDocument ?? {},
433
+ newDocument: entry.document
434
+ });
435
+ } catch (err) {
436
+ warnHookError("afterUpdate", err);
437
+ }
438
+ await ctx.orchestrator.replicateToWorkers({
439
+ type: "update",
440
+ indexName,
441
+ docId: entry.docId,
442
+ document: entry.document,
443
+ requestId: `replicate-update-${entry.docId}`,
444
+ removedSpellings
445
+ });
446
+ }
447
+ }
448
+ async function replayQueued(manager, indexName, waq, ctx) {
449
+ while (true) {
450
+ const entries = waq.drain();
451
+ if (entries.length === 0) break;
452
+ for (const entry of entries) {
453
+ try {
454
+ await replayEntry(manager, indexName, entry, ctx);
455
+ } catch (replayErr) {
456
+ const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
457
+ if (!isMissing) {
458
+ console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
459
+ }
460
+ }
461
+ }
462
+ }
463
+ }
464
+ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
465
+ if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
466
+ throw new NarsilError(
467
+ ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
468
+ `Target partition count must be a positive integer, got ${targetPartitionCount}`,
469
+ { targetPartitionCount }
470
+ );
471
+ }
472
+ const entry = ctx.requireIndex(indexName);
473
+ const maxPartitions = entry.config.partitions?.maxPartitions;
474
+ if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
475
+ throw new NarsilError(
476
+ ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
477
+ `Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
478
+ { targetPartitionCount, maxPartitions }
479
+ );
480
+ }
481
+ if (ctx.rebalancingIndexes.has(indexName)) {
482
+ throw new NarsilError(
483
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
484
+ `Index "${indexName}" is already being rebalanced`
485
+ );
486
+ }
487
+ const registered = ctx.requireIndex(indexName);
488
+ if (registered.pendingAnalysis !== null || registered.durableAnalysis !== null) {
489
+ throw new NarsilError(
490
+ ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
491
+ `Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
492
+ { indexName }
493
+ );
494
+ }
495
+ if (targetPartitionCount === manager.partitionCount) {
496
+ return;
497
+ }
498
+ const waq = createWriteAheadQueue();
499
+ ctx.waqMap.set(indexName, waq);
500
+ ctx.rebalancingIndexes.add(indexName);
501
+ ctx.rebalanceTargets.set(indexName, targetPartitionCount);
502
+ const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
503
+ let spill = null;
504
+ try {
505
+ const oldCount = manager.partitionCount;
506
+ spill = await rewriteSpillOf(ctx.durabilityManager?.rewriteStorage?.(), manager);
507
+ await createRebalancer().rebalance(
508
+ manager,
509
+ targetPartitionCount,
510
+ ctx.router,
511
+ (progress) => {
512
+ if (progress.phase === "complete") {
513
+ emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
514
+ indexName,
515
+ oldCount,
516
+ newCount: targetPartitionCount
517
+ });
518
+ }
519
+ },
520
+ spill
521
+ );
522
+ try {
523
+ await ctx.pluginRegistry.runHook("onPartitionSplit", {
524
+ indexName,
525
+ oldPartitionCount: oldCount,
526
+ newPartitionCount: targetPartitionCount
527
+ });
528
+ } catch (hookErr) {
529
+ console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
530
+ }
531
+ await replayQueued(manager, indexName, waq, ctx);
532
+ if (ctx.durabilityManager) {
533
+ await ctx.durabilityManager.persistMetadata(indexName);
534
+ await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
535
+ await spill?.removeUnread(manager);
536
+ await replayQueued(manager, indexName, waq, ctx);
537
+ }
538
+ await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
539
+ do {
540
+ await replayQueued(manager, indexName, waq, ctx);
541
+ } while (waq.size > 0);
542
+ } catch (err) {
543
+ await spill?.removeUnread(manager).catch(() => void 0);
544
+ try {
545
+ await replayQueued(manager, indexName, waq, ctx);
546
+ await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
547
+ do {
548
+ await replayQueued(manager, indexName, waq, ctx);
549
+ } while (waq.size > 0);
550
+ } catch (recoveryErr) {
551
+ console.warn(
552
+ `Rebalance failure recovery for index "${indexName}" did not complete:`,
553
+ recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)
554
+ );
555
+ }
556
+ throw err;
557
+ } finally {
558
+ ctx.rebalancingIndexes.delete(indexName);
559
+ ctx.rebalanceTargets.delete(indexName);
560
+ ctx.waqMap.delete(indexName);
561
+ waq.markReplayed();
562
+ }
563
+ ctx.checkWatermark(indexName);
564
+ }
565
+
171
566
  // src/engine/analysis-change/adopt-saved.ts
172
567
  function adoptSavedAnalysis(manager, entry, metadata) {
173
568
  const saved = reconstructSchemaFromMetadata(metadata);
@@ -259,10 +654,15 @@ async function buildAndSwitch(scope) {
259
654
  const wasPromoted = core.orchestrator.desyncIndex(indexName);
260
655
  try {
261
656
  const durable = core.durability?.manager;
657
+ let spill = null;
262
658
  const rebuilt = await rebuildTextIndexes(
263
659
  manager,
264
660
  {
265
661
  isCurrent,
662
+ rewriteStorage: () => durable?.rewriteStorage?.(),
663
+ onRewriteSpill(kept) {
664
+ spill = kept;
665
+ },
266
666
  onPartitionBuilt(built) {
267
667
  partitionsRebuilt = built;
268
668
  },
@@ -280,6 +680,7 @@ async function buildAndSwitch(scope) {
280
680
  textRebuildTargetOf(run.target.config, run.target.language, run.target.analysis)
281
681
  );
282
682
  if (!rebuilt) throw run.stopReason ?? stoppedError(core, indexName);
683
+ await spill?.removeUnread(manager);
283
684
  if (entry.durableAnalysis !== null) {
284
685
  throw new NarsilError(
285
686
  ErrorCodes.PERSISTENCE_SAVE_FAILED,
@@ -477,4 +878,4 @@ async function stampAnalysisGeneration(core, indexName, generation) {
477
878
  await (durable?.checkpointAfterSwitch?.(indexName, stamp) ?? Promise.resolve(stamp()));
478
879
  }
479
880
 
480
- export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, heldAnalysisIsBuilt, holdsNamedSettings, holdsSettings, namedSettingsOf, releaseHeldAnalysisChange, resolveAnalysisTarget, resumeAnalysisChange, stampAnalysisGeneration };
881
+ export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, holdsSettings, namedSettingsOf, releaseHeldAnalysisChange, resolveAnalysisTarget, resumeAnalysisChange, stampAnalysisGeneration };