@delali/narsil 0.3.1-next.140 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { requireAnalysisRebuildMemory } from './chunk-
|
|
2
|
-
import { requireConfigurationWithinSizeLimit, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms,
|
|
3
|
-
import { resolvePartialWordAnalysis } from './chunk-IBATGS3J.mjs';
|
|
4
|
-
import { ARRAY_POSITION_GAP } from './chunk-U4X4AWKL.mjs';
|
|
1
|
+
import { requireAnalysisRebuildMemory } from './chunk-VJEGOELZ.mjs';
|
|
2
|
+
import { requireConfigurationWithinSizeLimit, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, prepareStoredSynonyms, rulesOfStoredSynonyms, prepareConfiguredSynonyms, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue } from './chunk-IO5F7ZBE.mjs';
|
|
5
3
|
import { ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-EEBCDP36.mjs';
|
|
4
|
+
import { resolvePartialWordAnalysis } from './chunk-WJGO5PGV.mjs';
|
|
5
|
+
import { createPartitionIndex, ARRAY_POSITION_GAP } from './chunk-ZPWS2SKJ.mjs';
|
|
6
6
|
import { configUnderAnalysis, resolveIndexAnalysis, targetOfPending } from './chunk-O4LJQI4G.mjs';
|
|
7
7
|
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
8
8
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
@@ -168,6 +168,401 @@ function requireSynonymsUnderTarget(entry, target) {
|
|
|
168
168
|
prepareConfiguredSynonyms({ ...target.config, synonyms: rulesOfStoredSynonyms(entry.synonyms) }, target.language);
|
|
169
169
|
}
|
|
170
170
|
|
|
171
|
+
// src/partitioning/rebalancer.ts
|
|
172
|
+
function createRebalancer() {
|
|
173
|
+
const activeManagers = /* @__PURE__ */ new WeakSet();
|
|
174
|
+
let activeCount = 0;
|
|
175
|
+
async function rebalance(manager, newPartitionCount, router, onProgress, spill = null) {
|
|
176
|
+
if (activeManagers.has(manager)) {
|
|
177
|
+
throw new NarsilError(
|
|
178
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
179
|
+
"A rebalance operation is already in progress"
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
if (newPartitionCount <= 0) {
|
|
183
|
+
throw new NarsilError(
|
|
184
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
185
|
+
`New partition count must be greater than 0, received ${newPartitionCount}`,
|
|
186
|
+
{ newPartitionCount }
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
if (newPartitionCount === manager.partitionCount) {
|
|
190
|
+
throw new NarsilError(
|
|
191
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
192
|
+
`New partition count (${newPartitionCount}) is the same as the current count`,
|
|
193
|
+
{ newPartitionCount, currentPartitionCount: manager.partitionCount }
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
activeManagers.add(manager);
|
|
197
|
+
activeCount++;
|
|
198
|
+
try {
|
|
199
|
+
const currentPartitions = manager.getAllPartitions();
|
|
200
|
+
let documentsTotal = 0;
|
|
201
|
+
for (const partition of currentPartitions) {
|
|
202
|
+
documentsTotal += partition.count();
|
|
203
|
+
}
|
|
204
|
+
onProgress?.({
|
|
205
|
+
phase: "scanning",
|
|
206
|
+
documentsProcessed: 0,
|
|
207
|
+
documentsTotal
|
|
208
|
+
});
|
|
209
|
+
const newPartitions = [];
|
|
210
|
+
for (let i = 0; i < newPartitionCount; i++) {
|
|
211
|
+
newPartitions.push(spill?.newPartition(i) ?? createPartitionIndex(i, manager.config.trackPositions ?? true));
|
|
212
|
+
}
|
|
213
|
+
let documentsProcessed = 0;
|
|
214
|
+
let chunkFill = 0;
|
|
215
|
+
const insertOptions = {
|
|
216
|
+
validate: false,
|
|
217
|
+
skipClone: true,
|
|
218
|
+
stopWordOverride: manager.analysis.stopWords,
|
|
219
|
+
customTokenizer: manager.analysis.customTokenizer,
|
|
220
|
+
collectSurfaces: manager.config.surfaceForms !== false
|
|
221
|
+
};
|
|
222
|
+
for (const partition of currentPartitions) {
|
|
223
|
+
const docIds = [...partition.docIds()];
|
|
224
|
+
for (const docId of docIds) {
|
|
225
|
+
const document = partition.getRef(docId);
|
|
226
|
+
if (!document) continue;
|
|
227
|
+
const targetPartitionId = router.route(docId, newPartitionCount);
|
|
228
|
+
const targetPartition = newPartitions[targetPartitionId];
|
|
229
|
+
if (!targetPartition.has(docId)) {
|
|
230
|
+
targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
|
|
231
|
+
documentsProcessed++;
|
|
232
|
+
}
|
|
233
|
+
chunkFill++;
|
|
234
|
+
if (chunkFill >= REBALANCE_CHUNK_SIZE) {
|
|
235
|
+
chunkFill = 0;
|
|
236
|
+
onProgress?.({
|
|
237
|
+
phase: "moving",
|
|
238
|
+
documentsProcessed,
|
|
239
|
+
documentsTotal
|
|
240
|
+
});
|
|
241
|
+
for (let at = 0; at < newPartitions.length; at++) await spill?.spillIfFull(newPartitions[at], at);
|
|
242
|
+
await yieldToEventLoop();
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
if (chunkFill > 0) {
|
|
247
|
+
onProgress?.({
|
|
248
|
+
phase: "moving",
|
|
249
|
+
documentsProcessed,
|
|
250
|
+
documentsTotal
|
|
251
|
+
});
|
|
252
|
+
await yieldToEventLoop();
|
|
253
|
+
}
|
|
254
|
+
onProgress?.({
|
|
255
|
+
phase: "swapping",
|
|
256
|
+
documentsProcessed,
|
|
257
|
+
documentsTotal
|
|
258
|
+
});
|
|
259
|
+
manager.setPartitions(newPartitions);
|
|
260
|
+
onProgress?.({
|
|
261
|
+
phase: "complete",
|
|
262
|
+
documentsProcessed,
|
|
263
|
+
documentsTotal
|
|
264
|
+
});
|
|
265
|
+
} finally {
|
|
266
|
+
activeManagers.delete(manager);
|
|
267
|
+
activeCount--;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return {
|
|
271
|
+
rebalance,
|
|
272
|
+
isRebalancing() {
|
|
273
|
+
return activeCount > 0;
|
|
274
|
+
}
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// src/partitioning/write-ahead-queue.ts
|
|
279
|
+
function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
|
|
280
|
+
const entries = [];
|
|
281
|
+
const docStates = /* @__PURE__ */ new Map();
|
|
282
|
+
let nextSequence = 1;
|
|
283
|
+
let resolveReplayed = () => void 0;
|
|
284
|
+
const replayed = new Promise((resolve) => {
|
|
285
|
+
resolveReplayed = resolve;
|
|
286
|
+
});
|
|
287
|
+
return {
|
|
288
|
+
push(entry) {
|
|
289
|
+
if (entries.length >= maxSize) {
|
|
290
|
+
throw new NarsilError(
|
|
291
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
292
|
+
`Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
|
|
293
|
+
{ maxSize, currentSize: entries.length }
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
const seq = nextSequence++;
|
|
297
|
+
entries.push({ ...entry, sequenceNumber: seq });
|
|
298
|
+
docStates.set(entry.docId, entry.action === "remove" ? "absent" : "present");
|
|
299
|
+
return seq;
|
|
300
|
+
},
|
|
301
|
+
drain() {
|
|
302
|
+
const result = entries.splice(0);
|
|
303
|
+
result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
|
|
304
|
+
return result;
|
|
305
|
+
},
|
|
306
|
+
clear() {
|
|
307
|
+
entries.length = 0;
|
|
308
|
+
docStates.clear();
|
|
309
|
+
},
|
|
310
|
+
bufferedDocState(docId) {
|
|
311
|
+
return docStates.get(docId);
|
|
312
|
+
},
|
|
313
|
+
whenReplayed() {
|
|
314
|
+
return replayed;
|
|
315
|
+
},
|
|
316
|
+
markReplayed() {
|
|
317
|
+
resolveReplayed();
|
|
318
|
+
},
|
|
319
|
+
get size() {
|
|
320
|
+
return entries.length;
|
|
321
|
+
},
|
|
322
|
+
get isFull() {
|
|
323
|
+
return entries.length >= maxSize;
|
|
324
|
+
}
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// src/engine/rebalance-executor.ts
|
|
329
|
+
function warnHookError(hook, err) {
|
|
330
|
+
console.warn(`${hook} plugin hook error:`, err instanceof Error ? err.message : String(err));
|
|
331
|
+
}
|
|
332
|
+
function applyReplayUpdateVectors(entry, vectorFieldPaths) {
|
|
333
|
+
const updateVectors = /* @__PURE__ */ new Map();
|
|
334
|
+
for (const fieldPath of vectorFieldPaths) {
|
|
335
|
+
updateVectors.set(fieldPath, extractVectorFromDoc(entry.document, fieldPath));
|
|
336
|
+
}
|
|
337
|
+
let updateDoc = entry.document;
|
|
338
|
+
if (updateVectors.size > 0) {
|
|
339
|
+
updateDoc = structuredClone(entry.document);
|
|
340
|
+
for (const fieldPath of updateVectors.keys()) {
|
|
341
|
+
deleteNestedValue(updateDoc, fieldPath);
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
return { updateDoc, updateVectors };
|
|
345
|
+
}
|
|
346
|
+
async function replayEntry(manager, indexName, entry, ctx) {
|
|
347
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
348
|
+
const vectorFieldPaths = ctx.requireIndex(indexName).vectorFieldPaths;
|
|
349
|
+
if (entry.action === "insert" && entry.document) {
|
|
350
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
351
|
+
entry.document,
|
|
352
|
+
vectorFieldPaths
|
|
353
|
+
);
|
|
354
|
+
const overwrote = manager.has(entry.docId);
|
|
355
|
+
const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
|
|
356
|
+
if (overwrote) {
|
|
357
|
+
removeDocumentVectors(entry.docId, vecIndexes);
|
|
358
|
+
manager.remove(entry.docId, removedSpellings);
|
|
359
|
+
}
|
|
360
|
+
manager.insert(entry.docId, partitionDoc);
|
|
361
|
+
if (extractedVectors.size > 0) {
|
|
362
|
+
insertDocumentVectors(entry.docId, extractedVectors, vecIndexes, manager.partitionIdOf(entry.docId));
|
|
363
|
+
for (const fieldPath of extractedVectors.keys()) {
|
|
364
|
+
vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
try {
|
|
368
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: entry.docId, document: entry.document });
|
|
369
|
+
} catch (err) {
|
|
370
|
+
warnHookError("afterInsert", err);
|
|
371
|
+
}
|
|
372
|
+
const replicated = {
|
|
373
|
+
indexName,
|
|
374
|
+
docId: entry.docId,
|
|
375
|
+
document: entry.document,
|
|
376
|
+
requestId: `replicate-insert-${entry.docId}`
|
|
377
|
+
};
|
|
378
|
+
await ctx.orchestrator.replicateToWorkers(
|
|
379
|
+
overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
|
|
380
|
+
);
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
if (entry.action === "remove") {
|
|
384
|
+
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
385
|
+
manager.remove(entry.docId, removedSpellings);
|
|
386
|
+
removeDocumentVectors(entry.docId, vecIndexes);
|
|
387
|
+
try {
|
|
388
|
+
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
|
|
389
|
+
} catch (err) {
|
|
390
|
+
warnHookError("afterRemove", err);
|
|
391
|
+
}
|
|
392
|
+
await ctx.orchestrator.replicateToWorkers({
|
|
393
|
+
type: "remove",
|
|
394
|
+
indexName,
|
|
395
|
+
docId: entry.docId,
|
|
396
|
+
requestId: `replicate-remove-${entry.docId}`,
|
|
397
|
+
removedSpellings
|
|
398
|
+
});
|
|
399
|
+
return;
|
|
400
|
+
}
|
|
401
|
+
if (entry.action === "update" && entry.document) {
|
|
402
|
+
const oldDocument = manager.get(entry.docId);
|
|
403
|
+
const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths);
|
|
404
|
+
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
405
|
+
if (manager.has(entry.docId)) {
|
|
406
|
+
manager.update(entry.docId, updateDoc, void 0, removedSpellings);
|
|
407
|
+
} else {
|
|
408
|
+
manager.insert(entry.docId, updateDoc);
|
|
409
|
+
}
|
|
410
|
+
for (const [fieldPath, newVec] of updateVectors) {
|
|
411
|
+
const vecIndex = vecIndexes.get(fieldPath);
|
|
412
|
+
if (!vecIndex) continue;
|
|
413
|
+
if (newVec === null) {
|
|
414
|
+
if (vecIndex.has(entry.docId)) {
|
|
415
|
+
vecIndex.remove(entry.docId);
|
|
416
|
+
}
|
|
417
|
+
} else {
|
|
418
|
+
const oldVec = vecIndex.getVector(entry.docId);
|
|
419
|
+
if (!vectorsEqual(oldVec, newVec)) {
|
|
420
|
+
if (vecIndex.has(entry.docId)) {
|
|
421
|
+
vecIndex.remove(entry.docId);
|
|
422
|
+
}
|
|
423
|
+
vecIndex.insert(entry.docId, newVec);
|
|
424
|
+
vecIndex.scheduleBuild();
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
try {
|
|
429
|
+
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
430
|
+
indexName,
|
|
431
|
+
docId: entry.docId,
|
|
432
|
+
oldDocument: oldDocument ?? {},
|
|
433
|
+
newDocument: entry.document
|
|
434
|
+
});
|
|
435
|
+
} catch (err) {
|
|
436
|
+
warnHookError("afterUpdate", err);
|
|
437
|
+
}
|
|
438
|
+
await ctx.orchestrator.replicateToWorkers({
|
|
439
|
+
type: "update",
|
|
440
|
+
indexName,
|
|
441
|
+
docId: entry.docId,
|
|
442
|
+
document: entry.document,
|
|
443
|
+
requestId: `replicate-update-${entry.docId}`,
|
|
444
|
+
removedSpellings
|
|
445
|
+
});
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
async function replayQueued(manager, indexName, waq, ctx) {
|
|
449
|
+
while (true) {
|
|
450
|
+
const entries = waq.drain();
|
|
451
|
+
if (entries.length === 0) break;
|
|
452
|
+
for (const entry of entries) {
|
|
453
|
+
try {
|
|
454
|
+
await replayEntry(manager, indexName, entry, ctx);
|
|
455
|
+
} catch (replayErr) {
|
|
456
|
+
const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
|
|
457
|
+
if (!isMissing) {
|
|
458
|
+
console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
465
|
+
if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
|
|
466
|
+
throw new NarsilError(
|
|
467
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
468
|
+
`Target partition count must be a positive integer, got ${targetPartitionCount}`,
|
|
469
|
+
{ targetPartitionCount }
|
|
470
|
+
);
|
|
471
|
+
}
|
|
472
|
+
const entry = ctx.requireIndex(indexName);
|
|
473
|
+
const maxPartitions = entry.config.partitions?.maxPartitions;
|
|
474
|
+
if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
|
|
475
|
+
throw new NarsilError(
|
|
476
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
477
|
+
`Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
|
|
478
|
+
{ targetPartitionCount, maxPartitions }
|
|
479
|
+
);
|
|
480
|
+
}
|
|
481
|
+
if (ctx.rebalancingIndexes.has(indexName)) {
|
|
482
|
+
throw new NarsilError(
|
|
483
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
484
|
+
`Index "${indexName}" is already being rebalanced`
|
|
485
|
+
);
|
|
486
|
+
}
|
|
487
|
+
const registered = ctx.requireIndex(indexName);
|
|
488
|
+
if (registered.pendingAnalysis !== null || registered.durableAnalysis !== null) {
|
|
489
|
+
throw new NarsilError(
|
|
490
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
491
|
+
`Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
|
|
492
|
+
{ indexName }
|
|
493
|
+
);
|
|
494
|
+
}
|
|
495
|
+
if (targetPartitionCount === manager.partitionCount) {
|
|
496
|
+
return;
|
|
497
|
+
}
|
|
498
|
+
const waq = createWriteAheadQueue();
|
|
499
|
+
ctx.waqMap.set(indexName, waq);
|
|
500
|
+
ctx.rebalancingIndexes.add(indexName);
|
|
501
|
+
ctx.rebalanceTargets.set(indexName, targetPartitionCount);
|
|
502
|
+
const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
|
|
503
|
+
let spill = null;
|
|
504
|
+
try {
|
|
505
|
+
const oldCount = manager.partitionCount;
|
|
506
|
+
spill = await rewriteSpillOf(ctx.durabilityManager?.rewriteStorage?.(), manager);
|
|
507
|
+
await createRebalancer().rebalance(
|
|
508
|
+
manager,
|
|
509
|
+
targetPartitionCount,
|
|
510
|
+
ctx.router,
|
|
511
|
+
(progress) => {
|
|
512
|
+
if (progress.phase === "complete") {
|
|
513
|
+
emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
|
|
514
|
+
indexName,
|
|
515
|
+
oldCount,
|
|
516
|
+
newCount: targetPartitionCount
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
},
|
|
520
|
+
spill
|
|
521
|
+
);
|
|
522
|
+
try {
|
|
523
|
+
await ctx.pluginRegistry.runHook("onPartitionSplit", {
|
|
524
|
+
indexName,
|
|
525
|
+
oldPartitionCount: oldCount,
|
|
526
|
+
newPartitionCount: targetPartitionCount
|
|
527
|
+
});
|
|
528
|
+
} catch (hookErr) {
|
|
529
|
+
console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
|
|
530
|
+
}
|
|
531
|
+
await replayQueued(manager, indexName, waq, ctx);
|
|
532
|
+
if (ctx.durabilityManager) {
|
|
533
|
+
await ctx.durabilityManager.persistMetadata(indexName);
|
|
534
|
+
await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
|
|
535
|
+
await spill?.removeUnread(manager);
|
|
536
|
+
await replayQueued(manager, indexName, waq, ctx);
|
|
537
|
+
}
|
|
538
|
+
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
539
|
+
do {
|
|
540
|
+
await replayQueued(manager, indexName, waq, ctx);
|
|
541
|
+
} while (waq.size > 0);
|
|
542
|
+
} catch (err) {
|
|
543
|
+
await spill?.removeUnread(manager).catch(() => void 0);
|
|
544
|
+
try {
|
|
545
|
+
await replayQueued(manager, indexName, waq, ctx);
|
|
546
|
+
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
547
|
+
do {
|
|
548
|
+
await replayQueued(manager, indexName, waq, ctx);
|
|
549
|
+
} while (waq.size > 0);
|
|
550
|
+
} catch (recoveryErr) {
|
|
551
|
+
console.warn(
|
|
552
|
+
`Rebalance failure recovery for index "${indexName}" did not complete:`,
|
|
553
|
+
recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)
|
|
554
|
+
);
|
|
555
|
+
}
|
|
556
|
+
throw err;
|
|
557
|
+
} finally {
|
|
558
|
+
ctx.rebalancingIndexes.delete(indexName);
|
|
559
|
+
ctx.rebalanceTargets.delete(indexName);
|
|
560
|
+
ctx.waqMap.delete(indexName);
|
|
561
|
+
waq.markReplayed();
|
|
562
|
+
}
|
|
563
|
+
ctx.checkWatermark(indexName);
|
|
564
|
+
}
|
|
565
|
+
|
|
171
566
|
// src/engine/analysis-change/adopt-saved.ts
|
|
172
567
|
function adoptSavedAnalysis(manager, entry, metadata) {
|
|
173
568
|
const saved = reconstructSchemaFromMetadata(metadata);
|
|
@@ -259,10 +654,15 @@ async function buildAndSwitch(scope) {
|
|
|
259
654
|
const wasPromoted = core.orchestrator.desyncIndex(indexName);
|
|
260
655
|
try {
|
|
261
656
|
const durable = core.durability?.manager;
|
|
657
|
+
let spill = null;
|
|
262
658
|
const rebuilt = await rebuildTextIndexes(
|
|
263
659
|
manager,
|
|
264
660
|
{
|
|
265
661
|
isCurrent,
|
|
662
|
+
rewriteStorage: () => durable?.rewriteStorage?.(),
|
|
663
|
+
onRewriteSpill(kept) {
|
|
664
|
+
spill = kept;
|
|
665
|
+
},
|
|
266
666
|
onPartitionBuilt(built) {
|
|
267
667
|
partitionsRebuilt = built;
|
|
268
668
|
},
|
|
@@ -280,6 +680,7 @@ async function buildAndSwitch(scope) {
|
|
|
280
680
|
textRebuildTargetOf(run.target.config, run.target.language, run.target.analysis)
|
|
281
681
|
);
|
|
282
682
|
if (!rebuilt) throw run.stopReason ?? stoppedError(core, indexName);
|
|
683
|
+
await spill?.removeUnread(manager);
|
|
283
684
|
if (entry.durableAnalysis !== null) {
|
|
284
685
|
throw new NarsilError(
|
|
285
686
|
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
@@ -477,4 +878,4 @@ async function stampAnalysisGeneration(core, indexName, generation) {
|
|
|
477
878
|
await (durable?.checkpointAfterSwitch?.(indexName, stamp) ?? Promise.resolve(stamp()));
|
|
478
879
|
}
|
|
479
880
|
|
|
480
|
-
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, heldAnalysisIsBuilt, holdsNamedSettings, holdsSettings, namedSettingsOf, releaseHeldAnalysisChange, resolveAnalysisTarget, resumeAnalysisChange, stampAnalysisGeneration };
|
|
881
|
+
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, holdsSettings, namedSettingsOf, releaseHeldAnalysisChange, resolveAnalysisTarget, resumeAnalysisChange, stampAnalysisGeneration };
|