@delali/narsil 0.3.1-next.140 → 0.3.1-next.143
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,25 +1,25 @@
|
|
|
1
|
-
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
import './chunk-
|
|
4
|
-
import './chunk-
|
|
5
|
-
import './chunk-
|
|
1
|
+
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration } from './chunk-SRJBFG2T.mjs';
|
|
2
|
+
import './chunk-VJEGOELZ.mjs';
|
|
3
|
+
import './chunk-IO5F7ZBE.mjs';
|
|
4
|
+
import './chunk-VEFN6X6Q.mjs';
|
|
5
|
+
import './chunk-5WYVUPGD.mjs';
|
|
6
|
+
import './chunk-HJSN7UYG.mjs';
|
|
6
7
|
import './chunk-UIKN2OTQ.mjs';
|
|
7
8
|
import './chunk-AV4IJKD2.mjs';
|
|
8
|
-
import './chunk-
|
|
9
|
-
import './chunk-
|
|
9
|
+
import './chunk-EEBCDP36.mjs';
|
|
10
|
+
import './chunk-WJGO5PGV.mjs';
|
|
11
|
+
import './chunk-ZPWS2SKJ.mjs';
|
|
10
12
|
import './chunk-N37VMC62.mjs';
|
|
11
13
|
import './chunk-ZATRDAYU.mjs';
|
|
12
14
|
import './chunk-EBIHRL7B.mjs';
|
|
13
|
-
import './chunk-HJSN7UYG.mjs';
|
|
14
|
-
import './chunk-EEBCDP36.mjs';
|
|
15
15
|
import './chunk-KOKM4PQN.mjs';
|
|
16
16
|
import './chunk-QSK35UC2.mjs';
|
|
17
17
|
import './chunk-74XB2SXI.mjs';
|
|
18
18
|
import './chunk-RKAVHNRW.mjs';
|
|
19
19
|
import './chunk-WQBCBKY3.mjs';
|
|
20
20
|
import './chunk-JUKYYXSP.mjs';
|
|
21
|
-
import './chunk-
|
|
22
|
-
import './chunk-
|
|
21
|
+
import './chunk-DQNKVOJ2.mjs';
|
|
22
|
+
import './chunk-KGOXHLLZ.mjs';
|
|
23
23
|
import './chunk-U476IXKQ.mjs';
|
|
24
24
|
import './chunk-FQBFZAWG.mjs';
|
|
25
25
|
import './chunk-VJS4JFBQ.mjs';
|
|
@@ -35,8 +35,8 @@ import './chunk-4UMZ7KQL.mjs';
|
|
|
35
35
|
import './chunk-R6A773WK.mjs';
|
|
36
36
|
import './chunk-YTJCGKM4.mjs';
|
|
37
37
|
import './chunk-RIIUZJQ3.mjs';
|
|
38
|
-
import './chunk-
|
|
39
|
-
import './chunk-
|
|
38
|
+
import './chunk-QHPYS427.mjs';
|
|
39
|
+
import './chunk-GLZHNSGB.mjs';
|
|
40
40
|
import './chunk-22YPZVMG.mjs';
|
|
41
41
|
import './chunk-RFBLQLLZ.mjs';
|
|
42
42
|
import './chunk-L5ORED7V.mjs';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-
|
|
1
|
+
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-46NZOPHS.mjs';
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { requireAnalysisRebuildMemory } from './chunk-
|
|
2
|
-
import './chunk-
|
|
3
|
-
import { reconstructSchemaFromMetadata, getLanguage, resolveIndexAnalysis, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, emitEngineEvent, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, rebuildTextIndexes, textRebuildTargetOf, ARRAY_POSITION_GAP } from './chunk-ZAIZQOTM.mjs';
|
|
4
|
-
import './chunk-26A5JFT2.mjs';
|
|
5
|
-
import { compareCodePoints } from './chunk-4MFH226Q.mjs';
|
|
1
|
+
import { requireAnalysisRebuildMemory } from './chunk-R4WUDZQV.mjs';
|
|
2
|
+
import { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, rewriteSpillOf, emitEngineEvent, reconstructSchemaFromMetadata, getLanguage, resolveIndexAnalysis, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, createPartitionIndex, REBALANCE_CHUNK_SIZE, yieldToEventLoop, prepareDocumentVectors, removeDocumentVectors, insertDocumentVectors, vectorsEqual, rebuildTextIndexes, textRebuildTargetOf, extractVectorFromDoc, deleteNestedValue, ARRAY_POSITION_GAP } from './chunk-5DB2VDGI.mjs';
|
|
6
3
|
import './chunk-JPTQZWJQ.mjs';
|
|
4
|
+
import './chunk-26A5JFT2.mjs';
|
|
5
|
+
import './chunk-OIZS5KIX.mjs';
|
|
6
|
+
import { compareCodePoints } from './chunk-D33A5VMU.mjs';
|
|
7
7
|
import { ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-MZI3TEOW.mjs';
|
|
8
8
|
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
9
9
|
|
|
@@ -187,6 +187,401 @@ function requireSynonymsUnderTarget(entry, target) {
|
|
|
187
187
|
prepareConfiguredSynonyms({ ...target.config, synonyms: rulesOfStoredSynonyms(entry.synonyms) }, target.language);
|
|
188
188
|
}
|
|
189
189
|
|
|
190
|
+
// src/partitioning/rebalancer.ts
|
|
191
|
+
function createRebalancer() {
|
|
192
|
+
const activeManagers = /* @__PURE__ */ new WeakSet();
|
|
193
|
+
let activeCount = 0;
|
|
194
|
+
async function rebalance(manager, newPartitionCount, router, onProgress, spill = null) {
|
|
195
|
+
if (activeManagers.has(manager)) {
|
|
196
|
+
throw new NarsilError(
|
|
197
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
198
|
+
"A rebalance operation is already in progress"
|
|
199
|
+
);
|
|
200
|
+
}
|
|
201
|
+
if (newPartitionCount <= 0) {
|
|
202
|
+
throw new NarsilError(
|
|
203
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
204
|
+
`New partition count must be greater than 0, received ${newPartitionCount}`,
|
|
205
|
+
{ newPartitionCount }
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
if (newPartitionCount === manager.partitionCount) {
|
|
209
|
+
throw new NarsilError(
|
|
210
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
211
|
+
`New partition count (${newPartitionCount}) is the same as the current count`,
|
|
212
|
+
{ newPartitionCount, currentPartitionCount: manager.partitionCount }
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
activeManagers.add(manager);
|
|
216
|
+
activeCount++;
|
|
217
|
+
try {
|
|
218
|
+
const currentPartitions = manager.getAllPartitions();
|
|
219
|
+
let documentsTotal = 0;
|
|
220
|
+
for (const partition of currentPartitions) {
|
|
221
|
+
documentsTotal += partition.count();
|
|
222
|
+
}
|
|
223
|
+
onProgress?.({
|
|
224
|
+
phase: "scanning",
|
|
225
|
+
documentsProcessed: 0,
|
|
226
|
+
documentsTotal
|
|
227
|
+
});
|
|
228
|
+
const newPartitions = [];
|
|
229
|
+
for (let i = 0; i < newPartitionCount; i++) {
|
|
230
|
+
newPartitions.push(spill?.newPartition(i) ?? createPartitionIndex(i, manager.config.trackPositions ?? true));
|
|
231
|
+
}
|
|
232
|
+
let documentsProcessed = 0;
|
|
233
|
+
let chunkFill = 0;
|
|
234
|
+
const insertOptions = {
|
|
235
|
+
validate: false,
|
|
236
|
+
skipClone: true,
|
|
237
|
+
stopWordOverride: manager.analysis.stopWords,
|
|
238
|
+
customTokenizer: manager.analysis.customTokenizer,
|
|
239
|
+
collectSurfaces: manager.config.surfaceForms !== false
|
|
240
|
+
};
|
|
241
|
+
for (const partition of currentPartitions) {
|
|
242
|
+
const docIds = [...partition.docIds()];
|
|
243
|
+
for (const docId of docIds) {
|
|
244
|
+
const document = partition.getRef(docId);
|
|
245
|
+
if (!document) continue;
|
|
246
|
+
const targetPartitionId = router.route(docId, newPartitionCount);
|
|
247
|
+
const targetPartition = newPartitions[targetPartitionId];
|
|
248
|
+
if (!targetPartition.has(docId)) {
|
|
249
|
+
targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
|
|
250
|
+
documentsProcessed++;
|
|
251
|
+
}
|
|
252
|
+
chunkFill++;
|
|
253
|
+
if (chunkFill >= REBALANCE_CHUNK_SIZE) {
|
|
254
|
+
chunkFill = 0;
|
|
255
|
+
onProgress?.({
|
|
256
|
+
phase: "moving",
|
|
257
|
+
documentsProcessed,
|
|
258
|
+
documentsTotal
|
|
259
|
+
});
|
|
260
|
+
for (let at = 0; at < newPartitions.length; at++) await spill?.spillIfFull(newPartitions[at], at);
|
|
261
|
+
await yieldToEventLoop();
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
if (chunkFill > 0) {
|
|
266
|
+
onProgress?.({
|
|
267
|
+
phase: "moving",
|
|
268
|
+
documentsProcessed,
|
|
269
|
+
documentsTotal
|
|
270
|
+
});
|
|
271
|
+
await yieldToEventLoop();
|
|
272
|
+
}
|
|
273
|
+
onProgress?.({
|
|
274
|
+
phase: "swapping",
|
|
275
|
+
documentsProcessed,
|
|
276
|
+
documentsTotal
|
|
277
|
+
});
|
|
278
|
+
manager.setPartitions(newPartitions);
|
|
279
|
+
onProgress?.({
|
|
280
|
+
phase: "complete",
|
|
281
|
+
documentsProcessed,
|
|
282
|
+
documentsTotal
|
|
283
|
+
});
|
|
284
|
+
} finally {
|
|
285
|
+
activeManagers.delete(manager);
|
|
286
|
+
activeCount--;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
return {
|
|
290
|
+
rebalance,
|
|
291
|
+
isRebalancing() {
|
|
292
|
+
return activeCount > 0;
|
|
293
|
+
}
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// src/partitioning/write-ahead-queue.ts
|
|
298
|
+
function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
|
|
299
|
+
const entries = [];
|
|
300
|
+
const docStates = /* @__PURE__ */ new Map();
|
|
301
|
+
let nextSequence = 1;
|
|
302
|
+
let resolveReplayed = () => void 0;
|
|
303
|
+
const replayed = new Promise((resolve) => {
|
|
304
|
+
resolveReplayed = resolve;
|
|
305
|
+
});
|
|
306
|
+
return {
|
|
307
|
+
push(entry) {
|
|
308
|
+
if (entries.length >= maxSize) {
|
|
309
|
+
throw new NarsilError(
|
|
310
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
311
|
+
`Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
|
|
312
|
+
{ maxSize, currentSize: entries.length }
|
|
313
|
+
);
|
|
314
|
+
}
|
|
315
|
+
const seq = nextSequence++;
|
|
316
|
+
entries.push({ ...entry, sequenceNumber: seq });
|
|
317
|
+
docStates.set(entry.docId, entry.action === "remove" ? "absent" : "present");
|
|
318
|
+
return seq;
|
|
319
|
+
},
|
|
320
|
+
drain() {
|
|
321
|
+
const result = entries.splice(0);
|
|
322
|
+
result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
|
|
323
|
+
return result;
|
|
324
|
+
},
|
|
325
|
+
clear() {
|
|
326
|
+
entries.length = 0;
|
|
327
|
+
docStates.clear();
|
|
328
|
+
},
|
|
329
|
+
bufferedDocState(docId) {
|
|
330
|
+
return docStates.get(docId);
|
|
331
|
+
},
|
|
332
|
+
whenReplayed() {
|
|
333
|
+
return replayed;
|
|
334
|
+
},
|
|
335
|
+
markReplayed() {
|
|
336
|
+
resolveReplayed();
|
|
337
|
+
},
|
|
338
|
+
get size() {
|
|
339
|
+
return entries.length;
|
|
340
|
+
},
|
|
341
|
+
get isFull() {
|
|
342
|
+
return entries.length >= maxSize;
|
|
343
|
+
}
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// src/engine/rebalance-executor.ts
|
|
348
|
+
function warnHookError(hook, err) {
|
|
349
|
+
console.warn(`${hook} plugin hook error:`, err instanceof Error ? err.message : String(err));
|
|
350
|
+
}
|
|
351
|
+
function applyReplayUpdateVectors(entry, vectorFieldPaths) {
|
|
352
|
+
const updateVectors = /* @__PURE__ */ new Map();
|
|
353
|
+
for (const fieldPath of vectorFieldPaths) {
|
|
354
|
+
updateVectors.set(fieldPath, extractVectorFromDoc(entry.document, fieldPath));
|
|
355
|
+
}
|
|
356
|
+
let updateDoc = entry.document;
|
|
357
|
+
if (updateVectors.size > 0) {
|
|
358
|
+
updateDoc = structuredClone(entry.document);
|
|
359
|
+
for (const fieldPath of updateVectors.keys()) {
|
|
360
|
+
deleteNestedValue(updateDoc, fieldPath);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
return { updateDoc, updateVectors };
|
|
364
|
+
}
|
|
365
|
+
async function replayEntry(manager, indexName, entry, ctx) {
|
|
366
|
+
const vecIndexes = manager.getVectorIndexes();
|
|
367
|
+
const vectorFieldPaths = ctx.requireIndex(indexName).vectorFieldPaths;
|
|
368
|
+
if (entry.action === "insert" && entry.document) {
|
|
369
|
+
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
370
|
+
entry.document,
|
|
371
|
+
vectorFieldPaths
|
|
372
|
+
);
|
|
373
|
+
const overwrote = manager.has(entry.docId);
|
|
374
|
+
const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
|
|
375
|
+
if (overwrote) {
|
|
376
|
+
removeDocumentVectors(entry.docId, vecIndexes);
|
|
377
|
+
manager.remove(entry.docId, removedSpellings);
|
|
378
|
+
}
|
|
379
|
+
manager.insert(entry.docId, partitionDoc);
|
|
380
|
+
if (extractedVectors.size > 0) {
|
|
381
|
+
insertDocumentVectors(entry.docId, extractedVectors, vecIndexes, manager.partitionIdOf(entry.docId));
|
|
382
|
+
for (const fieldPath of extractedVectors.keys()) {
|
|
383
|
+
vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
try {
|
|
387
|
+
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: entry.docId, document: entry.document });
|
|
388
|
+
} catch (err) {
|
|
389
|
+
warnHookError("afterInsert", err);
|
|
390
|
+
}
|
|
391
|
+
const replicated = {
|
|
392
|
+
indexName,
|
|
393
|
+
docId: entry.docId,
|
|
394
|
+
document: entry.document,
|
|
395
|
+
requestId: `replicate-insert-${entry.docId}`
|
|
396
|
+
};
|
|
397
|
+
await ctx.orchestrator.replicateToWorkers(
|
|
398
|
+
overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
|
|
399
|
+
);
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
402
|
+
if (entry.action === "remove") {
|
|
403
|
+
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
404
|
+
manager.remove(entry.docId, removedSpellings);
|
|
405
|
+
removeDocumentVectors(entry.docId, vecIndexes);
|
|
406
|
+
try {
|
|
407
|
+
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
|
|
408
|
+
} catch (err) {
|
|
409
|
+
warnHookError("afterRemove", err);
|
|
410
|
+
}
|
|
411
|
+
await ctx.orchestrator.replicateToWorkers({
|
|
412
|
+
type: "remove",
|
|
413
|
+
indexName,
|
|
414
|
+
docId: entry.docId,
|
|
415
|
+
requestId: `replicate-remove-${entry.docId}`,
|
|
416
|
+
removedSpellings
|
|
417
|
+
});
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
if (entry.action === "update" && entry.document) {
|
|
421
|
+
const oldDocument = manager.get(entry.docId);
|
|
422
|
+
const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths);
|
|
423
|
+
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
424
|
+
if (manager.has(entry.docId)) {
|
|
425
|
+
manager.update(entry.docId, updateDoc, void 0, removedSpellings);
|
|
426
|
+
} else {
|
|
427
|
+
manager.insert(entry.docId, updateDoc);
|
|
428
|
+
}
|
|
429
|
+
for (const [fieldPath, newVec] of updateVectors) {
|
|
430
|
+
const vecIndex = vecIndexes.get(fieldPath);
|
|
431
|
+
if (!vecIndex) continue;
|
|
432
|
+
if (newVec === null) {
|
|
433
|
+
if (vecIndex.has(entry.docId)) {
|
|
434
|
+
vecIndex.remove(entry.docId);
|
|
435
|
+
}
|
|
436
|
+
} else {
|
|
437
|
+
const oldVec = vecIndex.getVector(entry.docId);
|
|
438
|
+
if (!vectorsEqual(oldVec, newVec)) {
|
|
439
|
+
if (vecIndex.has(entry.docId)) {
|
|
440
|
+
vecIndex.remove(entry.docId);
|
|
441
|
+
}
|
|
442
|
+
vecIndex.insert(entry.docId, newVec);
|
|
443
|
+
vecIndex.scheduleBuild();
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
try {
|
|
448
|
+
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
449
|
+
indexName,
|
|
450
|
+
docId: entry.docId,
|
|
451
|
+
oldDocument: oldDocument ?? {},
|
|
452
|
+
newDocument: entry.document
|
|
453
|
+
});
|
|
454
|
+
} catch (err) {
|
|
455
|
+
warnHookError("afterUpdate", err);
|
|
456
|
+
}
|
|
457
|
+
await ctx.orchestrator.replicateToWorkers({
|
|
458
|
+
type: "update",
|
|
459
|
+
indexName,
|
|
460
|
+
docId: entry.docId,
|
|
461
|
+
document: entry.document,
|
|
462
|
+
requestId: `replicate-update-${entry.docId}`,
|
|
463
|
+
removedSpellings
|
|
464
|
+
});
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
async function replayQueued(manager, indexName, waq, ctx) {
|
|
468
|
+
while (true) {
|
|
469
|
+
const entries = waq.drain();
|
|
470
|
+
if (entries.length === 0) break;
|
|
471
|
+
for (const entry of entries) {
|
|
472
|
+
try {
|
|
473
|
+
await replayEntry(manager, indexName, entry, ctx);
|
|
474
|
+
} catch (replayErr) {
|
|
475
|
+
const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
|
|
476
|
+
if (!isMissing) {
|
|
477
|
+
console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
484
|
+
if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
|
|
485
|
+
throw new NarsilError(
|
|
486
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
487
|
+
`Target partition count must be a positive integer, got ${targetPartitionCount}`,
|
|
488
|
+
{ targetPartitionCount }
|
|
489
|
+
);
|
|
490
|
+
}
|
|
491
|
+
const entry = ctx.requireIndex(indexName);
|
|
492
|
+
const maxPartitions = entry.config.partitions?.maxPartitions;
|
|
493
|
+
if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
|
|
494
|
+
throw new NarsilError(
|
|
495
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
496
|
+
`Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
|
|
497
|
+
{ targetPartitionCount, maxPartitions }
|
|
498
|
+
);
|
|
499
|
+
}
|
|
500
|
+
if (ctx.rebalancingIndexes.has(indexName)) {
|
|
501
|
+
throw new NarsilError(
|
|
502
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
503
|
+
`Index "${indexName}" is already being rebalanced`
|
|
504
|
+
);
|
|
505
|
+
}
|
|
506
|
+
const registered = ctx.requireIndex(indexName);
|
|
507
|
+
if (registered.pendingAnalysis !== null || registered.durableAnalysis !== null) {
|
|
508
|
+
throw new NarsilError(
|
|
509
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
510
|
+
`Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
|
|
511
|
+
{ indexName }
|
|
512
|
+
);
|
|
513
|
+
}
|
|
514
|
+
if (targetPartitionCount === manager.partitionCount) {
|
|
515
|
+
return;
|
|
516
|
+
}
|
|
517
|
+
const waq = createWriteAheadQueue();
|
|
518
|
+
ctx.waqMap.set(indexName, waq);
|
|
519
|
+
ctx.rebalancingIndexes.add(indexName);
|
|
520
|
+
ctx.rebalanceTargets.set(indexName, targetPartitionCount);
|
|
521
|
+
const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
|
|
522
|
+
let spill = null;
|
|
523
|
+
try {
|
|
524
|
+
const oldCount = manager.partitionCount;
|
|
525
|
+
spill = await rewriteSpillOf(ctx.durabilityManager?.rewriteStorage?.(), manager);
|
|
526
|
+
await createRebalancer().rebalance(
|
|
527
|
+
manager,
|
|
528
|
+
targetPartitionCount,
|
|
529
|
+
ctx.router,
|
|
530
|
+
(progress) => {
|
|
531
|
+
if (progress.phase === "complete") {
|
|
532
|
+
emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
|
|
533
|
+
indexName,
|
|
534
|
+
oldCount,
|
|
535
|
+
newCount: targetPartitionCount
|
|
536
|
+
});
|
|
537
|
+
}
|
|
538
|
+
},
|
|
539
|
+
spill
|
|
540
|
+
);
|
|
541
|
+
try {
|
|
542
|
+
await ctx.pluginRegistry.runHook("onPartitionSplit", {
|
|
543
|
+
indexName,
|
|
544
|
+
oldPartitionCount: oldCount,
|
|
545
|
+
newPartitionCount: targetPartitionCount
|
|
546
|
+
});
|
|
547
|
+
} catch (hookErr) {
|
|
548
|
+
console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
|
|
549
|
+
}
|
|
550
|
+
await replayQueued(manager, indexName, waq, ctx);
|
|
551
|
+
if (ctx.durabilityManager) {
|
|
552
|
+
await ctx.durabilityManager.persistMetadata(indexName);
|
|
553
|
+
await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
|
|
554
|
+
await spill?.removeUnread(manager);
|
|
555
|
+
await replayQueued(manager, indexName, waq, ctx);
|
|
556
|
+
}
|
|
557
|
+
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
558
|
+
do {
|
|
559
|
+
await replayQueued(manager, indexName, waq, ctx);
|
|
560
|
+
} while (waq.size > 0);
|
|
561
|
+
} catch (err) {
|
|
562
|
+
await spill?.removeUnread(manager).catch(() => void 0);
|
|
563
|
+
try {
|
|
564
|
+
await replayQueued(manager, indexName, waq, ctx);
|
|
565
|
+
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
566
|
+
do {
|
|
567
|
+
await replayQueued(manager, indexName, waq, ctx);
|
|
568
|
+
} while (waq.size > 0);
|
|
569
|
+
} catch (recoveryErr) {
|
|
570
|
+
console.warn(
|
|
571
|
+
`Rebalance failure recovery for index "${indexName}" did not complete:`,
|
|
572
|
+
recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)
|
|
573
|
+
);
|
|
574
|
+
}
|
|
575
|
+
throw err;
|
|
576
|
+
} finally {
|
|
577
|
+
ctx.rebalancingIndexes.delete(indexName);
|
|
578
|
+
ctx.rebalanceTargets.delete(indexName);
|
|
579
|
+
ctx.waqMap.delete(indexName);
|
|
580
|
+
waq.markReplayed();
|
|
581
|
+
}
|
|
582
|
+
ctx.checkWatermark(indexName);
|
|
583
|
+
}
|
|
584
|
+
|
|
190
585
|
// src/engine/analysis-change/adopt-saved.ts
|
|
191
586
|
function adoptSavedAnalysis(manager, entry, metadata) {
|
|
192
587
|
const saved = reconstructSchemaFromMetadata(metadata);
|
|
@@ -278,10 +673,15 @@ async function buildAndSwitch(scope) {
|
|
|
278
673
|
const wasPromoted = core.orchestrator.desyncIndex(indexName);
|
|
279
674
|
try {
|
|
280
675
|
const durable = core.durability?.manager;
|
|
676
|
+
let spill = null;
|
|
281
677
|
const rebuilt = await rebuildTextIndexes(
|
|
282
678
|
manager,
|
|
283
679
|
{
|
|
284
680
|
isCurrent,
|
|
681
|
+
rewriteStorage: () => durable?.rewriteStorage?.(),
|
|
682
|
+
onRewriteSpill(kept) {
|
|
683
|
+
spill = kept;
|
|
684
|
+
},
|
|
285
685
|
onPartitionBuilt(built) {
|
|
286
686
|
partitionsRebuilt = built;
|
|
287
687
|
},
|
|
@@ -299,6 +699,7 @@ async function buildAndSwitch(scope) {
|
|
|
299
699
|
textRebuildTargetOf(run.target.config, run.target.language, run.target.analysis)
|
|
300
700
|
);
|
|
301
701
|
if (!rebuilt) throw run.stopReason ?? stoppedError(core, indexName);
|
|
702
|
+
await spill?.removeUnread(manager);
|
|
302
703
|
if (entry.durableAnalysis !== null) {
|
|
303
704
|
throw new NarsilError(
|
|
304
705
|
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
@@ -496,4 +897,4 @@ async function stampAnalysisGeneration(core, indexName, generation) {
|
|
|
496
897
|
await (durable?.checkpointAfterSwitch?.(indexName, stamp) ?? Promise.resolve(stamp()));
|
|
497
898
|
}
|
|
498
899
|
|
|
499
|
-
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration };
|
|
900
|
+
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, executeRebalance, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration };
|
package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs}
RENAMED
|
@@ -1 +1 @@
|
|
|
1
|
-
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-
|
|
1
|
+
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-XNWPP5NS.mjs';
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
2
|
+
|
|
3
|
+
// src/vector/vector-store/vector-file.browser.ts
|
|
4
|
+
function filesystemUnavailable() {
|
|
5
|
+
throw new NarsilError(
|
|
6
|
+
ErrorCodes.CONFIG_INVALID,
|
|
7
|
+
"Reading vectors from a checkpoint file requires a Node.js runtime; keep the field in memory in the browser"
|
|
8
|
+
);
|
|
9
|
+
}
|
|
10
|
+
function createVectorFileReader() {
|
|
11
|
+
return {
|
|
12
|
+
readInto: () => filesystemUnavailable(),
|
|
13
|
+
readLater: () => filesystemUnavailable(),
|
|
14
|
+
retainOnly: () => void 0,
|
|
15
|
+
close: () => void 0
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export { createVectorFileReader };
|