@delali/narsil 0.3.1-next.140 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,25 +1,25 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-
|
|
2
|
-
import { resolveVectorText, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, lazyQueryBinding, executePreflight, executeSuggest, executeListDocuments, storedBM25Params, validateDocId } from './chunk-VEFN6X6Q.mjs';
|
|
2
|
+
import { resolveVectorText, bindEngineCore, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-5WYVUPGD.mjs';
|
|
3
|
+
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
3
4
|
import { splitWorkerBudget, resolveWorkerCount, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, resolveRequestThreadCount, workerResourceLimits, DEFAULT_BACKPRESSURE_LIMIT, SHUTDOWN_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH } from './chunk-UIKN2OTQ.mjs';
|
|
4
5
|
import { detectRuntime } from './chunk-AV4IJKD2.mjs';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
6
|
+
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-EEBCDP36.mjs';
|
|
7
|
+
import { resolvePartialWordAnalysis, resolvePartitionInsertOptions, isEmptySynonymSet, synonymModule, createDirectExecutor, createPartitionRouter, availableSynonymModule, mergeFrozenSegments, createSharedFrozenSegment, freezeSegmentShared, createFrozenSegment } from './chunk-WJGO5PGV.mjs';
|
|
8
|
+
import { ARRAY_POSITION_GAP, requirePatternSearchFor, requirePartialWordSearchFor, loadPhraseCodeFor, collectQueryTermStats, requireLoadedPhraseCode, createPartitionIndex, mergePartitionStats, oversampledFacetConfig, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, isCompositePartition, fnv1a, encodeSegmentState } from './chunk-ZPWS2SKJ.mjs';
|
|
7
9
|
import { generateId } from './chunk-N37VMC62.mjs';
|
|
8
10
|
import { readHeapStatistics } from './chunk-ZATRDAYU.mjs';
|
|
9
11
|
import { countEngineThread } from './chunk-EBIHRL7B.mjs';
|
|
10
|
-
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-HJSN7UYG.mjs';
|
|
11
|
-
import { MAX_INDEX_CONFIGURATION_BYTES, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, FOREIGN_STATS_TTL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE } from './chunk-EEBCDP36.mjs';
|
|
12
12
|
import { requireSearchBackendSettings } from './chunk-74XB2SXI.mjs';
|
|
13
13
|
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-RKAVHNRW.mjs';
|
|
14
|
-
import { getNestedValue } from './chunk-
|
|
15
|
-
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-
|
|
14
|
+
import { getNestedValue } from './chunk-DQNKVOJ2.mjs';
|
|
15
|
+
import { flattenSchema, raiseEarlierReleaseGraphSetting, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, validateTextStorage, schemaFieldsOf, extractVectorFieldsFromSchema, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KGOXHLLZ.mjs';
|
|
16
16
|
import { NATIVE_CRC32_MIN_BYTES, MAX_FILE_BYTES, CHECKSUM_YIELD_CHUNK_BYTES } from './chunk-U476IXKQ.mjs';
|
|
17
17
|
import { isSingleStringFieldType } from './chunk-ZSWHPKGM.mjs';
|
|
18
18
|
import { withOpenIndex, oneAtATime } from './chunk-O4LJQI4G.mjs';
|
|
19
19
|
import { getLanguage } from './chunk-WNBRW4Q2.mjs';
|
|
20
20
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
21
|
-
import { createRequestId, isValidWorkerResponse } from './chunk-
|
|
22
|
-
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-
|
|
21
|
+
import { createRequestId, isValidWorkerResponse } from './chunk-QHPYS427.mjs';
|
|
22
|
+
import { loadVectorCodeFor, requireLoadedVectorCode } from './chunk-GLZHNSGB.mjs';
|
|
23
23
|
import { noteThreadSlotLive, threadSlotOfWorker, noteThreadSlotGone } from './chunk-RFBLQLLZ.mjs';
|
|
24
24
|
import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
|
|
25
25
|
import { encode, decode } from '@msgpack/msgpack';
|
|
@@ -152,130 +152,6 @@ function requireValidPatternWorkCap(cap) {
|
|
|
152
152
|
);
|
|
153
153
|
}
|
|
154
154
|
|
|
155
|
-
// src/engine/orchestration/turn.ts
|
|
156
|
-
function afterCurrentTurn(callback) {
|
|
157
|
-
if (typeof setImmediate === "function") {
|
|
158
|
-
setImmediate(callback);
|
|
159
|
-
return;
|
|
160
|
-
}
|
|
161
|
-
setTimeout(callback, 0);
|
|
162
|
-
}
|
|
163
|
-
function yieldToEventLoop() {
|
|
164
|
-
return new Promise((resolve) => afterCurrentTurn(resolve));
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
// src/partitioning/constants.ts
|
|
168
|
-
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
169
|
-
var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
|
|
170
|
-
var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
|
|
171
|
-
var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
172
|
-
|
|
173
|
-
// src/partitioning/rebalancer.ts
|
|
174
|
-
function createRebalancer() {
|
|
175
|
-
const activeManagers = /* @__PURE__ */ new WeakSet();
|
|
176
|
-
let activeCount = 0;
|
|
177
|
-
async function rebalance(manager, newPartitionCount, router, onProgress) {
|
|
178
|
-
if (activeManagers.has(manager)) {
|
|
179
|
-
throw new NarsilError(
|
|
180
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
181
|
-
"A rebalance operation is already in progress"
|
|
182
|
-
);
|
|
183
|
-
}
|
|
184
|
-
if (newPartitionCount <= 0) {
|
|
185
|
-
throw new NarsilError(
|
|
186
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
187
|
-
`New partition count must be greater than 0, received ${newPartitionCount}`,
|
|
188
|
-
{ newPartitionCount }
|
|
189
|
-
);
|
|
190
|
-
}
|
|
191
|
-
if (newPartitionCount === manager.partitionCount) {
|
|
192
|
-
throw new NarsilError(
|
|
193
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
194
|
-
`New partition count (${newPartitionCount}) is the same as the current count`,
|
|
195
|
-
{ newPartitionCount, currentPartitionCount: manager.partitionCount }
|
|
196
|
-
);
|
|
197
|
-
}
|
|
198
|
-
activeManagers.add(manager);
|
|
199
|
-
activeCount++;
|
|
200
|
-
try {
|
|
201
|
-
const currentPartitions = manager.getAllPartitions();
|
|
202
|
-
let documentsTotal = 0;
|
|
203
|
-
for (const partition of currentPartitions) {
|
|
204
|
-
documentsTotal += partition.count();
|
|
205
|
-
}
|
|
206
|
-
onProgress?.({
|
|
207
|
-
phase: "scanning",
|
|
208
|
-
documentsProcessed: 0,
|
|
209
|
-
documentsTotal
|
|
210
|
-
});
|
|
211
|
-
const newPartitions = [];
|
|
212
|
-
for (let i = 0; i < newPartitionCount; i++) {
|
|
213
|
-
newPartitions.push(createPartitionIndex(i, manager.config.trackPositions ?? true));
|
|
214
|
-
}
|
|
215
|
-
let documentsProcessed = 0;
|
|
216
|
-
let chunkFill = 0;
|
|
217
|
-
const insertOptions = {
|
|
218
|
-
validate: false,
|
|
219
|
-
skipClone: true,
|
|
220
|
-
stopWordOverride: manager.analysis.stopWords,
|
|
221
|
-
customTokenizer: manager.analysis.customTokenizer,
|
|
222
|
-
collectSurfaces: manager.config.surfaceForms !== false
|
|
223
|
-
};
|
|
224
|
-
for (const partition of currentPartitions) {
|
|
225
|
-
const docIds = [...partition.docIds()];
|
|
226
|
-
for (const docId of docIds) {
|
|
227
|
-
const document = partition.getRef(docId);
|
|
228
|
-
if (!document) continue;
|
|
229
|
-
const targetPartitionId = router.route(docId, newPartitionCount);
|
|
230
|
-
const targetPartition = newPartitions[targetPartitionId];
|
|
231
|
-
if (!targetPartition.has(docId)) {
|
|
232
|
-
targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
|
|
233
|
-
documentsProcessed++;
|
|
234
|
-
}
|
|
235
|
-
chunkFill++;
|
|
236
|
-
if (chunkFill >= REBALANCE_CHUNK_SIZE) {
|
|
237
|
-
chunkFill = 0;
|
|
238
|
-
onProgress?.({
|
|
239
|
-
phase: "moving",
|
|
240
|
-
documentsProcessed,
|
|
241
|
-
documentsTotal
|
|
242
|
-
});
|
|
243
|
-
await yieldToEventLoop();
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
if (chunkFill > 0) {
|
|
248
|
-
onProgress?.({
|
|
249
|
-
phase: "moving",
|
|
250
|
-
documentsProcessed,
|
|
251
|
-
documentsTotal
|
|
252
|
-
});
|
|
253
|
-
await yieldToEventLoop();
|
|
254
|
-
}
|
|
255
|
-
onProgress?.({
|
|
256
|
-
phase: "swapping",
|
|
257
|
-
documentsProcessed,
|
|
258
|
-
documentsTotal
|
|
259
|
-
});
|
|
260
|
-
manager.setPartitions(newPartitions);
|
|
261
|
-
onProgress?.({
|
|
262
|
-
phase: "complete",
|
|
263
|
-
documentsProcessed,
|
|
264
|
-
documentsTotal
|
|
265
|
-
});
|
|
266
|
-
} finally {
|
|
267
|
-
activeManagers.delete(manager);
|
|
268
|
-
activeCount--;
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
return {
|
|
272
|
-
rebalance,
|
|
273
|
-
isRebalancing() {
|
|
274
|
-
return activeCount > 0;
|
|
275
|
-
}
|
|
276
|
-
};
|
|
277
|
-
}
|
|
278
|
-
|
|
279
155
|
// src/plugins/registry.ts
|
|
280
156
|
var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
|
|
281
157
|
function isThenable(value) {
|
|
@@ -479,6 +355,31 @@ function validateRequiredFieldsInSchema(required, schema) {
|
|
|
479
355
|
}
|
|
480
356
|
}
|
|
481
357
|
|
|
358
|
+
// src/engine/orchestration/turn.ts
|
|
359
|
+
function afterCurrentTurn(callback) {
|
|
360
|
+
if (typeof setImmediate === "function") {
|
|
361
|
+
setImmediate(callback);
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
setTimeout(callback, 0);
|
|
365
|
+
}
|
|
366
|
+
function yieldToEventLoop() {
|
|
367
|
+
return new Promise((resolve) => afterCurrentTurn(resolve));
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// src/partitioning/constants.ts
|
|
371
|
+
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
372
|
+
var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
|
|
373
|
+
var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
|
|
374
|
+
var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
375
|
+
|
|
376
|
+
// src/partitioning/rewrite-spill-of.ts
|
|
377
|
+
async function rewriteSpillOf(storage, manager, target) {
|
|
378
|
+
if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
|
|
379
|
+
const { openRewriteSpill } = await import('./module-IYPLWHGC.mjs');
|
|
380
|
+
return openRewriteSpill(storage, manager, target);
|
|
381
|
+
}
|
|
382
|
+
|
|
482
383
|
// src/partitioning/text-rebuild.ts
|
|
483
384
|
function textRebuildTargetOf(config, language, analysis) {
|
|
484
385
|
return { config, language, analysis, partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis) };
|
|
@@ -488,7 +389,7 @@ function interrupted(pass) {
|
|
|
488
389
|
}
|
|
489
390
|
async function buildPartition(pass, source, partitionId) {
|
|
490
391
|
const { manager, language, options } = pass;
|
|
491
|
-
const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
392
|
+
const target = pass.spill?.newPartition(partitionId) ?? createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
492
393
|
const docIds = [...source.docIds()];
|
|
493
394
|
target.beginBatch();
|
|
494
395
|
for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
|
|
@@ -497,6 +398,11 @@ async function buildPartition(pass, source, partitionId) {
|
|
|
497
398
|
const document = manager.getRef(docIds[i]);
|
|
498
399
|
if (document !== void 0) target.insert(docIds[i], document, manager.schema, language, options);
|
|
499
400
|
}
|
|
401
|
+
if (pass.spill !== null) {
|
|
402
|
+
target.endBatch();
|
|
403
|
+
await pass.spill.spillIfFull(target, partitionId);
|
|
404
|
+
target.beginBatch();
|
|
405
|
+
}
|
|
500
406
|
await yieldToEventLoop();
|
|
501
407
|
if (interrupted(pass)) return null;
|
|
502
408
|
}
|
|
@@ -558,12 +464,19 @@ async function rebuildTextIndexes(manager, hooks, target) {
|
|
|
558
464
|
);
|
|
559
465
|
const language = target?.language ?? manager.language;
|
|
560
466
|
const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
|
|
467
|
+
const spill = await rewriteSpillOf(hooks.rewriteStorage?.(), manager, target);
|
|
561
468
|
while (hooks.isCurrent()) {
|
|
562
469
|
const log = manager.openWriteLog();
|
|
470
|
+
let swapped = false;
|
|
563
471
|
try {
|
|
564
|
-
|
|
472
|
+
swapped = await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options, spill });
|
|
473
|
+
if (swapped) {
|
|
474
|
+
hooks.onRewriteSpill?.(spill);
|
|
475
|
+
return true;
|
|
476
|
+
}
|
|
565
477
|
} finally {
|
|
566
478
|
log.close();
|
|
479
|
+
if (!swapped) await spill?.abandon();
|
|
567
480
|
}
|
|
568
481
|
}
|
|
569
482
|
return false;
|
|
@@ -597,6 +510,7 @@ function toError(value) {
|
|
|
597
510
|
function wireAnalysisRebuild(wiring) {
|
|
598
511
|
return createAnalysisRebuildCoordinator(wiring.config, {
|
|
599
512
|
getManager: wiring.getManager,
|
|
513
|
+
rewriteStorage: () => wiring.durabilityManager?.rewriteStorage?.(),
|
|
600
514
|
desyncIndex: wiring.desyncIndex,
|
|
601
515
|
resyncIndex: wiring.resyncIndex,
|
|
602
516
|
hasPendingChange: wiring.hasPendingChange,
|
|
@@ -630,8 +544,13 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
630
544
|
const partitionCount = manager.partitionCount;
|
|
631
545
|
deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
|
|
632
546
|
const wasPromoted = deps.desyncIndex(indexName);
|
|
547
|
+
let spill = null;
|
|
633
548
|
try {
|
|
634
549
|
const rebuilt = await rebuildTextIndexes(manager, {
|
|
550
|
+
rewriteStorage: () => deps.rewriteStorage?.(),
|
|
551
|
+
onRewriteSpill(kept) {
|
|
552
|
+
spill = kept;
|
|
553
|
+
},
|
|
635
554
|
isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
|
|
636
555
|
onPartitionBuilt(partitionsBuilt) {
|
|
637
556
|
entry.partitionsRebuilt = partitionsBuilt;
|
|
@@ -646,6 +565,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
646
565
|
return;
|
|
647
566
|
}
|
|
648
567
|
await deps.persistAnalysisRevision(indexName);
|
|
568
|
+
await spill?.removeUnread(manager);
|
|
649
569
|
await deps.resyncIndex(indexName, wasPromoted);
|
|
650
570
|
deps.emit({
|
|
651
571
|
indexName,
|
|
@@ -919,6 +839,9 @@ function reconstructSchemaFromMetadata(metadata) {
|
|
|
919
839
|
if (metadata.trackPositions !== void 0) {
|
|
920
840
|
config.trackPositions = metadata.trackPositions;
|
|
921
841
|
}
|
|
842
|
+
if (metadata.textStorage !== void 0) {
|
|
843
|
+
config.textStorage = metadata.textStorage;
|
|
844
|
+
}
|
|
922
845
|
if (metadata.strict !== void 0) {
|
|
923
846
|
config.strict = metadata.strict;
|
|
924
847
|
}
|
|
@@ -997,7 +920,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
997
920
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
998
921
|
if (existing !== void 0) {
|
|
999
922
|
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
1000
|
-
const { adoptSavedAnalysis } = await import('./analysis-change-
|
|
923
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-HQUT2Z3I.mjs');
|
|
1001
924
|
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
1002
925
|
}
|
|
1003
926
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
@@ -1030,6 +953,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1030
953
|
const synonyms = prepared?.encoded ?? null;
|
|
1031
954
|
try {
|
|
1032
955
|
validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
|
|
956
|
+
validateTextStorage(indexConfig.textStorage, deps.filesystemDurability);
|
|
1033
957
|
if (loadData) {
|
|
1034
958
|
deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
|
|
1035
959
|
}
|
|
@@ -1781,6 +1705,9 @@ function checkHeapAfterRecovery(core) {
|
|
|
1781
1705
|
|
|
1782
1706
|
// src/core/partition/frozen/share.ts
|
|
1783
1707
|
function shareFrozenSegment(segment, analysis) {
|
|
1708
|
+
if (segment.paged !== null) {
|
|
1709
|
+
return { snapshot: segment.paged.snapshot, tombstonedDocIds: segment.tombstonedDocIds() };
|
|
1710
|
+
}
|
|
1784
1711
|
if (segment.sharedSnapshot !== null) {
|
|
1785
1712
|
const tombstonedDocIds = segment.tombstonedDocIds();
|
|
1786
1713
|
if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
|
|
@@ -2567,7 +2494,7 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
|
|
|
2567
2494
|
}
|
|
2568
2495
|
async function compactIndexSegments(state, indexName, policy) {
|
|
2569
2496
|
const manager = state.executor.getManager(indexName);
|
|
2570
|
-
if (!manager) return;
|
|
2497
|
+
if (!manager || manager.config.textStorage === "disk") return;
|
|
2571
2498
|
const partitionIds = /* @__PURE__ */ new Set();
|
|
2572
2499
|
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) partitionIds.add(partitionId);
|
|
2573
2500
|
for (const partitionId of state.segmentLedger.get(indexName)?.keys() ?? []) partitionIds.add(partitionId);
|
|
@@ -4041,6 +3968,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
4041
3968
|
scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
|
|
4042
3969
|
replicateToWorkers: replicate,
|
|
4043
3970
|
awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
|
|
3971
|
+
queueDiskAdoption: (action) => queueForCopies(state, action),
|
|
4044
3972
|
awaitWrites: (indexName) => awaitWritesApplied(state, indexName),
|
|
4045
3973
|
awaitCompactions: () => awaitCompactions(state),
|
|
4046
3974
|
openIndex,
|
|
@@ -4125,7 +4053,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4125
4053
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
4126
4054
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
4127
4055
|
}
|
|
4128
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
4056
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-324D2DDD.mjs');
|
|
4129
4057
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
4130
4058
|
const executor = createDirectExecutor({
|
|
4131
4059
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -4185,7 +4113,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4185
4113
|
},
|
|
4186
4114
|
vectorCopyPolicy
|
|
4187
4115
|
);
|
|
4188
|
-
const rebalancer = createRebalancer();
|
|
4189
4116
|
const rebalanceRouter = createPartitionRouter();
|
|
4190
4117
|
const waqMap = /* @__PURE__ */ new Map();
|
|
4191
4118
|
const rebalanceTargets = /* @__PURE__ */ new Map();
|
|
@@ -4245,6 +4172,8 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4245
4172
|
emitCheckpointHeld(report) {
|
|
4246
4173
|
emitEngineEvent(eventHandlers, "checkpointHeld", report);
|
|
4247
4174
|
},
|
|
4175
|
+
queueDiskAdoption: orchestrator.queueDiskAdoption,
|
|
4176
|
+
awaitCopiesOfIndex: (indexName) => orchestrator.awaitReplication(indexName),
|
|
4248
4177
|
publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve(),
|
|
4249
4178
|
recordCheckpoint(indexName, documentCount, partitionCount) {
|
|
4250
4179
|
const entry = indexRegistry.get(indexName);
|
|
@@ -4309,7 +4238,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4309
4238
|
requireManager,
|
|
4310
4239
|
resumeChange: async (indexName) => {
|
|
4311
4240
|
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
4312
|
-
await (await import('./analysis-change-entry-
|
|
4241
|
+
await (await import('./analysis-change-entry-TZTG4AZB.mjs')).resumeChangeOf(engineCore, indexName);
|
|
4313
4242
|
},
|
|
4314
4243
|
onClose: hooks?.onIndexClose,
|
|
4315
4244
|
onOpen: (indexName) => {
|
|
@@ -4339,7 +4268,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4339
4268
|
checkHeapPressure: heapPressureNotifier.check
|
|
4340
4269
|
};
|
|
4341
4270
|
const rebalanceCtx = {
|
|
4342
|
-
rebalancer,
|
|
4343
4271
|
router: rebalanceRouter,
|
|
4344
4272
|
waqMap,
|
|
4345
4273
|
rebalancingIndexes,
|
|
@@ -4366,7 +4294,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
4366
4294
|
shutdownState,
|
|
4367
4295
|
abortController,
|
|
4368
4296
|
orchestrator,
|
|
4369
|
-
rebalancer,
|
|
4370
4297
|
rebalanceRouter,
|
|
4371
4298
|
rebalancingIndexes,
|
|
4372
4299
|
waqMap,
|
|
@@ -4393,7 +4320,7 @@ async function startEngineCore(core, config) {
|
|
|
4393
4320
|
checkHeapAfterRecovery(core);
|
|
4394
4321
|
}
|
|
4395
4322
|
if (core.invalidation) await core.invalidation.start();
|
|
4396
|
-
if (core.durability) await (await import('./analysis-change-entry-
|
|
4323
|
+
if (core.durability) await (await import('./analysis-change-entry-TZTG4AZB.mjs')).resumeEveryChange(core);
|
|
4397
4324
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
4398
4325
|
} catch (err) {
|
|
4399
4326
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -4536,6 +4463,7 @@ async function checksumOfChunks(chunks) {
|
|
|
4536
4463
|
// src/serialization/header.ts
|
|
4537
4464
|
var MAGIC_BYTES = new Uint8Array([78, 82, 83, 76]);
|
|
4538
4465
|
var HEADER_SIZE = 32;
|
|
4466
|
+
var RESERVED_SIZE = 14;
|
|
4539
4467
|
var MAX_PAYLOAD_LENGTH = 4294967295;
|
|
4540
4468
|
var COMPRESSION_ALGORITHMS = {
|
|
4541
4469
|
0: "none",
|
|
@@ -4648,9 +4576,12 @@ function requireWithinFileLimit(byteLength, what) {
|
|
|
4648
4576
|
);
|
|
4649
4577
|
}
|
|
4650
4578
|
async function packChecksummedEnvelope(payloadChunks, envelopeFormatVersion, what) {
|
|
4579
|
+
requireWithinFileLimit(HEADER_SIZE + payloadByteLength(payloadChunks), what);
|
|
4580
|
+
return packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, await checksumOfChunks(payloadChunks));
|
|
4581
|
+
}
|
|
4582
|
+
function packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, checksum) {
|
|
4651
4583
|
const payloadLength = payloadByteLength(payloadChunks);
|
|
4652
4584
|
requireWithinFileLimit(HEADER_SIZE + payloadLength, what);
|
|
4653
|
-
const checksum = await checksumOfChunks(payloadChunks);
|
|
4654
4585
|
const [major, minor, patch] = engineVersionParts();
|
|
4655
4586
|
const header = writeHeader({
|
|
4656
4587
|
envelopeFormatVersion,
|
|
@@ -4812,6 +4743,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
|
|
|
4812
4743
|
...config.partitions !== void 0 ? { partitionConfig: config.partitions } : {},
|
|
4813
4744
|
...config.defaultScoring !== void 0 ? { defaultScoring: config.defaultScoring } : {},
|
|
4814
4745
|
...config.trackPositions !== void 0 ? { trackPositions: config.trackPositions } : {},
|
|
4746
|
+
...config.textStorage !== void 0 ? { textStorage: config.textStorage } : {},
|
|
4815
4747
|
...config.strict !== void 0 ? { strict: config.strict } : {},
|
|
4816
4748
|
...config.required !== void 0 ? { required: config.required } : {},
|
|
4817
4749
|
...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
|
|
@@ -4900,6 +4832,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
4900
4832
|
requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
|
|
4901
4833
|
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
4902
4834
|
validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
|
|
4835
|
+
validateTextStorage(indexConfig.textStorage, core.filesystemDurability);
|
|
4903
4836
|
if (indexConfig.bm25 !== void 0) {
|
|
4904
4837
|
validateBM25Params(
|
|
4905
4838
|
indexConfig.bm25,
|
|
@@ -4991,7 +4924,7 @@ async function dropEngineIndex(core, name) {
|
|
|
4991
4924
|
const entry = core.requireIndex(name);
|
|
4992
4925
|
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
4993
4926
|
const abandonChange = async () => {
|
|
4994
|
-
await (await import('./analysis-change-entry-
|
|
4927
|
+
await (await import('./analysis-change-entry-TZTG4AZB.mjs')).abandonChangeOf(core, name);
|
|
4995
4928
|
};
|
|
4996
4929
|
if (changeRecorded()) await abandonChange();
|
|
4997
4930
|
await core.indexState.drop(name, async () => {
|
|
@@ -6246,284 +6179,6 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
6246
6179
|
return { succeeded, failed };
|
|
6247
6180
|
}
|
|
6248
6181
|
|
|
6249
|
-
// src/partitioning/write-ahead-queue.ts
|
|
6250
|
-
function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
|
|
6251
|
-
const entries = [];
|
|
6252
|
-
const docStates = /* @__PURE__ */ new Map();
|
|
6253
|
-
let nextSequence = 1;
|
|
6254
|
-
let resolveReplayed = () => void 0;
|
|
6255
|
-
const replayed = new Promise((resolve) => {
|
|
6256
|
-
resolveReplayed = resolve;
|
|
6257
|
-
});
|
|
6258
|
-
return {
|
|
6259
|
-
push(entry) {
|
|
6260
|
-
if (entries.length >= maxSize) {
|
|
6261
|
-
throw new NarsilError(
|
|
6262
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
6263
|
-
`Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
|
|
6264
|
-
{ maxSize, currentSize: entries.length }
|
|
6265
|
-
);
|
|
6266
|
-
}
|
|
6267
|
-
const seq = nextSequence++;
|
|
6268
|
-
entries.push({ ...entry, sequenceNumber: seq });
|
|
6269
|
-
docStates.set(entry.docId, entry.action === "remove" ? "absent" : "present");
|
|
6270
|
-
return seq;
|
|
6271
|
-
},
|
|
6272
|
-
drain() {
|
|
6273
|
-
const result = entries.splice(0);
|
|
6274
|
-
result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
|
|
6275
|
-
return result;
|
|
6276
|
-
},
|
|
6277
|
-
clear() {
|
|
6278
|
-
entries.length = 0;
|
|
6279
|
-
docStates.clear();
|
|
6280
|
-
},
|
|
6281
|
-
bufferedDocState(docId) {
|
|
6282
|
-
return docStates.get(docId);
|
|
6283
|
-
},
|
|
6284
|
-
whenReplayed() {
|
|
6285
|
-
return replayed;
|
|
6286
|
-
},
|
|
6287
|
-
markReplayed() {
|
|
6288
|
-
resolveReplayed();
|
|
6289
|
-
},
|
|
6290
|
-
get size() {
|
|
6291
|
-
return entries.length;
|
|
6292
|
-
},
|
|
6293
|
-
get isFull() {
|
|
6294
|
-
return entries.length >= maxSize;
|
|
6295
|
-
}
|
|
6296
|
-
};
|
|
6297
|
-
}
|
|
6298
|
-
|
|
6299
|
-
// src/engine/rebalance-executor.ts
|
|
6300
|
-
function warnHookError(hook, err) {
|
|
6301
|
-
console.warn(`${hook} plugin hook error:`, err instanceof Error ? err.message : String(err));
|
|
6302
|
-
}
|
|
6303
|
-
function applyReplayUpdateVectors(entry, vectorFieldPaths) {
|
|
6304
|
-
const updateVectors = /* @__PURE__ */ new Map();
|
|
6305
|
-
for (const fieldPath of vectorFieldPaths) {
|
|
6306
|
-
updateVectors.set(fieldPath, extractVectorFromDoc(entry.document, fieldPath));
|
|
6307
|
-
}
|
|
6308
|
-
let updateDoc = entry.document;
|
|
6309
|
-
if (updateVectors.size > 0) {
|
|
6310
|
-
updateDoc = structuredClone(entry.document);
|
|
6311
|
-
for (const fieldPath of updateVectors.keys()) {
|
|
6312
|
-
deleteNestedValue(updateDoc, fieldPath);
|
|
6313
|
-
}
|
|
6314
|
-
}
|
|
6315
|
-
return { updateDoc, updateVectors };
|
|
6316
|
-
}
|
|
6317
|
-
async function replayEntry(manager, indexName, entry, ctx) {
|
|
6318
|
-
const vecIndexes = manager.getVectorIndexes();
|
|
6319
|
-
const vectorFieldPaths = ctx.requireIndex(indexName).vectorFieldPaths;
|
|
6320
|
-
if (entry.action === "insert" && entry.document) {
|
|
6321
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
6322
|
-
entry.document,
|
|
6323
|
-
vectorFieldPaths
|
|
6324
|
-
);
|
|
6325
|
-
const overwrote = manager.has(entry.docId);
|
|
6326
|
-
const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
|
|
6327
|
-
if (overwrote) {
|
|
6328
|
-
removeDocumentVectors(entry.docId, vecIndexes);
|
|
6329
|
-
manager.remove(entry.docId, removedSpellings);
|
|
6330
|
-
}
|
|
6331
|
-
manager.insert(entry.docId, partitionDoc);
|
|
6332
|
-
if (extractedVectors.size > 0) {
|
|
6333
|
-
insertDocumentVectors(entry.docId, extractedVectors, vecIndexes, manager.partitionIdOf(entry.docId));
|
|
6334
|
-
for (const fieldPath of extractedVectors.keys()) {
|
|
6335
|
-
vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
6336
|
-
}
|
|
6337
|
-
}
|
|
6338
|
-
try {
|
|
6339
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: entry.docId, document: entry.document });
|
|
6340
|
-
} catch (err) {
|
|
6341
|
-
warnHookError("afterInsert", err);
|
|
6342
|
-
}
|
|
6343
|
-
const replicated = {
|
|
6344
|
-
indexName,
|
|
6345
|
-
docId: entry.docId,
|
|
6346
|
-
document: entry.document,
|
|
6347
|
-
requestId: `replicate-insert-${entry.docId}`
|
|
6348
|
-
};
|
|
6349
|
-
await ctx.orchestrator.replicateToWorkers(
|
|
6350
|
-
overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
|
|
6351
|
-
);
|
|
6352
|
-
return;
|
|
6353
|
-
}
|
|
6354
|
-
if (entry.action === "remove") {
|
|
6355
|
-
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
6356
|
-
manager.remove(entry.docId, removedSpellings);
|
|
6357
|
-
removeDocumentVectors(entry.docId, vecIndexes);
|
|
6358
|
-
try {
|
|
6359
|
-
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
|
|
6360
|
-
} catch (err) {
|
|
6361
|
-
warnHookError("afterRemove", err);
|
|
6362
|
-
}
|
|
6363
|
-
await ctx.orchestrator.replicateToWorkers({
|
|
6364
|
-
type: "remove",
|
|
6365
|
-
indexName,
|
|
6366
|
-
docId: entry.docId,
|
|
6367
|
-
requestId: `replicate-remove-${entry.docId}`,
|
|
6368
|
-
removedSpellings
|
|
6369
|
-
});
|
|
6370
|
-
return;
|
|
6371
|
-
}
|
|
6372
|
-
if (entry.action === "update" && entry.document) {
|
|
6373
|
-
const oldDocument = manager.get(entry.docId);
|
|
6374
|
-
const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths);
|
|
6375
|
-
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
6376
|
-
if (manager.has(entry.docId)) {
|
|
6377
|
-
manager.update(entry.docId, updateDoc, void 0, removedSpellings);
|
|
6378
|
-
} else {
|
|
6379
|
-
manager.insert(entry.docId, updateDoc);
|
|
6380
|
-
}
|
|
6381
|
-
for (const [fieldPath, newVec] of updateVectors) {
|
|
6382
|
-
const vecIndex = vecIndexes.get(fieldPath);
|
|
6383
|
-
if (!vecIndex) continue;
|
|
6384
|
-
if (newVec === null) {
|
|
6385
|
-
if (vecIndex.has(entry.docId)) {
|
|
6386
|
-
vecIndex.remove(entry.docId);
|
|
6387
|
-
}
|
|
6388
|
-
} else {
|
|
6389
|
-
const oldVec = vecIndex.getVector(entry.docId);
|
|
6390
|
-
if (!vectorsEqual(oldVec, newVec)) {
|
|
6391
|
-
if (vecIndex.has(entry.docId)) {
|
|
6392
|
-
vecIndex.remove(entry.docId);
|
|
6393
|
-
}
|
|
6394
|
-
vecIndex.insert(entry.docId, newVec);
|
|
6395
|
-
vecIndex.scheduleBuild();
|
|
6396
|
-
}
|
|
6397
|
-
}
|
|
6398
|
-
}
|
|
6399
|
-
try {
|
|
6400
|
-
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
6401
|
-
indexName,
|
|
6402
|
-
docId: entry.docId,
|
|
6403
|
-
oldDocument: oldDocument ?? {},
|
|
6404
|
-
newDocument: entry.document
|
|
6405
|
-
});
|
|
6406
|
-
} catch (err) {
|
|
6407
|
-
warnHookError("afterUpdate", err);
|
|
6408
|
-
}
|
|
6409
|
-
await ctx.orchestrator.replicateToWorkers({
|
|
6410
|
-
type: "update",
|
|
6411
|
-
indexName,
|
|
6412
|
-
docId: entry.docId,
|
|
6413
|
-
document: entry.document,
|
|
6414
|
-
requestId: `replicate-update-${entry.docId}`,
|
|
6415
|
-
removedSpellings
|
|
6416
|
-
});
|
|
6417
|
-
}
|
|
6418
|
-
}
|
|
6419
|
-
async function replayQueued(manager, indexName, waq, ctx) {
|
|
6420
|
-
while (true) {
|
|
6421
|
-
const entries = waq.drain();
|
|
6422
|
-
if (entries.length === 0) break;
|
|
6423
|
-
for (const entry of entries) {
|
|
6424
|
-
try {
|
|
6425
|
-
await replayEntry(manager, indexName, entry, ctx);
|
|
6426
|
-
} catch (replayErr) {
|
|
6427
|
-
const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
|
|
6428
|
-
if (!isMissing) {
|
|
6429
|
-
console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
|
|
6430
|
-
}
|
|
6431
|
-
}
|
|
6432
|
-
}
|
|
6433
|
-
}
|
|
6434
|
-
}
|
|
6435
|
-
async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
6436
|
-
if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
|
|
6437
|
-
throw new NarsilError(
|
|
6438
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
6439
|
-
`Target partition count must be a positive integer, got ${targetPartitionCount}`,
|
|
6440
|
-
{ targetPartitionCount }
|
|
6441
|
-
);
|
|
6442
|
-
}
|
|
6443
|
-
const entry = ctx.requireIndex(indexName);
|
|
6444
|
-
const maxPartitions = entry.config.partitions?.maxPartitions;
|
|
6445
|
-
if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
|
|
6446
|
-
throw new NarsilError(
|
|
6447
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
6448
|
-
`Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
|
|
6449
|
-
{ targetPartitionCount, maxPartitions }
|
|
6450
|
-
);
|
|
6451
|
-
}
|
|
6452
|
-
if (ctx.rebalancingIndexes.has(indexName)) {
|
|
6453
|
-
throw new NarsilError(
|
|
6454
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
6455
|
-
`Index "${indexName}" is already being rebalanced`
|
|
6456
|
-
);
|
|
6457
|
-
}
|
|
6458
|
-
const registered = ctx.requireIndex(indexName);
|
|
6459
|
-
if (registered.pendingAnalysis !== null || registered.durableAnalysis !== null) {
|
|
6460
|
-
throw new NarsilError(
|
|
6461
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
6462
|
-
`Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
|
|
6463
|
-
{ indexName }
|
|
6464
|
-
);
|
|
6465
|
-
}
|
|
6466
|
-
if (targetPartitionCount === manager.partitionCount) {
|
|
6467
|
-
return;
|
|
6468
|
-
}
|
|
6469
|
-
const waq = createWriteAheadQueue();
|
|
6470
|
-
ctx.waqMap.set(indexName, waq);
|
|
6471
|
-
ctx.rebalancingIndexes.add(indexName);
|
|
6472
|
-
ctx.rebalanceTargets.set(indexName, targetPartitionCount);
|
|
6473
|
-
const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
|
|
6474
|
-
try {
|
|
6475
|
-
const oldCount = manager.partitionCount;
|
|
6476
|
-
await ctx.rebalancer.rebalance(manager, targetPartitionCount, ctx.router, (progress) => {
|
|
6477
|
-
if (progress.phase === "complete") {
|
|
6478
|
-
emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
|
|
6479
|
-
indexName,
|
|
6480
|
-
oldCount,
|
|
6481
|
-
newCount: targetPartitionCount
|
|
6482
|
-
});
|
|
6483
|
-
}
|
|
6484
|
-
});
|
|
6485
|
-
try {
|
|
6486
|
-
await ctx.pluginRegistry.runHook("onPartitionSplit", {
|
|
6487
|
-
indexName,
|
|
6488
|
-
oldPartitionCount: oldCount,
|
|
6489
|
-
newPartitionCount: targetPartitionCount
|
|
6490
|
-
});
|
|
6491
|
-
} catch (hookErr) {
|
|
6492
|
-
console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
|
|
6493
|
-
}
|
|
6494
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
6495
|
-
if (ctx.durabilityManager) {
|
|
6496
|
-
await ctx.durabilityManager.persistMetadata(indexName);
|
|
6497
|
-
await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
|
|
6498
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
6499
|
-
}
|
|
6500
|
-
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
6501
|
-
do {
|
|
6502
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
6503
|
-
} while (waq.size > 0);
|
|
6504
|
-
} catch (err) {
|
|
6505
|
-
try {
|
|
6506
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
6507
|
-
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
6508
|
-
do {
|
|
6509
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
6510
|
-
} while (waq.size > 0);
|
|
6511
|
-
} catch (recoveryErr) {
|
|
6512
|
-
console.warn(
|
|
6513
|
-
`Rebalance failure recovery for index "${indexName}" did not complete:`,
|
|
6514
|
-
recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)
|
|
6515
|
-
);
|
|
6516
|
-
}
|
|
6517
|
-
throw err;
|
|
6518
|
-
} finally {
|
|
6519
|
-
ctx.rebalancingIndexes.delete(indexName);
|
|
6520
|
-
ctx.rebalanceTargets.delete(indexName);
|
|
6521
|
-
ctx.waqMap.delete(indexName);
|
|
6522
|
-
waq.markReplayed();
|
|
6523
|
-
}
|
|
6524
|
-
ctx.checkWatermark(indexName);
|
|
6525
|
-
}
|
|
6526
|
-
|
|
6527
6182
|
// src/engine/vector-maintenance.ts
|
|
6528
6183
|
function resolveVectorIndexes(manager, indexName, fieldName) {
|
|
6529
6184
|
const all = manager.getVectorIndexes();
|
|
@@ -6563,6 +6218,7 @@ function getVectorMaintenanceStatus(manager) {
|
|
|
6563
6218
|
bufferSize: status.bufferSize,
|
|
6564
6219
|
building: status.building,
|
|
6565
6220
|
waitingForMemory: status.waitingForMemory,
|
|
6221
|
+
loadingFromFiles: status.loadingFromFiles === true,
|
|
6566
6222
|
estimatedCompactMs: status.estimatedCompactMs,
|
|
6567
6223
|
estimatedOptimizeMs: status.estimatedOptimizeMs
|
|
6568
6224
|
});
|
|
@@ -6677,14 +6333,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
6677
6333
|
return {
|
|
6678
6334
|
snapshot(indexName) {
|
|
6679
6335
|
return withOpenIndex2(indexName, async () => {
|
|
6680
|
-
const { createSnapshot } = await import('./snapshot-
|
|
6336
|
+
const { createSnapshot } = await import('./snapshot-CGM65VCD.mjs');
|
|
6681
6337
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
6682
6338
|
});
|
|
6683
6339
|
},
|
|
6684
6340
|
async restore(indexName, data) {
|
|
6685
6341
|
core.guardShutdown();
|
|
6686
6342
|
validateIndexName(indexName);
|
|
6687
|
-
const restored = import('./restore-queue-
|
|
6343
|
+
const restored = import('./restore-queue-FEYAC4SB.mjs').then(
|
|
6688
6344
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
6689
6345
|
);
|
|
6690
6346
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -6693,8 +6349,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
6693
6349
|
exportDocuments(indexName) {
|
|
6694
6350
|
return withOpenIndex2(indexName, async () => {
|
|
6695
6351
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
6696
|
-
import('./snapshot-
|
|
6697
|
-
import('./write-
|
|
6352
|
+
import('./snapshot-CGM65VCD.mjs'),
|
|
6353
|
+
import('./write-2DZPEQL2.mjs')
|
|
6698
6354
|
]);
|
|
6699
6355
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
6700
6356
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -6702,7 +6358,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
6702
6358
|
},
|
|
6703
6359
|
importDocumentExport(indexName, data) {
|
|
6704
6360
|
core.guardShutdown();
|
|
6705
|
-
const imported = import('./import-
|
|
6361
|
+
const imported = import('./import-OOBUW3RR.mjs').then(
|
|
6706
6362
|
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
6707
6363
|
);
|
|
6708
6364
|
return emitWriteEventAfter(core, indexName, imported);
|
|
@@ -6951,8 +6607,11 @@ function createNarsilFromCore(core, config) {
|
|
|
6951
6607
|
release();
|
|
6952
6608
|
}
|
|
6953
6609
|
}
|
|
6610
|
+
const rebalancesStarting = /* @__PURE__ */ new Map();
|
|
6954
6611
|
function withOpenIndexWrite(indexName, action) {
|
|
6955
|
-
|
|
6612
|
+
const starting = rebalancesStarting.get(indexName);
|
|
6613
|
+
const written = starting === void 0 ? withOpenIndex2(indexName, action) : starting.then(() => withOpenIndex2(indexName, action));
|
|
6614
|
+
return emitWriteEventAfter(core, indexName, written);
|
|
6956
6615
|
}
|
|
6957
6616
|
const narsil = {
|
|
6958
6617
|
createIndex(name, indexConfig) {
|
|
@@ -7080,16 +6739,17 @@ function createNarsilFromCore(core, config) {
|
|
|
7080
6739
|
const entry = requireIndex(indexName);
|
|
7081
6740
|
await withOpenIndexWrite(indexName, async () => {
|
|
7082
6741
|
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
7083
|
-
return (await import('./analysis-change-
|
|
6742
|
+
return (await import('./analysis-change-HQUT2Z3I.mjs')).resumeAnalysisChange(core, indexName);
|
|
7084
6743
|
});
|
|
7085
6744
|
},
|
|
7086
6745
|
async changeAnalysis(indexName, change) {
|
|
7087
6746
|
guardShutdown();
|
|
7088
6747
|
requireIndex(indexName);
|
|
7089
|
-
|
|
6748
|
+
const changed = withOpenIndex2(
|
|
7090
6749
|
indexName,
|
|
7091
|
-
async () => (await import('./analysis-change-
|
|
6750
|
+
async () => (await import('./analysis-change-HQUT2Z3I.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
7092
6751
|
);
|
|
6752
|
+
await emitWriteEventAfter(core, indexName, changed);
|
|
7093
6753
|
},
|
|
7094
6754
|
...portabilityOperations(core, config, () => narsil),
|
|
7095
6755
|
...synonymOperations(core),
|
|
@@ -7108,15 +6768,29 @@ function createNarsilFromCore(core, config) {
|
|
|
7108
6768
|
await executor.execute({ type: "clear", indexName, requestId: indexName });
|
|
7109
6769
|
await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
|
|
7110
6770
|
core.watermarkNotifier.forget(indexName);
|
|
6771
|
+
const durable = core.durability?.manager;
|
|
6772
|
+
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
7111
6773
|
});
|
|
7112
6774
|
},
|
|
7113
6775
|
async rebalance(indexName, targetPartitionCount) {
|
|
7114
6776
|
guardShutdown();
|
|
7115
6777
|
requireIndex(indexName);
|
|
7116
|
-
|
|
7117
|
-
|
|
7118
|
-
|
|
7119
|
-
);
|
|
6778
|
+
let writesMayFollow = () => void 0;
|
|
6779
|
+
const started = new Promise((resolve) => {
|
|
6780
|
+
writesMayFollow = resolve;
|
|
6781
|
+
});
|
|
6782
|
+
function releaseWrites() {
|
|
6783
|
+
writesMayFollow();
|
|
6784
|
+
if (rebalancesStarting.get(indexName) === started) rebalancesStarting.delete(indexName);
|
|
6785
|
+
}
|
|
6786
|
+
rebalancesStarting.set(indexName, started);
|
|
6787
|
+
const rebalanced = withOpenIndex2(indexName, async () => {
|
|
6788
|
+
const { executeRebalance } = await import('./analysis-change-HQUT2Z3I.mjs');
|
|
6789
|
+
const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
|
|
6790
|
+
releaseWrites();
|
|
6791
|
+
return running;
|
|
6792
|
+
});
|
|
6793
|
+
return emitWriteEventAfter(core, indexName, rebalanced).finally(releaseWrites);
|
|
7120
6794
|
},
|
|
7121
6795
|
async updatePartitionConfig(indexName, partitionConfig) {
|
|
7122
6796
|
guardShutdown();
|
|
@@ -7235,4 +6909,4 @@ async function createNarsil(config) {
|
|
|
7235
6909
|
// src/index.ts
|
|
7236
6910
|
var VERSION = "0.1.0";
|
|
7237
6911
|
|
|
7238
|
-
export { HEADER_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineVersionParts, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, resolveRunnableConfig, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, validateEmbeddingConfig, validateRequiredFieldsInSchema, writeHeader, yieldToEventLoop };
|
|
6912
|
+
export { DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, REBALANCE_CHUNK_SIZE, RESERVED_SIZE, VERSION, applyStoredIndexBm25, arrayPositionsAreStale, checksumOfChunks, crc322 as crc32, crc32Final, crc32Init, crc32Update, createEngineCore, createEngineIndex, createNarsil, createNarsilFromCore, decodeIndexConfiguration, deleteNestedValue, emitEngineEvent, emptySerializablePartition, encodeIndexConfiguration, engineVersionParts, extractVectorFromDoc, hasNrslMagic, insertDocumentVectors, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, readHeader, rebuildTextIndexes, reconstructSchemaFromMetadata, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireKnownConfig, requireWithinFileLimit, resolveRunnableConfig, rewriteSpillOf, rulesOfStoredSynonyms, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, startEngineCore, storedArrayPositionGap, textRebuildTargetOf, validateEmbeddingConfig, validateRequiredFieldsInSchema, vectorsEqual, writeHeader, yieldToEventLoop };
|