@delali/narsil 0.3.1-next.140 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, LOOKUP_KEY_SCRATCH_BYTES, bitsetHas, bitsetSet, createBitSet, bitsetIsEmpty, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
|
|
2
|
-
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-4MFH226Q.mjs';
|
|
3
1
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
|
|
2
|
+
import { TARGET_BLOCK_ENTRIES, bitsetHas, createBitSet, LOOKUP_KEY_SCRATCH_BYTES, GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, bitsetIsEmpty, bitsetSet, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
|
|
3
|
+
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, CHECKSUM_YIELD_CHUNK_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, toReducedSortValue, compareComparableValues, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, defaultSortMode, resolveRequestThreadCount, readSortField, holdsUnpairedSurrogate, truncateSortString, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-D33A5VMU.mjs';
|
|
4
4
|
import { MAX_INDEX_CONFIGURATION_BYTES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-MZI3TEOW.mjs';
|
|
5
5
|
import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-35B42UFM.mjs';
|
|
6
6
|
import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
|
|
@@ -2287,6 +2287,162 @@ var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith",
|
|
|
2287
2287
|
var PATTERN_EXPRESSION_OPERATORS = ["wildcard", "regex"];
|
|
2288
2288
|
var PATTERN_ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
|
|
2289
2289
|
|
|
2290
|
+
// src/core/hash.ts
|
|
2291
|
+
var FNV_OFFSET_BASIS = 2166136261;
|
|
2292
|
+
var FNV_PRIME = 16777619;
|
|
2293
|
+
var encoder = new TextEncoder();
|
|
2294
|
+
function fnv1aBytes(bytes) {
|
|
2295
|
+
return fnv1aContinue(FNV_OFFSET_BASIS, bytes, bytes.length) >>> 0;
|
|
2296
|
+
}
|
|
2297
|
+
function fnv1aContinue(start, bytes, length) {
|
|
2298
|
+
let hash = start;
|
|
2299
|
+
for (let i = 0; i < length; i++) {
|
|
2300
|
+
hash = Math.imul(hash ^ bytes[i], FNV_PRIME);
|
|
2301
|
+
}
|
|
2302
|
+
return hash;
|
|
2303
|
+
}
|
|
2304
|
+
var scratchNumber = new Float64Array(1);
|
|
2305
|
+
var scratchBytes = new Uint8Array(scratchNumber.buffer);
|
|
2306
|
+
var HOST_IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
2307
|
+
function fnv1aContinueTaggedNumbers(start, tag, numbers) {
|
|
2308
|
+
let hash = start;
|
|
2309
|
+
const b = scratchBytes;
|
|
2310
|
+
for (let i = 0; i < numbers.length; i++) {
|
|
2311
|
+
scratchNumber[0] = Math.fround(numbers[i]);
|
|
2312
|
+
hash = Math.imul(hash ^ tag, FNV_PRIME);
|
|
2313
|
+
if (HOST_IS_LITTLE_ENDIAN) {
|
|
2314
|
+
hash = Math.imul(hash ^ b[7], FNV_PRIME);
|
|
2315
|
+
hash = Math.imul(hash ^ b[6], FNV_PRIME);
|
|
2316
|
+
hash = Math.imul(hash ^ b[5], FNV_PRIME);
|
|
2317
|
+
hash = Math.imul(hash ^ b[4], FNV_PRIME);
|
|
2318
|
+
hash = Math.imul(hash ^ b[3], FNV_PRIME);
|
|
2319
|
+
hash = Math.imul(hash ^ b[2], FNV_PRIME);
|
|
2320
|
+
hash = Math.imul(hash ^ b[1], FNV_PRIME);
|
|
2321
|
+
hash = Math.imul(hash ^ b[0], FNV_PRIME);
|
|
2322
|
+
} else {
|
|
2323
|
+
for (let at = 0; at < 8; at++) hash = Math.imul(hash ^ b[at], FNV_PRIME);
|
|
2324
|
+
}
|
|
2325
|
+
}
|
|
2326
|
+
return hash;
|
|
2327
|
+
}
|
|
2328
|
+
function fnv1a(input) {
|
|
2329
|
+
return fnv1aBytes(encoder.encode(input));
|
|
2330
|
+
}
|
|
2331
|
+
|
|
2332
|
+
// src/partitioning/router.ts
|
|
2333
|
+
function createPartitionRouter() {
|
|
2334
|
+
function route(docId, partitionCount) {
|
|
2335
|
+
if (partitionCount <= 0) {
|
|
2336
|
+
throw new NarsilError(
|
|
2337
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
2338
|
+
`Partition count must be greater than 0, received ${partitionCount}`,
|
|
2339
|
+
{ partitionCount }
|
|
2340
|
+
);
|
|
2341
|
+
}
|
|
2342
|
+
return fnv1a(docId) % partitionCount;
|
|
2343
|
+
}
|
|
2344
|
+
function routeBatch(docIds, partitionCount) {
|
|
2345
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2346
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
2347
|
+
const pid = route(docIds[i], partitionCount);
|
|
2348
|
+
let group = groups.get(pid);
|
|
2349
|
+
if (!group) {
|
|
2350
|
+
group = [];
|
|
2351
|
+
groups.set(pid, group);
|
|
2352
|
+
}
|
|
2353
|
+
group.push(docIds[i]);
|
|
2354
|
+
}
|
|
2355
|
+
return groups;
|
|
2356
|
+
}
|
|
2357
|
+
return { route, routeBatch };
|
|
2358
|
+
}
|
|
2359
|
+
|
|
2360
|
+
// src/plugins/registry.ts
|
|
2361
|
+
var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
|
|
2362
|
+
function isThenable(value) {
|
|
2363
|
+
return value !== null && typeof value === "object" && typeof value.then === "function";
|
|
2364
|
+
}
|
|
2365
|
+
function thrownText(err, plugin, hookName) {
|
|
2366
|
+
if (err instanceof Error) return err.message;
|
|
2367
|
+
try {
|
|
2368
|
+
return String(err);
|
|
2369
|
+
} catch {
|
|
2370
|
+
return `The ${hookName} hook of plugin "${plugin}" rejects the write`;
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
function asWriteRejection(err, plugin, hookName) {
|
|
2374
|
+
if (err instanceof NarsilError) return err;
|
|
2375
|
+
return new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, thrownText(err, plugin, hookName), {
|
|
2376
|
+
plugin,
|
|
2377
|
+
hook: hookName
|
|
2378
|
+
});
|
|
2379
|
+
}
|
|
2380
|
+
function invoke(hook, hookName, context) {
|
|
2381
|
+
const rejectsWrites = WRITE_REJECTING_HOOKS.has(hookName);
|
|
2382
|
+
let result;
|
|
2383
|
+
try {
|
|
2384
|
+
result = hook.run(context);
|
|
2385
|
+
} catch (err) {
|
|
2386
|
+
throw rejectsWrites ? asWriteRejection(err, hook.plugin, hookName) : err;
|
|
2387
|
+
}
|
|
2388
|
+
if (!isThenable(result)) return null;
|
|
2389
|
+
if (!rejectsWrites) return result;
|
|
2390
|
+
return Promise.resolve(result).catch((err) => {
|
|
2391
|
+
throw asWriteRejection(err, hook.plugin, hookName);
|
|
2392
|
+
});
|
|
2393
|
+
}
|
|
2394
|
+
async function continueAsync(pending, hooks, startIndex, hookName, context) {
|
|
2395
|
+
await pending;
|
|
2396
|
+
for (let i = startIndex; i < hooks.length; i++) {
|
|
2397
|
+
await invoke(hooks[i], hookName, context);
|
|
2398
|
+
}
|
|
2399
|
+
}
|
|
2400
|
+
function createPluginRegistry() {
|
|
2401
|
+
const plugins = [];
|
|
2402
|
+
return {
|
|
2403
|
+
register(plugin) {
|
|
2404
|
+
plugins.push(plugin);
|
|
2405
|
+
},
|
|
2406
|
+
hasHooks(hookName) {
|
|
2407
|
+
return plugins.some((p) => typeof p[hookName] === "function");
|
|
2408
|
+
},
|
|
2409
|
+
runHook(hookName, context) {
|
|
2410
|
+
const hooks = [];
|
|
2411
|
+
for (const plugin of plugins) {
|
|
2412
|
+
const hook = plugin[hookName];
|
|
2413
|
+
if (typeof hook === "function") {
|
|
2414
|
+
hooks.push({ plugin: plugin.name, run: hook.bind(plugin) });
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
if (hooks.length === 0) return;
|
|
2418
|
+
for (let i = 0; i < hooks.length; i++) {
|
|
2419
|
+
const pending = invoke(hooks[i], hookName, context);
|
|
2420
|
+
if (pending !== null) {
|
|
2421
|
+
return continueAsync(pending, hooks, i + 1, hookName, context);
|
|
2422
|
+
}
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2425
|
+
};
|
|
2426
|
+
}
|
|
2427
|
+
|
|
2428
|
+
// src/rerank/registry.ts
|
|
2429
|
+
function validateRerankAdapter(name, adapter) {
|
|
2430
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
2431
|
+
throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, "A rerank adapter needs a non-empty string name");
|
|
2432
|
+
}
|
|
2433
|
+
const rerank = typeof adapter === "object" && adapter !== null ? adapter.rerank : void 0;
|
|
2434
|
+
if (typeof rerank !== "function") {
|
|
2435
|
+
throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, `Rerank adapter "${name}" must provide a rerank function`, {
|
|
2436
|
+
adapter: name
|
|
2437
|
+
});
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
function registerEngineRerankAdapter(core, name, adapter) {
|
|
2441
|
+
core.guardShutdown();
|
|
2442
|
+
validateRerankAdapter(name, adapter);
|
|
2443
|
+
core.rerankAdapters.set(name, adapter);
|
|
2444
|
+
}
|
|
2445
|
+
|
|
2290
2446
|
// src/geo/constants.ts
|
|
2291
2447
|
var DEG_TO_RAD = Math.PI / 180;
|
|
2292
2448
|
var MAX_LATITUDE_DEGREES = 90;
|
|
@@ -2776,6 +2932,10 @@ function validateVectorStorage(config, filesystemDurability) {
|
|
|
2776
2932
|
});
|
|
2777
2933
|
}
|
|
2778
2934
|
}
|
|
2935
|
+
function validateTextStorage(textStorage, filesystemDurability) {
|
|
2936
|
+
if (textStorage === void 0 || textStorage === "memory" || textStorage === "disk" && filesystemDurability) return;
|
|
2937
|
+
fail("textStorage must be 'memory', or 'disk' with filesystem durability", { textStorage });
|
|
2938
|
+
}
|
|
2779
2939
|
function raiseEarlierReleaseGraphSetting(config) {
|
|
2780
2940
|
if (typeof config !== "object" || config === null) return config;
|
|
2781
2941
|
const hnsw = config.hnswConfig;
|
|
@@ -2812,91 +2972,213 @@ function validateVectorPromotion(config) {
|
|
|
2812
2972
|
}
|
|
2813
2973
|
}
|
|
2814
2974
|
|
|
2815
|
-
// src/
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
2825
|
-
function getNestedValue(obj, path) {
|
|
2826
|
-
const parts = path.split(".");
|
|
2827
|
-
let current = obj;
|
|
2828
|
-
for (let i = 0; i < parts.length; i++) {
|
|
2829
|
-
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
2830
|
-
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
2831
|
-
current = current[parts[i]];
|
|
2832
|
-
}
|
|
2833
|
-
return current;
|
|
2834
|
-
}
|
|
2835
|
-
function stringArraysEqual(a, b) {
|
|
2836
|
-
if (a === b) return true;
|
|
2837
|
-
if (!a || !b) return false;
|
|
2838
|
-
if (a.length !== b.length) return false;
|
|
2839
|
-
for (let i = 0; i < a.length; i++) {
|
|
2840
|
-
if (a[i] !== b[i]) return false;
|
|
2975
|
+
// src/schema/embedding-validator.ts
|
|
2976
|
+
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
2977
|
+
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
2978
|
+
const fieldEntries = Object.keys(embeddingConfig.fields);
|
|
2979
|
+
if (fieldEntries.length === 0) {
|
|
2980
|
+
throw new NarsilError(
|
|
2981
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
2982
|
+
"Embedding config must define at least one field mapping"
|
|
2983
|
+
);
|
|
2841
2984
|
}
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
const newVal = getNestedValue(newDoc, path);
|
|
2849
|
-
if (base === "string") {
|
|
2850
|
-
if (oldVal !== newVal) return true;
|
|
2851
|
-
} else {
|
|
2852
|
-
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
2853
|
-
}
|
|
2985
|
+
if (typeof embeddingConfig.adapter === "string") {
|
|
2986
|
+
throw new NarsilError(
|
|
2987
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
2988
|
+
`Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
|
|
2989
|
+
{ adapter: embeddingConfig.adapter }
|
|
2990
|
+
);
|
|
2854
2991
|
}
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
function parseVectorDimension(fieldType) {
|
|
2858
|
-
const match = VECTOR_PATTERN2.exec(fieldType);
|
|
2859
|
-
return match ? Number.parseInt(match[1], 10) : null;
|
|
2860
|
-
}
|
|
2861
|
-
function tokenizeOptions(options) {
|
|
2862
|
-
return {
|
|
2863
|
-
stem: true,
|
|
2864
|
-
removeStopWords: true,
|
|
2865
|
-
collectSurfaces: options?.collectSurfaces === true,
|
|
2866
|
-
stopWordOverride: options?.stopWordOverride,
|
|
2867
|
-
customTokenizer: options?.customTokenizer
|
|
2868
|
-
};
|
|
2869
|
-
}
|
|
2870
|
-
function getFlatSchema(state, schema) {
|
|
2871
|
-
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
2872
|
-
state.flatSchemaCache = flattenSchema(schema);
|
|
2873
|
-
state.lastSchemaRef = schema;
|
|
2874
|
-
return state.flatSchemaCache;
|
|
2875
|
-
}
|
|
2876
|
-
|
|
2877
|
-
// src/core/projection.ts
|
|
2878
|
-
function requireFieldPathList(paths, option) {
|
|
2879
|
-
if (paths === void 0) return;
|
|
2880
|
-
if (!Array.isArray(paths)) {
|
|
2992
|
+
const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
|
|
2993
|
+
if (!resolvedAdapter) {
|
|
2881
2994
|
throw new NarsilError(
|
|
2882
|
-
ErrorCodes.
|
|
2883
|
-
|
|
2884
|
-
{ option: `document.${option}`, received: paths === null ? "null" : typeof paths }
|
|
2995
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
2996
|
+
"Embedding config requires an adapter (set on the index or the instance)"
|
|
2885
2997
|
);
|
|
2886
2998
|
}
|
|
2887
|
-
|
|
2888
|
-
const entry = paths[position];
|
|
2889
|
-
if (typeof entry === "string") continue;
|
|
2999
|
+
if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
|
|
2890
3000
|
throw new NarsilError(
|
|
2891
|
-
ErrorCodes.
|
|
2892
|
-
`
|
|
2893
|
-
{ option: `document.${option}`, position, received: entry === null ? "null" : typeof entry }
|
|
3001
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3002
|
+
`Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
|
|
2894
3003
|
);
|
|
2895
3004
|
}
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
3005
|
+
const flatFields = flattenSchema(schema);
|
|
3006
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
3007
|
+
const targetType = flatFields[targetField];
|
|
3008
|
+
if (targetType === void 0) {
|
|
3009
|
+
throw new NarsilError(
|
|
3010
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3011
|
+
`Embedding target field "${targetField}" does not exist in the schema`,
|
|
3012
|
+
{ field: targetField }
|
|
3013
|
+
);
|
|
3014
|
+
}
|
|
3015
|
+
const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
|
|
3016
|
+
if (!vectorMatch) {
|
|
3017
|
+
throw new NarsilError(
|
|
3018
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3019
|
+
`Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
|
|
3020
|
+
{ field: targetField, type: targetType }
|
|
3021
|
+
);
|
|
3022
|
+
}
|
|
3023
|
+
const schemaDimension = Number.parseInt(vectorMatch[1], 10);
|
|
3024
|
+
if (schemaDimension !== resolvedAdapter.dimensions) {
|
|
3025
|
+
throw new NarsilError(
|
|
3026
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
3027
|
+
`Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
|
|
3028
|
+
{ field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
|
|
3029
|
+
);
|
|
3030
|
+
}
|
|
3031
|
+
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
3032
|
+
if (sources.length === 0) {
|
|
3033
|
+
throw new NarsilError(
|
|
3034
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3035
|
+
`Embedding target field "${targetField}" must have at least one source field`,
|
|
3036
|
+
{ field: targetField }
|
|
3037
|
+
);
|
|
3038
|
+
}
|
|
3039
|
+
for (const source of sources) {
|
|
3040
|
+
const sourceType = flatFields[source];
|
|
3041
|
+
if (sourceType === void 0) {
|
|
3042
|
+
throw new NarsilError(
|
|
3043
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3044
|
+
`Embedding source field "${source}" does not exist in the schema`,
|
|
3045
|
+
{ field: source, targetField }
|
|
3046
|
+
);
|
|
3047
|
+
}
|
|
3048
|
+
if (!isSingleStringFieldType(sourceType)) {
|
|
3049
|
+
throw new NarsilError(
|
|
3050
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3051
|
+
`Embedding source field "${source}" must be a string type, got "${sourceType}"`,
|
|
3052
|
+
{ field: source, type: sourceType, targetField }
|
|
3053
|
+
);
|
|
3054
|
+
}
|
|
3055
|
+
}
|
|
3056
|
+
}
|
|
3057
|
+
return resolvedAdapter;
|
|
3058
|
+
}
|
|
3059
|
+
function validateRegisteredAdapter(name, adapter) {
|
|
3060
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
3061
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
|
|
3062
|
+
}
|
|
3063
|
+
if (typeof adapter?.embed !== "function") {
|
|
3064
|
+
throw new NarsilError(
|
|
3065
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3066
|
+
`Embedding adapter "${name}" must provide an embed function`,
|
|
3067
|
+
{ adapter: name }
|
|
3068
|
+
);
|
|
3069
|
+
}
|
|
3070
|
+
if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
|
|
3071
|
+
throw new NarsilError(
|
|
3072
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3073
|
+
`Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
|
|
3074
|
+
{ adapter: name, dimensions: adapter.dimensions }
|
|
3075
|
+
);
|
|
3076
|
+
}
|
|
3077
|
+
}
|
|
3078
|
+
function validateRequiredFieldsInSchema(required, schema) {
|
|
3079
|
+
if (required.length === 0) return;
|
|
3080
|
+
const flatFields = flattenSchema(schema);
|
|
3081
|
+
for (const field of required) {
|
|
3082
|
+
if (flatFields[field] === void 0) {
|
|
3083
|
+
throw new NarsilError(
|
|
3084
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3085
|
+
`Required field "${field}" does not exist in the schema`,
|
|
3086
|
+
{ field }
|
|
3087
|
+
);
|
|
3088
|
+
}
|
|
3089
|
+
}
|
|
3090
|
+
}
|
|
3091
|
+
|
|
3092
|
+
// src/runtime/heap-statistics.browser.ts
|
|
3093
|
+
function readHeapStatistics() {
|
|
3094
|
+
return null;
|
|
3095
|
+
}
|
|
3096
|
+
|
|
3097
|
+
// src/core/partition/utils.ts
|
|
3098
|
+
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
3099
|
+
const existing = table.indexMap.get(fieldName);
|
|
3100
|
+
if (existing !== void 0) return existing;
|
|
3101
|
+
const idx = table.names.length;
|
|
3102
|
+
table.names.push(fieldName);
|
|
3103
|
+
table.indexMap.set(fieldName, idx);
|
|
3104
|
+
return idx;
|
|
3105
|
+
}
|
|
3106
|
+
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
3107
|
+
function getNestedValue(obj, path) {
|
|
3108
|
+
const parts = path.split(".");
|
|
3109
|
+
let current = obj;
|
|
3110
|
+
for (let i = 0; i < parts.length; i++) {
|
|
3111
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
3112
|
+
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
3113
|
+
current = current[parts[i]];
|
|
3114
|
+
}
|
|
3115
|
+
return current;
|
|
3116
|
+
}
|
|
3117
|
+
function stringArraysEqual(a, b) {
|
|
3118
|
+
if (a === b) return true;
|
|
3119
|
+
if (!a || !b) return false;
|
|
3120
|
+
if (a.length !== b.length) return false;
|
|
3121
|
+
for (let i = 0; i < a.length; i++) {
|
|
3122
|
+
if (a[i] !== b[i]) return false;
|
|
3123
|
+
}
|
|
3124
|
+
return true;
|
|
3125
|
+
}
|
|
3126
|
+
function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
3127
|
+
for (const { path, base } of fields) {
|
|
3128
|
+
if (!isWordIndexedBase(base)) continue;
|
|
3129
|
+
const oldVal = getNestedValue(oldDoc, path);
|
|
3130
|
+
const newVal = getNestedValue(newDoc, path);
|
|
3131
|
+
if (base === "string") {
|
|
3132
|
+
if (oldVal !== newVal) return true;
|
|
3133
|
+
} else {
|
|
3134
|
+
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
3135
|
+
}
|
|
3136
|
+
}
|
|
3137
|
+
return false;
|
|
3138
|
+
}
|
|
3139
|
+
function parseVectorDimension(fieldType) {
|
|
3140
|
+
const match = VECTOR_PATTERN2.exec(fieldType);
|
|
3141
|
+
return match ? Number.parseInt(match[1], 10) : null;
|
|
3142
|
+
}
|
|
3143
|
+
function tokenizeOptions(options) {
|
|
3144
|
+
return {
|
|
3145
|
+
stem: true,
|
|
3146
|
+
removeStopWords: true,
|
|
3147
|
+
collectSurfaces: options?.collectSurfaces === true,
|
|
3148
|
+
stopWordOverride: options?.stopWordOverride,
|
|
3149
|
+
customTokenizer: options?.customTokenizer
|
|
3150
|
+
};
|
|
3151
|
+
}
|
|
3152
|
+
function getFlatSchema(state, schema) {
|
|
3153
|
+
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
3154
|
+
state.flatSchemaCache = flattenSchema(schema);
|
|
3155
|
+
state.lastSchemaRef = schema;
|
|
3156
|
+
return state.flatSchemaCache;
|
|
3157
|
+
}
|
|
3158
|
+
|
|
3159
|
+
// src/core/projection.ts
|
|
3160
|
+
function requireFieldPathList(paths, option) {
|
|
3161
|
+
if (paths === void 0) return;
|
|
3162
|
+
if (!Array.isArray(paths)) {
|
|
3163
|
+
throw new NarsilError(
|
|
3164
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
3165
|
+
`The "document.${option}" option takes an array of field paths`,
|
|
3166
|
+
{ option: `document.${option}`, received: paths === null ? "null" : typeof paths }
|
|
3167
|
+
);
|
|
3168
|
+
}
|
|
3169
|
+
for (let position = 0; position < paths.length; position++) {
|
|
3170
|
+
const entry = paths[position];
|
|
3171
|
+
if (typeof entry === "string") continue;
|
|
3172
|
+
throw new NarsilError(
|
|
3173
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
3174
|
+
`The "document.${option}" option takes field paths as strings, and entry ${position} is not a string`,
|
|
3175
|
+
{ option: `document.${option}`, position, received: entry === null ? "null" : typeof entry }
|
|
3176
|
+
);
|
|
3177
|
+
}
|
|
3178
|
+
}
|
|
3179
|
+
function splitPaths(paths) {
|
|
3180
|
+
if (!paths) return [];
|
|
3181
|
+
const split = [];
|
|
2900
3182
|
for (const path of paths) {
|
|
2901
3183
|
const segments = path.split(".").filter((segment) => segment.length > 0);
|
|
2902
3184
|
if (segments.length === 0) continue;
|
|
@@ -8128,7 +8410,7 @@ function fieldLengthBound(columns, fieldIndex, internalId) {
|
|
|
8128
8410
|
function entriesCovered(bounds) {
|
|
8129
8411
|
return bounds.blockCount === 0 ? 0 : bounds.entryEnd[bounds.blockCount - 1];
|
|
8130
8412
|
}
|
|
8131
|
-
function
|
|
8413
|
+
function scanPostingBlocks(list, columns, previous, keepBlocks) {
|
|
8132
8414
|
const { docIds, termFrequencies, fieldNameIndices, length } = list;
|
|
8133
8415
|
const startEntry = previous !== null && keepBlocks > 0 ? previous.entryEnd[keepBlocks - 1] : 0;
|
|
8134
8416
|
const capacity = keepBlocks + Math.ceil((length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
|
|
@@ -8202,12 +8484,13 @@ function blockBoundsFor(list, columns) {
|
|
|
8202
8484
|
const covered = entriesCovered(existing);
|
|
8203
8485
|
if (covered === list.length) return existing;
|
|
8204
8486
|
if (list.ordered && covered < list.length) {
|
|
8205
|
-
const extended =
|
|
8487
|
+
const extended = scanPostingBlocks(list, columns, existing, Math.max(0, existing.blockCount - 1));
|
|
8206
8488
|
boundsByList.set(list, extended);
|
|
8207
8489
|
return extended;
|
|
8208
8490
|
}
|
|
8209
8491
|
}
|
|
8210
|
-
const
|
|
8492
|
+
const stored = list.storedBlockBounds?.() ?? null;
|
|
8493
|
+
const built = stored === null ? scanPostingBlocks(list, columns, null, 0) : { ...stored, structureRevision: list.structureRevision };
|
|
8211
8494
|
boundsByList.set(list, built);
|
|
8212
8495
|
return built;
|
|
8213
8496
|
}
|
|
@@ -8400,33 +8683,33 @@ function boostsKeepScoresNonNegative(fields) {
|
|
|
8400
8683
|
}
|
|
8401
8684
|
return true;
|
|
8402
8685
|
}
|
|
8403
|
-
function samePairUpperBound(
|
|
8404
|
-
if (
|
|
8405
|
-
const bounds = blockBoundsFor(list,
|
|
8686
|
+
function samePairUpperBound(scan, list, maxBoost, maxAverageLength) {
|
|
8687
|
+
if (scan.scoresAreZero) return 0;
|
|
8688
|
+
const bounds = blockBoundsFor(list, scan.fields.lengthColumns);
|
|
8406
8689
|
let best = 0;
|
|
8407
8690
|
for (let block = 0; block < bounds.blockCount; block++) {
|
|
8408
8691
|
const bestEntry = computeBM25WithIDF(
|
|
8409
8692
|
bounds.maxTermFrequency[block],
|
|
8410
|
-
|
|
8693
|
+
scan.idf,
|
|
8411
8694
|
bounds.minFieldLength[block],
|
|
8412
8695
|
maxAverageLength,
|
|
8413
|
-
|
|
8414
|
-
|
|
8696
|
+
scan.k1,
|
|
8697
|
+
scan.b
|
|
8415
8698
|
);
|
|
8416
8699
|
const blockBound = bestEntry * maxBoost * bounds.maxEntriesPerDocument[block];
|
|
8417
8700
|
if (blockBound > best) best = blockBound;
|
|
8418
8701
|
}
|
|
8419
8702
|
return best;
|
|
8420
8703
|
}
|
|
8421
|
-
function fieldPairUpperBound(
|
|
8422
|
-
if (
|
|
8423
|
-
const bounds = blockBoundsFor(list,
|
|
8704
|
+
function fieldPairUpperBound(scan, list, maxBoost, maxAverageLength) {
|
|
8705
|
+
if (scan.scoresAreZero) return 0;
|
|
8706
|
+
const bounds = blockBoundsFor(list, scan.fields.lengthColumns);
|
|
8424
8707
|
let best = 0;
|
|
8425
8708
|
for (let block = 0; block < bounds.blockCount; block++) {
|
|
8426
8709
|
const bestEntry = blockEntryBound(
|
|
8427
|
-
|
|
8710
|
+
scan.fields,
|
|
8428
8711
|
bounds.maxTermFrequency[block],
|
|
8429
|
-
|
|
8712
|
+
scan.idf,
|
|
8430
8713
|
bounds.minFieldLength[block],
|
|
8431
8714
|
maxAverageLength
|
|
8432
8715
|
);
|
|
@@ -8435,22 +8718,22 @@ function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
|
8435
8718
|
}
|
|
8436
8719
|
return best;
|
|
8437
8720
|
}
|
|
8438
|
-
function samePairEntryScore(
|
|
8439
|
-
const { fields, columns } =
|
|
8721
|
+
function samePairEntryScore(scan, entry, internalId) {
|
|
8722
|
+
const { fields, columns } = scan;
|
|
8440
8723
|
const fieldIndex = columns.fieldNameIndices[entry];
|
|
8441
8724
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
8442
8725
|
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
8443
|
-
const bm25 =
|
|
8726
|
+
const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan.idf, fieldLength, averageLength, scan.k1, scan.b);
|
|
8444
8727
|
return bm25 * fields.boosts[fieldIndex];
|
|
8445
8728
|
}
|
|
8446
|
-
function fieldPairEntryScore(
|
|
8447
|
-
const { fields, columns } =
|
|
8729
|
+
function fieldPairEntryScore(scan, entry, internalId) {
|
|
8730
|
+
const { fields, columns } = scan;
|
|
8448
8731
|
const fieldIndex = columns.fieldNameIndices[entry];
|
|
8449
8732
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
8450
8733
|
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
8451
|
-
const bm25 =
|
|
8734
|
+
const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8452
8735
|
columns.termFrequencies[entry],
|
|
8453
|
-
|
|
8736
|
+
scan.idf,
|
|
8454
8737
|
fieldLength,
|
|
8455
8738
|
averageLength,
|
|
8456
8739
|
fields.k1s[fieldIndex],
|
|
@@ -8458,48 +8741,48 @@ function fieldPairEntryScore(scan2, entry, internalId) {
|
|
|
8458
8741
|
);
|
|
8459
8742
|
return bm25 * fields.boosts[fieldIndex];
|
|
8460
8743
|
}
|
|
8461
|
-
function samePairScoreEveryDocument(
|
|
8462
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
8463
|
-
const searchable =
|
|
8744
|
+
function samePairScoreEveryDocument(scan, buffer) {
|
|
8745
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8746
|
+
const searchable = scan.fields.searchable;
|
|
8464
8747
|
for (let entry = 0; entry < count; entry++) {
|
|
8465
8748
|
const internalId = docIds[entry];
|
|
8466
8749
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8467
8750
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8468
|
-
addScore(buffer, internalId, samePairEntryScore(
|
|
8751
|
+
addScore(buffer, internalId, samePairEntryScore(scan, entry, internalId));
|
|
8469
8752
|
}
|
|
8470
8753
|
}
|
|
8471
|
-
function fieldPairScoreEveryDocument(
|
|
8472
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
8473
|
-
const searchable =
|
|
8754
|
+
function fieldPairScoreEveryDocument(scan, buffer) {
|
|
8755
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8756
|
+
const searchable = scan.fields.searchable;
|
|
8474
8757
|
for (let entry = 0; entry < count; entry++) {
|
|
8475
8758
|
const internalId = docIds[entry];
|
|
8476
8759
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8477
8760
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8478
|
-
addScore(buffer, internalId, fieldPairEntryScore(
|
|
8761
|
+
addScore(buffer, internalId, fieldPairEntryScore(scan, entry, internalId));
|
|
8479
8762
|
}
|
|
8480
8763
|
}
|
|
8481
|
-
function samePairScoreKnownDocuments(
|
|
8482
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
8483
|
-
const searchable =
|
|
8764
|
+
function samePairScoreKnownDocuments(scan, buffer) {
|
|
8765
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8766
|
+
const searchable = scan.fields.searchable;
|
|
8484
8767
|
let counted = 0;
|
|
8485
8768
|
for (let entry = 0; entry < count; entry++) {
|
|
8486
8769
|
const internalId = docIds[entry];
|
|
8487
8770
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8488
8771
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8489
|
-
if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(
|
|
8772
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan, entry, internalId));
|
|
8490
8773
|
else if (markCounted(buffer, internalId)) counted++;
|
|
8491
8774
|
}
|
|
8492
8775
|
return counted;
|
|
8493
8776
|
}
|
|
8494
|
-
function fieldPairScoreKnownDocuments(
|
|
8495
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
8496
|
-
const searchable =
|
|
8777
|
+
function fieldPairScoreKnownDocuments(scan, buffer) {
|
|
8778
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8779
|
+
const searchable = scan.fields.searchable;
|
|
8497
8780
|
let counted = 0;
|
|
8498
8781
|
for (let entry = 0; entry < count; entry++) {
|
|
8499
8782
|
const internalId = docIds[entry];
|
|
8500
8783
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8501
8784
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8502
|
-
if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(
|
|
8785
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan, entry, internalId));
|
|
8503
8786
|
else if (markCounted(buffer, internalId)) counted++;
|
|
8504
8787
|
}
|
|
8505
8788
|
return counted;
|
|
@@ -8516,24 +8799,24 @@ function firstEntryOf(docIds, count, internalId) {
|
|
|
8516
8799
|
}
|
|
8517
8800
|
function samePairQueryOrderScore(scans, internalId) {
|
|
8518
8801
|
let score = 0;
|
|
8519
|
-
for (const
|
|
8520
|
-
const { docIds, deletedDocs, hasDeleted, count } =
|
|
8802
|
+
for (const scan of scans) {
|
|
8803
|
+
const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8521
8804
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8522
8805
|
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
8523
|
-
if (
|
|
8524
|
-
score += samePairEntryScore(
|
|
8806
|
+
if (scan.fields.searchable[scan.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
8807
|
+
score += samePairEntryScore(scan, entry, internalId);
|
|
8525
8808
|
}
|
|
8526
8809
|
}
|
|
8527
8810
|
return score;
|
|
8528
8811
|
}
|
|
8529
8812
|
function fieldPairQueryOrderScore(scans, internalId) {
|
|
8530
8813
|
let score = 0;
|
|
8531
|
-
for (const
|
|
8532
|
-
const { docIds, deletedDocs, hasDeleted, count } =
|
|
8814
|
+
for (const scan of scans) {
|
|
8815
|
+
const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8533
8816
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8534
8817
|
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
8535
|
-
if (
|
|
8536
|
-
score += fieldPairEntryScore(
|
|
8818
|
+
if (scan.fields.searchable[scan.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
8819
|
+
score += fieldPairEntryScore(scan, entry, internalId);
|
|
8537
8820
|
}
|
|
8538
8821
|
}
|
|
8539
8822
|
return score;
|
|
@@ -8613,7 +8896,7 @@ function multiTermTopK(request) {
|
|
|
8613
8896
|
}
|
|
8614
8897
|
const maxBoost = bestSearchable(fields.searchable, fields.boosts);
|
|
8615
8898
|
const maxAverageLength = bestSearchable(fields.searchable, fields.averageLengths);
|
|
8616
|
-
const upperBounds = scans.map((
|
|
8899
|
+
const upperBounds = scans.map((scan, term) => scorer.upperBound(scan, terms[term].list, maxBoost, maxAverageLength));
|
|
8617
8900
|
const order = scans.map((_, term) => term).sort((left, right) => upperBounds[right] - upperBounds[left]);
|
|
8618
8901
|
const remainingBound = new Float64Array(order.length + 1);
|
|
8619
8902
|
for (let position2 = order.length - 1; position2 >= 0; position2--) {
|
|
@@ -9764,570 +10047,104 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9764
10047
|
const next = indexDocument(state, docId, document, fields, language, options);
|
|
9765
10048
|
state.docStore.store(docId, document, next.fieldLengths);
|
|
9766
10049
|
state.stats.addDocument(next.fieldLengths, next.tokensByField);
|
|
9767
|
-
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, next.partialSpellings, 1);
|
|
9768
|
-
recordSortValues(state, internalId, document);
|
|
9769
|
-
} else {
|
|
9770
|
-
updateFieldIndexOnly(state, docId, stored.fields, document, fields);
|
|
9771
|
-
state.docStore.store(docId, document, stored.fieldLengths);
|
|
9772
|
-
const internalId = state.docStore.getInternalId(docId);
|
|
9773
|
-
if (internalId !== void 0) recordSortValues(state, internalId, document);
|
|
9774
|
-
}
|
|
9775
|
-
},
|
|
9776
|
-
get(docId, projection) {
|
|
9777
|
-
const stored = state.docStore.get(docId);
|
|
9778
|
-
if (!stored) return void 0;
|
|
9779
|
-
return cloneProjected(stored.fields, projection);
|
|
9780
|
-
},
|
|
9781
|
-
getRef(docId) {
|
|
9782
|
-
const stored = state.docStore.get(docId);
|
|
9783
|
-
if (!stored) return void 0;
|
|
9784
|
-
return stored.fields;
|
|
9785
|
-
},
|
|
9786
|
-
fieldValues(docId, fieldPaths) {
|
|
9787
|
-
const stored = state.docStore.get(docId);
|
|
9788
|
-
if (!stored) return void 0;
|
|
9789
|
-
return fieldPaths.map((fieldPath) => getNestedValue(stored.fields, fieldPath));
|
|
9790
|
-
},
|
|
9791
|
-
has(docId) {
|
|
9792
|
-
return state.docStore.has(docId);
|
|
9793
|
-
},
|
|
9794
|
-
count() {
|
|
9795
|
-
return state.docStore.count();
|
|
9796
|
-
},
|
|
9797
|
-
*docIds() {
|
|
9798
|
-
for (const [id] of state.docStore.all()) {
|
|
9799
|
-
yield id;
|
|
9800
|
-
}
|
|
9801
|
-
},
|
|
9802
|
-
sortedDocIds() {
|
|
9803
|
-
return state.docStore.sortedDocIds();
|
|
9804
|
-
},
|
|
9805
|
-
releaseSortedDocIds() {
|
|
9806
|
-
state.docStore.releaseSortedDocIds();
|
|
9807
|
-
},
|
|
9808
|
-
clear: clearAll,
|
|
9809
|
-
estimateMemoryBytes() {
|
|
9810
|
-
return estimatePartitionBytes(state);
|
|
9811
|
-
},
|
|
9812
|
-
readStates() {
|
|
9813
|
-
return [state];
|
|
9814
|
-
},
|
|
9815
|
-
searchFulltext(params) {
|
|
9816
|
-
return searchFulltext(state, params);
|
|
9817
|
-
},
|
|
9818
|
-
searchFulltextMatches(params) {
|
|
9819
|
-
return searchFulltextMatches(state, params);
|
|
9820
|
-
},
|
|
9821
|
-
sortedPage(request) {
|
|
9822
|
-
return sortedPageOf(state, request);
|
|
9823
|
-
},
|
|
9824
|
-
sortValues(docId, fields, fieldTypes, modes) {
|
|
9825
|
-
return sortValuesOf(state, docId, fields, fieldTypes, modes);
|
|
9826
|
-
},
|
|
9827
|
-
applyFilters(filters, schema, meter) {
|
|
9828
|
-
return applyPartitionFilters(state, filters, schema, meter);
|
|
9829
|
-
},
|
|
9830
|
-
applyFiltersBitset(filters, schema, meter) {
|
|
9831
|
-
return applyPartitionFiltersBitset(state, filters, schema, meter);
|
|
9832
|
-
},
|
|
9833
|
-
filterMatches(filters, schema, meter) {
|
|
9834
|
-
return partitionFilterMatches(state, filters, schema, meter);
|
|
9835
|
-
},
|
|
9836
|
-
computeFacets(matched, config, schema) {
|
|
9837
|
-
return computeFacets(state, matched, config, schema);
|
|
9838
|
-
},
|
|
9839
|
-
suggestTerms(surfacePrefix, stemmedPrefix, limit) {
|
|
9840
|
-
return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
|
|
9841
|
-
},
|
|
9842
|
-
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
9843
|
-
return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
|
|
9844
|
-
},
|
|
9845
|
-
expandPartialWords(analysis, requests) {
|
|
9846
|
-
return expandPartialWords([state], analysis, requests);
|
|
9847
|
-
},
|
|
9848
|
-
preparePartialWords(analysis) {
|
|
9849
|
-
expandPartialWords([state], analysis, []);
|
|
9850
|
-
},
|
|
9851
|
-
serialize(indexName, totalPartitions, language, schema) {
|
|
9852
|
-
return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
|
|
9853
|
-
},
|
|
9854
|
-
deserialize(data, schema) {
|
|
9855
|
-
deserializePartition(state, data, clearAll, schema);
|
|
9856
|
-
}
|
|
9857
|
-
};
|
|
9858
|
-
statesByPartition.set(partition, state);
|
|
9859
|
-
return partition;
|
|
9860
|
-
}
|
|
9861
|
-
|
|
9862
|
-
// src/engine/orchestration/turn.ts
|
|
9863
|
-
function afterCurrentTurn(callback) {
|
|
9864
|
-
if (typeof setImmediate === "function") {
|
|
9865
|
-
setImmediate(callback);
|
|
9866
|
-
return;
|
|
9867
|
-
}
|
|
9868
|
-
setTimeout(callback, 0);
|
|
9869
|
-
}
|
|
9870
|
-
function yieldToEventLoop() {
|
|
9871
|
-
return new Promise((resolve) => afterCurrentTurn(resolve));
|
|
9872
|
-
}
|
|
9873
|
-
|
|
9874
|
-
// src/partitioning/constants.ts
|
|
9875
|
-
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
9876
|
-
var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
|
|
9877
|
-
var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
|
|
9878
|
-
var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
9879
|
-
|
|
9880
|
-
// src/partitioning/rebalancer.ts
|
|
9881
|
-
function createRebalancer() {
|
|
9882
|
-
const activeManagers = /* @__PURE__ */ new WeakSet();
|
|
9883
|
-
let activeCount = 0;
|
|
9884
|
-
async function rebalance(manager, newPartitionCount, router, onProgress) {
|
|
9885
|
-
if (activeManagers.has(manager)) {
|
|
9886
|
-
throw new NarsilError(
|
|
9887
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
9888
|
-
"A rebalance operation is already in progress"
|
|
9889
|
-
);
|
|
9890
|
-
}
|
|
9891
|
-
if (newPartitionCount <= 0) {
|
|
9892
|
-
throw new NarsilError(
|
|
9893
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
9894
|
-
`New partition count must be greater than 0, received ${newPartitionCount}`,
|
|
9895
|
-
{ newPartitionCount }
|
|
9896
|
-
);
|
|
9897
|
-
}
|
|
9898
|
-
if (newPartitionCount === manager.partitionCount) {
|
|
9899
|
-
throw new NarsilError(
|
|
9900
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
9901
|
-
`New partition count (${newPartitionCount}) is the same as the current count`,
|
|
9902
|
-
{ newPartitionCount, currentPartitionCount: manager.partitionCount }
|
|
9903
|
-
);
|
|
9904
|
-
}
|
|
9905
|
-
activeManagers.add(manager);
|
|
9906
|
-
activeCount++;
|
|
9907
|
-
try {
|
|
9908
|
-
const currentPartitions = manager.getAllPartitions();
|
|
9909
|
-
let documentsTotal = 0;
|
|
9910
|
-
for (const partition of currentPartitions) {
|
|
9911
|
-
documentsTotal += partition.count();
|
|
9912
|
-
}
|
|
9913
|
-
onProgress?.({
|
|
9914
|
-
phase: "scanning",
|
|
9915
|
-
documentsProcessed: 0,
|
|
9916
|
-
documentsTotal
|
|
9917
|
-
});
|
|
9918
|
-
const newPartitions = [];
|
|
9919
|
-
for (let i = 0; i < newPartitionCount; i++) {
|
|
9920
|
-
newPartitions.push(createPartitionIndex(i, manager.config.trackPositions ?? true));
|
|
9921
|
-
}
|
|
9922
|
-
let documentsProcessed = 0;
|
|
9923
|
-
let chunkFill = 0;
|
|
9924
|
-
const insertOptions = {
|
|
9925
|
-
validate: false,
|
|
9926
|
-
skipClone: true,
|
|
9927
|
-
stopWordOverride: manager.analysis.stopWords,
|
|
9928
|
-
customTokenizer: manager.analysis.customTokenizer,
|
|
9929
|
-
collectSurfaces: manager.config.surfaceForms !== false
|
|
9930
|
-
};
|
|
9931
|
-
for (const partition of currentPartitions) {
|
|
9932
|
-
const docIds = [...partition.docIds()];
|
|
9933
|
-
for (const docId of docIds) {
|
|
9934
|
-
const document = partition.getRef(docId);
|
|
9935
|
-
if (!document) continue;
|
|
9936
|
-
const targetPartitionId = router.route(docId, newPartitionCount);
|
|
9937
|
-
const targetPartition = newPartitions[targetPartitionId];
|
|
9938
|
-
if (!targetPartition.has(docId)) {
|
|
9939
|
-
targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
|
|
9940
|
-
documentsProcessed++;
|
|
9941
|
-
}
|
|
9942
|
-
chunkFill++;
|
|
9943
|
-
if (chunkFill >= REBALANCE_CHUNK_SIZE) {
|
|
9944
|
-
chunkFill = 0;
|
|
9945
|
-
onProgress?.({
|
|
9946
|
-
phase: "moving",
|
|
9947
|
-
documentsProcessed,
|
|
9948
|
-
documentsTotal
|
|
9949
|
-
});
|
|
9950
|
-
await yieldToEventLoop();
|
|
9951
|
-
}
|
|
9952
|
-
}
|
|
9953
|
-
}
|
|
9954
|
-
if (chunkFill > 0) {
|
|
9955
|
-
onProgress?.({
|
|
9956
|
-
phase: "moving",
|
|
9957
|
-
documentsProcessed,
|
|
9958
|
-
documentsTotal
|
|
9959
|
-
});
|
|
9960
|
-
await yieldToEventLoop();
|
|
9961
|
-
}
|
|
9962
|
-
onProgress?.({
|
|
9963
|
-
phase: "swapping",
|
|
9964
|
-
documentsProcessed,
|
|
9965
|
-
documentsTotal
|
|
9966
|
-
});
|
|
9967
|
-
manager.setPartitions(newPartitions);
|
|
9968
|
-
onProgress?.({
|
|
9969
|
-
phase: "complete",
|
|
9970
|
-
documentsProcessed,
|
|
9971
|
-
documentsTotal
|
|
9972
|
-
});
|
|
9973
|
-
} finally {
|
|
9974
|
-
activeManagers.delete(manager);
|
|
9975
|
-
activeCount--;
|
|
9976
|
-
}
|
|
9977
|
-
}
|
|
9978
|
-
return {
|
|
9979
|
-
rebalance,
|
|
9980
|
-
isRebalancing() {
|
|
9981
|
-
return activeCount > 0;
|
|
9982
|
-
}
|
|
9983
|
-
};
|
|
9984
|
-
}
|
|
9985
|
-
|
|
9986
|
-
// src/core/hash.ts
|
|
9987
|
-
var FNV_OFFSET_BASIS = 2166136261;
|
|
9988
|
-
var FNV_PRIME = 16777619;
|
|
9989
|
-
var encoder = new TextEncoder();
|
|
9990
|
-
function fnv1aBytes(bytes) {
|
|
9991
|
-
return fnv1aContinue(FNV_OFFSET_BASIS, bytes, bytes.length) >>> 0;
|
|
9992
|
-
}
|
|
9993
|
-
function fnv1aContinue(start, bytes, length) {
|
|
9994
|
-
let hash = start;
|
|
9995
|
-
for (let i = 0; i < length; i++) {
|
|
9996
|
-
hash = Math.imul(hash ^ bytes[i], FNV_PRIME);
|
|
9997
|
-
}
|
|
9998
|
-
return hash;
|
|
9999
|
-
}
|
|
10000
|
-
var scratchNumber = new Float64Array(1);
|
|
10001
|
-
var scratchBytes = new Uint8Array(scratchNumber.buffer);
|
|
10002
|
-
var HOST_IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
10003
|
-
function fnv1aContinueTaggedNumbers(start, tag, numbers) {
|
|
10004
|
-
let hash = start;
|
|
10005
|
-
const b = scratchBytes;
|
|
10006
|
-
for (let i = 0; i < numbers.length; i++) {
|
|
10007
|
-
scratchNumber[0] = Math.fround(numbers[i]);
|
|
10008
|
-
hash = Math.imul(hash ^ tag, FNV_PRIME);
|
|
10009
|
-
if (HOST_IS_LITTLE_ENDIAN) {
|
|
10010
|
-
hash = Math.imul(hash ^ b[7], FNV_PRIME);
|
|
10011
|
-
hash = Math.imul(hash ^ b[6], FNV_PRIME);
|
|
10012
|
-
hash = Math.imul(hash ^ b[5], FNV_PRIME);
|
|
10013
|
-
hash = Math.imul(hash ^ b[4], FNV_PRIME);
|
|
10014
|
-
hash = Math.imul(hash ^ b[3], FNV_PRIME);
|
|
10015
|
-
hash = Math.imul(hash ^ b[2], FNV_PRIME);
|
|
10016
|
-
hash = Math.imul(hash ^ b[1], FNV_PRIME);
|
|
10017
|
-
hash = Math.imul(hash ^ b[0], FNV_PRIME);
|
|
10018
|
-
} else {
|
|
10019
|
-
for (let at = 0; at < 8; at++) hash = Math.imul(hash ^ b[at], FNV_PRIME);
|
|
10020
|
-
}
|
|
10021
|
-
}
|
|
10022
|
-
return hash;
|
|
10023
|
-
}
|
|
10024
|
-
function fnv1a(input) {
|
|
10025
|
-
return fnv1aBytes(encoder.encode(input));
|
|
10026
|
-
}
|
|
10027
|
-
|
|
10028
|
-
// src/partitioning/router.ts
|
|
10029
|
-
function createPartitionRouter() {
|
|
10030
|
-
function route(docId, partitionCount) {
|
|
10031
|
-
if (partitionCount <= 0) {
|
|
10032
|
-
throw new NarsilError(
|
|
10033
|
-
ErrorCodes.INDEX_NOT_FOUND,
|
|
10034
|
-
`Partition count must be greater than 0, received ${partitionCount}`,
|
|
10035
|
-
{ partitionCount }
|
|
10036
|
-
);
|
|
10037
|
-
}
|
|
10038
|
-
return fnv1a(docId) % partitionCount;
|
|
10039
|
-
}
|
|
10040
|
-
function routeBatch(docIds, partitionCount) {
|
|
10041
|
-
const groups = /* @__PURE__ */ new Map();
|
|
10042
|
-
for (let i = 0; i < docIds.length; i++) {
|
|
10043
|
-
const pid = route(docIds[i], partitionCount);
|
|
10044
|
-
let group = groups.get(pid);
|
|
10045
|
-
if (!group) {
|
|
10046
|
-
group = [];
|
|
10047
|
-
groups.set(pid, group);
|
|
10048
|
-
}
|
|
10049
|
-
group.push(docIds[i]);
|
|
10050
|
-
}
|
|
10051
|
-
return groups;
|
|
10052
|
-
}
|
|
10053
|
-
return { route, routeBatch };
|
|
10054
|
-
}
|
|
10055
|
-
|
|
10056
|
-
// src/plugins/registry.ts
|
|
10057
|
-
var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
|
|
10058
|
-
function isThenable(value) {
|
|
10059
|
-
return value !== null && typeof value === "object" && typeof value.then === "function";
|
|
10060
|
-
}
|
|
10061
|
-
function thrownText(err, plugin, hookName) {
|
|
10062
|
-
if (err instanceof Error) return err.message;
|
|
10063
|
-
try {
|
|
10064
|
-
return String(err);
|
|
10065
|
-
} catch {
|
|
10066
|
-
return `The ${hookName} hook of plugin "${plugin}" rejects the write`;
|
|
10067
|
-
}
|
|
10068
|
-
}
|
|
10069
|
-
function asWriteRejection(err, plugin, hookName) {
|
|
10070
|
-
if (err instanceof NarsilError) return err;
|
|
10071
|
-
return new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, thrownText(err, plugin, hookName), {
|
|
10072
|
-
plugin,
|
|
10073
|
-
hook: hookName
|
|
10074
|
-
});
|
|
10075
|
-
}
|
|
10076
|
-
function invoke(hook, hookName, context) {
|
|
10077
|
-
const rejectsWrites = WRITE_REJECTING_HOOKS.has(hookName);
|
|
10078
|
-
let result;
|
|
10079
|
-
try {
|
|
10080
|
-
result = hook.run(context);
|
|
10081
|
-
} catch (err) {
|
|
10082
|
-
throw rejectsWrites ? asWriteRejection(err, hook.plugin, hookName) : err;
|
|
10083
|
-
}
|
|
10084
|
-
if (!isThenable(result)) return null;
|
|
10085
|
-
if (!rejectsWrites) return result;
|
|
10086
|
-
return Promise.resolve(result).catch((err) => {
|
|
10087
|
-
throw asWriteRejection(err, hook.plugin, hookName);
|
|
10088
|
-
});
|
|
10089
|
-
}
|
|
10090
|
-
async function continueAsync(pending, hooks, startIndex, hookName, context) {
|
|
10091
|
-
await pending;
|
|
10092
|
-
for (let i = startIndex; i < hooks.length; i++) {
|
|
10093
|
-
await invoke(hooks[i], hookName, context);
|
|
10094
|
-
}
|
|
10095
|
-
}
|
|
10096
|
-
function createPluginRegistry() {
|
|
10097
|
-
const plugins = [];
|
|
10098
|
-
return {
|
|
10099
|
-
register(plugin) {
|
|
10100
|
-
plugins.push(plugin);
|
|
10101
|
-
},
|
|
10102
|
-
hasHooks(hookName) {
|
|
10103
|
-
return plugins.some((p) => typeof p[hookName] === "function");
|
|
10104
|
-
},
|
|
10105
|
-
runHook(hookName, context) {
|
|
10106
|
-
const hooks = [];
|
|
10107
|
-
for (const plugin of plugins) {
|
|
10108
|
-
const hook = plugin[hookName];
|
|
10109
|
-
if (typeof hook === "function") {
|
|
10110
|
-
hooks.push({ plugin: plugin.name, run: hook.bind(plugin) });
|
|
10111
|
-
}
|
|
10112
|
-
}
|
|
10113
|
-
if (hooks.length === 0) return;
|
|
10114
|
-
for (let i = 0; i < hooks.length; i++) {
|
|
10115
|
-
const pending = invoke(hooks[i], hookName, context);
|
|
10116
|
-
if (pending !== null) {
|
|
10117
|
-
return continueAsync(pending, hooks, i + 1, hookName, context);
|
|
10118
|
-
}
|
|
10119
|
-
}
|
|
10120
|
-
}
|
|
10121
|
-
};
|
|
10122
|
-
}
|
|
10123
|
-
|
|
10124
|
-
// src/rerank/registry.ts
|
|
10125
|
-
function validateRerankAdapter(name, adapter) {
|
|
10126
|
-
if (typeof name !== "string" || name.trim().length === 0) {
|
|
10127
|
-
throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, "A rerank adapter needs a non-empty string name");
|
|
10128
|
-
}
|
|
10129
|
-
const rerank = typeof adapter === "object" && adapter !== null ? adapter.rerank : void 0;
|
|
10130
|
-
if (typeof rerank !== "function") {
|
|
10131
|
-
throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, `Rerank adapter "${name}" must provide a rerank function`, {
|
|
10132
|
-
adapter: name
|
|
10133
|
-
});
|
|
10134
|
-
}
|
|
10135
|
-
}
|
|
10136
|
-
function registerEngineRerankAdapter(core, name, adapter) {
|
|
10137
|
-
core.guardShutdown();
|
|
10138
|
-
validateRerankAdapter(name, adapter);
|
|
10139
|
-
core.rerankAdapters.set(name, adapter);
|
|
10140
|
-
}
|
|
10141
|
-
|
|
10142
|
-
// src/schema/embedding-validator.ts
|
|
10143
|
-
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
10144
|
-
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
10145
|
-
const fieldEntries = Object.keys(embeddingConfig.fields);
|
|
10146
|
-
if (fieldEntries.length === 0) {
|
|
10147
|
-
throw new NarsilError(
|
|
10148
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10149
|
-
"Embedding config must define at least one field mapping"
|
|
10150
|
-
);
|
|
10151
|
-
}
|
|
10152
|
-
if (typeof embeddingConfig.adapter === "string") {
|
|
10153
|
-
throw new NarsilError(
|
|
10154
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10155
|
-
`Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
|
|
10156
|
-
{ adapter: embeddingConfig.adapter }
|
|
10157
|
-
);
|
|
10158
|
-
}
|
|
10159
|
-
const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
|
|
10160
|
-
if (!resolvedAdapter) {
|
|
10161
|
-
throw new NarsilError(
|
|
10162
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10163
|
-
"Embedding config requires an adapter (set on the index or the instance)"
|
|
10164
|
-
);
|
|
10165
|
-
}
|
|
10166
|
-
if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
|
|
10167
|
-
throw new NarsilError(
|
|
10168
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10169
|
-
`Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
|
|
10170
|
-
);
|
|
10171
|
-
}
|
|
10172
|
-
const flatFields = flattenSchema(schema);
|
|
10173
|
-
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
10174
|
-
const targetType = flatFields[targetField];
|
|
10175
|
-
if (targetType === void 0) {
|
|
10176
|
-
throw new NarsilError(
|
|
10177
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10178
|
-
`Embedding target field "${targetField}" does not exist in the schema`,
|
|
10179
|
-
{ field: targetField }
|
|
10180
|
-
);
|
|
10181
|
-
}
|
|
10182
|
-
const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
|
|
10183
|
-
if (!vectorMatch) {
|
|
10184
|
-
throw new NarsilError(
|
|
10185
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10186
|
-
`Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
|
|
10187
|
-
{ field: targetField, type: targetType }
|
|
10188
|
-
);
|
|
10189
|
-
}
|
|
10190
|
-
const schemaDimension = Number.parseInt(vectorMatch[1], 10);
|
|
10191
|
-
if (schemaDimension !== resolvedAdapter.dimensions) {
|
|
10192
|
-
throw new NarsilError(
|
|
10193
|
-
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
10194
|
-
`Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
|
|
10195
|
-
{ field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
|
|
10196
|
-
);
|
|
10197
|
-
}
|
|
10198
|
-
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
10199
|
-
if (sources.length === 0) {
|
|
10200
|
-
throw new NarsilError(
|
|
10201
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10202
|
-
`Embedding target field "${targetField}" must have at least one source field`,
|
|
10203
|
-
{ field: targetField }
|
|
10204
|
-
);
|
|
10205
|
-
}
|
|
10206
|
-
for (const source of sources) {
|
|
10207
|
-
const sourceType = flatFields[source];
|
|
10208
|
-
if (sourceType === void 0) {
|
|
10209
|
-
throw new NarsilError(
|
|
10210
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10211
|
-
`Embedding source field "${source}" does not exist in the schema`,
|
|
10212
|
-
{ field: source, targetField }
|
|
10213
|
-
);
|
|
10214
|
-
}
|
|
10215
|
-
if (!isSingleStringFieldType(sourceType)) {
|
|
10216
|
-
throw new NarsilError(
|
|
10217
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10218
|
-
`Embedding source field "${source}" must be a string type, got "${sourceType}"`,
|
|
10219
|
-
{ field: source, type: sourceType, targetField }
|
|
10220
|
-
);
|
|
10221
|
-
}
|
|
10222
|
-
}
|
|
10223
|
-
}
|
|
10224
|
-
return resolvedAdapter;
|
|
10225
|
-
}
|
|
10226
|
-
function validateRegisteredAdapter(name, adapter) {
|
|
10227
|
-
if (typeof name !== "string" || name.trim().length === 0) {
|
|
10228
|
-
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
|
|
10229
|
-
}
|
|
10230
|
-
if (typeof adapter?.embed !== "function") {
|
|
10231
|
-
throw new NarsilError(
|
|
10232
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10233
|
-
`Embedding adapter "${name}" must provide an embed function`,
|
|
10234
|
-
{ adapter: name }
|
|
10235
|
-
);
|
|
10236
|
-
}
|
|
10237
|
-
if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
|
|
10238
|
-
throw new NarsilError(
|
|
10239
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10240
|
-
`Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
|
|
10241
|
-
{ adapter: name, dimensions: adapter.dimensions }
|
|
10242
|
-
);
|
|
10243
|
-
}
|
|
10244
|
-
}
|
|
10245
|
-
function validateRequiredFieldsInSchema(required, schema) {
|
|
10246
|
-
if (required.length === 0) return;
|
|
10247
|
-
const flatFields = flattenSchema(schema);
|
|
10248
|
-
for (const field of required) {
|
|
10249
|
-
if (flatFields[field] === void 0) {
|
|
10250
|
-
throw new NarsilError(
|
|
10251
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10252
|
-
`Required field "${field}" does not exist in the schema`,
|
|
10253
|
-
{ field }
|
|
10254
|
-
);
|
|
10255
|
-
}
|
|
10256
|
-
}
|
|
10257
|
-
}
|
|
10258
|
-
|
|
10259
|
-
// src/runtime/heap-statistics.browser.ts
|
|
10260
|
-
function readHeapStatistics() {
|
|
10261
|
-
return null;
|
|
10262
|
-
}
|
|
10263
|
-
|
|
10264
|
-
// src/search/oversample.ts
|
|
10265
|
-
function oversampledShardSize(size) {
|
|
10266
|
-
return Math.ceil(size * 1.5) + 10;
|
|
10267
|
-
}
|
|
10268
|
-
|
|
10269
|
-
// src/search/facets.ts
|
|
10270
|
-
function keptValueCount(fieldConfig) {
|
|
10271
|
-
return keptFacetValues(fieldConfig?.limit);
|
|
10272
|
-
}
|
|
10273
|
-
function everyValueFacetConfig(config) {
|
|
10274
|
-
const widened = {};
|
|
10275
|
-
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
10276
|
-
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
10277
|
-
}
|
|
10278
|
-
return widened;
|
|
10279
|
-
}
|
|
10280
|
-
function oversampledFacetConfig(config) {
|
|
10281
|
-
const widened = {};
|
|
10282
|
-
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
10283
|
-
if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
|
|
10284
|
-
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
10285
|
-
continue;
|
|
10286
|
-
}
|
|
10287
|
-
const kept = keptValueCount(fieldConfig);
|
|
10288
|
-
widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
|
|
10289
|
-
}
|
|
10290
|
-
return widened;
|
|
10291
|
-
}
|
|
10292
|
-
function mergeFacets(partitionFacets, config) {
|
|
10293
|
-
const merged = /* @__PURE__ */ new Map();
|
|
10294
|
-
const bounds = /* @__PURE__ */ new Map();
|
|
10295
|
-
for (const partition of partitionFacets) {
|
|
10296
|
-
for (const [field, facetResult] of Object.entries(partition)) {
|
|
10297
|
-
let fieldMap = merged.get(field);
|
|
10298
|
-
if (!fieldMap) {
|
|
10299
|
-
fieldMap = /* @__PURE__ */ new Map();
|
|
10300
|
-
merged.set(field, fieldMap);
|
|
10050
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, next.partialSpellings, 1);
|
|
10051
|
+
recordSortValues(state, internalId, document);
|
|
10052
|
+
} else {
|
|
10053
|
+
updateFieldIndexOnly(state, docId, stored.fields, document, fields);
|
|
10054
|
+
state.docStore.store(docId, document, stored.fieldLengths);
|
|
10055
|
+
const internalId = state.docStore.getInternalId(docId);
|
|
10056
|
+
if (internalId !== void 0) recordSortValues(state, internalId, document);
|
|
10301
10057
|
}
|
|
10302
|
-
|
|
10303
|
-
|
|
10304
|
-
|
|
10058
|
+
},
|
|
10059
|
+
get(docId, projection) {
|
|
10060
|
+
const stored = state.docStore.get(docId);
|
|
10061
|
+
if (!stored) return void 0;
|
|
10062
|
+
return cloneProjected(stored.fields, projection);
|
|
10063
|
+
},
|
|
10064
|
+
getRef(docId) {
|
|
10065
|
+
const stored = state.docStore.get(docId);
|
|
10066
|
+
if (!stored) return void 0;
|
|
10067
|
+
return stored.fields;
|
|
10068
|
+
},
|
|
10069
|
+
fieldValues(docId, fieldPaths) {
|
|
10070
|
+
const stored = state.docStore.get(docId);
|
|
10071
|
+
if (!stored) return void 0;
|
|
10072
|
+
return fieldPaths.map((fieldPath) => getNestedValue(stored.fields, fieldPath));
|
|
10073
|
+
},
|
|
10074
|
+
has(docId) {
|
|
10075
|
+
return state.docStore.has(docId);
|
|
10076
|
+
},
|
|
10077
|
+
count() {
|
|
10078
|
+
return state.docStore.count();
|
|
10079
|
+
},
|
|
10080
|
+
*docIds() {
|
|
10081
|
+
for (const [id] of state.docStore.all()) {
|
|
10082
|
+
yield id;
|
|
10305
10083
|
}
|
|
10084
|
+
},
|
|
10085
|
+
sortedDocIds() {
|
|
10086
|
+
return state.docStore.sortedDocIds();
|
|
10087
|
+
},
|
|
10088
|
+
releaseSortedDocIds() {
|
|
10089
|
+
state.docStore.releaseSortedDocIds();
|
|
10090
|
+
},
|
|
10091
|
+
clear: clearAll,
|
|
10092
|
+
estimateMemoryBytes() {
|
|
10093
|
+
return estimatePartitionBytes(state);
|
|
10094
|
+
},
|
|
10095
|
+
readStates() {
|
|
10096
|
+
return [state];
|
|
10097
|
+
},
|
|
10098
|
+
searchFulltext(params) {
|
|
10099
|
+
return searchFulltext(state, params);
|
|
10100
|
+
},
|
|
10101
|
+
searchFulltextMatches(params) {
|
|
10102
|
+
return searchFulltextMatches(state, params);
|
|
10103
|
+
},
|
|
10104
|
+
sortedPage(request) {
|
|
10105
|
+
return sortedPageOf(state, request);
|
|
10106
|
+
},
|
|
10107
|
+
sortValues(docId, fields, fieldTypes, modes) {
|
|
10108
|
+
return sortValuesOf(state, docId, fields, fieldTypes, modes);
|
|
10109
|
+
},
|
|
10110
|
+
applyFilters(filters, schema, meter) {
|
|
10111
|
+
return applyPartitionFilters(state, filters, schema, meter);
|
|
10112
|
+
},
|
|
10113
|
+
applyFiltersBitset(filters, schema, meter) {
|
|
10114
|
+
return applyPartitionFiltersBitset(state, filters, schema, meter);
|
|
10115
|
+
},
|
|
10116
|
+
filterMatches(filters, schema, meter) {
|
|
10117
|
+
return partitionFilterMatches(state, filters, schema, meter);
|
|
10118
|
+
},
|
|
10119
|
+
computeFacets(matched, config, schema) {
|
|
10120
|
+
return computeFacets(state, matched, config, schema);
|
|
10121
|
+
},
|
|
10122
|
+
suggestTerms(surfacePrefix, stemmedPrefix, limit) {
|
|
10123
|
+
return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
|
|
10124
|
+
},
|
|
10125
|
+
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
10126
|
+
return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
|
|
10127
|
+
},
|
|
10128
|
+
expandPartialWords(analysis, requests) {
|
|
10129
|
+
return expandPartialWords([state], analysis, requests);
|
|
10130
|
+
},
|
|
10131
|
+
preparePartialWords(analysis) {
|
|
10132
|
+
expandPartialWords([state], analysis, []);
|
|
10133
|
+
},
|
|
10134
|
+
serialize(indexName, totalPartitions, language, schema) {
|
|
10135
|
+
return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
|
|
10136
|
+
},
|
|
10137
|
+
deserialize(data, schema) {
|
|
10138
|
+
deserializePartition(state, data, clearAll, schema);
|
|
10306
10139
|
}
|
|
10307
|
-
}
|
|
10308
|
-
|
|
10309
|
-
|
|
10310
|
-
const fieldConfig = config[field];
|
|
10311
|
-
const ascending = fieldConfig?.sort === "asc";
|
|
10312
|
-
const ordered = Array.from(valueMap.entries());
|
|
10313
|
-
ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
10314
|
-
const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
|
|
10315
|
-
let largestLeftOut = 0;
|
|
10316
|
-
for (let index = kept; index < ordered.length; index++) {
|
|
10317
|
-
if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
|
|
10318
|
-
}
|
|
10319
|
-
const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
|
|
10320
|
-
const values = {};
|
|
10321
|
-
for (let index = 0; index < kept; index++) {
|
|
10322
|
-
values[ordered[index][0]] = ordered[index][1];
|
|
10323
|
-
}
|
|
10324
|
-
result[field] = { values, count: kept, errorBound };
|
|
10325
|
-
}
|
|
10326
|
-
return result;
|
|
10140
|
+
};
|
|
10141
|
+
statesByPartition.set(partition, state);
|
|
10142
|
+
return partition;
|
|
10327
10143
|
}
|
|
10328
10144
|
|
|
10329
10145
|
// src/core/partition/frozen/memory.ts
|
|
10330
10146
|
function frozenSegmentBytes(segment) {
|
|
10147
|
+
if (segment.paged !== null) return segment.paged.heldBytes(segment);
|
|
10331
10148
|
const arrays = segment.arrays;
|
|
10332
10149
|
let bytes = arrays.tokenTable.bytes + arrays.idTable.bytes;
|
|
10333
10150
|
bytes += arrays.postingOffsets.byteLength;
|
|
@@ -10439,6 +10256,71 @@ function compositeSortedDocIds(frozen, live) {
|
|
|
10439
10256
|
return merged;
|
|
10440
10257
|
}
|
|
10441
10258
|
|
|
10259
|
+
// src/search/oversample.ts
|
|
10260
|
+
function oversampledShardSize(size) {
|
|
10261
|
+
return Math.ceil(size * 1.5) + 10;
|
|
10262
|
+
}
|
|
10263
|
+
|
|
10264
|
+
// src/search/facets.ts
|
|
10265
|
+
function keptValueCount(fieldConfig) {
|
|
10266
|
+
return keptFacetValues(fieldConfig?.limit);
|
|
10267
|
+
}
|
|
10268
|
+
function everyValueFacetConfig(config) {
|
|
10269
|
+
const widened = {};
|
|
10270
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
10271
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
10272
|
+
}
|
|
10273
|
+
return widened;
|
|
10274
|
+
}
|
|
10275
|
+
function oversampledFacetConfig(config) {
|
|
10276
|
+
const widened = {};
|
|
10277
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
10278
|
+
if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
|
|
10279
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
10280
|
+
continue;
|
|
10281
|
+
}
|
|
10282
|
+
const kept = keptValueCount(fieldConfig);
|
|
10283
|
+
widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
|
|
10284
|
+
}
|
|
10285
|
+
return widened;
|
|
10286
|
+
}
|
|
10287
|
+
function mergeFacets(partitionFacets, config) {
|
|
10288
|
+
const merged = /* @__PURE__ */ new Map();
|
|
10289
|
+
const bounds = /* @__PURE__ */ new Map();
|
|
10290
|
+
for (const partition of partitionFacets) {
|
|
10291
|
+
for (const [field, facetResult] of Object.entries(partition)) {
|
|
10292
|
+
let fieldMap = merged.get(field);
|
|
10293
|
+
if (!fieldMap) {
|
|
10294
|
+
fieldMap = /* @__PURE__ */ new Map();
|
|
10295
|
+
merged.set(field, fieldMap);
|
|
10296
|
+
}
|
|
10297
|
+
bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
|
|
10298
|
+
for (const [value, count] of Object.entries(facetResult.values)) {
|
|
10299
|
+
fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
|
|
10300
|
+
}
|
|
10301
|
+
}
|
|
10302
|
+
}
|
|
10303
|
+
const result = {};
|
|
10304
|
+
for (const [field, valueMap] of merged) {
|
|
10305
|
+
const fieldConfig = config[field];
|
|
10306
|
+
const ascending = fieldConfig?.sort === "asc";
|
|
10307
|
+
const ordered = Array.from(valueMap.entries());
|
|
10308
|
+
ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
10309
|
+
const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
|
|
10310
|
+
let largestLeftOut = 0;
|
|
10311
|
+
for (let index = kept; index < ordered.length; index++) {
|
|
10312
|
+
if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
|
|
10313
|
+
}
|
|
10314
|
+
const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
|
|
10315
|
+
const values = {};
|
|
10316
|
+
for (let index = 0; index < kept; index++) {
|
|
10317
|
+
values[ordered[index][0]] = ordered[index][1];
|
|
10318
|
+
}
|
|
10319
|
+
result[field] = { values, count: kept, errorBound };
|
|
10320
|
+
}
|
|
10321
|
+
return result;
|
|
10322
|
+
}
|
|
10323
|
+
|
|
10442
10324
|
// src/core/partition/composite/filters.ts
|
|
10443
10325
|
var WORD_BITS = 32;
|
|
10444
10326
|
function alignToWord(value) {
|
|
@@ -10519,6 +10401,29 @@ function subMatchesView(matches, layout, subIndex) {
|
|
|
10519
10401
|
};
|
|
10520
10402
|
}
|
|
10521
10403
|
|
|
10404
|
+
// src/core/partition/composite/facets.ts
|
|
10405
|
+
function compositeFacets(subs, layout, matched, config, schema) {
|
|
10406
|
+
const everyValue = everyValueFacetConfig(config);
|
|
10407
|
+
if ("ordinalBitset" in matched) {
|
|
10408
|
+
const ordinalLayout = layout();
|
|
10409
|
+
return mergeFacets(
|
|
10410
|
+
subs.map(
|
|
10411
|
+
(sub, index) => computeFacets(
|
|
10412
|
+
sub,
|
|
10413
|
+
{ ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
|
|
10414
|
+
everyValue,
|
|
10415
|
+
schema
|
|
10416
|
+
)
|
|
10417
|
+
),
|
|
10418
|
+
config
|
|
10419
|
+
);
|
|
10420
|
+
}
|
|
10421
|
+
return mergeFacets(
|
|
10422
|
+
subs.map((sub) => computeFacets(sub, matched, everyValue, schema)),
|
|
10423
|
+
config
|
|
10424
|
+
);
|
|
10425
|
+
}
|
|
10426
|
+
|
|
10522
10427
|
// src/core/partition/scored-merge.ts
|
|
10523
10428
|
function kWayMerge(arrays) {
|
|
10524
10429
|
const nonEmpty = arrays.filter((a) => a.length > 0);
|
|
@@ -10898,7 +10803,8 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10898
10803
|
}
|
|
10899
10804
|
function cachedDocFrequencies() {
|
|
10900
10805
|
if (docFrequenciesCache === null) {
|
|
10901
|
-
|
|
10806
|
+
const paged = frozen.find((segment) => segment.paged !== null)?.paged;
|
|
10807
|
+
docFrequenciesCache = paged ? paged.termFrequenciesAcross(subs) : mergeDocFrequencies(subs());
|
|
10902
10808
|
}
|
|
10903
10809
|
return docFrequenciesCache;
|
|
10904
10810
|
}
|
|
@@ -10969,6 +10875,20 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10969
10875
|
frozen.push(segment);
|
|
10970
10876
|
bindSegments();
|
|
10971
10877
|
},
|
|
10878
|
+
frozenSegments() {
|
|
10879
|
+
return frozen;
|
|
10880
|
+
},
|
|
10881
|
+
replaceFrozenSegments(segments) {
|
|
10882
|
+
frozen.length = 0;
|
|
10883
|
+
frozen.push(...segments);
|
|
10884
|
+
bindSegments();
|
|
10885
|
+
},
|
|
10886
|
+
documentIdsHeldInMemory() {
|
|
10887
|
+
return compositeDocIds(
|
|
10888
|
+
frozen.filter((segment) => segment.paged === null),
|
|
10889
|
+
live
|
|
10890
|
+
);
|
|
10891
|
+
},
|
|
10972
10892
|
mergeSegment(segment) {
|
|
10973
10893
|
invalidateDocFrequencies();
|
|
10974
10894
|
live.mergeSegment(segment);
|
|
@@ -10991,6 +10911,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10991
10911
|
live.deserialize(data, schema);
|
|
10992
10912
|
},
|
|
10993
10913
|
clear() {
|
|
10914
|
+
for (const segment of frozen) segment.paged?.close();
|
|
10994
10915
|
frozen.length = 0;
|
|
10995
10916
|
invalidateDocFrequencies();
|
|
10996
10917
|
live.clear();
|
|
@@ -11057,25 +10978,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
11057
10978
|
return compositeFilterMatches(subs(), layout(), filters, schema, meter);
|
|
11058
10979
|
},
|
|
11059
10980
|
computeFacets(matched, config, schema) {
|
|
11060
|
-
|
|
11061
|
-
if ("ordinalBitset" in matched) {
|
|
11062
|
-
const ordinalLayout = layout();
|
|
11063
|
-
return mergeFacets(
|
|
11064
|
-
subs().map(
|
|
11065
|
-
(sub, index) => computeFacets(
|
|
11066
|
-
sub,
|
|
11067
|
-
{ ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
|
|
11068
|
-
everyValue,
|
|
11069
|
-
schema
|
|
11070
|
-
)
|
|
11071
|
-
),
|
|
11072
|
-
config
|
|
11073
|
-
);
|
|
11074
|
-
}
|
|
11075
|
-
return mergeFacets(
|
|
11076
|
-
subs().map((sub) => computeFacets(sub, matched, everyValue, schema)),
|
|
11077
|
-
config
|
|
11078
|
-
);
|
|
10981
|
+
return compositeFacets(subs(), layout, matched, config, schema);
|
|
11079
10982
|
},
|
|
11080
10983
|
sortedPage(request) {
|
|
11081
10984
|
return compositeSortedPage(subs(), layout(), request);
|
|
@@ -11203,15 +11106,19 @@ function createCapacityChecks(source) {
|
|
|
11203
11106
|
}
|
|
11204
11107
|
|
|
11205
11108
|
// src/partitioning/manager/doc-locations.ts
|
|
11109
|
+
function documentIdsHeldInMemory(partition) {
|
|
11110
|
+
return isCompositePartition(partition) ? partition.documentIdsHeldInMemory() : partition.docIds();
|
|
11111
|
+
}
|
|
11206
11112
|
function rebuildDocLocations(locations, partitions) {
|
|
11207
11113
|
locations.clear();
|
|
11208
11114
|
for (let i = 0; i < partitions.length; i++) {
|
|
11209
|
-
for (const docId of partitions[i]
|
|
11115
|
+
for (const docId of documentIdsHeldInMemory(partitions[i])) {
|
|
11210
11116
|
locations.set(docId, i);
|
|
11211
11117
|
}
|
|
11212
11118
|
}
|
|
11213
11119
|
}
|
|
11214
11120
|
function recordSegmentDocLocations(locations, partitionId, segment) {
|
|
11121
|
+
if (segment.paged !== null) return;
|
|
11215
11122
|
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11216
11123
|
const docId = segment.docStore.getExternalId(internalId);
|
|
11217
11124
|
if (docId !== void 0) locations.set(docId, partitionId);
|
|
@@ -11223,7 +11130,7 @@ function rebuildPartitionDocLocations(locations, partitionId, partition) {
|
|
|
11223
11130
|
locations.delete(docId);
|
|
11224
11131
|
}
|
|
11225
11132
|
}
|
|
11226
|
-
for (const docId of partition
|
|
11133
|
+
for (const docId of documentIdsHeldInMemory(partition)) {
|
|
11227
11134
|
locations.set(docId, partitionId);
|
|
11228
11135
|
}
|
|
11229
11136
|
}
|
|
@@ -11266,6 +11173,109 @@ function requireNewSegmentDocuments(segment, docPartitionMap) {
|
|
|
11266
11173
|
}
|
|
11267
11174
|
}
|
|
11268
11175
|
|
|
11176
|
+
// src/partitioning/manager/write-log.ts
|
|
11177
|
+
var REANALYSIS_WRITE_LOG = 0;
|
|
11178
|
+
var CHECKPOINT_WRITE_LOG = 1;
|
|
11179
|
+
function createWriteLogRecorder() {
|
|
11180
|
+
const current = [null, null];
|
|
11181
|
+
return {
|
|
11182
|
+
open(owner) {
|
|
11183
|
+
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
11184
|
+
current[owner] = log;
|
|
11185
|
+
return {
|
|
11186
|
+
get partitionsReplaced() {
|
|
11187
|
+
return log.replaced;
|
|
11188
|
+
},
|
|
11189
|
+
get pendingDocuments() {
|
|
11190
|
+
return log.docIds.size;
|
|
11191
|
+
},
|
|
11192
|
+
takeWrittenDocIds() {
|
|
11193
|
+
const taken = log.docIds;
|
|
11194
|
+
log.docIds = /* @__PURE__ */ new Set();
|
|
11195
|
+
return taken;
|
|
11196
|
+
},
|
|
11197
|
+
close() {
|
|
11198
|
+
if (current[owner] === log) current[owner] = null;
|
|
11199
|
+
}
|
|
11200
|
+
};
|
|
11201
|
+
},
|
|
11202
|
+
documentWritten(docId) {
|
|
11203
|
+
current[0]?.docIds.add(docId);
|
|
11204
|
+
current[1]?.docIds.add(docId);
|
|
11205
|
+
},
|
|
11206
|
+
segmentWritten(segment) {
|
|
11207
|
+
for (const log of current) {
|
|
11208
|
+
if (log === null) continue;
|
|
11209
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11210
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
11211
|
+
if (docId !== void 0) log.docIds.add(docId);
|
|
11212
|
+
}
|
|
11213
|
+
}
|
|
11214
|
+
},
|
|
11215
|
+
partitionsReplaced() {
|
|
11216
|
+
for (const log of current) {
|
|
11217
|
+
if (log !== null) log.replaced = true;
|
|
11218
|
+
}
|
|
11219
|
+
}
|
|
11220
|
+
};
|
|
11221
|
+
}
|
|
11222
|
+
|
|
11223
|
+
// src/partitioning/manager/segment-operations.ts
|
|
11224
|
+
function createSegmentOperations(source) {
|
|
11225
|
+
const { docPartitionMap, writeLogs, validatePartitionId, asCompositePartition, resolveInsertOptions } = source;
|
|
11226
|
+
return {
|
|
11227
|
+
mergeSegment(partitionId, payload, documents) {
|
|
11228
|
+
validatePartitionId(partitionId);
|
|
11229
|
+
const partition = source.partitionAt(partitionId);
|
|
11230
|
+
partition.beginBatch();
|
|
11231
|
+
partition.mergeSegmentPayload(payload, documents);
|
|
11232
|
+
partition.endBatch();
|
|
11233
|
+
for (const docId of payload.docIds) {
|
|
11234
|
+
docPartitionMap.set(docId, partitionId);
|
|
11235
|
+
writeLogs.documentWritten(docId);
|
|
11236
|
+
}
|
|
11237
|
+
},
|
|
11238
|
+
attachFrozenSegment(partitionId, segment) {
|
|
11239
|
+
validatePartitionId(partitionId);
|
|
11240
|
+
if (segment.paged === null) requireNewSegmentDocuments(segment, docPartitionMap);
|
|
11241
|
+
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
11242
|
+
if (segment.paged !== null) return;
|
|
11243
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
11244
|
+
writeLogs.segmentWritten(segment);
|
|
11245
|
+
},
|
|
11246
|
+
openCheckpointChangeLog() {
|
|
11247
|
+
return writeLogs.open(CHECKPOINT_WRITE_LOG);
|
|
11248
|
+
},
|
|
11249
|
+
diskSegmentAccess(partitionId) {
|
|
11250
|
+
validatePartitionId(partitionId);
|
|
11251
|
+
return {
|
|
11252
|
+
composite: asCompositePartition(partitionId),
|
|
11253
|
+
docPartitionMap,
|
|
11254
|
+
insertOptions: resolveInsertOptions()
|
|
11255
|
+
};
|
|
11256
|
+
},
|
|
11257
|
+
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
11258
|
+
validatePartitionId(partitionId);
|
|
11259
|
+
const applied = asCompositePartition(partitionId).swapFrozenSegments(
|
|
11260
|
+
dropSegmentIds,
|
|
11261
|
+
replacement,
|
|
11262
|
+
removedSpellings
|
|
11263
|
+
);
|
|
11264
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
|
|
11265
|
+
return applied;
|
|
11266
|
+
},
|
|
11267
|
+
freezeLiveTail(partitionId, freeze) {
|
|
11268
|
+
validatePartitionId(partitionId);
|
|
11269
|
+
return asCompositePartition(partitionId).freezeLiveTail(freeze);
|
|
11270
|
+
},
|
|
11271
|
+
replaceLiveTail(partitionId, segment) {
|
|
11272
|
+
validatePartitionId(partitionId);
|
|
11273
|
+
asCompositePartition(partitionId).replaceLiveTail(segment);
|
|
11274
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
11275
|
+
}
|
|
11276
|
+
};
|
|
11277
|
+
}
|
|
11278
|
+
|
|
11269
11279
|
// src/partitioning/manager/stats.ts
|
|
11270
11280
|
function aggregateStatsOf(partitions) {
|
|
11271
11281
|
let totalDocuments = 0;
|
|
@@ -11301,47 +11311,13 @@ function partitionStatsOf(partitions) {
|
|
|
11301
11311
|
}));
|
|
11302
11312
|
}
|
|
11303
11313
|
|
|
11304
|
-
// src/partitioning/manager/write-log.ts
|
|
11305
|
-
function createWriteLogRecorder() {
|
|
11306
|
-
let current = null;
|
|
11307
|
-
return {
|
|
11308
|
-
open() {
|
|
11309
|
-
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
11310
|
-
current = log;
|
|
11311
|
-
return {
|
|
11312
|
-
get partitionsReplaced() {
|
|
11313
|
-
return log.replaced;
|
|
11314
|
-
},
|
|
11315
|
-
get pendingDocuments() {
|
|
11316
|
-
return log.docIds.size;
|
|
11317
|
-
},
|
|
11318
|
-
takeWrittenDocIds() {
|
|
11319
|
-
const taken = log.docIds;
|
|
11320
|
-
log.docIds = /* @__PURE__ */ new Set();
|
|
11321
|
-
return taken;
|
|
11322
|
-
},
|
|
11323
|
-
close() {
|
|
11324
|
-
if (current === log) current = null;
|
|
11325
|
-
}
|
|
11326
|
-
};
|
|
11327
|
-
},
|
|
11328
|
-
documentWritten(docId) {
|
|
11329
|
-
if (current !== null) current.docIds.add(docId);
|
|
11330
|
-
},
|
|
11331
|
-
segmentWritten(segment) {
|
|
11332
|
-
if (current === null) return;
|
|
11333
|
-
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11334
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
11335
|
-
if (docId !== void 0) current.docIds.add(docId);
|
|
11336
|
-
}
|
|
11337
|
-
},
|
|
11338
|
-
partitionsReplaced() {
|
|
11339
|
-
if (current !== null) current.replaced = true;
|
|
11340
|
-
}
|
|
11341
|
-
};
|
|
11342
|
-
}
|
|
11343
|
-
|
|
11344
11314
|
// src/partitioning/manager/index.ts
|
|
11315
|
+
function closePagedSegmentsLeftBehind(earlier, kept) {
|
|
11316
|
+
for (const partition of earlier) {
|
|
11317
|
+
if (!isCompositePartition(partition) || kept.includes(partition)) continue;
|
|
11318
|
+
for (const segment of partition.frozenSegments()) segment.paged?.close();
|
|
11319
|
+
}
|
|
11320
|
+
}
|
|
11345
11321
|
function createPartitionManager(indexName, initialConfig, initialLanguage, router, initialPartitionCount, vectorIndexes) {
|
|
11346
11322
|
let config = initialConfig;
|
|
11347
11323
|
let language = initialLanguage;
|
|
@@ -11355,7 +11331,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
11355
11331
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
11356
11332
|
let partitions = [];
|
|
11357
11333
|
const docPartitionMap = /* @__PURE__ */ new Map();
|
|
11358
|
-
const
|
|
11334
|
+
const writeLogs = createWriteLogRecorder();
|
|
11359
11335
|
for (let i = 0; i < count; i++) {
|
|
11360
11336
|
partitions.push(createPartitionIndex(i, trackPositions));
|
|
11361
11337
|
}
|
|
@@ -11375,7 +11351,14 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
11375
11351
|
if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
|
|
11376
11352
|
}
|
|
11377
11353
|
function locateDocument(docId) {
|
|
11378
|
-
|
|
11354
|
+
const known = docPartitionMap.get(docId);
|
|
11355
|
+
if (known !== void 0 || config.textStorage !== "disk") return known;
|
|
11356
|
+
const routed = router.route(docId, partitions.length);
|
|
11357
|
+
if (partitions[routed]?.has(docId)) return routed;
|
|
11358
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
11359
|
+
if (i !== routed && partitions[i].has(docId)) return i;
|
|
11360
|
+
}
|
|
11361
|
+
return void 0;
|
|
11379
11362
|
}
|
|
11380
11363
|
function asCompositePartition(partitionId) {
|
|
11381
11364
|
const existing = partitions[partitionId];
|
|
@@ -11421,12 +11404,14 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
11421
11404
|
return [...partitions];
|
|
11422
11405
|
},
|
|
11423
11406
|
setPartitions(newPartitions) {
|
|
11424
|
-
|
|
11407
|
+
writeLogs.partitionsReplaced();
|
|
11408
|
+
const earlier = partitions;
|
|
11425
11409
|
partitions = [...newPartitions];
|
|
11410
|
+
closePagedSegmentsLeftBehind(earlier, partitions);
|
|
11426
11411
|
rebuildDocLocations(docPartitionMap, partitions);
|
|
11427
11412
|
},
|
|
11428
11413
|
openWriteLog() {
|
|
11429
|
-
return
|
|
11414
|
+
return writeLogs.open(REANALYSIS_WRITE_LOG);
|
|
11430
11415
|
},
|
|
11431
11416
|
adoptRebuiltPartitions(rebuilt) {
|
|
11432
11417
|
if (rebuilt.length !== partitions.length) {
|
|
@@ -11436,7 +11421,9 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
11436
11421
|
{ indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
|
|
11437
11422
|
);
|
|
11438
11423
|
}
|
|
11424
|
+
const earlier = partitions;
|
|
11439
11425
|
partitions = [...rebuilt];
|
|
11426
|
+
closePagedSegmentsLeftBehind(earlier, partitions);
|
|
11440
11427
|
},
|
|
11441
11428
|
adoptAnalysis(change) {
|
|
11442
11429
|
config = change.config;
|
|
@@ -11456,13 +11443,13 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
11456
11443
|
);
|
|
11457
11444
|
}
|
|
11458
11445
|
const partition = createPartitionIndex(partitions.length, trackPositions);
|
|
11459
|
-
|
|
11446
|
+
writeLogs.partitionsReplaced();
|
|
11460
11447
|
partitions.push(partition);
|
|
11461
11448
|
return partition;
|
|
11462
11449
|
},
|
|
11463
11450
|
removePartition(partitionId) {
|
|
11464
11451
|
validatePartitionId(partitionId);
|
|
11465
|
-
|
|
11452
|
+
writeLogs.partitionsReplaced();
|
|
11466
11453
|
partitions.splice(partitionId, 1);
|
|
11467
11454
|
rebuildDocLocations(docPartitionMap, partitions);
|
|
11468
11455
|
},
|
|
@@ -11476,7 +11463,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
11476
11463
|
if (newCount >= partitions.length) {
|
|
11477
11464
|
return;
|
|
11478
11465
|
}
|
|
11479
|
-
|
|
11466
|
+
writeLogs.partitionsReplaced();
|
|
11480
11467
|
for (const [docId, pid] of docPartitionMap) {
|
|
11481
11468
|
if (pid >= newCount) {
|
|
11482
11469
|
docPartitionMap.delete(docId);
|
|
@@ -11499,7 +11486,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
11499
11486
|
const insertOpts = resolveInsertOptions(options);
|
|
11500
11487
|
partitions[pid].insert(docId, document, config.schema, language, insertOpts);
|
|
11501
11488
|
docPartitionMap.set(docId, pid);
|
|
11502
|
-
|
|
11489
|
+
writeLogs.documentWritten(docId);
|
|
11503
11490
|
},
|
|
11504
11491
|
removedSpellingsOf(docId) {
|
|
11505
11492
|
const pid = locateDocument(docId);
|
|
@@ -11513,7 +11500,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
11513
11500
|
}
|
|
11514
11501
|
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
|
|
11515
11502
|
docPartitionMap.delete(docId);
|
|
11516
|
-
|
|
11503
|
+
writeLogs.documentWritten(docId);
|
|
11517
11504
|
},
|
|
11518
11505
|
beginBatchRemove() {
|
|
11519
11506
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -11533,7 +11520,7 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
11533
11520
|
const updateOpts = resolveInsertOptions(options);
|
|
11534
11521
|
partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
|
|
11535
11522
|
docPartitionMap.set(docId, pid);
|
|
11536
|
-
|
|
11523
|
+
writeLogs.documentWritten(docId);
|
|
11537
11524
|
},
|
|
11538
11525
|
get(docId, projection) {
|
|
11539
11526
|
const pid = locateDocument(docId);
|
|
@@ -11573,46 +11560,17 @@ function createPartitionManager(indexName, initialConfig, initialLanguage, route
|
|
|
11573
11560
|
validatePartitionId(partitionId);
|
|
11574
11561
|
return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
|
|
11575
11562
|
},
|
|
11576
|
-
|
|
11577
|
-
|
|
11578
|
-
|
|
11579
|
-
|
|
11580
|
-
|
|
11581
|
-
|
|
11582
|
-
|
|
11583
|
-
|
|
11584
|
-
writeLog.documentWritten(docId);
|
|
11585
|
-
}
|
|
11586
|
-
},
|
|
11587
|
-
attachFrozenSegment(partitionId, segment) {
|
|
11588
|
-
validatePartitionId(partitionId);
|
|
11589
|
-
requireNewSegmentDocuments(segment, docPartitionMap);
|
|
11590
|
-
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
11591
|
-
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
11592
|
-
writeLog.segmentWritten(segment);
|
|
11593
|
-
},
|
|
11594
|
-
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
11595
|
-
validatePartitionId(partitionId);
|
|
11596
|
-
const applied = asCompositePartition(partitionId).swapFrozenSegments(
|
|
11597
|
-
dropSegmentIds,
|
|
11598
|
-
replacement,
|
|
11599
|
-
removedSpellings
|
|
11600
|
-
);
|
|
11601
|
-
recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
|
|
11602
|
-
return applied;
|
|
11603
|
-
},
|
|
11604
|
-
freezeLiveTail(partitionId, freeze) {
|
|
11605
|
-
validatePartitionId(partitionId);
|
|
11606
|
-
return asCompositePartition(partitionId).freezeLiveTail(freeze);
|
|
11607
|
-
},
|
|
11608
|
-
replaceLiveTail(partitionId, segment) {
|
|
11609
|
-
validatePartitionId(partitionId);
|
|
11610
|
-
asCompositePartition(partitionId).replaceLiveTail(segment);
|
|
11611
|
-
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
11612
|
-
},
|
|
11563
|
+
...createSegmentOperations({
|
|
11564
|
+
docPartitionMap,
|
|
11565
|
+
writeLogs,
|
|
11566
|
+
partitionAt: (partitionId) => partitions[partitionId],
|
|
11567
|
+
validatePartitionId,
|
|
11568
|
+
asCompositePartition,
|
|
11569
|
+
resolveInsertOptions: () => resolveInsertOptions()
|
|
11570
|
+
}),
|
|
11613
11571
|
deserializePartition(partitionId, data) {
|
|
11614
11572
|
validatePartitionId(partitionId);
|
|
11615
|
-
|
|
11573
|
+
writeLogs.partitionsReplaced();
|
|
11616
11574
|
partitions[partitionId].deserialize(data, config.schema);
|
|
11617
11575
|
preparePartialWords(partitionId);
|
|
11618
11576
|
rebuildPartitionDocLocations(docPartitionMap, partitionId, partitions[partitionId]);
|
|
@@ -13074,11 +13032,17 @@ function encodeDocumentTableData(documents) {
|
|
|
13074
13032
|
}
|
|
13075
13033
|
function wrapEncodedDocumentTable(data) {
|
|
13076
13034
|
const { blob, offsets } = data;
|
|
13077
|
-
|
|
13078
|
-
|
|
13035
|
+
return createEncodedDocumentSource(
|
|
13036
|
+
offsets.length - 1,
|
|
13037
|
+
blob.byteLength + offsets.byteLength,
|
|
13038
|
+
(ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]),
|
|
13039
|
+
data.discardedFields
|
|
13040
|
+
);
|
|
13041
|
+
}
|
|
13042
|
+
function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields) {
|
|
13043
|
+
const discarded = (discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
|
|
13079
13044
|
const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
|
|
13080
|
-
const
|
|
13081
|
-
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
|
|
13045
|
+
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), discardedFields));
|
|
13082
13046
|
function projectedPathAt(ordinal, path) {
|
|
13083
13047
|
const segments = encodedProjectionPathOf(path);
|
|
13084
13048
|
if (segments === null) return { found: false, value: void 0 };
|
|
@@ -13088,10 +13052,10 @@ function wrapEncodedDocumentTable(data) {
|
|
|
13088
13052
|
}
|
|
13089
13053
|
return {
|
|
13090
13054
|
count,
|
|
13091
|
-
byteLength
|
|
13055
|
+
byteLength,
|
|
13092
13056
|
docAt,
|
|
13093
13057
|
readAt(ordinal) {
|
|
13094
|
-
return decodeStoredDocument(bytesAt(ordinal),
|
|
13058
|
+
return decodeStoredDocument(bytesAt(ordinal), discardedFields);
|
|
13095
13059
|
},
|
|
13096
13060
|
fieldReader(fieldPath) {
|
|
13097
13061
|
const path = fieldPath.split(".");
|
|
@@ -13254,6 +13218,9 @@ function wrapExternalIdTable(data) {
|
|
|
13254
13218
|
const ordinal = sortedOrdinals[lo];
|
|
13255
13219
|
return compareBytes(blob, offsets[ordinal], offsets[ordinal + 1], query) === 0 ? ordinal : -1;
|
|
13256
13220
|
},
|
|
13221
|
+
sortedOrdinalAt(position) {
|
|
13222
|
+
return sortedOrdinals[position];
|
|
13223
|
+
},
|
|
13257
13224
|
collectSortedIds(excluded) {
|
|
13258
13225
|
const ids = [];
|
|
13259
13226
|
for (let position = 0; position < count; position++) {
|
|
@@ -13273,6 +13240,7 @@ function buildExternalIdTable(docIds) {
|
|
|
13273
13240
|
characters += docIds[ordinal].length;
|
|
13274
13241
|
}
|
|
13275
13242
|
let sortedIds = null;
|
|
13243
|
+
let sortedOrdinals = null;
|
|
13276
13244
|
return {
|
|
13277
13245
|
count: docIds.length,
|
|
13278
13246
|
bytes: characters,
|
|
@@ -13282,6 +13250,10 @@ function buildExternalIdTable(docIds) {
|
|
|
13282
13250
|
ordinalOf(docId) {
|
|
13283
13251
|
return ordinals.get(docId) ?? -1;
|
|
13284
13252
|
},
|
|
13253
|
+
sortedOrdinalAt(position) {
|
|
13254
|
+
if (sortedOrdinals === null) sortedOrdinals = codePointOrder(docIds);
|
|
13255
|
+
return sortedOrdinals[position];
|
|
13256
|
+
},
|
|
13285
13257
|
collectSortedIds(excluded) {
|
|
13286
13258
|
if (sortedIds === null) {
|
|
13287
13259
|
const ids = [...docIds];
|
|
@@ -13853,6 +13825,10 @@ function createFrozenRemovalPass(documents, surfaces) {
|
|
|
13853
13825
|
});
|
|
13854
13826
|
return spellings;
|
|
13855
13827
|
}
|
|
13828
|
+
function subtract(ordinal, analysis) {
|
|
13829
|
+
if (surfaces.tableStaysOnDisk()) surfaces.subtractOnceLoaded(ordinal, analysis);
|
|
13830
|
+
else surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, analysis));
|
|
13831
|
+
}
|
|
13856
13832
|
return {
|
|
13857
13833
|
get settled() {
|
|
13858
13834
|
return pending.length === 0;
|
|
@@ -13861,17 +13837,17 @@ function createFrozenRemovalPass(documents, surfaces) {
|
|
|
13861
13837
|
if (bound !== null) return;
|
|
13862
13838
|
const scoped = { ...analysis, fields: documents.textFields };
|
|
13863
13839
|
bound = scoped;
|
|
13864
|
-
for (const ordinal of pending)
|
|
13840
|
+
for (const ordinal of pending) subtract(ordinal, scoped);
|
|
13865
13841
|
pending = [];
|
|
13866
13842
|
},
|
|
13867
13843
|
spellingsOf(ordinal) {
|
|
13868
|
-
return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
|
|
13844
|
+
return bound === null || surfaces.tableStaysOnDisk() ? void 0 : heldSpellingsOf(ordinal, bound);
|
|
13869
13845
|
},
|
|
13870
13846
|
afterRemoval(ordinal, spellings) {
|
|
13871
13847
|
if (spellings !== void 0) {
|
|
13872
13848
|
surfaces.subtractSpellings(ordinal, spellings);
|
|
13873
13849
|
} else if (bound !== null) {
|
|
13874
|
-
|
|
13850
|
+
subtract(ordinal, bound);
|
|
13875
13851
|
} else {
|
|
13876
13852
|
pending.push(ordinal);
|
|
13877
13853
|
}
|
|
@@ -13895,15 +13871,32 @@ function addForms(target, forms) {
|
|
|
13895
13871
|
if (Array.isArray(value)) target.add(surface, value[1], value[0]);
|
|
13896
13872
|
}
|
|
13897
13873
|
}
|
|
13898
|
-
function createFrozenSurfaceSlot(load, documents) {
|
|
13874
|
+
function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
|
|
13899
13875
|
let registry = null;
|
|
13900
13876
|
let uncounted = null;
|
|
13877
|
+
let table = tableOnDisk;
|
|
13878
|
+
let waitingRemovals = [];
|
|
13879
|
+
let waitingAnalysis = null;
|
|
13880
|
+
function subtractWaitingRemovals(target) {
|
|
13881
|
+
if (waitingAnalysis === null || target.size() === 0) return;
|
|
13882
|
+
for (const ordinal of waitingRemovals) {
|
|
13883
|
+
const spellings = [];
|
|
13884
|
+
visitDocumentSurfaceSpellings(documents.documentAt(ordinal), waitingAnalysis, (spelling) => {
|
|
13885
|
+
if (target.holds(spelling)) spellings.push(spelling);
|
|
13886
|
+
});
|
|
13887
|
+
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
13888
|
+
}
|
|
13889
|
+
}
|
|
13901
13890
|
function loaded3() {
|
|
13902
13891
|
if (registry === null) {
|
|
13903
13892
|
const built = createSurfaceRegistry();
|
|
13904
13893
|
const forms = load();
|
|
13905
13894
|
if (forms !== null) built.deserialize(forms);
|
|
13895
|
+
subtractWaitingRemovals(built);
|
|
13906
13896
|
registry = built;
|
|
13897
|
+
table = void 0;
|
|
13898
|
+
waitingRemovals = [];
|
|
13899
|
+
waitingAnalysis = null;
|
|
13907
13900
|
}
|
|
13908
13901
|
return registry;
|
|
13909
13902
|
}
|
|
@@ -13911,12 +13904,22 @@ function createFrozenSurfaceSlot(load, documents) {
|
|
|
13911
13904
|
return uncounted === null || uncounted[ordinal] === 0;
|
|
13912
13905
|
}
|
|
13913
13906
|
return {
|
|
13907
|
+
heldBytes: () => table?.heldBytes() ?? 0,
|
|
13914
13908
|
reader: {
|
|
13915
|
-
candidatesForPrefix: (prefix) => loaded3().candidatesForPrefix(prefix),
|
|
13909
|
+
candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) : loaded3().candidatesForPrefix(prefix),
|
|
13916
13910
|
stemChangedTotalFor: (token) => loaded3().stemChangedTotalFor(token),
|
|
13917
13911
|
size: () => loaded3().size(),
|
|
13918
13912
|
serialize: () => loaded3().serialize()
|
|
13919
13913
|
},
|
|
13914
|
+
tableStaysOnDisk() {
|
|
13915
|
+
return registry === null && tableOnDisk !== void 0;
|
|
13916
|
+
},
|
|
13917
|
+
subtractOnceLoaded(ordinal, analysis) {
|
|
13918
|
+
uncounted ??= new Uint8Array(documents.count);
|
|
13919
|
+
uncounted[ordinal] = 1;
|
|
13920
|
+
waitingAnalysis = analysis;
|
|
13921
|
+
waitingRemovals.push(ordinal);
|
|
13922
|
+
},
|
|
13920
13923
|
holdsAny() {
|
|
13921
13924
|
return loaded3().size() > 0;
|
|
13922
13925
|
},
|
|
@@ -14064,6 +14067,9 @@ function wrapFrozenTokenTable(data) {
|
|
|
14064
14067
|
}
|
|
14065
14068
|
return token;
|
|
14066
14069
|
},
|
|
14070
|
+
tokenBytesAt(sortedIndex) {
|
|
14071
|
+
return blob.subarray(offsets[sortedIndex], offsets[sortedIndex + 1]);
|
|
14072
|
+
},
|
|
14067
14073
|
byteLengthAt(sortedIndex) {
|
|
14068
14074
|
return offsets[sortedIndex + 1] - offsets[sortedIndex];
|
|
14069
14075
|
},
|
|
@@ -14338,10 +14344,10 @@ function buildGeoReaders(entries) {
|
|
|
14338
14344
|
}
|
|
14339
14345
|
return readers;
|
|
14340
14346
|
}
|
|
14341
|
-
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
|
|
14347
|
+
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms, pagedParts) {
|
|
14342
14348
|
const tombstones = createFrozenTombstones();
|
|
14343
|
-
const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
|
|
14344
|
-
const postingViews = createFrozenPostingViews(source, tombstones, counts);
|
|
14349
|
+
const counts = pagedParts?.createCounts(tombstones) ?? createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
|
|
14350
|
+
const postingViews = pagedParts?.createViews(tombstones, counts) ?? createFrozenPostingViews(source, tombstones, counts);
|
|
14345
14351
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
14346
14352
|
for (let i = 0; i < source.fieldLengthNames.length; i++) {
|
|
14347
14353
|
lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
|
|
@@ -14358,7 +14364,11 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
14358
14364
|
documentAt: (ordinal) => documentSource.readAt(ordinal)
|
|
14359
14365
|
};
|
|
14360
14366
|
const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
|
|
14361
|
-
const surfaces = createFrozenSurfaceSlot(
|
|
14367
|
+
const surfaces = createFrozenSurfaceSlot(
|
|
14368
|
+
loadSurfaceForms,
|
|
14369
|
+
textDocuments,
|
|
14370
|
+
survivingSurfaceForms === void 0 ? source.surfaceTableOnDisk : void 0
|
|
14371
|
+
);
|
|
14362
14372
|
const removals = createFrozenRemovalPass(textDocuments, surfaces);
|
|
14363
14373
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
14364
14374
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
@@ -14380,6 +14390,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
14380
14390
|
segmentId,
|
|
14381
14391
|
documentSource,
|
|
14382
14392
|
sharedSnapshot,
|
|
14393
|
+
paged: pagedParts?.access ?? null,
|
|
14383
14394
|
arrays: source,
|
|
14384
14395
|
invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
|
|
14385
14396
|
docStore,
|
|
@@ -14395,7 +14406,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
14395
14406
|
sizeIndexes,
|
|
14396
14407
|
fieldNameTable,
|
|
14397
14408
|
partialWords,
|
|
14398
|
-
trackPositions: source.positionOffsets !== null,
|
|
14409
|
+
trackPositions: pagedParts?.access.snapshot.tracksPositions ?? source.positionOffsets !== null,
|
|
14399
14410
|
flatSchemaCache: null,
|
|
14400
14411
|
lastSchemaRef: null,
|
|
14401
14412
|
sortColumns: null,
|
|
@@ -14439,6 +14450,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
14439
14450
|
};
|
|
14440
14451
|
return segment;
|
|
14441
14452
|
}
|
|
14453
|
+
function assemblePagedFrozenSegment(segmentId, source, documentSource, partialWordTable, parts) {
|
|
14454
|
+
return assembleFrozenSegment(segmentId, source, documentSource, null, partialWordTable, void 0, parts);
|
|
14455
|
+
}
|
|
14442
14456
|
function createFrozenSegment(payload, documents, segmentId) {
|
|
14443
14457
|
const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
|
|
14444
14458
|
return assembleFrozenSegment(
|
|
@@ -15302,7 +15316,20 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
15302
15316
|
|
|
15303
15317
|
// src/workers/segment-actions.ts
|
|
15304
15318
|
function isSegmentAction(action) {
|
|
15305
|
-
return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail";
|
|
15319
|
+
return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail" || action.type === "adoptDiskSegments";
|
|
15320
|
+
}
|
|
15321
|
+
function isPagedSnapshot(snapshot) {
|
|
15322
|
+
return "path" in snapshot;
|
|
15323
|
+
}
|
|
15324
|
+
async function attachSegments(entry, action) {
|
|
15325
|
+
for (const segment of action.segments) {
|
|
15326
|
+
growPartitionsTo(entry.manager, segment.partitionId);
|
|
15327
|
+
const surfaceForms = segment.survivingSurfaceForms;
|
|
15328
|
+
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
15329
|
+
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-BUFNZMEU.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
15330
|
+
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
15331
|
+
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
15332
|
+
}
|
|
15306
15333
|
}
|
|
15307
15334
|
function growPartitionsTo(manager, partitionId) {
|
|
15308
15335
|
while (manager.partitionCount <= partitionId) {
|
|
@@ -15351,17 +15378,10 @@ function runSegmentAction(entry, action) {
|
|
|
15351
15378
|
}
|
|
15352
15379
|
return void 0;
|
|
15353
15380
|
}
|
|
15354
|
-
case "attachSegments":
|
|
15355
|
-
|
|
15356
|
-
|
|
15357
|
-
|
|
15358
|
-
const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
15359
|
-
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
15360
|
-
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
15361
|
-
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
15362
|
-
}
|
|
15363
|
-
return void 0;
|
|
15364
|
-
}
|
|
15381
|
+
case "attachSegments":
|
|
15382
|
+
return attachSegments(entry, action);
|
|
15383
|
+
case "adoptDiskSegments":
|
|
15384
|
+
return import('./module-BUFNZMEU.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
|
|
15365
15385
|
case "compactSegments": {
|
|
15366
15386
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
15367
15387
|
return mergeFrozenSegments(
|
|
@@ -15409,7 +15429,7 @@ function requireLoadedVectorCode() {
|
|
|
15409
15429
|
function loadVectorCode() {
|
|
15410
15430
|
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
15411
15431
|
if (loading2 === null) {
|
|
15412
|
-
loading2 = import('./module-
|
|
15432
|
+
loading2 = import('./module-BVP4TKZU.mjs').then(
|
|
15413
15433
|
(code) => {
|
|
15414
15434
|
loaded2 = code;
|
|
15415
15435
|
return code;
|
|
@@ -15485,7 +15505,7 @@ function closeHeldVectorCopies(held) {
|
|
|
15485
15505
|
held.fields.clear();
|
|
15486
15506
|
held.searchers.clear();
|
|
15487
15507
|
}
|
|
15488
|
-
function
|
|
15508
|
+
function yieldToEventLoop() {
|
|
15489
15509
|
return new Promise((resolve) => {
|
|
15490
15510
|
if (typeof setImmediate === "function") setImmediate(resolve);
|
|
15491
15511
|
else setTimeout(resolve, 0);
|
|
@@ -15496,7 +15516,7 @@ async function insertIntoHeldGraph(held, fieldName, handle, ordinals) {
|
|
|
15496
15516
|
if (field === void 0) return null;
|
|
15497
15517
|
for (const ordinal of ordinals) {
|
|
15498
15518
|
field.view.insertOrdinal(ordinal);
|
|
15499
|
-
await
|
|
15519
|
+
await yieldToEventLoop();
|
|
15500
15520
|
}
|
|
15501
15521
|
return field.view.takeOutcome();
|
|
15502
15522
|
}
|
|
@@ -15804,6 +15824,31 @@ function createDirectExecutor(options) {
|
|
|
15804
15824
|
};
|
|
15805
15825
|
}
|
|
15806
15826
|
|
|
15827
|
+
// src/engine/orchestration/turn.ts
|
|
15828
|
+
function afterCurrentTurn(callback) {
|
|
15829
|
+
if (typeof setImmediate === "function") {
|
|
15830
|
+
setImmediate(callback);
|
|
15831
|
+
return;
|
|
15832
|
+
}
|
|
15833
|
+
setTimeout(callback, 0);
|
|
15834
|
+
}
|
|
15835
|
+
function yieldToEventLoop2() {
|
|
15836
|
+
return new Promise((resolve) => afterCurrentTurn(resolve));
|
|
15837
|
+
}
|
|
15838
|
+
|
|
15839
|
+
// src/partitioning/constants.ts
|
|
15840
|
+
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
15841
|
+
var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
|
|
15842
|
+
var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
|
|
15843
|
+
var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
15844
|
+
|
|
15845
|
+
// src/partitioning/rewrite-spill-of.ts
|
|
15846
|
+
async function rewriteSpillOf(storage, manager, target) {
|
|
15847
|
+
if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
|
|
15848
|
+
const { openRewriteSpill } = await import('./module-BUFNZMEU.mjs');
|
|
15849
|
+
return openRewriteSpill(storage, manager, target);
|
|
15850
|
+
}
|
|
15851
|
+
|
|
15807
15852
|
// src/partitioning/text-rebuild.ts
|
|
15808
15853
|
function textRebuildTargetOf(config, language, analysis) {
|
|
15809
15854
|
return { config, language, analysis, partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis) };
|
|
@@ -15813,7 +15858,7 @@ function interrupted(pass) {
|
|
|
15813
15858
|
}
|
|
15814
15859
|
async function buildPartition(pass, source, partitionId) {
|
|
15815
15860
|
const { manager, language, options } = pass;
|
|
15816
|
-
const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
15861
|
+
const target = pass.spill?.newPartition(partitionId) ?? createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
15817
15862
|
const docIds = [...source.docIds()];
|
|
15818
15863
|
target.beginBatch();
|
|
15819
15864
|
for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
|
|
@@ -15822,7 +15867,12 @@ async function buildPartition(pass, source, partitionId) {
|
|
|
15822
15867
|
const document = manager.getRef(docIds[i]);
|
|
15823
15868
|
if (document !== void 0) target.insert(docIds[i], document, manager.schema, language, options);
|
|
15824
15869
|
}
|
|
15825
|
-
|
|
15870
|
+
if (pass.spill !== null) {
|
|
15871
|
+
target.endBatch();
|
|
15872
|
+
await pass.spill.spillIfFull(target, partitionId);
|
|
15873
|
+
target.beginBatch();
|
|
15874
|
+
}
|
|
15875
|
+
await yieldToEventLoop2();
|
|
15826
15876
|
if (interrupted(pass)) return null;
|
|
15827
15877
|
}
|
|
15828
15878
|
target.endBatch();
|
|
@@ -15854,7 +15904,7 @@ async function rebuildOnce(pass) {
|
|
|
15854
15904
|
}
|
|
15855
15905
|
while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
|
|
15856
15906
|
applyWrittenDocuments(pass, built);
|
|
15857
|
-
await
|
|
15907
|
+
await yieldToEventLoop2();
|
|
15858
15908
|
if (interrupted(pass)) return false;
|
|
15859
15909
|
}
|
|
15860
15910
|
const keepCurrent = () => {
|
|
@@ -15883,12 +15933,19 @@ async function rebuildTextIndexes(manager, hooks, target) {
|
|
|
15883
15933
|
);
|
|
15884
15934
|
const language = target?.language ?? manager.language;
|
|
15885
15935
|
const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
|
|
15936
|
+
const spill = await rewriteSpillOf(hooks.rewriteStorage?.(), manager, target);
|
|
15886
15937
|
while (hooks.isCurrent()) {
|
|
15887
15938
|
const log = manager.openWriteLog();
|
|
15939
|
+
let swapped = false;
|
|
15888
15940
|
try {
|
|
15889
|
-
|
|
15941
|
+
swapped = await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options, spill });
|
|
15942
|
+
if (swapped) {
|
|
15943
|
+
hooks.onRewriteSpill?.(spill);
|
|
15944
|
+
return true;
|
|
15945
|
+
}
|
|
15890
15946
|
} finally {
|
|
15891
15947
|
log.close();
|
|
15948
|
+
if (!swapped) await spill?.abandon();
|
|
15892
15949
|
}
|
|
15893
15950
|
}
|
|
15894
15951
|
return false;
|
|
@@ -15922,6 +15979,7 @@ function toError(value) {
|
|
|
15922
15979
|
function wireAnalysisRebuild(wiring) {
|
|
15923
15980
|
return createAnalysisRebuildCoordinator(wiring.config, {
|
|
15924
15981
|
getManager: wiring.getManager,
|
|
15982
|
+
rewriteStorage: () => wiring.durabilityManager?.rewriteStorage?.(),
|
|
15925
15983
|
desyncIndex: wiring.desyncIndex,
|
|
15926
15984
|
resyncIndex: wiring.resyncIndex,
|
|
15927
15985
|
hasPendingChange: wiring.hasPendingChange,
|
|
@@ -15955,8 +16013,13 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
15955
16013
|
const partitionCount = manager.partitionCount;
|
|
15956
16014
|
deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
|
|
15957
16015
|
const wasPromoted = deps.desyncIndex(indexName);
|
|
16016
|
+
let spill = null;
|
|
15958
16017
|
try {
|
|
15959
16018
|
const rebuilt = await rebuildTextIndexes(manager, {
|
|
16019
|
+
rewriteStorage: () => deps.rewriteStorage?.(),
|
|
16020
|
+
onRewriteSpill(kept) {
|
|
16021
|
+
spill = kept;
|
|
16022
|
+
},
|
|
15960
16023
|
isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
|
|
15961
16024
|
onPartitionBuilt(partitionsBuilt) {
|
|
15962
16025
|
entry.partitionsRebuilt = partitionsBuilt;
|
|
@@ -15971,6 +16034,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
15971
16034
|
return;
|
|
15972
16035
|
}
|
|
15973
16036
|
await deps.persistAnalysisRevision(indexName);
|
|
16037
|
+
await spill?.removeUnread(manager);
|
|
15974
16038
|
await deps.resyncIndex(indexName, wasPromoted);
|
|
15975
16039
|
deps.emit({
|
|
15976
16040
|
indexName,
|
|
@@ -16244,6 +16308,9 @@ function reconstructSchemaFromMetadata(metadata) {
|
|
|
16244
16308
|
if (metadata.trackPositions !== void 0) {
|
|
16245
16309
|
config.trackPositions = metadata.trackPositions;
|
|
16246
16310
|
}
|
|
16311
|
+
if (metadata.textStorage !== void 0) {
|
|
16312
|
+
config.textStorage = metadata.textStorage;
|
|
16313
|
+
}
|
|
16247
16314
|
if (metadata.strict !== void 0) {
|
|
16248
16315
|
config.strict = metadata.strict;
|
|
16249
16316
|
}
|
|
@@ -16465,7 +16532,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
16465
16532
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
16466
16533
|
if (existing !== void 0) {
|
|
16467
16534
|
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
16468
|
-
const { adoptSavedAnalysis } = await import('./analysis-change-
|
|
16535
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-JPV775LR.mjs');
|
|
16469
16536
|
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
16470
16537
|
}
|
|
16471
16538
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
@@ -16498,6 +16565,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
16498
16565
|
const synonyms = prepared?.encoded ?? null;
|
|
16499
16566
|
try {
|
|
16500
16567
|
validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
|
|
16568
|
+
validateTextStorage(indexConfig.textStorage, deps.filesystemDurability);
|
|
16501
16569
|
if (loadData) {
|
|
16502
16570
|
deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
|
|
16503
16571
|
}
|
|
@@ -17249,6 +17317,9 @@ function checkHeapAfterRecovery(core) {
|
|
|
17249
17317
|
|
|
17250
17318
|
// src/core/partition/frozen/share.ts
|
|
17251
17319
|
function shareFrozenSegment(segment, analysis) {
|
|
17320
|
+
if (segment.paged !== null) {
|
|
17321
|
+
return { snapshot: segment.paged.snapshot, tombstonedDocIds: segment.tombstonedDocIds() };
|
|
17322
|
+
}
|
|
17252
17323
|
if (segment.sharedSnapshot !== null) {
|
|
17253
17324
|
const tombstonedDocIds = segment.tombstonedDocIds();
|
|
17254
17325
|
if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
|
|
@@ -17847,7 +17918,7 @@ async function replicateToWorkers(state, action) {
|
|
|
17847
17918
|
const loading3 = state.copyLoadBuffers.get(action.indexName);
|
|
17848
17919
|
if (loading3 !== void 0) {
|
|
17849
17920
|
queueForCopies(state, detached);
|
|
17850
|
-
if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await
|
|
17921
|
+
if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop2();
|
|
17851
17922
|
return;
|
|
17852
17923
|
}
|
|
17853
17924
|
const queue = state.replicationQueues.get(action.indexName);
|
|
@@ -18055,7 +18126,7 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
|
|
|
18055
18126
|
}
|
|
18056
18127
|
async function compactIndexSegments(state, indexName, policy) {
|
|
18057
18128
|
const manager = state.executor.getManager(indexName);
|
|
18058
|
-
if (!manager) return;
|
|
18129
|
+
if (!manager || manager.config.textStorage === "disk") return;
|
|
18059
18130
|
const partitionIds = /* @__PURE__ */ new Set();
|
|
18060
18131
|
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) partitionIds.add(partitionId);
|
|
18061
18132
|
for (const partitionId of state.segmentLedger.get(indexName)?.keys() ?? []) partitionIds.add(partitionId);
|
|
@@ -19506,6 +19577,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
19506
19577
|
scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
|
|
19507
19578
|
replicateToWorkers: replicate,
|
|
19508
19579
|
awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
|
|
19580
|
+
queueDiskAdoption: (action) => queueForCopies(state, action),
|
|
19509
19581
|
awaitWrites: (indexName) => awaitWritesApplied(state, indexName),
|
|
19510
19582
|
awaitCompactions: () => awaitCompactions(state),
|
|
19511
19583
|
openIndex,
|
|
@@ -19590,7 +19662,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19590
19662
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
19591
19663
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
19592
19664
|
}
|
|
19593
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
19665
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-SNOWIT6Z.mjs');
|
|
19594
19666
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
19595
19667
|
const executor = createDirectExecutor({
|
|
19596
19668
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -19650,7 +19722,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19650
19722
|
},
|
|
19651
19723
|
vectorCopyPolicy
|
|
19652
19724
|
);
|
|
19653
|
-
const rebalancer = createRebalancer();
|
|
19654
19725
|
const rebalanceRouter = createPartitionRouter();
|
|
19655
19726
|
const waqMap = /* @__PURE__ */ new Map();
|
|
19656
19727
|
const rebalanceTargets = /* @__PURE__ */ new Map();
|
|
@@ -19710,6 +19781,8 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19710
19781
|
emitCheckpointHeld(report) {
|
|
19711
19782
|
emitEngineEvent(eventHandlers, "checkpointHeld", report);
|
|
19712
19783
|
},
|
|
19784
|
+
queueDiskAdoption: orchestrator.queueDiskAdoption,
|
|
19785
|
+
awaitCopiesOfIndex: (indexName) => orchestrator.awaitReplication(indexName),
|
|
19713
19786
|
publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve(),
|
|
19714
19787
|
recordCheckpoint(indexName, documentCount, partitionCount) {
|
|
19715
19788
|
const entry = indexRegistry.get(indexName);
|
|
@@ -19774,7 +19847,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19774
19847
|
requireManager,
|
|
19775
19848
|
resumeChange: async (indexName) => {
|
|
19776
19849
|
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
19777
|
-
await (await import('./analysis-change-entry-
|
|
19850
|
+
await (await import('./analysis-change-entry-7HR4LWL3.mjs')).resumeChangeOf(engineCore, indexName);
|
|
19778
19851
|
},
|
|
19779
19852
|
onClose: hooks?.onIndexClose,
|
|
19780
19853
|
onOpen: (indexName) => {
|
|
@@ -19804,7 +19877,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19804
19877
|
checkHeapPressure: heapPressureNotifier.check
|
|
19805
19878
|
};
|
|
19806
19879
|
const rebalanceCtx = {
|
|
19807
|
-
rebalancer,
|
|
19808
19880
|
router: rebalanceRouter,
|
|
19809
19881
|
waqMap,
|
|
19810
19882
|
rebalancingIndexes,
|
|
@@ -19831,7 +19903,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19831
19903
|
shutdownState,
|
|
19832
19904
|
abortController,
|
|
19833
19905
|
orchestrator,
|
|
19834
|
-
rebalancer,
|
|
19835
19906
|
rebalanceRouter,
|
|
19836
19907
|
rebalancingIndexes,
|
|
19837
19908
|
waqMap,
|
|
@@ -19858,7 +19929,7 @@ async function startEngineCore(core, config) {
|
|
|
19858
19929
|
checkHeapAfterRecovery(core);
|
|
19859
19930
|
}
|
|
19860
19931
|
if (core.invalidation) await core.invalidation.start();
|
|
19861
|
-
if (core.durability) await (await import('./analysis-change-entry-
|
|
19932
|
+
if (core.durability) await (await import('./analysis-change-entry-7HR4LWL3.mjs')).resumeEveryChange(core);
|
|
19862
19933
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
19863
19934
|
} catch (err) {
|
|
19864
19935
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -20206,9 +20277,12 @@ function requireWithinFileLimit(byteLength, what) {
|
|
|
20206
20277
|
);
|
|
20207
20278
|
}
|
|
20208
20279
|
async function packChecksummedEnvelope(payloadChunks, envelopeFormatVersion, what) {
|
|
20280
|
+
requireWithinFileLimit(HEADER_SIZE + payloadByteLength(payloadChunks), what);
|
|
20281
|
+
return packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, await checksumOfChunks(payloadChunks));
|
|
20282
|
+
}
|
|
20283
|
+
function packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, checksum) {
|
|
20209
20284
|
const payloadLength = payloadByteLength(payloadChunks);
|
|
20210
20285
|
requireWithinFileLimit(HEADER_SIZE + payloadLength, what);
|
|
20211
|
-
const checksum = await checksumOfChunks(payloadChunks);
|
|
20212
20286
|
const [major, minor, patch] = engineVersionParts();
|
|
20213
20287
|
const header = writeHeader({
|
|
20214
20288
|
envelopeFormatVersion,
|
|
@@ -20370,6 +20444,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
|
|
|
20370
20444
|
...config.partitions !== void 0 ? { partitionConfig: config.partitions } : {},
|
|
20371
20445
|
...config.defaultScoring !== void 0 ? { defaultScoring: config.defaultScoring } : {},
|
|
20372
20446
|
...config.trackPositions !== void 0 ? { trackPositions: config.trackPositions } : {},
|
|
20447
|
+
...config.textStorage !== void 0 ? { textStorage: config.textStorage } : {},
|
|
20373
20448
|
...config.strict !== void 0 ? { strict: config.strict } : {},
|
|
20374
20449
|
...config.required !== void 0 ? { required: config.required } : {},
|
|
20375
20450
|
...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
|
|
@@ -20458,6 +20533,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
20458
20533
|
requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
|
|
20459
20534
|
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
20460
20535
|
validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
|
|
20536
|
+
validateTextStorage(indexConfig.textStorage, core.filesystemDurability);
|
|
20461
20537
|
if (indexConfig.bm25 !== void 0) {
|
|
20462
20538
|
validateBM25Params(
|
|
20463
20539
|
indexConfig.bm25,
|
|
@@ -20549,7 +20625,7 @@ async function dropEngineIndex(core, name) {
|
|
|
20549
20625
|
const entry = core.requireIndex(name);
|
|
20550
20626
|
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
20551
20627
|
const abandonChange = async () => {
|
|
20552
|
-
await (await import('./analysis-change-entry-
|
|
20628
|
+
await (await import('./analysis-change-entry-7HR4LWL3.mjs')).abandonChangeOf(core, name);
|
|
20553
20629
|
};
|
|
20554
20630
|
if (changeRecorded()) await abandonChange();
|
|
20555
20631
|
await core.indexState.drop(name, async () => {
|
|
@@ -21386,7 +21462,7 @@ function snapshotSegments(segments) {
|
|
|
21386
21462
|
}
|
|
21387
21463
|
return shared2;
|
|
21388
21464
|
}
|
|
21389
|
-
function
|
|
21465
|
+
function attachSegments2(orchestrator, indexName, segments) {
|
|
21390
21466
|
return orchestrator.replicateToWorkers({
|
|
21391
21467
|
type: "attachSegments",
|
|
21392
21468
|
indexName,
|
|
@@ -21397,11 +21473,11 @@ function attachSegments(orchestrator, indexName, segments) {
|
|
|
21397
21473
|
async function broadcastBuiltSegments(orchestrator, indexName, segments, skipClone) {
|
|
21398
21474
|
const frozen = tryFreezeSegmentsForAttach(segments);
|
|
21399
21475
|
if (frozen !== null) {
|
|
21400
|
-
await
|
|
21476
|
+
await attachSegments2(orchestrator, indexName, frozen);
|
|
21401
21477
|
return;
|
|
21402
21478
|
}
|
|
21403
21479
|
const shared2 = snapshotSegments(segments);
|
|
21404
|
-
if (shared2.length > 0) await
|
|
21480
|
+
if (shared2.length > 0) await attachSegments2(orchestrator, indexName, shared2);
|
|
21405
21481
|
const plain = payloadSegments(segments);
|
|
21406
21482
|
if (plain.length === 0) return;
|
|
21407
21483
|
await orchestrator.replicateToWorkers({
|
|
@@ -22157,284 +22233,6 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
22157
22233
|
return { succeeded, failed };
|
|
22158
22234
|
}
|
|
22159
22235
|
|
|
22160
|
-
// src/partitioning/write-ahead-queue.ts
|
|
22161
|
-
function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
|
|
22162
|
-
const entries = [];
|
|
22163
|
-
const docStates = /* @__PURE__ */ new Map();
|
|
22164
|
-
let nextSequence = 1;
|
|
22165
|
-
let resolveReplayed = () => void 0;
|
|
22166
|
-
const replayed = new Promise((resolve) => {
|
|
22167
|
-
resolveReplayed = resolve;
|
|
22168
|
-
});
|
|
22169
|
-
return {
|
|
22170
|
-
push(entry) {
|
|
22171
|
-
if (entries.length >= maxSize) {
|
|
22172
|
-
throw new NarsilError(
|
|
22173
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
22174
|
-
`Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
|
|
22175
|
-
{ maxSize, currentSize: entries.length }
|
|
22176
|
-
);
|
|
22177
|
-
}
|
|
22178
|
-
const seq = nextSequence++;
|
|
22179
|
-
entries.push({ ...entry, sequenceNumber: seq });
|
|
22180
|
-
docStates.set(entry.docId, entry.action === "remove" ? "absent" : "present");
|
|
22181
|
-
return seq;
|
|
22182
|
-
},
|
|
22183
|
-
drain() {
|
|
22184
|
-
const result = entries.splice(0);
|
|
22185
|
-
result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
|
|
22186
|
-
return result;
|
|
22187
|
-
},
|
|
22188
|
-
clear() {
|
|
22189
|
-
entries.length = 0;
|
|
22190
|
-
docStates.clear();
|
|
22191
|
-
},
|
|
22192
|
-
bufferedDocState(docId) {
|
|
22193
|
-
return docStates.get(docId);
|
|
22194
|
-
},
|
|
22195
|
-
whenReplayed() {
|
|
22196
|
-
return replayed;
|
|
22197
|
-
},
|
|
22198
|
-
markReplayed() {
|
|
22199
|
-
resolveReplayed();
|
|
22200
|
-
},
|
|
22201
|
-
get size() {
|
|
22202
|
-
return entries.length;
|
|
22203
|
-
},
|
|
22204
|
-
get isFull() {
|
|
22205
|
-
return entries.length >= maxSize;
|
|
22206
|
-
}
|
|
22207
|
-
};
|
|
22208
|
-
}
|
|
22209
|
-
|
|
22210
|
-
// src/engine/rebalance-executor.ts
|
|
22211
|
-
function warnHookError(hook, err) {
|
|
22212
|
-
console.warn(`${hook} plugin hook error:`, err instanceof Error ? err.message : String(err));
|
|
22213
|
-
}
|
|
22214
|
-
function applyReplayUpdateVectors(entry, vectorFieldPaths2) {
|
|
22215
|
-
const updateVectors = /* @__PURE__ */ new Map();
|
|
22216
|
-
for (const fieldPath of vectorFieldPaths2) {
|
|
22217
|
-
updateVectors.set(fieldPath, extractVectorFromDoc(entry.document, fieldPath));
|
|
22218
|
-
}
|
|
22219
|
-
let updateDoc = entry.document;
|
|
22220
|
-
if (updateVectors.size > 0) {
|
|
22221
|
-
updateDoc = structuredClone(entry.document);
|
|
22222
|
-
for (const fieldPath of updateVectors.keys()) {
|
|
22223
|
-
deleteNestedValue(updateDoc, fieldPath);
|
|
22224
|
-
}
|
|
22225
|
-
}
|
|
22226
|
-
return { updateDoc, updateVectors };
|
|
22227
|
-
}
|
|
22228
|
-
async function replayEntry(manager, indexName, entry, ctx) {
|
|
22229
|
-
const vecIndexes = manager.getVectorIndexes();
|
|
22230
|
-
const vectorFieldPaths2 = ctx.requireIndex(indexName).vectorFieldPaths;
|
|
22231
|
-
if (entry.action === "insert" && entry.document) {
|
|
22232
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
22233
|
-
entry.document,
|
|
22234
|
-
vectorFieldPaths2
|
|
22235
|
-
);
|
|
22236
|
-
const overwrote = manager.has(entry.docId);
|
|
22237
|
-
const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
|
|
22238
|
-
if (overwrote) {
|
|
22239
|
-
removeDocumentVectors(entry.docId, vecIndexes);
|
|
22240
|
-
manager.remove(entry.docId, removedSpellings);
|
|
22241
|
-
}
|
|
22242
|
-
manager.insert(entry.docId, partitionDoc);
|
|
22243
|
-
if (extractedVectors.size > 0) {
|
|
22244
|
-
insertDocumentVectors(entry.docId, extractedVectors, vecIndexes, manager.partitionIdOf(entry.docId));
|
|
22245
|
-
for (const fieldPath of extractedVectors.keys()) {
|
|
22246
|
-
vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
22247
|
-
}
|
|
22248
|
-
}
|
|
22249
|
-
try {
|
|
22250
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: entry.docId, document: entry.document });
|
|
22251
|
-
} catch (err) {
|
|
22252
|
-
warnHookError("afterInsert", err);
|
|
22253
|
-
}
|
|
22254
|
-
const replicated = {
|
|
22255
|
-
indexName,
|
|
22256
|
-
docId: entry.docId,
|
|
22257
|
-
document: entry.document,
|
|
22258
|
-
requestId: `replicate-insert-${entry.docId}`
|
|
22259
|
-
};
|
|
22260
|
-
await ctx.orchestrator.replicateToWorkers(
|
|
22261
|
-
overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
|
|
22262
|
-
);
|
|
22263
|
-
return;
|
|
22264
|
-
}
|
|
22265
|
-
if (entry.action === "remove") {
|
|
22266
|
-
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
22267
|
-
manager.remove(entry.docId, removedSpellings);
|
|
22268
|
-
removeDocumentVectors(entry.docId, vecIndexes);
|
|
22269
|
-
try {
|
|
22270
|
-
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
|
|
22271
|
-
} catch (err) {
|
|
22272
|
-
warnHookError("afterRemove", err);
|
|
22273
|
-
}
|
|
22274
|
-
await ctx.orchestrator.replicateToWorkers({
|
|
22275
|
-
type: "remove",
|
|
22276
|
-
indexName,
|
|
22277
|
-
docId: entry.docId,
|
|
22278
|
-
requestId: `replicate-remove-${entry.docId}`,
|
|
22279
|
-
removedSpellings
|
|
22280
|
-
});
|
|
22281
|
-
return;
|
|
22282
|
-
}
|
|
22283
|
-
if (entry.action === "update" && entry.document) {
|
|
22284
|
-
const oldDocument = manager.get(entry.docId);
|
|
22285
|
-
const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths2);
|
|
22286
|
-
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
22287
|
-
if (manager.has(entry.docId)) {
|
|
22288
|
-
manager.update(entry.docId, updateDoc, void 0, removedSpellings);
|
|
22289
|
-
} else {
|
|
22290
|
-
manager.insert(entry.docId, updateDoc);
|
|
22291
|
-
}
|
|
22292
|
-
for (const [fieldPath, newVec] of updateVectors) {
|
|
22293
|
-
const vecIndex = vecIndexes.get(fieldPath);
|
|
22294
|
-
if (!vecIndex) continue;
|
|
22295
|
-
if (newVec === null) {
|
|
22296
|
-
if (vecIndex.has(entry.docId)) {
|
|
22297
|
-
vecIndex.remove(entry.docId);
|
|
22298
|
-
}
|
|
22299
|
-
} else {
|
|
22300
|
-
const oldVec = vecIndex.getVector(entry.docId);
|
|
22301
|
-
if (!vectorsEqual(oldVec, newVec)) {
|
|
22302
|
-
if (vecIndex.has(entry.docId)) {
|
|
22303
|
-
vecIndex.remove(entry.docId);
|
|
22304
|
-
}
|
|
22305
|
-
vecIndex.insert(entry.docId, newVec);
|
|
22306
|
-
vecIndex.scheduleBuild();
|
|
22307
|
-
}
|
|
22308
|
-
}
|
|
22309
|
-
}
|
|
22310
|
-
try {
|
|
22311
|
-
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
22312
|
-
indexName,
|
|
22313
|
-
docId: entry.docId,
|
|
22314
|
-
oldDocument: oldDocument ?? {},
|
|
22315
|
-
newDocument: entry.document
|
|
22316
|
-
});
|
|
22317
|
-
} catch (err) {
|
|
22318
|
-
warnHookError("afterUpdate", err);
|
|
22319
|
-
}
|
|
22320
|
-
await ctx.orchestrator.replicateToWorkers({
|
|
22321
|
-
type: "update",
|
|
22322
|
-
indexName,
|
|
22323
|
-
docId: entry.docId,
|
|
22324
|
-
document: entry.document,
|
|
22325
|
-
requestId: `replicate-update-${entry.docId}`,
|
|
22326
|
-
removedSpellings
|
|
22327
|
-
});
|
|
22328
|
-
}
|
|
22329
|
-
}
|
|
22330
|
-
async function replayQueued(manager, indexName, waq, ctx) {
|
|
22331
|
-
while (true) {
|
|
22332
|
-
const entries = waq.drain();
|
|
22333
|
-
if (entries.length === 0) break;
|
|
22334
|
-
for (const entry of entries) {
|
|
22335
|
-
try {
|
|
22336
|
-
await replayEntry(manager, indexName, entry, ctx);
|
|
22337
|
-
} catch (replayErr) {
|
|
22338
|
-
const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
|
|
22339
|
-
if (!isMissing) {
|
|
22340
|
-
console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
|
|
22341
|
-
}
|
|
22342
|
-
}
|
|
22343
|
-
}
|
|
22344
|
-
}
|
|
22345
|
-
}
|
|
22346
|
-
async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
22347
|
-
if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
|
|
22348
|
-
throw new NarsilError(
|
|
22349
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
22350
|
-
`Target partition count must be a positive integer, got ${targetPartitionCount}`,
|
|
22351
|
-
{ targetPartitionCount }
|
|
22352
|
-
);
|
|
22353
|
-
}
|
|
22354
|
-
const entry = ctx.requireIndex(indexName);
|
|
22355
|
-
const maxPartitions = entry.config.partitions?.maxPartitions;
|
|
22356
|
-
if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
|
|
22357
|
-
throw new NarsilError(
|
|
22358
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
22359
|
-
`Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
|
|
22360
|
-
{ targetPartitionCount, maxPartitions }
|
|
22361
|
-
);
|
|
22362
|
-
}
|
|
22363
|
-
if (ctx.rebalancingIndexes.has(indexName)) {
|
|
22364
|
-
throw new NarsilError(
|
|
22365
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
22366
|
-
`Index "${indexName}" is already being rebalanced`
|
|
22367
|
-
);
|
|
22368
|
-
}
|
|
22369
|
-
const registered4 = ctx.requireIndex(indexName);
|
|
22370
|
-
if (registered4.pendingAnalysis !== null || registered4.durableAnalysis !== null) {
|
|
22371
|
-
throw new NarsilError(
|
|
22372
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
22373
|
-
`Index "${indexName}" is changing its analysis settings, so try the rebalance again once the change ends`,
|
|
22374
|
-
{ indexName }
|
|
22375
|
-
);
|
|
22376
|
-
}
|
|
22377
|
-
if (targetPartitionCount === manager.partitionCount) {
|
|
22378
|
-
return;
|
|
22379
|
-
}
|
|
22380
|
-
const waq = createWriteAheadQueue();
|
|
22381
|
-
ctx.waqMap.set(indexName, waq);
|
|
22382
|
-
ctx.rebalancingIndexes.add(indexName);
|
|
22383
|
-
ctx.rebalanceTargets.set(indexName, targetPartitionCount);
|
|
22384
|
-
const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
|
|
22385
|
-
try {
|
|
22386
|
-
const oldCount = manager.partitionCount;
|
|
22387
|
-
await ctx.rebalancer.rebalance(manager, targetPartitionCount, ctx.router, (progress) => {
|
|
22388
|
-
if (progress.phase === "complete") {
|
|
22389
|
-
emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
|
|
22390
|
-
indexName,
|
|
22391
|
-
oldCount,
|
|
22392
|
-
newCount: targetPartitionCount
|
|
22393
|
-
});
|
|
22394
|
-
}
|
|
22395
|
-
});
|
|
22396
|
-
try {
|
|
22397
|
-
await ctx.pluginRegistry.runHook("onPartitionSplit", {
|
|
22398
|
-
indexName,
|
|
22399
|
-
oldPartitionCount: oldCount,
|
|
22400
|
-
newPartitionCount: targetPartitionCount
|
|
22401
|
-
});
|
|
22402
|
-
} catch (hookErr) {
|
|
22403
|
-
console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
|
|
22404
|
-
}
|
|
22405
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
22406
|
-
if (ctx.durabilityManager) {
|
|
22407
|
-
await ctx.durabilityManager.persistMetadata(indexName);
|
|
22408
|
-
await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
|
|
22409
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
22410
|
-
}
|
|
22411
|
-
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
22412
|
-
do {
|
|
22413
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
22414
|
-
} while (waq.size > 0);
|
|
22415
|
-
} catch (err) {
|
|
22416
|
-
try {
|
|
22417
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
22418
|
-
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
22419
|
-
do {
|
|
22420
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
22421
|
-
} while (waq.size > 0);
|
|
22422
|
-
} catch (recoveryErr) {
|
|
22423
|
-
console.warn(
|
|
22424
|
-
`Rebalance failure recovery for index "${indexName}" did not complete:`,
|
|
22425
|
-
recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)
|
|
22426
|
-
);
|
|
22427
|
-
}
|
|
22428
|
-
throw err;
|
|
22429
|
-
} finally {
|
|
22430
|
-
ctx.rebalancingIndexes.delete(indexName);
|
|
22431
|
-
ctx.rebalanceTargets.delete(indexName);
|
|
22432
|
-
ctx.waqMap.delete(indexName);
|
|
22433
|
-
waq.markReplayed();
|
|
22434
|
-
}
|
|
22435
|
-
ctx.checkWatermark(indexName);
|
|
22436
|
-
}
|
|
22437
|
-
|
|
22438
22236
|
// src/engine/vector-maintenance.ts
|
|
22439
22237
|
function resolveVectorIndexes(manager, indexName, fieldName) {
|
|
22440
22238
|
const all = manager.getVectorIndexes();
|
|
@@ -22474,6 +22272,7 @@ function getVectorMaintenanceStatus(manager) {
|
|
|
22474
22272
|
bufferSize: status.bufferSize,
|
|
22475
22273
|
building: status.building,
|
|
22476
22274
|
waitingForMemory: status.waitingForMemory,
|
|
22275
|
+
loadingFromFiles: status.loadingFromFiles === true,
|
|
22477
22276
|
estimatedCompactMs: status.estimatedCompactMs,
|
|
22478
22277
|
estimatedOptimizeMs: status.estimatedOptimizeMs
|
|
22479
22278
|
});
|
|
@@ -22611,14 +22410,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
22611
22410
|
return {
|
|
22612
22411
|
snapshot(indexName) {
|
|
22613
22412
|
return withOpenIndex2(indexName, async () => {
|
|
22614
|
-
const { createSnapshot } = await import('./snapshot-
|
|
22413
|
+
const { createSnapshot } = await import('./snapshot-QXBKHJAG.mjs');
|
|
22615
22414
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
22616
22415
|
});
|
|
22617
22416
|
},
|
|
22618
22417
|
async restore(indexName, data) {
|
|
22619
22418
|
core.guardShutdown();
|
|
22620
22419
|
validateIndexName(indexName);
|
|
22621
|
-
const restored = import('./restore-queue-
|
|
22420
|
+
const restored = import('./restore-queue-ONYNYUDE.mjs').then(
|
|
22622
22421
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
22623
22422
|
);
|
|
22624
22423
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -22627,8 +22426,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
22627
22426
|
exportDocuments(indexName) {
|
|
22628
22427
|
return withOpenIndex2(indexName, async () => {
|
|
22629
22428
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
22630
|
-
import('./snapshot-
|
|
22631
|
-
import('./write-
|
|
22429
|
+
import('./snapshot-QXBKHJAG.mjs'),
|
|
22430
|
+
import('./write-WHFBDWQS.mjs')
|
|
22632
22431
|
]);
|
|
22633
22432
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
22634
22433
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -22636,7 +22435,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
22636
22435
|
},
|
|
22637
22436
|
importDocumentExport(indexName, data) {
|
|
22638
22437
|
core.guardShutdown();
|
|
22639
|
-
const imported = import('./import-
|
|
22438
|
+
const imported = import('./import-TLB2A5LJ.mjs').then(
|
|
22640
22439
|
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
22641
22440
|
);
|
|
22642
22441
|
return emitWriteEventAfter(core, indexName, imported);
|
|
@@ -24302,7 +24101,7 @@ function pickDisplaySurface(surfaceOccurrences) {
|
|
|
24302
24101
|
// src/narsil/reads.ts
|
|
24303
24102
|
async function enterRead(core, indexName) {
|
|
24304
24103
|
core.guardShutdown();
|
|
24305
|
-
if (core.rebalancingIndexes.has(indexName)) await
|
|
24104
|
+
if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop2();
|
|
24306
24105
|
}
|
|
24307
24106
|
async function readUnderOneAnalysis(core, indexName, read) {
|
|
24308
24107
|
const manager = core.requireManager(indexName);
|
|
@@ -24507,8 +24306,11 @@ function createNarsilFromCore(core, config) {
|
|
|
24507
24306
|
release();
|
|
24508
24307
|
}
|
|
24509
24308
|
}
|
|
24309
|
+
const rebalancesStarting = /* @__PURE__ */ new Map();
|
|
24510
24310
|
function withOpenIndexWrite(indexName, action) {
|
|
24511
|
-
|
|
24311
|
+
const starting = rebalancesStarting.get(indexName);
|
|
24312
|
+
const written = starting === void 0 ? withOpenIndex2(indexName, action) : starting.then(() => withOpenIndex2(indexName, action));
|
|
24313
|
+
return emitWriteEventAfter(core, indexName, written);
|
|
24512
24314
|
}
|
|
24513
24315
|
const narsil = {
|
|
24514
24316
|
createIndex(name, indexConfig) {
|
|
@@ -24636,16 +24438,17 @@ function createNarsilFromCore(core, config) {
|
|
|
24636
24438
|
const entry = requireIndex(indexName);
|
|
24637
24439
|
await withOpenIndexWrite(indexName, async () => {
|
|
24638
24440
|
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
24639
|
-
return (await import('./analysis-change-
|
|
24441
|
+
return (await import('./analysis-change-JPV775LR.mjs')).resumeAnalysisChange(core, indexName);
|
|
24640
24442
|
});
|
|
24641
24443
|
},
|
|
24642
24444
|
async changeAnalysis(indexName, change) {
|
|
24643
24445
|
guardShutdown();
|
|
24644
24446
|
requireIndex(indexName);
|
|
24645
|
-
|
|
24447
|
+
const changed = withOpenIndex2(
|
|
24646
24448
|
indexName,
|
|
24647
|
-
async () => (await import('./analysis-change-
|
|
24449
|
+
async () => (await import('./analysis-change-JPV775LR.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
24648
24450
|
);
|
|
24451
|
+
await emitWriteEventAfter(core, indexName, changed);
|
|
24649
24452
|
},
|
|
24650
24453
|
...portabilityOperations(core, config, () => narsil),
|
|
24651
24454
|
...synonymOperations(core),
|
|
@@ -24664,15 +24467,29 @@ function createNarsilFromCore(core, config) {
|
|
|
24664
24467
|
await executor.execute({ type: "clear", indexName, requestId: indexName });
|
|
24665
24468
|
await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
|
|
24666
24469
|
core.watermarkNotifier.forget(indexName);
|
|
24470
|
+
const durable = core.durability?.manager;
|
|
24471
|
+
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
24667
24472
|
});
|
|
24668
24473
|
},
|
|
24669
24474
|
async rebalance(indexName, targetPartitionCount) {
|
|
24670
24475
|
guardShutdown();
|
|
24671
24476
|
requireIndex(indexName);
|
|
24672
|
-
|
|
24673
|
-
|
|
24674
|
-
|
|
24675
|
-
);
|
|
24477
|
+
let writesMayFollow = () => void 0;
|
|
24478
|
+
const started = new Promise((resolve) => {
|
|
24479
|
+
writesMayFollow = resolve;
|
|
24480
|
+
});
|
|
24481
|
+
function releaseWrites() {
|
|
24482
|
+
writesMayFollow();
|
|
24483
|
+
if (rebalancesStarting.get(indexName) === started) rebalancesStarting.delete(indexName);
|
|
24484
|
+
}
|
|
24485
|
+
rebalancesStarting.set(indexName, started);
|
|
24486
|
+
const rebalanced = withOpenIndex2(indexName, async () => {
|
|
24487
|
+
const { executeRebalance } = await import('./analysis-change-JPV775LR.mjs');
|
|
24488
|
+
const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
|
|
24489
|
+
releaseWrites();
|
|
24490
|
+
return running;
|
|
24491
|
+
});
|
|
24492
|
+
return emitWriteEventAfter(core, indexName, rebalanced).finally(releaseWrites);
|
|
24676
24493
|
},
|
|
24677
24494
|
async updatePartitionConfig(indexName, partitionConfig) {
|
|
24678
24495
|
guardShutdown();
|
|
@@ -24791,4 +24608,4 @@ async function createNarsil(config) {
|
|
|
24791
24608
|
// src/index.ts
|
|
24792
24609
|
var VERSION = "0.1.0";
|
|
24793
24610
|
|
|
24794
|
-
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, HEADER_SIZE, MAX_DOCUMENT_NESTING_DEPTH, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, emitEngineEvent, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, rebuildTextIndexes, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, storedArrayPositionGap, survivorDocumentsOf, textRebuildTargetOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
|
|
24611
|
+
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, MAX_DOCUMENT_NESTING_DEPTH, REBALANCE_CHUNK_SIZE, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, assemblePagedFrozenSegment, buildPartialWordTableFrom, buildSegmentPayload, checksumOfChunks, clearNormalizationCache, configureNormalizationCache, crc32, crc32Final, crc32Init, crc32Update, createCompositePartition, createEncodedDocumentSource, createFrozenSegment, createNarsil, createPartitionIndex, createPartitionManager, createPartitionRouter, createRequestId, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, deleteNestedValue, emitEngineEvent, encodeDocument, encodeIndexConfiguration, encodeSurfaceTable, endOfValue, engineVersionParts, extractVectorFieldsFromSchema, extractVectorFromDoc, fieldIndexRemapOf, flattenSchema, fnv1a, freezeSegmentShared, generateId, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, holdsADiscardedField, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeDocFrequencies, mergeFieldIndexes2 as mergeFieldIndexes, mergeFieldLengths, mergePartialWords, mergeSegmentContents, mergeSurfaceForms2 as mergeSurfaceForms, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, rebuildTextIndexes, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireSurfaceFormsForPartialFields, requireValidFieldTypes, requireWithinFileLimit, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, rewriteSpillOf, rulesOfStoredSynonyms, scanPostingBlocks, segmentArraysOf, selectionKeeps, shareSegmentArrays, storedArrayPositionGap, survivorDocumentsOf, textRebuildTargetOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, vectorsEqual, wrapFrozenTokenTable, writeHeader, yieldToEventLoop2 as yieldToEventLoop };
|