@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, LOOKUP_KEY_SCRATCH_BYTES, bitsetHas, bitsetSet, createBitSet, bitsetIsEmpty, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
|
|
2
|
-
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-AIIJDWMG.mjs';
|
|
3
1
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
2
|
+
import { TARGET_BLOCK_ENTRIES, bitsetHas, createBitSet, LOOKUP_KEY_SCRATCH_BYTES, GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, bitsetIsEmpty, bitsetSet, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
|
|
3
|
+
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, CHECKSUM_YIELD_CHUNK_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, toReducedSortValue, compareComparableValues, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, defaultSortMode, resolveRequestThreadCount, readSortField, holdsUnpairedSurrogate, truncateSortString, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-D33A5VMU.mjs';
|
|
4
|
+
import { MAX_INDEX_CONFIGURATION_BYTES, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-MZI3TEOW.mjs';
|
|
5
|
+
import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-35B42UFM.mjs';
|
|
6
6
|
import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
|
|
7
7
|
|
|
8
8
|
// src/analysis/registry.ts
|
|
@@ -2287,6 +2287,162 @@ var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith",
|
|
|
2287
2287
|
var PATTERN_EXPRESSION_OPERATORS = ["wildcard", "regex"];
|
|
2288
2288
|
var PATTERN_ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
|
|
2289
2289
|
|
|
2290
|
+
// src/core/hash.ts
|
|
2291
|
+
var FNV_OFFSET_BASIS = 2166136261;
|
|
2292
|
+
var FNV_PRIME = 16777619;
|
|
2293
|
+
var encoder = new TextEncoder();
|
|
2294
|
+
function fnv1aBytes(bytes) {
|
|
2295
|
+
return fnv1aContinue(FNV_OFFSET_BASIS, bytes, bytes.length) >>> 0;
|
|
2296
|
+
}
|
|
2297
|
+
function fnv1aContinue(start, bytes, length) {
|
|
2298
|
+
let hash = start;
|
|
2299
|
+
for (let i = 0; i < length; i++) {
|
|
2300
|
+
hash = Math.imul(hash ^ bytes[i], FNV_PRIME);
|
|
2301
|
+
}
|
|
2302
|
+
return hash;
|
|
2303
|
+
}
|
|
2304
|
+
var scratchNumber = new Float64Array(1);
|
|
2305
|
+
var scratchBytes = new Uint8Array(scratchNumber.buffer);
|
|
2306
|
+
var HOST_IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
2307
|
+
function fnv1aContinueTaggedNumbers(start, tag, numbers) {
|
|
2308
|
+
let hash = start;
|
|
2309
|
+
const b = scratchBytes;
|
|
2310
|
+
for (let i = 0; i < numbers.length; i++) {
|
|
2311
|
+
scratchNumber[0] = Math.fround(numbers[i]);
|
|
2312
|
+
hash = Math.imul(hash ^ tag, FNV_PRIME);
|
|
2313
|
+
if (HOST_IS_LITTLE_ENDIAN) {
|
|
2314
|
+
hash = Math.imul(hash ^ b[7], FNV_PRIME);
|
|
2315
|
+
hash = Math.imul(hash ^ b[6], FNV_PRIME);
|
|
2316
|
+
hash = Math.imul(hash ^ b[5], FNV_PRIME);
|
|
2317
|
+
hash = Math.imul(hash ^ b[4], FNV_PRIME);
|
|
2318
|
+
hash = Math.imul(hash ^ b[3], FNV_PRIME);
|
|
2319
|
+
hash = Math.imul(hash ^ b[2], FNV_PRIME);
|
|
2320
|
+
hash = Math.imul(hash ^ b[1], FNV_PRIME);
|
|
2321
|
+
hash = Math.imul(hash ^ b[0], FNV_PRIME);
|
|
2322
|
+
} else {
|
|
2323
|
+
for (let at = 0; at < 8; at++) hash = Math.imul(hash ^ b[at], FNV_PRIME);
|
|
2324
|
+
}
|
|
2325
|
+
}
|
|
2326
|
+
return hash;
|
|
2327
|
+
}
|
|
2328
|
+
function fnv1a(input) {
|
|
2329
|
+
return fnv1aBytes(encoder.encode(input));
|
|
2330
|
+
}
|
|
2331
|
+
|
|
2332
|
+
// src/partitioning/router.ts
|
|
2333
|
+
function createPartitionRouter() {
|
|
2334
|
+
function route(docId, partitionCount) {
|
|
2335
|
+
if (partitionCount <= 0) {
|
|
2336
|
+
throw new NarsilError(
|
|
2337
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
2338
|
+
`Partition count must be greater than 0, received ${partitionCount}`,
|
|
2339
|
+
{ partitionCount }
|
|
2340
|
+
);
|
|
2341
|
+
}
|
|
2342
|
+
return fnv1a(docId) % partitionCount;
|
|
2343
|
+
}
|
|
2344
|
+
function routeBatch(docIds, partitionCount) {
|
|
2345
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2346
|
+
for (let i = 0; i < docIds.length; i++) {
|
|
2347
|
+
const pid = route(docIds[i], partitionCount);
|
|
2348
|
+
let group = groups.get(pid);
|
|
2349
|
+
if (!group) {
|
|
2350
|
+
group = [];
|
|
2351
|
+
groups.set(pid, group);
|
|
2352
|
+
}
|
|
2353
|
+
group.push(docIds[i]);
|
|
2354
|
+
}
|
|
2355
|
+
return groups;
|
|
2356
|
+
}
|
|
2357
|
+
return { route, routeBatch };
|
|
2358
|
+
}
|
|
2359
|
+
|
|
2360
|
+
// src/plugins/registry.ts
|
|
2361
|
+
var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
|
|
2362
|
+
function isThenable(value) {
|
|
2363
|
+
return value !== null && typeof value === "object" && typeof value.then === "function";
|
|
2364
|
+
}
|
|
2365
|
+
function thrownText(err, plugin, hookName) {
|
|
2366
|
+
if (err instanceof Error) return err.message;
|
|
2367
|
+
try {
|
|
2368
|
+
return String(err);
|
|
2369
|
+
} catch {
|
|
2370
|
+
return `The ${hookName} hook of plugin "${plugin}" rejects the write`;
|
|
2371
|
+
}
|
|
2372
|
+
}
|
|
2373
|
+
function asWriteRejection(err, plugin, hookName) {
|
|
2374
|
+
if (err instanceof NarsilError) return err;
|
|
2375
|
+
return new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, thrownText(err, plugin, hookName), {
|
|
2376
|
+
plugin,
|
|
2377
|
+
hook: hookName
|
|
2378
|
+
});
|
|
2379
|
+
}
|
|
2380
|
+
function invoke(hook, hookName, context) {
|
|
2381
|
+
const rejectsWrites = WRITE_REJECTING_HOOKS.has(hookName);
|
|
2382
|
+
let result;
|
|
2383
|
+
try {
|
|
2384
|
+
result = hook.run(context);
|
|
2385
|
+
} catch (err) {
|
|
2386
|
+
throw rejectsWrites ? asWriteRejection(err, hook.plugin, hookName) : err;
|
|
2387
|
+
}
|
|
2388
|
+
if (!isThenable(result)) return null;
|
|
2389
|
+
if (!rejectsWrites) return result;
|
|
2390
|
+
return Promise.resolve(result).catch((err) => {
|
|
2391
|
+
throw asWriteRejection(err, hook.plugin, hookName);
|
|
2392
|
+
});
|
|
2393
|
+
}
|
|
2394
|
+
async function continueAsync(pending, hooks, startIndex, hookName, context) {
|
|
2395
|
+
await pending;
|
|
2396
|
+
for (let i = startIndex; i < hooks.length; i++) {
|
|
2397
|
+
await invoke(hooks[i], hookName, context);
|
|
2398
|
+
}
|
|
2399
|
+
}
|
|
2400
|
+
function createPluginRegistry() {
|
|
2401
|
+
const plugins = [];
|
|
2402
|
+
return {
|
|
2403
|
+
register(plugin) {
|
|
2404
|
+
plugins.push(plugin);
|
|
2405
|
+
},
|
|
2406
|
+
hasHooks(hookName) {
|
|
2407
|
+
return plugins.some((p) => typeof p[hookName] === "function");
|
|
2408
|
+
},
|
|
2409
|
+
runHook(hookName, context) {
|
|
2410
|
+
const hooks = [];
|
|
2411
|
+
for (const plugin of plugins) {
|
|
2412
|
+
const hook = plugin[hookName];
|
|
2413
|
+
if (typeof hook === "function") {
|
|
2414
|
+
hooks.push({ plugin: plugin.name, run: hook.bind(plugin) });
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
if (hooks.length === 0) return;
|
|
2418
|
+
for (let i = 0; i < hooks.length; i++) {
|
|
2419
|
+
const pending = invoke(hooks[i], hookName, context);
|
|
2420
|
+
if (pending !== null) {
|
|
2421
|
+
return continueAsync(pending, hooks, i + 1, hookName, context);
|
|
2422
|
+
}
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2425
|
+
};
|
|
2426
|
+
}
|
|
2427
|
+
|
|
2428
|
+
// src/rerank/registry.ts
|
|
2429
|
+
function validateRerankAdapter(name, adapter) {
|
|
2430
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
2431
|
+
throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, "A rerank adapter needs a non-empty string name");
|
|
2432
|
+
}
|
|
2433
|
+
const rerank = typeof adapter === "object" && adapter !== null ? adapter.rerank : void 0;
|
|
2434
|
+
if (typeof rerank !== "function") {
|
|
2435
|
+
throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, `Rerank adapter "${name}" must provide a rerank function`, {
|
|
2436
|
+
adapter: name
|
|
2437
|
+
});
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
function registerEngineRerankAdapter(core, name, adapter) {
|
|
2441
|
+
core.guardShutdown();
|
|
2442
|
+
validateRerankAdapter(name, adapter);
|
|
2443
|
+
core.rerankAdapters.set(name, adapter);
|
|
2444
|
+
}
|
|
2445
|
+
|
|
2290
2446
|
// src/geo/constants.ts
|
|
2291
2447
|
var DEG_TO_RAD = Math.PI / 180;
|
|
2292
2448
|
var MAX_LATITUDE_DEGREES = 90;
|
|
@@ -2776,6 +2932,10 @@ function validateVectorStorage(config, filesystemDurability) {
|
|
|
2776
2932
|
});
|
|
2777
2933
|
}
|
|
2778
2934
|
}
|
|
2935
|
+
function validateTextStorage(textStorage, filesystemDurability) {
|
|
2936
|
+
if (textStorage === void 0 || textStorage === "memory" || textStorage === "disk" && filesystemDurability) return;
|
|
2937
|
+
fail("textStorage must be 'memory', or 'disk' with filesystem durability", { textStorage });
|
|
2938
|
+
}
|
|
2779
2939
|
function raiseEarlierReleaseGraphSetting(config) {
|
|
2780
2940
|
if (typeof config !== "object" || config === null) return config;
|
|
2781
2941
|
const hnsw = config.hnswConfig;
|
|
@@ -2812,91 +2972,213 @@ function validateVectorPromotion(config) {
|
|
|
2812
2972
|
}
|
|
2813
2973
|
}
|
|
2814
2974
|
|
|
2815
|
-
// src/
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
2825
|
-
function getNestedValue(obj, path) {
|
|
2826
|
-
const parts = path.split(".");
|
|
2827
|
-
let current = obj;
|
|
2828
|
-
for (let i = 0; i < parts.length; i++) {
|
|
2829
|
-
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
2830
|
-
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
2831
|
-
current = current[parts[i]];
|
|
2832
|
-
}
|
|
2833
|
-
return current;
|
|
2834
|
-
}
|
|
2835
|
-
function stringArraysEqual(a, b) {
|
|
2836
|
-
if (a === b) return true;
|
|
2837
|
-
if (!a || !b) return false;
|
|
2838
|
-
if (a.length !== b.length) return false;
|
|
2839
|
-
for (let i = 0; i < a.length; i++) {
|
|
2840
|
-
if (a[i] !== b[i]) return false;
|
|
2975
|
+
// src/schema/embedding-validator.ts
|
|
2976
|
+
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
2977
|
+
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
2978
|
+
const fieldEntries = Object.keys(embeddingConfig.fields);
|
|
2979
|
+
if (fieldEntries.length === 0) {
|
|
2980
|
+
throw new NarsilError(
|
|
2981
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
2982
|
+
"Embedding config must define at least one field mapping"
|
|
2983
|
+
);
|
|
2841
2984
|
}
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
const newVal = getNestedValue(newDoc, path);
|
|
2849
|
-
if (base === "string") {
|
|
2850
|
-
if (oldVal !== newVal) return true;
|
|
2851
|
-
} else {
|
|
2852
|
-
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
2853
|
-
}
|
|
2985
|
+
if (typeof embeddingConfig.adapter === "string") {
|
|
2986
|
+
throw new NarsilError(
|
|
2987
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
2988
|
+
`Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
|
|
2989
|
+
{ adapter: embeddingConfig.adapter }
|
|
2990
|
+
);
|
|
2854
2991
|
}
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
function parseVectorDimension(fieldType) {
|
|
2858
|
-
const match = VECTOR_PATTERN2.exec(fieldType);
|
|
2859
|
-
return match ? Number.parseInt(match[1], 10) : null;
|
|
2860
|
-
}
|
|
2861
|
-
function tokenizeOptions(options) {
|
|
2862
|
-
return {
|
|
2863
|
-
stem: true,
|
|
2864
|
-
removeStopWords: true,
|
|
2865
|
-
collectSurfaces: options?.collectSurfaces === true,
|
|
2866
|
-
stopWordOverride: options?.stopWordOverride,
|
|
2867
|
-
customTokenizer: options?.customTokenizer
|
|
2868
|
-
};
|
|
2869
|
-
}
|
|
2870
|
-
function getFlatSchema(state, schema) {
|
|
2871
|
-
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
2872
|
-
state.flatSchemaCache = flattenSchema(schema);
|
|
2873
|
-
state.lastSchemaRef = schema;
|
|
2874
|
-
return state.flatSchemaCache;
|
|
2875
|
-
}
|
|
2876
|
-
|
|
2877
|
-
// src/core/projection.ts
|
|
2878
|
-
function requireFieldPathList(paths, option) {
|
|
2879
|
-
if (paths === void 0) return;
|
|
2880
|
-
if (!Array.isArray(paths)) {
|
|
2992
|
+
const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
|
|
2993
|
+
if (!resolvedAdapter) {
|
|
2881
2994
|
throw new NarsilError(
|
|
2882
|
-
ErrorCodes.
|
|
2883
|
-
|
|
2884
|
-
{ option: `document.${option}`, received: paths === null ? "null" : typeof paths }
|
|
2995
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
2996
|
+
"Embedding config requires an adapter (set on the index or the instance)"
|
|
2885
2997
|
);
|
|
2886
2998
|
}
|
|
2887
|
-
|
|
2888
|
-
const entry = paths[position];
|
|
2889
|
-
if (typeof entry === "string") continue;
|
|
2999
|
+
if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
|
|
2890
3000
|
throw new NarsilError(
|
|
2891
|
-
ErrorCodes.
|
|
2892
|
-
`
|
|
2893
|
-
{ option: `document.${option}`, position, received: entry === null ? "null" : typeof entry }
|
|
3001
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3002
|
+
`Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
|
|
2894
3003
|
);
|
|
2895
3004
|
}
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
|
|
2899
|
-
|
|
3005
|
+
const flatFields = flattenSchema(schema);
|
|
3006
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
3007
|
+
const targetType = flatFields[targetField];
|
|
3008
|
+
if (targetType === void 0) {
|
|
3009
|
+
throw new NarsilError(
|
|
3010
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3011
|
+
`Embedding target field "${targetField}" does not exist in the schema`,
|
|
3012
|
+
{ field: targetField }
|
|
3013
|
+
);
|
|
3014
|
+
}
|
|
3015
|
+
const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
|
|
3016
|
+
if (!vectorMatch) {
|
|
3017
|
+
throw new NarsilError(
|
|
3018
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3019
|
+
`Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
|
|
3020
|
+
{ field: targetField, type: targetType }
|
|
3021
|
+
);
|
|
3022
|
+
}
|
|
3023
|
+
const schemaDimension = Number.parseInt(vectorMatch[1], 10);
|
|
3024
|
+
if (schemaDimension !== resolvedAdapter.dimensions) {
|
|
3025
|
+
throw new NarsilError(
|
|
3026
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
3027
|
+
`Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
|
|
3028
|
+
{ field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
|
|
3029
|
+
);
|
|
3030
|
+
}
|
|
3031
|
+
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
3032
|
+
if (sources.length === 0) {
|
|
3033
|
+
throw new NarsilError(
|
|
3034
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3035
|
+
`Embedding target field "${targetField}" must have at least one source field`,
|
|
3036
|
+
{ field: targetField }
|
|
3037
|
+
);
|
|
3038
|
+
}
|
|
3039
|
+
for (const source of sources) {
|
|
3040
|
+
const sourceType = flatFields[source];
|
|
3041
|
+
if (sourceType === void 0) {
|
|
3042
|
+
throw new NarsilError(
|
|
3043
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3044
|
+
`Embedding source field "${source}" does not exist in the schema`,
|
|
3045
|
+
{ field: source, targetField }
|
|
3046
|
+
);
|
|
3047
|
+
}
|
|
3048
|
+
if (!isSingleStringFieldType(sourceType)) {
|
|
3049
|
+
throw new NarsilError(
|
|
3050
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3051
|
+
`Embedding source field "${source}" must be a string type, got "${sourceType}"`,
|
|
3052
|
+
{ field: source, type: sourceType, targetField }
|
|
3053
|
+
);
|
|
3054
|
+
}
|
|
3055
|
+
}
|
|
3056
|
+
}
|
|
3057
|
+
return resolvedAdapter;
|
|
3058
|
+
}
|
|
3059
|
+
function validateRegisteredAdapter(name, adapter) {
|
|
3060
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
3061
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
|
|
3062
|
+
}
|
|
3063
|
+
if (typeof adapter?.embed !== "function") {
|
|
3064
|
+
throw new NarsilError(
|
|
3065
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3066
|
+
`Embedding adapter "${name}" must provide an embed function`,
|
|
3067
|
+
{ adapter: name }
|
|
3068
|
+
);
|
|
3069
|
+
}
|
|
3070
|
+
if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
|
|
3071
|
+
throw new NarsilError(
|
|
3072
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3073
|
+
`Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
|
|
3074
|
+
{ adapter: name, dimensions: adapter.dimensions }
|
|
3075
|
+
);
|
|
3076
|
+
}
|
|
3077
|
+
}
|
|
3078
|
+
function validateRequiredFieldsInSchema(required, schema) {
|
|
3079
|
+
if (required.length === 0) return;
|
|
3080
|
+
const flatFields = flattenSchema(schema);
|
|
3081
|
+
for (const field of required) {
|
|
3082
|
+
if (flatFields[field] === void 0) {
|
|
3083
|
+
throw new NarsilError(
|
|
3084
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
3085
|
+
`Required field "${field}" does not exist in the schema`,
|
|
3086
|
+
{ field }
|
|
3087
|
+
);
|
|
3088
|
+
}
|
|
3089
|
+
}
|
|
3090
|
+
}
|
|
3091
|
+
|
|
3092
|
+
// src/runtime/heap-statistics.browser.ts
|
|
3093
|
+
function readHeapStatistics() {
|
|
3094
|
+
return null;
|
|
3095
|
+
}
|
|
3096
|
+
|
|
3097
|
+
// src/core/partition/utils.ts
|
|
3098
|
+
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
3099
|
+
const existing = table.indexMap.get(fieldName);
|
|
3100
|
+
if (existing !== void 0) return existing;
|
|
3101
|
+
const idx = table.names.length;
|
|
3102
|
+
table.names.push(fieldName);
|
|
3103
|
+
table.indexMap.set(fieldName, idx);
|
|
3104
|
+
return idx;
|
|
3105
|
+
}
|
|
3106
|
+
var VECTOR_PATTERN2 = /^vector\[(\d+)]$/;
|
|
3107
|
+
function getNestedValue(obj, path) {
|
|
3108
|
+
const parts = path.split(".");
|
|
3109
|
+
let current = obj;
|
|
3110
|
+
for (let i = 0; i < parts.length; i++) {
|
|
3111
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
3112
|
+
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
3113
|
+
current = current[parts[i]];
|
|
3114
|
+
}
|
|
3115
|
+
return current;
|
|
3116
|
+
}
|
|
3117
|
+
function stringArraysEqual(a, b) {
|
|
3118
|
+
if (a === b) return true;
|
|
3119
|
+
if (!a || !b) return false;
|
|
3120
|
+
if (a.length !== b.length) return false;
|
|
3121
|
+
for (let i = 0; i < a.length; i++) {
|
|
3122
|
+
if (a[i] !== b[i]) return false;
|
|
3123
|
+
}
|
|
3124
|
+
return true;
|
|
3125
|
+
}
|
|
3126
|
+
function textFieldsChanged(oldDoc, newDoc, fields) {
|
|
3127
|
+
for (const { path, base } of fields) {
|
|
3128
|
+
if (!isWordIndexedBase(base)) continue;
|
|
3129
|
+
const oldVal = getNestedValue(oldDoc, path);
|
|
3130
|
+
const newVal = getNestedValue(newDoc, path);
|
|
3131
|
+
if (base === "string") {
|
|
3132
|
+
if (oldVal !== newVal) return true;
|
|
3133
|
+
} else {
|
|
3134
|
+
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
3135
|
+
}
|
|
3136
|
+
}
|
|
3137
|
+
return false;
|
|
3138
|
+
}
|
|
3139
|
+
function parseVectorDimension(fieldType) {
|
|
3140
|
+
const match = VECTOR_PATTERN2.exec(fieldType);
|
|
3141
|
+
return match ? Number.parseInt(match[1], 10) : null;
|
|
3142
|
+
}
|
|
3143
|
+
function tokenizeOptions(options) {
|
|
3144
|
+
return {
|
|
3145
|
+
stem: true,
|
|
3146
|
+
removeStopWords: true,
|
|
3147
|
+
collectSurfaces: options?.collectSurfaces === true,
|
|
3148
|
+
stopWordOverride: options?.stopWordOverride,
|
|
3149
|
+
customTokenizer: options?.customTokenizer
|
|
3150
|
+
};
|
|
3151
|
+
}
|
|
3152
|
+
function getFlatSchema(state, schema) {
|
|
3153
|
+
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
3154
|
+
state.flatSchemaCache = flattenSchema(schema);
|
|
3155
|
+
state.lastSchemaRef = schema;
|
|
3156
|
+
return state.flatSchemaCache;
|
|
3157
|
+
}
|
|
3158
|
+
|
|
3159
|
+
// src/core/projection.ts
|
|
3160
|
+
function requireFieldPathList(paths, option) {
|
|
3161
|
+
if (paths === void 0) return;
|
|
3162
|
+
if (!Array.isArray(paths)) {
|
|
3163
|
+
throw new NarsilError(
|
|
3164
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
3165
|
+
`The "document.${option}" option takes an array of field paths`,
|
|
3166
|
+
{ option: `document.${option}`, received: paths === null ? "null" : typeof paths }
|
|
3167
|
+
);
|
|
3168
|
+
}
|
|
3169
|
+
for (let position = 0; position < paths.length; position++) {
|
|
3170
|
+
const entry = paths[position];
|
|
3171
|
+
if (typeof entry === "string") continue;
|
|
3172
|
+
throw new NarsilError(
|
|
3173
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
3174
|
+
`The "document.${option}" option takes field paths as strings, and entry ${position} is not a string`,
|
|
3175
|
+
{ option: `document.${option}`, position, received: entry === null ? "null" : typeof entry }
|
|
3176
|
+
);
|
|
3177
|
+
}
|
|
3178
|
+
}
|
|
3179
|
+
function splitPaths(paths) {
|
|
3180
|
+
if (!paths) return [];
|
|
3181
|
+
const split = [];
|
|
2900
3182
|
for (const path of paths) {
|
|
2901
3183
|
const segments = path.split(".").filter((segment) => segment.length > 0);
|
|
2902
3184
|
if (segments.length === 0) continue;
|
|
@@ -7538,7 +7820,7 @@ function requireLoadedPhraseCode() {
|
|
|
7538
7820
|
}
|
|
7539
7821
|
function loadPhraseCode() {
|
|
7540
7822
|
if (loading === null) {
|
|
7541
|
-
loading = import('./phrase-module-
|
|
7823
|
+
loading = import('./phrase-module-HGVBO24W.mjs').then(
|
|
7542
7824
|
(code) => {
|
|
7543
7825
|
loaded = code;
|
|
7544
7826
|
return code;
|
|
@@ -8128,7 +8410,7 @@ function fieldLengthBound(columns, fieldIndex, internalId) {
|
|
|
8128
8410
|
function entriesCovered(bounds) {
|
|
8129
8411
|
return bounds.blockCount === 0 ? 0 : bounds.entryEnd[bounds.blockCount - 1];
|
|
8130
8412
|
}
|
|
8131
|
-
function
|
|
8413
|
+
function scanPostingBlocks(list, columns, previous, keepBlocks) {
|
|
8132
8414
|
const { docIds, termFrequencies, fieldNameIndices, length } = list;
|
|
8133
8415
|
const startEntry = previous !== null && keepBlocks > 0 ? previous.entryEnd[keepBlocks - 1] : 0;
|
|
8134
8416
|
const capacity = keepBlocks + Math.ceil((length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
|
|
@@ -8202,12 +8484,13 @@ function blockBoundsFor(list, columns) {
|
|
|
8202
8484
|
const covered = entriesCovered(existing);
|
|
8203
8485
|
if (covered === list.length) return existing;
|
|
8204
8486
|
if (list.ordered && covered < list.length) {
|
|
8205
|
-
const extended =
|
|
8487
|
+
const extended = scanPostingBlocks(list, columns, existing, Math.max(0, existing.blockCount - 1));
|
|
8206
8488
|
boundsByList.set(list, extended);
|
|
8207
8489
|
return extended;
|
|
8208
8490
|
}
|
|
8209
8491
|
}
|
|
8210
|
-
const
|
|
8492
|
+
const stored = list.storedBlockBounds?.() ?? null;
|
|
8493
|
+
const built = stored === null ? scanPostingBlocks(list, columns, null, 0) : { ...stored, structureRevision: list.structureRevision };
|
|
8211
8494
|
boundsByList.set(list, built);
|
|
8212
8495
|
return built;
|
|
8213
8496
|
}
|
|
@@ -8400,33 +8683,33 @@ function boostsKeepScoresNonNegative(fields) {
|
|
|
8400
8683
|
}
|
|
8401
8684
|
return true;
|
|
8402
8685
|
}
|
|
8403
|
-
function samePairUpperBound(
|
|
8404
|
-
if (
|
|
8405
|
-
const bounds = blockBoundsFor(list,
|
|
8686
|
+
function samePairUpperBound(scan, list, maxBoost, maxAverageLength) {
|
|
8687
|
+
if (scan.scoresAreZero) return 0;
|
|
8688
|
+
const bounds = blockBoundsFor(list, scan.fields.lengthColumns);
|
|
8406
8689
|
let best = 0;
|
|
8407
8690
|
for (let block = 0; block < bounds.blockCount; block++) {
|
|
8408
8691
|
const bestEntry = computeBM25WithIDF(
|
|
8409
8692
|
bounds.maxTermFrequency[block],
|
|
8410
|
-
|
|
8693
|
+
scan.idf,
|
|
8411
8694
|
bounds.minFieldLength[block],
|
|
8412
8695
|
maxAverageLength,
|
|
8413
|
-
|
|
8414
|
-
|
|
8696
|
+
scan.k1,
|
|
8697
|
+
scan.b
|
|
8415
8698
|
);
|
|
8416
8699
|
const blockBound = bestEntry * maxBoost * bounds.maxEntriesPerDocument[block];
|
|
8417
8700
|
if (blockBound > best) best = blockBound;
|
|
8418
8701
|
}
|
|
8419
8702
|
return best;
|
|
8420
8703
|
}
|
|
8421
|
-
function fieldPairUpperBound(
|
|
8422
|
-
if (
|
|
8423
|
-
const bounds = blockBoundsFor(list,
|
|
8704
|
+
function fieldPairUpperBound(scan, list, maxBoost, maxAverageLength) {
|
|
8705
|
+
if (scan.scoresAreZero) return 0;
|
|
8706
|
+
const bounds = blockBoundsFor(list, scan.fields.lengthColumns);
|
|
8424
8707
|
let best = 0;
|
|
8425
8708
|
for (let block = 0; block < bounds.blockCount; block++) {
|
|
8426
8709
|
const bestEntry = blockEntryBound(
|
|
8427
|
-
|
|
8710
|
+
scan.fields,
|
|
8428
8711
|
bounds.maxTermFrequency[block],
|
|
8429
|
-
|
|
8712
|
+
scan.idf,
|
|
8430
8713
|
bounds.minFieldLength[block],
|
|
8431
8714
|
maxAverageLength
|
|
8432
8715
|
);
|
|
@@ -8435,22 +8718,22 @@ function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
|
8435
8718
|
}
|
|
8436
8719
|
return best;
|
|
8437
8720
|
}
|
|
8438
|
-
function samePairEntryScore(
|
|
8439
|
-
const { fields, columns } =
|
|
8721
|
+
function samePairEntryScore(scan, entry, internalId) {
|
|
8722
|
+
const { fields, columns } = scan;
|
|
8440
8723
|
const fieldIndex = columns.fieldNameIndices[entry];
|
|
8441
8724
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
8442
8725
|
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
8443
|
-
const bm25 =
|
|
8726
|
+
const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan.idf, fieldLength, averageLength, scan.k1, scan.b);
|
|
8444
8727
|
return bm25 * fields.boosts[fieldIndex];
|
|
8445
8728
|
}
|
|
8446
|
-
function fieldPairEntryScore(
|
|
8447
|
-
const { fields, columns } =
|
|
8729
|
+
function fieldPairEntryScore(scan, entry, internalId) {
|
|
8730
|
+
const { fields, columns } = scan;
|
|
8448
8731
|
const fieldIndex = columns.fieldNameIndices[entry];
|
|
8449
8732
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
8450
8733
|
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
8451
|
-
const bm25 =
|
|
8734
|
+
const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8452
8735
|
columns.termFrequencies[entry],
|
|
8453
|
-
|
|
8736
|
+
scan.idf,
|
|
8454
8737
|
fieldLength,
|
|
8455
8738
|
averageLength,
|
|
8456
8739
|
fields.k1s[fieldIndex],
|
|
@@ -8458,48 +8741,48 @@ function fieldPairEntryScore(scan2, entry, internalId) {
|
|
|
8458
8741
|
);
|
|
8459
8742
|
return bm25 * fields.boosts[fieldIndex];
|
|
8460
8743
|
}
|
|
8461
|
-
function samePairScoreEveryDocument(
|
|
8462
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
8463
|
-
const searchable =
|
|
8744
|
+
function samePairScoreEveryDocument(scan, buffer) {
|
|
8745
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8746
|
+
const searchable = scan.fields.searchable;
|
|
8464
8747
|
for (let entry = 0; entry < count; entry++) {
|
|
8465
8748
|
const internalId = docIds[entry];
|
|
8466
8749
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8467
8750
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8468
|
-
addScore(buffer, internalId, samePairEntryScore(
|
|
8751
|
+
addScore(buffer, internalId, samePairEntryScore(scan, entry, internalId));
|
|
8469
8752
|
}
|
|
8470
8753
|
}
|
|
8471
|
-
function fieldPairScoreEveryDocument(
|
|
8472
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
8473
|
-
const searchable =
|
|
8754
|
+
function fieldPairScoreEveryDocument(scan, buffer) {
|
|
8755
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8756
|
+
const searchable = scan.fields.searchable;
|
|
8474
8757
|
for (let entry = 0; entry < count; entry++) {
|
|
8475
8758
|
const internalId = docIds[entry];
|
|
8476
8759
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8477
8760
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8478
|
-
addScore(buffer, internalId, fieldPairEntryScore(
|
|
8761
|
+
addScore(buffer, internalId, fieldPairEntryScore(scan, entry, internalId));
|
|
8479
8762
|
}
|
|
8480
8763
|
}
|
|
8481
|
-
function samePairScoreKnownDocuments(
|
|
8482
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
8483
|
-
const searchable =
|
|
8764
|
+
function samePairScoreKnownDocuments(scan, buffer) {
|
|
8765
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8766
|
+
const searchable = scan.fields.searchable;
|
|
8484
8767
|
let counted = 0;
|
|
8485
8768
|
for (let entry = 0; entry < count; entry++) {
|
|
8486
8769
|
const internalId = docIds[entry];
|
|
8487
8770
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8488
8771
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8489
|
-
if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(
|
|
8772
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan, entry, internalId));
|
|
8490
8773
|
else if (markCounted(buffer, internalId)) counted++;
|
|
8491
8774
|
}
|
|
8492
8775
|
return counted;
|
|
8493
8776
|
}
|
|
8494
|
-
function fieldPairScoreKnownDocuments(
|
|
8495
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
8496
|
-
const searchable =
|
|
8777
|
+
function fieldPairScoreKnownDocuments(scan, buffer) {
|
|
8778
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8779
|
+
const searchable = scan.fields.searchable;
|
|
8497
8780
|
let counted = 0;
|
|
8498
8781
|
for (let entry = 0; entry < count; entry++) {
|
|
8499
8782
|
const internalId = docIds[entry];
|
|
8500
8783
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8501
8784
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8502
|
-
if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(
|
|
8785
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan, entry, internalId));
|
|
8503
8786
|
else if (markCounted(buffer, internalId)) counted++;
|
|
8504
8787
|
}
|
|
8505
8788
|
return counted;
|
|
@@ -8516,24 +8799,24 @@ function firstEntryOf(docIds, count, internalId) {
|
|
|
8516
8799
|
}
|
|
8517
8800
|
function samePairQueryOrderScore(scans, internalId) {
|
|
8518
8801
|
let score = 0;
|
|
8519
|
-
for (const
|
|
8520
|
-
const { docIds, deletedDocs, hasDeleted, count } =
|
|
8802
|
+
for (const scan of scans) {
|
|
8803
|
+
const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8521
8804
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8522
8805
|
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
8523
|
-
if (
|
|
8524
|
-
score += samePairEntryScore(
|
|
8806
|
+
if (scan.fields.searchable[scan.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
8807
|
+
score += samePairEntryScore(scan, entry, internalId);
|
|
8525
8808
|
}
|
|
8526
8809
|
}
|
|
8527
8810
|
return score;
|
|
8528
8811
|
}
|
|
8529
8812
|
function fieldPairQueryOrderScore(scans, internalId) {
|
|
8530
8813
|
let score = 0;
|
|
8531
|
-
for (const
|
|
8532
|
-
const { docIds, deletedDocs, hasDeleted, count } =
|
|
8814
|
+
for (const scan of scans) {
|
|
8815
|
+
const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
|
|
8533
8816
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8534
8817
|
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
8535
|
-
if (
|
|
8536
|
-
score += fieldPairEntryScore(
|
|
8818
|
+
if (scan.fields.searchable[scan.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
8819
|
+
score += fieldPairEntryScore(scan, entry, internalId);
|
|
8537
8820
|
}
|
|
8538
8821
|
}
|
|
8539
8822
|
return score;
|
|
@@ -8613,7 +8896,7 @@ function multiTermTopK(request) {
|
|
|
8613
8896
|
}
|
|
8614
8897
|
const maxBoost = bestSearchable(fields.searchable, fields.boosts);
|
|
8615
8898
|
const maxAverageLength = bestSearchable(fields.searchable, fields.averageLengths);
|
|
8616
|
-
const upperBounds = scans.map((
|
|
8899
|
+
const upperBounds = scans.map((scan, term) => scorer.upperBound(scan, terms[term].list, maxBoost, maxAverageLength));
|
|
8617
8900
|
const order = scans.map((_, term) => term).sort((left, right) => upperBounds[right] - upperBounds[left]);
|
|
8618
8901
|
const remainingBound = new Float64Array(order.length + 1);
|
|
8619
8902
|
for (let position2 = order.length - 1; position2 >= 0; position2--) {
|
|
@@ -9822,512 +10105,46 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
9822
10105
|
return sortedPageOf(state, request);
|
|
9823
10106
|
},
|
|
9824
10107
|
sortValues(docId, fields, fieldTypes, modes) {
|
|
9825
|
-
return sortValuesOf(state, docId, fields, fieldTypes, modes);
|
|
9826
|
-
},
|
|
9827
|
-
applyFilters(filters, schema, meter) {
|
|
9828
|
-
return applyPartitionFilters(state, filters, schema, meter);
|
|
9829
|
-
},
|
|
9830
|
-
applyFiltersBitset(filters, schema, meter) {
|
|
9831
|
-
return applyPartitionFiltersBitset(state, filters, schema, meter);
|
|
9832
|
-
},
|
|
9833
|
-
filterMatches(filters, schema, meter) {
|
|
9834
|
-
return partitionFilterMatches(state, filters, schema, meter);
|
|
9835
|
-
},
|
|
9836
|
-
computeFacets(matched, config, schema) {
|
|
9837
|
-
return computeFacets(state, matched, config, schema);
|
|
9838
|
-
},
|
|
9839
|
-
suggestTerms(surfacePrefix, stemmedPrefix, limit) {
|
|
9840
|
-
return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
|
|
9841
|
-
},
|
|
9842
|
-
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
9843
|
-
return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
|
|
9844
|
-
},
|
|
9845
|
-
expandPartialWords(analysis, requests) {
|
|
9846
|
-
return expandPartialWords([state], analysis, requests);
|
|
9847
|
-
},
|
|
9848
|
-
preparePartialWords(analysis) {
|
|
9849
|
-
expandPartialWords([state], analysis, []);
|
|
9850
|
-
},
|
|
9851
|
-
serialize(indexName, totalPartitions, language, schema) {
|
|
9852
|
-
return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
|
|
9853
|
-
},
|
|
9854
|
-
deserialize(data, schema) {
|
|
9855
|
-
deserializePartition(state, data, clearAll, schema);
|
|
9856
|
-
}
|
|
9857
|
-
};
|
|
9858
|
-
statesByPartition.set(partition, state);
|
|
9859
|
-
return partition;
|
|
9860
|
-
}
|
|
9861
|
-
|
|
9862
|
-
// src/engine/orchestration/turn.ts
|
|
9863
|
-
function afterCurrentTurn(callback) {
|
|
9864
|
-
if (typeof setImmediate === "function") {
|
|
9865
|
-
setImmediate(callback);
|
|
9866
|
-
return;
|
|
9867
|
-
}
|
|
9868
|
-
setTimeout(callback, 0);
|
|
9869
|
-
}
|
|
9870
|
-
function yieldToEventLoop() {
|
|
9871
|
-
return new Promise((resolve) => afterCurrentTurn(resolve));
|
|
9872
|
-
}
|
|
9873
|
-
|
|
9874
|
-
// src/partitioning/constants.ts
|
|
9875
|
-
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
9876
|
-
var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
|
|
9877
|
-
var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
|
|
9878
|
-
var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
9879
|
-
|
|
9880
|
-
// src/partitioning/rebalancer.ts
|
|
9881
|
-
function createRebalancer() {
|
|
9882
|
-
const activeManagers = /* @__PURE__ */ new WeakSet();
|
|
9883
|
-
let activeCount = 0;
|
|
9884
|
-
async function rebalance(manager, newPartitionCount, router, onProgress) {
|
|
9885
|
-
if (activeManagers.has(manager)) {
|
|
9886
|
-
throw new NarsilError(
|
|
9887
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
9888
|
-
"A rebalance operation is already in progress"
|
|
9889
|
-
);
|
|
9890
|
-
}
|
|
9891
|
-
if (newPartitionCount <= 0) {
|
|
9892
|
-
throw new NarsilError(
|
|
9893
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
9894
|
-
`New partition count must be greater than 0, received ${newPartitionCount}`,
|
|
9895
|
-
{ newPartitionCount }
|
|
9896
|
-
);
|
|
9897
|
-
}
|
|
9898
|
-
if (newPartitionCount === manager.partitionCount) {
|
|
9899
|
-
throw new NarsilError(
|
|
9900
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
9901
|
-
`New partition count (${newPartitionCount}) is the same as the current count`,
|
|
9902
|
-
{ newPartitionCount, currentPartitionCount: manager.partitionCount }
|
|
9903
|
-
);
|
|
9904
|
-
}
|
|
9905
|
-
activeManagers.add(manager);
|
|
9906
|
-
activeCount++;
|
|
9907
|
-
try {
|
|
9908
|
-
const currentPartitions = manager.getAllPartitions();
|
|
9909
|
-
let documentsTotal = 0;
|
|
9910
|
-
for (const partition of currentPartitions) {
|
|
9911
|
-
documentsTotal += partition.count();
|
|
9912
|
-
}
|
|
9913
|
-
onProgress?.({
|
|
9914
|
-
phase: "scanning",
|
|
9915
|
-
documentsProcessed: 0,
|
|
9916
|
-
documentsTotal
|
|
9917
|
-
});
|
|
9918
|
-
const newPartitions = [];
|
|
9919
|
-
for (let i = 0; i < newPartitionCount; i++) {
|
|
9920
|
-
newPartitions.push(createPartitionIndex(i, manager.config.trackPositions ?? true));
|
|
9921
|
-
}
|
|
9922
|
-
let documentsProcessed = 0;
|
|
9923
|
-
let chunkFill = 0;
|
|
9924
|
-
const insertOptions = {
|
|
9925
|
-
validate: false,
|
|
9926
|
-
skipClone: true,
|
|
9927
|
-
stopWordOverride: manager.analysis.stopWords,
|
|
9928
|
-
customTokenizer: manager.analysis.customTokenizer,
|
|
9929
|
-
collectSurfaces: manager.config.surfaceForms !== false
|
|
9930
|
-
};
|
|
9931
|
-
for (const partition of currentPartitions) {
|
|
9932
|
-
const docIds = [...partition.docIds()];
|
|
9933
|
-
for (const docId of docIds) {
|
|
9934
|
-
const document = partition.getRef(docId);
|
|
9935
|
-
if (!document) continue;
|
|
9936
|
-
const targetPartitionId = router.route(docId, newPartitionCount);
|
|
9937
|
-
const targetPartition = newPartitions[targetPartitionId];
|
|
9938
|
-
if (!targetPartition.has(docId)) {
|
|
9939
|
-
targetPartition.insert(docId, document, manager.schema, manager.language, insertOptions);
|
|
9940
|
-
documentsProcessed++;
|
|
9941
|
-
}
|
|
9942
|
-
chunkFill++;
|
|
9943
|
-
if (chunkFill >= REBALANCE_CHUNK_SIZE) {
|
|
9944
|
-
chunkFill = 0;
|
|
9945
|
-
onProgress?.({
|
|
9946
|
-
phase: "moving",
|
|
9947
|
-
documentsProcessed,
|
|
9948
|
-
documentsTotal
|
|
9949
|
-
});
|
|
9950
|
-
await yieldToEventLoop();
|
|
9951
|
-
}
|
|
9952
|
-
}
|
|
9953
|
-
}
|
|
9954
|
-
if (chunkFill > 0) {
|
|
9955
|
-
onProgress?.({
|
|
9956
|
-
phase: "moving",
|
|
9957
|
-
documentsProcessed,
|
|
9958
|
-
documentsTotal
|
|
9959
|
-
});
|
|
9960
|
-
await yieldToEventLoop();
|
|
9961
|
-
}
|
|
9962
|
-
onProgress?.({
|
|
9963
|
-
phase: "swapping",
|
|
9964
|
-
documentsProcessed,
|
|
9965
|
-
documentsTotal
|
|
9966
|
-
});
|
|
9967
|
-
manager.setPartitions(newPartitions);
|
|
9968
|
-
onProgress?.({
|
|
9969
|
-
phase: "complete",
|
|
9970
|
-
documentsProcessed,
|
|
9971
|
-
documentsTotal
|
|
9972
|
-
});
|
|
9973
|
-
} finally {
|
|
9974
|
-
activeManagers.delete(manager);
|
|
9975
|
-
activeCount--;
|
|
9976
|
-
}
|
|
9977
|
-
}
|
|
9978
|
-
return {
|
|
9979
|
-
rebalance,
|
|
9980
|
-
isRebalancing() {
|
|
9981
|
-
return activeCount > 0;
|
|
9982
|
-
}
|
|
9983
|
-
};
|
|
9984
|
-
}
|
|
9985
|
-
|
|
9986
|
-
// src/core/hash.ts
|
|
9987
|
-
var FNV_OFFSET_BASIS = 2166136261;
|
|
9988
|
-
var FNV_PRIME = 16777619;
|
|
9989
|
-
var encoder = new TextEncoder();
|
|
9990
|
-
function fnv1aBytes(bytes) {
|
|
9991
|
-
return fnv1aContinue(FNV_OFFSET_BASIS, bytes, bytes.length) >>> 0;
|
|
9992
|
-
}
|
|
9993
|
-
function fnv1aContinue(start, bytes, length) {
|
|
9994
|
-
let hash = start;
|
|
9995
|
-
for (let i = 0; i < length; i++) {
|
|
9996
|
-
hash = Math.imul(hash ^ bytes[i], FNV_PRIME);
|
|
9997
|
-
}
|
|
9998
|
-
return hash;
|
|
9999
|
-
}
|
|
10000
|
-
var scratchNumber = new Float64Array(1);
|
|
10001
|
-
var scratchBytes = new Uint8Array(scratchNumber.buffer);
|
|
10002
|
-
var HOST_IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
10003
|
-
function fnv1aContinueTaggedNumbers(start, tag, numbers) {
|
|
10004
|
-
let hash = start;
|
|
10005
|
-
const b = scratchBytes;
|
|
10006
|
-
for (let i = 0; i < numbers.length; i++) {
|
|
10007
|
-
scratchNumber[0] = Math.fround(numbers[i]);
|
|
10008
|
-
hash = Math.imul(hash ^ tag, FNV_PRIME);
|
|
10009
|
-
if (HOST_IS_LITTLE_ENDIAN) {
|
|
10010
|
-
hash = Math.imul(hash ^ b[7], FNV_PRIME);
|
|
10011
|
-
hash = Math.imul(hash ^ b[6], FNV_PRIME);
|
|
10012
|
-
hash = Math.imul(hash ^ b[5], FNV_PRIME);
|
|
10013
|
-
hash = Math.imul(hash ^ b[4], FNV_PRIME);
|
|
10014
|
-
hash = Math.imul(hash ^ b[3], FNV_PRIME);
|
|
10015
|
-
hash = Math.imul(hash ^ b[2], FNV_PRIME);
|
|
10016
|
-
hash = Math.imul(hash ^ b[1], FNV_PRIME);
|
|
10017
|
-
hash = Math.imul(hash ^ b[0], FNV_PRIME);
|
|
10018
|
-
} else {
|
|
10019
|
-
for (let at = 0; at < 8; at++) hash = Math.imul(hash ^ b[at], FNV_PRIME);
|
|
10020
|
-
}
|
|
10021
|
-
}
|
|
10022
|
-
return hash;
|
|
10023
|
-
}
|
|
10024
|
-
function fnv1a(input) {
|
|
10025
|
-
return fnv1aBytes(encoder.encode(input));
|
|
10026
|
-
}
|
|
10027
|
-
|
|
10028
|
-
// src/partitioning/router.ts
|
|
10029
|
-
function createPartitionRouter() {
|
|
10030
|
-
function route(docId, partitionCount) {
|
|
10031
|
-
if (partitionCount <= 0) {
|
|
10032
|
-
throw new NarsilError(
|
|
10033
|
-
ErrorCodes.INDEX_NOT_FOUND,
|
|
10034
|
-
`Partition count must be greater than 0, received ${partitionCount}`,
|
|
10035
|
-
{ partitionCount }
|
|
10036
|
-
);
|
|
10037
|
-
}
|
|
10038
|
-
return fnv1a(docId) % partitionCount;
|
|
10039
|
-
}
|
|
10040
|
-
function routeBatch(docIds, partitionCount) {
|
|
10041
|
-
const groups = /* @__PURE__ */ new Map();
|
|
10042
|
-
for (let i = 0; i < docIds.length; i++) {
|
|
10043
|
-
const pid = route(docIds[i], partitionCount);
|
|
10044
|
-
let group = groups.get(pid);
|
|
10045
|
-
if (!group) {
|
|
10046
|
-
group = [];
|
|
10047
|
-
groups.set(pid, group);
|
|
10048
|
-
}
|
|
10049
|
-
group.push(docIds[i]);
|
|
10050
|
-
}
|
|
10051
|
-
return groups;
|
|
10052
|
-
}
|
|
10053
|
-
return { route, routeBatch };
|
|
10054
|
-
}
|
|
10055
|
-
|
|
10056
|
-
// src/plugins/registry.ts
|
|
10057
|
-
var WRITE_REJECTING_HOOKS = /* @__PURE__ */ new Set(["beforeInsert", "beforeUpdate", "beforeRemove"]);
|
|
10058
|
-
function isThenable(value) {
|
|
10059
|
-
return value !== null && typeof value === "object" && typeof value.then === "function";
|
|
10060
|
-
}
|
|
10061
|
-
function thrownText(err, plugin, hookName) {
|
|
10062
|
-
if (err instanceof Error) return err.message;
|
|
10063
|
-
try {
|
|
10064
|
-
return String(err);
|
|
10065
|
-
} catch {
|
|
10066
|
-
return `The ${hookName} hook of plugin "${plugin}" rejects the write`;
|
|
10067
|
-
}
|
|
10068
|
-
}
|
|
10069
|
-
function asWriteRejection(err, plugin, hookName) {
|
|
10070
|
-
if (err instanceof NarsilError) return err;
|
|
10071
|
-
return new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, thrownText(err, plugin, hookName), {
|
|
10072
|
-
plugin,
|
|
10073
|
-
hook: hookName
|
|
10074
|
-
});
|
|
10075
|
-
}
|
|
10076
|
-
function invoke(hook, hookName, context) {
|
|
10077
|
-
const rejectsWrites = WRITE_REJECTING_HOOKS.has(hookName);
|
|
10078
|
-
let result;
|
|
10079
|
-
try {
|
|
10080
|
-
result = hook.run(context);
|
|
10081
|
-
} catch (err) {
|
|
10082
|
-
throw rejectsWrites ? asWriteRejection(err, hook.plugin, hookName) : err;
|
|
10083
|
-
}
|
|
10084
|
-
if (!isThenable(result)) return null;
|
|
10085
|
-
if (!rejectsWrites) return result;
|
|
10086
|
-
return Promise.resolve(result).catch((err) => {
|
|
10087
|
-
throw asWriteRejection(err, hook.plugin, hookName);
|
|
10088
|
-
});
|
|
10089
|
-
}
|
|
10090
|
-
async function continueAsync(pending, hooks, startIndex, hookName, context) {
|
|
10091
|
-
await pending;
|
|
10092
|
-
for (let i = startIndex; i < hooks.length; i++) {
|
|
10093
|
-
await invoke(hooks[i], hookName, context);
|
|
10094
|
-
}
|
|
10095
|
-
}
|
|
10096
|
-
function createPluginRegistry() {
|
|
10097
|
-
const plugins = [];
|
|
10098
|
-
return {
|
|
10099
|
-
register(plugin) {
|
|
10100
|
-
plugins.push(plugin);
|
|
10101
|
-
},
|
|
10102
|
-
hasHooks(hookName) {
|
|
10103
|
-
return plugins.some((p) => typeof p[hookName] === "function");
|
|
10104
|
-
},
|
|
10105
|
-
runHook(hookName, context) {
|
|
10106
|
-
const hooks = [];
|
|
10107
|
-
for (const plugin of plugins) {
|
|
10108
|
-
const hook = plugin[hookName];
|
|
10109
|
-
if (typeof hook === "function") {
|
|
10110
|
-
hooks.push({ plugin: plugin.name, run: hook.bind(plugin) });
|
|
10111
|
-
}
|
|
10112
|
-
}
|
|
10113
|
-
if (hooks.length === 0) return;
|
|
10114
|
-
for (let i = 0; i < hooks.length; i++) {
|
|
10115
|
-
const pending = invoke(hooks[i], hookName, context);
|
|
10116
|
-
if (pending !== null) {
|
|
10117
|
-
return continueAsync(pending, hooks, i + 1, hookName, context);
|
|
10118
|
-
}
|
|
10119
|
-
}
|
|
10120
|
-
}
|
|
10121
|
-
};
|
|
10122
|
-
}
|
|
10123
|
-
|
|
10124
|
-
// src/rerank/registry.ts
|
|
10125
|
-
function validateRerankAdapter(name, adapter) {
|
|
10126
|
-
if (typeof name !== "string" || name.trim().length === 0) {
|
|
10127
|
-
throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, "A rerank adapter needs a non-empty string name");
|
|
10128
|
-
}
|
|
10129
|
-
const rerank = typeof adapter === "object" && adapter !== null ? adapter.rerank : void 0;
|
|
10130
|
-
if (typeof rerank !== "function") {
|
|
10131
|
-
throw new NarsilError(ErrorCodes.RERANK_CONFIG_INVALID, `Rerank adapter "${name}" must provide a rerank function`, {
|
|
10132
|
-
adapter: name
|
|
10133
|
-
});
|
|
10134
|
-
}
|
|
10135
|
-
}
|
|
10136
|
-
function registerEngineRerankAdapter(core, name, adapter) {
|
|
10137
|
-
core.guardShutdown();
|
|
10138
|
-
validateRerankAdapter(name, adapter);
|
|
10139
|
-
core.rerankAdapters.set(name, adapter);
|
|
10140
|
-
}
|
|
10141
|
-
|
|
10142
|
-
// src/schema/embedding-validator.ts
|
|
10143
|
-
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
10144
|
-
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
10145
|
-
const fieldEntries = Object.keys(embeddingConfig.fields);
|
|
10146
|
-
if (fieldEntries.length === 0) {
|
|
10147
|
-
throw new NarsilError(
|
|
10148
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10149
|
-
"Embedding config must define at least one field mapping"
|
|
10150
|
-
);
|
|
10151
|
-
}
|
|
10152
|
-
if (typeof embeddingConfig.adapter === "string") {
|
|
10153
|
-
throw new NarsilError(
|
|
10154
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10155
|
-
`Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
|
|
10156
|
-
{ adapter: embeddingConfig.adapter }
|
|
10157
|
-
);
|
|
10158
|
-
}
|
|
10159
|
-
const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
|
|
10160
|
-
if (!resolvedAdapter) {
|
|
10161
|
-
throw new NarsilError(
|
|
10162
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10163
|
-
"Embedding config requires an adapter (set on the index or the instance)"
|
|
10164
|
-
);
|
|
10165
|
-
}
|
|
10166
|
-
if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
|
|
10167
|
-
throw new NarsilError(
|
|
10168
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10169
|
-
`Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
|
|
10170
|
-
);
|
|
10171
|
-
}
|
|
10172
|
-
const flatFields = flattenSchema(schema);
|
|
10173
|
-
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
10174
|
-
const targetType = flatFields[targetField];
|
|
10175
|
-
if (targetType === void 0) {
|
|
10176
|
-
throw new NarsilError(
|
|
10177
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10178
|
-
`Embedding target field "${targetField}" does not exist in the schema`,
|
|
10179
|
-
{ field: targetField }
|
|
10180
|
-
);
|
|
10181
|
-
}
|
|
10182
|
-
const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
|
|
10183
|
-
if (!vectorMatch) {
|
|
10184
|
-
throw new NarsilError(
|
|
10185
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10186
|
-
`Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
|
|
10187
|
-
{ field: targetField, type: targetType }
|
|
10188
|
-
);
|
|
10189
|
-
}
|
|
10190
|
-
const schemaDimension = Number.parseInt(vectorMatch[1], 10);
|
|
10191
|
-
if (schemaDimension !== resolvedAdapter.dimensions) {
|
|
10192
|
-
throw new NarsilError(
|
|
10193
|
-
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
10194
|
-
`Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
|
|
10195
|
-
{ field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
|
|
10196
|
-
);
|
|
10197
|
-
}
|
|
10198
|
-
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
10199
|
-
if (sources.length === 0) {
|
|
10200
|
-
throw new NarsilError(
|
|
10201
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10202
|
-
`Embedding target field "${targetField}" must have at least one source field`,
|
|
10203
|
-
{ field: targetField }
|
|
10204
|
-
);
|
|
10205
|
-
}
|
|
10206
|
-
for (const source of sources) {
|
|
10207
|
-
const sourceType = flatFields[source];
|
|
10208
|
-
if (sourceType === void 0) {
|
|
10209
|
-
throw new NarsilError(
|
|
10210
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10211
|
-
`Embedding source field "${source}" does not exist in the schema`,
|
|
10212
|
-
{ field: source, targetField }
|
|
10213
|
-
);
|
|
10214
|
-
}
|
|
10215
|
-
if (!isSingleStringFieldType(sourceType)) {
|
|
10216
|
-
throw new NarsilError(
|
|
10217
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10218
|
-
`Embedding source field "${source}" must be a string type, got "${sourceType}"`,
|
|
10219
|
-
{ field: source, type: sourceType, targetField }
|
|
10220
|
-
);
|
|
10221
|
-
}
|
|
10222
|
-
}
|
|
10223
|
-
}
|
|
10224
|
-
return resolvedAdapter;
|
|
10225
|
-
}
|
|
10226
|
-
function validateRegisteredAdapter(name, adapter) {
|
|
10227
|
-
if (typeof name !== "string" || name.trim().length === 0) {
|
|
10228
|
-
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
|
|
10229
|
-
}
|
|
10230
|
-
if (typeof adapter?.embed !== "function") {
|
|
10231
|
-
throw new NarsilError(
|
|
10232
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10233
|
-
`Embedding adapter "${name}" must provide an embed function`,
|
|
10234
|
-
{ adapter: name }
|
|
10235
|
-
);
|
|
10236
|
-
}
|
|
10237
|
-
if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
|
|
10238
|
-
throw new NarsilError(
|
|
10239
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10240
|
-
`Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
|
|
10241
|
-
{ adapter: name, dimensions: adapter.dimensions }
|
|
10242
|
-
);
|
|
10243
|
-
}
|
|
10244
|
-
}
|
|
10245
|
-
function validateRequiredFieldsInSchema(required, schema) {
|
|
10246
|
-
if (required.length === 0) return;
|
|
10247
|
-
const flatFields = flattenSchema(schema);
|
|
10248
|
-
for (const field of required) {
|
|
10249
|
-
if (flatFields[field] === void 0) {
|
|
10250
|
-
throw new NarsilError(
|
|
10251
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
10252
|
-
`Required field "${field}" does not exist in the schema`,
|
|
10253
|
-
{ field }
|
|
10254
|
-
);
|
|
10255
|
-
}
|
|
10256
|
-
}
|
|
10257
|
-
}
|
|
10258
|
-
|
|
10259
|
-
// src/runtime/heap-statistics.browser.ts
|
|
10260
|
-
function readHeapStatistics() {
|
|
10261
|
-
return null;
|
|
10262
|
-
}
|
|
10263
|
-
|
|
10264
|
-
// src/search/oversample.ts
|
|
10265
|
-
function oversampledShardSize(size) {
|
|
10266
|
-
return Math.ceil(size * 1.5) + 10;
|
|
10267
|
-
}
|
|
10268
|
-
|
|
10269
|
-
// src/search/facets.ts
|
|
10270
|
-
function keptValueCount(fieldConfig) {
|
|
10271
|
-
return keptFacetValues(fieldConfig?.limit);
|
|
10272
|
-
}
|
|
10273
|
-
function everyValueFacetConfig(config) {
|
|
10274
|
-
const widened = {};
|
|
10275
|
-
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
10276
|
-
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
10277
|
-
}
|
|
10278
|
-
return widened;
|
|
10279
|
-
}
|
|
10280
|
-
function oversampledFacetConfig(config) {
|
|
10281
|
-
const widened = {};
|
|
10282
|
-
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
10283
|
-
if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
|
|
10284
|
-
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
10285
|
-
continue;
|
|
10286
|
-
}
|
|
10287
|
-
const kept = keptValueCount(fieldConfig);
|
|
10288
|
-
widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
|
|
10289
|
-
}
|
|
10290
|
-
return widened;
|
|
10291
|
-
}
|
|
10292
|
-
function mergeFacets(partitionFacets, config) {
|
|
10293
|
-
const merged = /* @__PURE__ */ new Map();
|
|
10294
|
-
const bounds = /* @__PURE__ */ new Map();
|
|
10295
|
-
for (const partition of partitionFacets) {
|
|
10296
|
-
for (const [field, facetResult] of Object.entries(partition)) {
|
|
10297
|
-
let fieldMap = merged.get(field);
|
|
10298
|
-
if (!fieldMap) {
|
|
10299
|
-
fieldMap = /* @__PURE__ */ new Map();
|
|
10300
|
-
merged.set(field, fieldMap);
|
|
10301
|
-
}
|
|
10302
|
-
bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
|
|
10303
|
-
for (const [value, count] of Object.entries(facetResult.values)) {
|
|
10304
|
-
fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
|
|
10305
|
-
}
|
|
10306
|
-
}
|
|
10307
|
-
}
|
|
10308
|
-
const result = {};
|
|
10309
|
-
for (const [field, valueMap] of merged) {
|
|
10310
|
-
const fieldConfig = config[field];
|
|
10311
|
-
const ascending = fieldConfig?.sort === "asc";
|
|
10312
|
-
const ordered = Array.from(valueMap.entries());
|
|
10313
|
-
ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
10314
|
-
const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
|
|
10315
|
-
let largestLeftOut = 0;
|
|
10316
|
-
for (let index = kept; index < ordered.length; index++) {
|
|
10317
|
-
if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
|
|
10318
|
-
}
|
|
10319
|
-
const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
|
|
10320
|
-
const values = {};
|
|
10321
|
-
for (let index = 0; index < kept; index++) {
|
|
10322
|
-
values[ordered[index][0]] = ordered[index][1];
|
|
10108
|
+
return sortValuesOf(state, docId, fields, fieldTypes, modes);
|
|
10109
|
+
},
|
|
10110
|
+
applyFilters(filters, schema, meter) {
|
|
10111
|
+
return applyPartitionFilters(state, filters, schema, meter);
|
|
10112
|
+
},
|
|
10113
|
+
applyFiltersBitset(filters, schema, meter) {
|
|
10114
|
+
return applyPartitionFiltersBitset(state, filters, schema, meter);
|
|
10115
|
+
},
|
|
10116
|
+
filterMatches(filters, schema, meter) {
|
|
10117
|
+
return partitionFilterMatches(state, filters, schema, meter);
|
|
10118
|
+
},
|
|
10119
|
+
computeFacets(matched, config, schema) {
|
|
10120
|
+
return computeFacets(state, matched, config, schema);
|
|
10121
|
+
},
|
|
10122
|
+
suggestTerms(surfacePrefix, stemmedPrefix, limit) {
|
|
10123
|
+
return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
|
|
10124
|
+
},
|
|
10125
|
+
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
10126
|
+
return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
|
|
10127
|
+
},
|
|
10128
|
+
expandPartialWords(analysis, requests) {
|
|
10129
|
+
return expandPartialWords([state], analysis, requests);
|
|
10130
|
+
},
|
|
10131
|
+
preparePartialWords(analysis) {
|
|
10132
|
+
expandPartialWords([state], analysis, []);
|
|
10133
|
+
},
|
|
10134
|
+
serialize(indexName, totalPartitions, language, schema) {
|
|
10135
|
+
return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
|
|
10136
|
+
},
|
|
10137
|
+
deserialize(data, schema) {
|
|
10138
|
+
deserializePartition(state, data, clearAll, schema);
|
|
10323
10139
|
}
|
|
10324
|
-
|
|
10325
|
-
|
|
10326
|
-
return
|
|
10140
|
+
};
|
|
10141
|
+
statesByPartition.set(partition, state);
|
|
10142
|
+
return partition;
|
|
10327
10143
|
}
|
|
10328
10144
|
|
|
10329
10145
|
// src/core/partition/frozen/memory.ts
|
|
10330
10146
|
function frozenSegmentBytes(segment) {
|
|
10147
|
+
if (segment.paged !== null) return segment.paged.heldBytes(segment);
|
|
10331
10148
|
const arrays = segment.arrays;
|
|
10332
10149
|
let bytes = arrays.tokenTable.bytes + arrays.idTable.bytes;
|
|
10333
10150
|
bytes += arrays.postingOffsets.byteLength;
|
|
@@ -10439,6 +10256,71 @@ function compositeSortedDocIds(frozen, live) {
|
|
|
10439
10256
|
return merged;
|
|
10440
10257
|
}
|
|
10441
10258
|
|
|
10259
|
+
// src/search/oversample.ts
|
|
10260
|
+
function oversampledShardSize(size) {
|
|
10261
|
+
return Math.ceil(size * 1.5) + 10;
|
|
10262
|
+
}
|
|
10263
|
+
|
|
10264
|
+
// src/search/facets.ts
|
|
10265
|
+
function keptValueCount(fieldConfig) {
|
|
10266
|
+
return keptFacetValues(fieldConfig?.limit);
|
|
10267
|
+
}
|
|
10268
|
+
function everyValueFacetConfig(config) {
|
|
10269
|
+
const widened = {};
|
|
10270
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
10271
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
10272
|
+
}
|
|
10273
|
+
return widened;
|
|
10274
|
+
}
|
|
10275
|
+
function oversampledFacetConfig(config) {
|
|
10276
|
+
const widened = {};
|
|
10277
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
10278
|
+
if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
|
|
10279
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
10280
|
+
continue;
|
|
10281
|
+
}
|
|
10282
|
+
const kept = keptValueCount(fieldConfig);
|
|
10283
|
+
widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
|
|
10284
|
+
}
|
|
10285
|
+
return widened;
|
|
10286
|
+
}
|
|
10287
|
+
function mergeFacets(partitionFacets, config) {
|
|
10288
|
+
const merged = /* @__PURE__ */ new Map();
|
|
10289
|
+
const bounds = /* @__PURE__ */ new Map();
|
|
10290
|
+
for (const partition of partitionFacets) {
|
|
10291
|
+
for (const [field, facetResult] of Object.entries(partition)) {
|
|
10292
|
+
let fieldMap = merged.get(field);
|
|
10293
|
+
if (!fieldMap) {
|
|
10294
|
+
fieldMap = /* @__PURE__ */ new Map();
|
|
10295
|
+
merged.set(field, fieldMap);
|
|
10296
|
+
}
|
|
10297
|
+
bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
|
|
10298
|
+
for (const [value, count] of Object.entries(facetResult.values)) {
|
|
10299
|
+
fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
|
|
10300
|
+
}
|
|
10301
|
+
}
|
|
10302
|
+
}
|
|
10303
|
+
const result = {};
|
|
10304
|
+
for (const [field, valueMap] of merged) {
|
|
10305
|
+
const fieldConfig = config[field];
|
|
10306
|
+
const ascending = fieldConfig?.sort === "asc";
|
|
10307
|
+
const ordered = Array.from(valueMap.entries());
|
|
10308
|
+
ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
10309
|
+
const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
|
|
10310
|
+
let largestLeftOut = 0;
|
|
10311
|
+
for (let index = kept; index < ordered.length; index++) {
|
|
10312
|
+
if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
|
|
10313
|
+
}
|
|
10314
|
+
const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
|
|
10315
|
+
const values = {};
|
|
10316
|
+
for (let index = 0; index < kept; index++) {
|
|
10317
|
+
values[ordered[index][0]] = ordered[index][1];
|
|
10318
|
+
}
|
|
10319
|
+
result[field] = { values, count: kept, errorBound };
|
|
10320
|
+
}
|
|
10321
|
+
return result;
|
|
10322
|
+
}
|
|
10323
|
+
|
|
10442
10324
|
// src/core/partition/composite/filters.ts
|
|
10443
10325
|
var WORD_BITS = 32;
|
|
10444
10326
|
function alignToWord(value) {
|
|
@@ -10519,6 +10401,29 @@ function subMatchesView(matches, layout, subIndex) {
|
|
|
10519
10401
|
};
|
|
10520
10402
|
}
|
|
10521
10403
|
|
|
10404
|
+
// src/core/partition/composite/facets.ts
|
|
10405
|
+
function compositeFacets(subs, layout, matched, config, schema) {
|
|
10406
|
+
const everyValue = everyValueFacetConfig(config);
|
|
10407
|
+
if ("ordinalBitset" in matched) {
|
|
10408
|
+
const ordinalLayout = layout();
|
|
10409
|
+
return mergeFacets(
|
|
10410
|
+
subs.map(
|
|
10411
|
+
(sub, index) => computeFacets(
|
|
10412
|
+
sub,
|
|
10413
|
+
{ ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
|
|
10414
|
+
everyValue,
|
|
10415
|
+
schema
|
|
10416
|
+
)
|
|
10417
|
+
),
|
|
10418
|
+
config
|
|
10419
|
+
);
|
|
10420
|
+
}
|
|
10421
|
+
return mergeFacets(
|
|
10422
|
+
subs.map((sub) => computeFacets(sub, matched, everyValue, schema)),
|
|
10423
|
+
config
|
|
10424
|
+
);
|
|
10425
|
+
}
|
|
10426
|
+
|
|
10522
10427
|
// src/core/partition/scored-merge.ts
|
|
10523
10428
|
function kWayMerge(arrays) {
|
|
10524
10429
|
const nonEmpty = arrays.filter((a) => a.length > 0);
|
|
@@ -10898,7 +10803,8 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10898
10803
|
}
|
|
10899
10804
|
function cachedDocFrequencies() {
|
|
10900
10805
|
if (docFrequenciesCache === null) {
|
|
10901
|
-
|
|
10806
|
+
const paged = frozen.find((segment) => segment.paged !== null)?.paged;
|
|
10807
|
+
docFrequenciesCache = paged ? paged.termFrequenciesAcross(subs) : mergeDocFrequencies(subs());
|
|
10902
10808
|
}
|
|
10903
10809
|
return docFrequenciesCache;
|
|
10904
10810
|
}
|
|
@@ -10969,6 +10875,20 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10969
10875
|
frozen.push(segment);
|
|
10970
10876
|
bindSegments();
|
|
10971
10877
|
},
|
|
10878
|
+
frozenSegments() {
|
|
10879
|
+
return frozen;
|
|
10880
|
+
},
|
|
10881
|
+
replaceFrozenSegments(segments) {
|
|
10882
|
+
frozen.length = 0;
|
|
10883
|
+
frozen.push(...segments);
|
|
10884
|
+
bindSegments();
|
|
10885
|
+
},
|
|
10886
|
+
documentIdsHeldInMemory() {
|
|
10887
|
+
return compositeDocIds(
|
|
10888
|
+
frozen.filter((segment) => segment.paged === null),
|
|
10889
|
+
live
|
|
10890
|
+
);
|
|
10891
|
+
},
|
|
10972
10892
|
mergeSegment(segment) {
|
|
10973
10893
|
invalidateDocFrequencies();
|
|
10974
10894
|
live.mergeSegment(segment);
|
|
@@ -10991,6 +10911,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10991
10911
|
live.deserialize(data, schema);
|
|
10992
10912
|
},
|
|
10993
10913
|
clear() {
|
|
10914
|
+
for (const segment of frozen) segment.paged?.close();
|
|
10994
10915
|
frozen.length = 0;
|
|
10995
10916
|
invalidateDocFrequencies();
|
|
10996
10917
|
live.clear();
|
|
@@ -11057,25 +10978,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
11057
10978
|
return compositeFilterMatches(subs(), layout(), filters, schema, meter);
|
|
11058
10979
|
},
|
|
11059
10980
|
computeFacets(matched, config, schema) {
|
|
11060
|
-
|
|
11061
|
-
if ("ordinalBitset" in matched) {
|
|
11062
|
-
const ordinalLayout = layout();
|
|
11063
|
-
return mergeFacets(
|
|
11064
|
-
subs().map(
|
|
11065
|
-
(sub, index) => computeFacets(
|
|
11066
|
-
sub,
|
|
11067
|
-
{ ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
|
|
11068
|
-
everyValue,
|
|
11069
|
-
schema
|
|
11070
|
-
)
|
|
11071
|
-
),
|
|
11072
|
-
config
|
|
11073
|
-
);
|
|
11074
|
-
}
|
|
11075
|
-
return mergeFacets(
|
|
11076
|
-
subs().map((sub) => computeFacets(sub, matched, everyValue, schema)),
|
|
11077
|
-
config
|
|
11078
|
-
);
|
|
10981
|
+
return compositeFacets(subs(), layout, matched, config, schema);
|
|
11079
10982
|
},
|
|
11080
10983
|
sortedPage(request) {
|
|
11081
10984
|
return compositeSortedPage(subs(), layout(), request);
|
|
@@ -11202,6 +11105,36 @@ function createCapacityChecks(source) {
|
|
|
11202
11105
|
};
|
|
11203
11106
|
}
|
|
11204
11107
|
|
|
11108
|
+
// src/partitioning/manager/doc-locations.ts
|
|
11109
|
+
function documentIdsHeldInMemory(partition) {
|
|
11110
|
+
return isCompositePartition(partition) ? partition.documentIdsHeldInMemory() : partition.docIds();
|
|
11111
|
+
}
|
|
11112
|
+
function rebuildDocLocations(locations, partitions) {
|
|
11113
|
+
locations.clear();
|
|
11114
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
11115
|
+
for (const docId of documentIdsHeldInMemory(partitions[i])) {
|
|
11116
|
+
locations.set(docId, i);
|
|
11117
|
+
}
|
|
11118
|
+
}
|
|
11119
|
+
}
|
|
11120
|
+
function recordSegmentDocLocations(locations, partitionId, segment) {
|
|
11121
|
+
if (segment.paged !== null) return;
|
|
11122
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11123
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
11124
|
+
if (docId !== void 0) locations.set(docId, partitionId);
|
|
11125
|
+
}
|
|
11126
|
+
}
|
|
11127
|
+
function rebuildPartitionDocLocations(locations, partitionId, partition) {
|
|
11128
|
+
for (const [docId, pid] of locations) {
|
|
11129
|
+
if (pid === partitionId) {
|
|
11130
|
+
locations.delete(docId);
|
|
11131
|
+
}
|
|
11132
|
+
}
|
|
11133
|
+
for (const docId of documentIdsHeldInMemory(partition)) {
|
|
11134
|
+
locations.set(docId, partitionId);
|
|
11135
|
+
}
|
|
11136
|
+
}
|
|
11137
|
+
|
|
11205
11138
|
// src/partitioning/manager/nested-values.ts
|
|
11206
11139
|
function setNestedValue(obj, path, value) {
|
|
11207
11140
|
if (!path.includes(".")) {
|
|
@@ -11240,6 +11173,109 @@ function requireNewSegmentDocuments(segment, docPartitionMap) {
|
|
|
11240
11173
|
}
|
|
11241
11174
|
}
|
|
11242
11175
|
|
|
11176
|
+
// src/partitioning/manager/write-log.ts
|
|
11177
|
+
var REANALYSIS_WRITE_LOG = 0;
|
|
11178
|
+
var CHECKPOINT_WRITE_LOG = 1;
|
|
11179
|
+
function createWriteLogRecorder() {
|
|
11180
|
+
const current = [null, null];
|
|
11181
|
+
return {
|
|
11182
|
+
open(owner) {
|
|
11183
|
+
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
11184
|
+
current[owner] = log;
|
|
11185
|
+
return {
|
|
11186
|
+
get partitionsReplaced() {
|
|
11187
|
+
return log.replaced;
|
|
11188
|
+
},
|
|
11189
|
+
get pendingDocuments() {
|
|
11190
|
+
return log.docIds.size;
|
|
11191
|
+
},
|
|
11192
|
+
takeWrittenDocIds() {
|
|
11193
|
+
const taken = log.docIds;
|
|
11194
|
+
log.docIds = /* @__PURE__ */ new Set();
|
|
11195
|
+
return taken;
|
|
11196
|
+
},
|
|
11197
|
+
close() {
|
|
11198
|
+
if (current[owner] === log) current[owner] = null;
|
|
11199
|
+
}
|
|
11200
|
+
};
|
|
11201
|
+
},
|
|
11202
|
+
documentWritten(docId) {
|
|
11203
|
+
current[0]?.docIds.add(docId);
|
|
11204
|
+
current[1]?.docIds.add(docId);
|
|
11205
|
+
},
|
|
11206
|
+
segmentWritten(segment) {
|
|
11207
|
+
for (const log of current) {
|
|
11208
|
+
if (log === null) continue;
|
|
11209
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11210
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
11211
|
+
if (docId !== void 0) log.docIds.add(docId);
|
|
11212
|
+
}
|
|
11213
|
+
}
|
|
11214
|
+
},
|
|
11215
|
+
partitionsReplaced() {
|
|
11216
|
+
for (const log of current) {
|
|
11217
|
+
if (log !== null) log.replaced = true;
|
|
11218
|
+
}
|
|
11219
|
+
}
|
|
11220
|
+
};
|
|
11221
|
+
}
|
|
11222
|
+
|
|
11223
|
+
// src/partitioning/manager/segment-operations.ts
|
|
11224
|
+
function createSegmentOperations(source) {
|
|
11225
|
+
const { docPartitionMap, writeLogs, validatePartitionId, asCompositePartition, resolveInsertOptions } = source;
|
|
11226
|
+
return {
|
|
11227
|
+
mergeSegment(partitionId, payload, documents) {
|
|
11228
|
+
validatePartitionId(partitionId);
|
|
11229
|
+
const partition = source.partitionAt(partitionId);
|
|
11230
|
+
partition.beginBatch();
|
|
11231
|
+
partition.mergeSegmentPayload(payload, documents);
|
|
11232
|
+
partition.endBatch();
|
|
11233
|
+
for (const docId of payload.docIds) {
|
|
11234
|
+
docPartitionMap.set(docId, partitionId);
|
|
11235
|
+
writeLogs.documentWritten(docId);
|
|
11236
|
+
}
|
|
11237
|
+
},
|
|
11238
|
+
attachFrozenSegment(partitionId, segment) {
|
|
11239
|
+
validatePartitionId(partitionId);
|
|
11240
|
+
if (segment.paged === null) requireNewSegmentDocuments(segment, docPartitionMap);
|
|
11241
|
+
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
11242
|
+
if (segment.paged !== null) return;
|
|
11243
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
11244
|
+
writeLogs.segmentWritten(segment);
|
|
11245
|
+
},
|
|
11246
|
+
openCheckpointChangeLog() {
|
|
11247
|
+
return writeLogs.open(CHECKPOINT_WRITE_LOG);
|
|
11248
|
+
},
|
|
11249
|
+
diskSegmentAccess(partitionId) {
|
|
11250
|
+
validatePartitionId(partitionId);
|
|
11251
|
+
return {
|
|
11252
|
+
composite: asCompositePartition(partitionId),
|
|
11253
|
+
docPartitionMap,
|
|
11254
|
+
insertOptions: resolveInsertOptions()
|
|
11255
|
+
};
|
|
11256
|
+
},
|
|
11257
|
+
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
11258
|
+
validatePartitionId(partitionId);
|
|
11259
|
+
const applied = asCompositePartition(partitionId).swapFrozenSegments(
|
|
11260
|
+
dropSegmentIds,
|
|
11261
|
+
replacement,
|
|
11262
|
+
removedSpellings
|
|
11263
|
+
);
|
|
11264
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, replacement);
|
|
11265
|
+
return applied;
|
|
11266
|
+
},
|
|
11267
|
+
freezeLiveTail(partitionId, freeze) {
|
|
11268
|
+
validatePartitionId(partitionId);
|
|
11269
|
+
return asCompositePartition(partitionId).freezeLiveTail(freeze);
|
|
11270
|
+
},
|
|
11271
|
+
replaceLiveTail(partitionId, segment) {
|
|
11272
|
+
validatePartitionId(partitionId);
|
|
11273
|
+
asCompositePartition(partitionId).replaceLiveTail(segment);
|
|
11274
|
+
recordSegmentDocLocations(docPartitionMap, partitionId, segment);
|
|
11275
|
+
}
|
|
11276
|
+
};
|
|
11277
|
+
}
|
|
11278
|
+
|
|
11243
11279
|
// src/partitioning/manager/stats.ts
|
|
11244
11280
|
function aggregateStatsOf(partitions) {
|
|
11245
11281
|
let totalDocuments = 0;
|
|
@@ -11275,59 +11311,27 @@ function partitionStatsOf(partitions) {
|
|
|
11275
11311
|
}));
|
|
11276
11312
|
}
|
|
11277
11313
|
|
|
11278
|
-
// src/partitioning/manager/write-log.ts
|
|
11279
|
-
function createWriteLogRecorder() {
|
|
11280
|
-
let current = null;
|
|
11281
|
-
return {
|
|
11282
|
-
open() {
|
|
11283
|
-
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
11284
|
-
current = log;
|
|
11285
|
-
return {
|
|
11286
|
-
get partitionsReplaced() {
|
|
11287
|
-
return log.replaced;
|
|
11288
|
-
},
|
|
11289
|
-
get pendingDocuments() {
|
|
11290
|
-
return log.docIds.size;
|
|
11291
|
-
},
|
|
11292
|
-
takeWrittenDocIds() {
|
|
11293
|
-
const taken = log.docIds;
|
|
11294
|
-
log.docIds = /* @__PURE__ */ new Set();
|
|
11295
|
-
return taken;
|
|
11296
|
-
},
|
|
11297
|
-
close() {
|
|
11298
|
-
if (current === log) current = null;
|
|
11299
|
-
}
|
|
11300
|
-
};
|
|
11301
|
-
},
|
|
11302
|
-
documentWritten(docId) {
|
|
11303
|
-
if (current !== null) current.docIds.add(docId);
|
|
11304
|
-
},
|
|
11305
|
-
segmentWritten(segment) {
|
|
11306
|
-
if (current === null) return;
|
|
11307
|
-
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11308
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
11309
|
-
if (docId !== void 0) current.docIds.add(docId);
|
|
11310
|
-
}
|
|
11311
|
-
},
|
|
11312
|
-
partitionsReplaced() {
|
|
11313
|
-
if (current !== null) current.replaced = true;
|
|
11314
|
-
}
|
|
11315
|
-
};
|
|
11316
|
-
}
|
|
11317
|
-
|
|
11318
11314
|
// src/partitioning/manager/index.ts
|
|
11319
|
-
function
|
|
11315
|
+
function closePagedSegmentsLeftBehind(earlier, kept) {
|
|
11316
|
+
for (const partition of earlier) {
|
|
11317
|
+
if (!isCompositePartition(partition) || kept.includes(partition)) continue;
|
|
11318
|
+
for (const segment of partition.frozenSegments()) segment.paged?.close();
|
|
11319
|
+
}
|
|
11320
|
+
}
|
|
11321
|
+
function createPartitionManager(indexName, initialConfig, initialLanguage, router, initialPartitionCount, vectorIndexes) {
|
|
11322
|
+
let config = initialConfig;
|
|
11323
|
+
let language = initialLanguage;
|
|
11320
11324
|
requirePatternSearchFor(config.schema);
|
|
11321
11325
|
requirePartialWordSearchFor(config.schema);
|
|
11322
11326
|
const count = initialPartitionCount ?? 1;
|
|
11323
|
-
|
|
11324
|
-
|
|
11325
|
-
|
|
11327
|
+
let analysis = resolveIndexAnalysis(config);
|
|
11328
|
+
let partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
11329
|
+
let textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
11326
11330
|
const trackPositions = config.trackPositions ?? true;
|
|
11327
11331
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
11328
11332
|
let partitions = [];
|
|
11329
11333
|
const docPartitionMap = /* @__PURE__ */ new Map();
|
|
11330
|
-
const
|
|
11334
|
+
const writeLogs = createWriteLogRecorder();
|
|
11331
11335
|
for (let i = 0; i < count; i++) {
|
|
11332
11336
|
partitions.push(createPartitionIndex(i, trackPositions));
|
|
11333
11337
|
}
|
|
@@ -11347,38 +11351,21 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11347
11351
|
if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
|
|
11348
11352
|
}
|
|
11349
11353
|
function locateDocument(docId) {
|
|
11350
|
-
|
|
11351
|
-
|
|
11352
|
-
|
|
11353
|
-
|
|
11354
|
-
if (isCompositePartition(existing)) return existing;
|
|
11355
|
-
const composite = createCompositePartition(partitionId, trackPositions, existing, textAnalysis);
|
|
11356
|
-
partitions[partitionId] = composite;
|
|
11357
|
-
return composite;
|
|
11358
|
-
}
|
|
11359
|
-
function rebuildDocPartitionMap() {
|
|
11360
|
-
docPartitionMap.clear();
|
|
11354
|
+
const known = docPartitionMap.get(docId);
|
|
11355
|
+
if (known !== void 0 || config.textStorage !== "disk") return known;
|
|
11356
|
+
const routed = router.route(docId, partitions.length);
|
|
11357
|
+
if (partitions[routed]?.has(docId)) return routed;
|
|
11361
11358
|
for (let i = 0; i < partitions.length; i++) {
|
|
11362
|
-
|
|
11363
|
-
docPartitionMap.set(docId, i);
|
|
11364
|
-
}
|
|
11365
|
-
}
|
|
11366
|
-
}
|
|
11367
|
-
function registerSegmentDocuments(partitionId, segment) {
|
|
11368
|
-
for (const internalId of segment.docStore.allInternalIds()) {
|
|
11369
|
-
const docId = segment.docStore.getExternalId(internalId);
|
|
11370
|
-
if (docId !== void 0) docPartitionMap.set(docId, partitionId);
|
|
11359
|
+
if (i !== routed && partitions[i].has(docId)) return i;
|
|
11371
11360
|
}
|
|
11361
|
+
return void 0;
|
|
11372
11362
|
}
|
|
11373
|
-
function
|
|
11374
|
-
|
|
11375
|
-
|
|
11376
|
-
|
|
11377
|
-
|
|
11378
|
-
|
|
11379
|
-
for (const docId of partitions[partitionId].docIds()) {
|
|
11380
|
-
docPartitionMap.set(docId, partitionId);
|
|
11381
|
-
}
|
|
11363
|
+
function asCompositePartition(partitionId) {
|
|
11364
|
+
const existing = partitions[partitionId];
|
|
11365
|
+
if (isCompositePartition(existing)) return existing;
|
|
11366
|
+
const composite = createCompositePartition(partitionId, trackPositions, existing, textAnalysis);
|
|
11367
|
+
partitions[partitionId] = composite;
|
|
11368
|
+
return composite;
|
|
11382
11369
|
}
|
|
11383
11370
|
const manager = {
|
|
11384
11371
|
get partitionCount() {
|
|
@@ -11417,12 +11404,14 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11417
11404
|
return [...partitions];
|
|
11418
11405
|
},
|
|
11419
11406
|
setPartitions(newPartitions) {
|
|
11420
|
-
|
|
11407
|
+
writeLogs.partitionsReplaced();
|
|
11408
|
+
const earlier = partitions;
|
|
11421
11409
|
partitions = [...newPartitions];
|
|
11422
|
-
|
|
11410
|
+
closePagedSegmentsLeftBehind(earlier, partitions);
|
|
11411
|
+
rebuildDocLocations(docPartitionMap, partitions);
|
|
11423
11412
|
},
|
|
11424
11413
|
openWriteLog() {
|
|
11425
|
-
return
|
|
11414
|
+
return writeLogs.open(REANALYSIS_WRITE_LOG);
|
|
11426
11415
|
},
|
|
11427
11416
|
adoptRebuiltPartitions(rebuilt) {
|
|
11428
11417
|
if (rebuilt.length !== partitions.length) {
|
|
@@ -11432,7 +11421,17 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11432
11421
|
{ indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
|
|
11433
11422
|
);
|
|
11434
11423
|
}
|
|
11424
|
+
const earlier = partitions;
|
|
11435
11425
|
partitions = [...rebuilt];
|
|
11426
|
+
closePagedSegmentsLeftBehind(earlier, partitions);
|
|
11427
|
+
},
|
|
11428
|
+
adoptAnalysis(change) {
|
|
11429
|
+
config = change.config;
|
|
11430
|
+
language = change.language;
|
|
11431
|
+
analysis = change.analysis;
|
|
11432
|
+
partialWordAnalysis = change.partialWordAnalysis;
|
|
11433
|
+
textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
11434
|
+
manager.synonyms = change.synonyms;
|
|
11436
11435
|
},
|
|
11437
11436
|
addPartition() {
|
|
11438
11437
|
const currentMaxPartitions = config.partitions?.maxPartitions;
|
|
@@ -11444,15 +11443,15 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11444
11443
|
);
|
|
11445
11444
|
}
|
|
11446
11445
|
const partition = createPartitionIndex(partitions.length, trackPositions);
|
|
11447
|
-
|
|
11446
|
+
writeLogs.partitionsReplaced();
|
|
11448
11447
|
partitions.push(partition);
|
|
11449
11448
|
return partition;
|
|
11450
11449
|
},
|
|
11451
11450
|
removePartition(partitionId) {
|
|
11452
11451
|
validatePartitionId(partitionId);
|
|
11453
|
-
|
|
11452
|
+
writeLogs.partitionsReplaced();
|
|
11454
11453
|
partitions.splice(partitionId, 1);
|
|
11455
|
-
|
|
11454
|
+
rebuildDocLocations(docPartitionMap, partitions);
|
|
11456
11455
|
},
|
|
11457
11456
|
trimPartitions(newCount) {
|
|
11458
11457
|
if (newCount < 1) {
|
|
@@ -11464,7 +11463,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11464
11463
|
if (newCount >= partitions.length) {
|
|
11465
11464
|
return;
|
|
11466
11465
|
}
|
|
11467
|
-
|
|
11466
|
+
writeLogs.partitionsReplaced();
|
|
11468
11467
|
for (const [docId, pid] of docPartitionMap) {
|
|
11469
11468
|
if (pid >= newCount) {
|
|
11470
11469
|
docPartitionMap.delete(docId);
|
|
@@ -11474,7 +11473,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11474
11473
|
},
|
|
11475
11474
|
...createCapacityChecks({
|
|
11476
11475
|
indexName,
|
|
11477
|
-
config,
|
|
11476
|
+
config: initialConfig,
|
|
11478
11477
|
partitionCount: () => partitions.length,
|
|
11479
11478
|
documentCount: () => manager.countDocuments(),
|
|
11480
11479
|
getPartition: (partitionId) => manager.getPartition(partitionId)
|
|
@@ -11487,7 +11486,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11487
11486
|
const insertOpts = resolveInsertOptions(options);
|
|
11488
11487
|
partitions[pid].insert(docId, document, config.schema, language, insertOpts);
|
|
11489
11488
|
docPartitionMap.set(docId, pid);
|
|
11490
|
-
|
|
11489
|
+
writeLogs.documentWritten(docId);
|
|
11491
11490
|
},
|
|
11492
11491
|
removedSpellingsOf(docId) {
|
|
11493
11492
|
const pid = locateDocument(docId);
|
|
@@ -11501,7 +11500,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11501
11500
|
}
|
|
11502
11501
|
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
|
|
11503
11502
|
docPartitionMap.delete(docId);
|
|
11504
|
-
|
|
11503
|
+
writeLogs.documentWritten(docId);
|
|
11505
11504
|
},
|
|
11506
11505
|
beginBatchRemove() {
|
|
11507
11506
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -11521,7 +11520,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11521
11520
|
const updateOpts = resolveInsertOptions(options);
|
|
11522
11521
|
partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
|
|
11523
11522
|
docPartitionMap.set(docId, pid);
|
|
11524
|
-
|
|
11523
|
+
writeLogs.documentWritten(docId);
|
|
11525
11524
|
},
|
|
11526
11525
|
get(docId, projection) {
|
|
11527
11526
|
const pid = locateDocument(docId);
|
|
@@ -11561,49 +11560,20 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
11561
11560
|
validatePartitionId(partitionId);
|
|
11562
11561
|
return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
|
|
11563
11562
|
},
|
|
11564
|
-
|
|
11565
|
-
|
|
11566
|
-
|
|
11567
|
-
|
|
11568
|
-
|
|
11569
|
-
|
|
11570
|
-
|
|
11571
|
-
|
|
11572
|
-
writeLog.documentWritten(docId);
|
|
11573
|
-
}
|
|
11574
|
-
},
|
|
11575
|
-
attachFrozenSegment(partitionId, segment) {
|
|
11576
|
-
validatePartitionId(partitionId);
|
|
11577
|
-
requireNewSegmentDocuments(segment, docPartitionMap);
|
|
11578
|
-
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
11579
|
-
registerSegmentDocuments(partitionId, segment);
|
|
11580
|
-
writeLog.segmentWritten(segment);
|
|
11581
|
-
},
|
|
11582
|
-
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
11583
|
-
validatePartitionId(partitionId);
|
|
11584
|
-
const applied = asCompositePartition(partitionId).swapFrozenSegments(
|
|
11585
|
-
dropSegmentIds,
|
|
11586
|
-
replacement,
|
|
11587
|
-
removedSpellings
|
|
11588
|
-
);
|
|
11589
|
-
registerSegmentDocuments(partitionId, replacement);
|
|
11590
|
-
return applied;
|
|
11591
|
-
},
|
|
11592
|
-
freezeLiveTail(partitionId, freeze) {
|
|
11593
|
-
validatePartitionId(partitionId);
|
|
11594
|
-
return asCompositePartition(partitionId).freezeLiveTail(freeze);
|
|
11595
|
-
},
|
|
11596
|
-
replaceLiveTail(partitionId, segment) {
|
|
11597
|
-
validatePartitionId(partitionId);
|
|
11598
|
-
asCompositePartition(partitionId).replaceLiveTail(segment);
|
|
11599
|
-
registerSegmentDocuments(partitionId, segment);
|
|
11600
|
-
},
|
|
11563
|
+
...createSegmentOperations({
|
|
11564
|
+
docPartitionMap,
|
|
11565
|
+
writeLogs,
|
|
11566
|
+
partitionAt: (partitionId) => partitions[partitionId],
|
|
11567
|
+
validatePartitionId,
|
|
11568
|
+
asCompositePartition,
|
|
11569
|
+
resolveInsertOptions: () => resolveInsertOptions()
|
|
11570
|
+
}),
|
|
11601
11571
|
deserializePartition(partitionId, data) {
|
|
11602
11572
|
validatePartitionId(partitionId);
|
|
11603
|
-
|
|
11573
|
+
writeLogs.partitionsReplaced();
|
|
11604
11574
|
partitions[partitionId].deserialize(data, config.schema);
|
|
11605
11575
|
preparePartialWords(partitionId);
|
|
11606
|
-
|
|
11576
|
+
rebuildPartitionDocLocations(docPartitionMap, partitionId, partitions[partitionId]);
|
|
11607
11577
|
},
|
|
11608
11578
|
getAggregateStats() {
|
|
11609
11579
|
return aggregateStatsOf(partitions);
|
|
@@ -13062,11 +13032,17 @@ function encodeDocumentTableData(documents) {
|
|
|
13062
13032
|
}
|
|
13063
13033
|
function wrapEncodedDocumentTable(data) {
|
|
13064
13034
|
const { blob, offsets } = data;
|
|
13065
|
-
|
|
13066
|
-
|
|
13035
|
+
return createEncodedDocumentSource(
|
|
13036
|
+
offsets.length - 1,
|
|
13037
|
+
blob.byteLength + offsets.byteLength,
|
|
13038
|
+
(ordinal) => blob.subarray(offsets[ordinal], offsets[ordinal + 1]),
|
|
13039
|
+
data.discardedFields
|
|
13040
|
+
);
|
|
13041
|
+
}
|
|
13042
|
+
function createEncodedDocumentSource(count, byteLength, bytesAt, discardedFields) {
|
|
13043
|
+
const discarded = (discardedFields ?? []).map((fieldPath) => fieldPath.split("."));
|
|
13067
13044
|
const touchesDiscarded = (path) => discarded.some((discardedPath) => sharesPrefix(discardedPath, path));
|
|
13068
|
-
const
|
|
13069
|
-
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), data.discardedFields));
|
|
13045
|
+
const docAt = (ordinal) => documentWithOwnBuffers(decodeStoredDocument(bytesAt(ordinal), discardedFields));
|
|
13070
13046
|
function projectedPathAt(ordinal, path) {
|
|
13071
13047
|
const segments = encodedProjectionPathOf(path);
|
|
13072
13048
|
if (segments === null) return { found: false, value: void 0 };
|
|
@@ -13076,10 +13052,10 @@ function wrapEncodedDocumentTable(data) {
|
|
|
13076
13052
|
}
|
|
13077
13053
|
return {
|
|
13078
13054
|
count,
|
|
13079
|
-
byteLength
|
|
13055
|
+
byteLength,
|
|
13080
13056
|
docAt,
|
|
13081
13057
|
readAt(ordinal) {
|
|
13082
|
-
return decodeStoredDocument(bytesAt(ordinal),
|
|
13058
|
+
return decodeStoredDocument(bytesAt(ordinal), discardedFields);
|
|
13083
13059
|
},
|
|
13084
13060
|
fieldReader(fieldPath) {
|
|
13085
13061
|
const path = fieldPath.split(".");
|
|
@@ -13159,9 +13135,9 @@ function basicPlaneUtf8Bytes(text) {
|
|
|
13159
13135
|
return bytes;
|
|
13160
13136
|
}
|
|
13161
13137
|
function encodeStringBlob(strings) {
|
|
13162
|
-
const
|
|
13163
|
-
if (SURROGATE.test(
|
|
13164
|
-
const blob = encoder3.encode(
|
|
13138
|
+
const joined = strings.join("");
|
|
13139
|
+
if (SURROGATE.test(joined)) return encodeOneByOne(strings);
|
|
13140
|
+
const blob = encoder3.encode(joined);
|
|
13165
13141
|
const offsets = new Uint32Array(strings.length + 1);
|
|
13166
13142
|
let cursor = 0;
|
|
13167
13143
|
for (let i = 0; i < strings.length; i++) {
|
|
@@ -13242,6 +13218,9 @@ function wrapExternalIdTable(data) {
|
|
|
13242
13218
|
const ordinal = sortedOrdinals[lo];
|
|
13243
13219
|
return compareBytes(blob, offsets[ordinal], offsets[ordinal + 1], query) === 0 ? ordinal : -1;
|
|
13244
13220
|
},
|
|
13221
|
+
sortedOrdinalAt(position) {
|
|
13222
|
+
return sortedOrdinals[position];
|
|
13223
|
+
},
|
|
13245
13224
|
collectSortedIds(excluded) {
|
|
13246
13225
|
const ids = [];
|
|
13247
13226
|
for (let position = 0; position < count; position++) {
|
|
@@ -13261,6 +13240,7 @@ function buildExternalIdTable(docIds) {
|
|
|
13261
13240
|
characters += docIds[ordinal].length;
|
|
13262
13241
|
}
|
|
13263
13242
|
let sortedIds = null;
|
|
13243
|
+
let sortedOrdinals = null;
|
|
13264
13244
|
return {
|
|
13265
13245
|
count: docIds.length,
|
|
13266
13246
|
bytes: characters,
|
|
@@ -13270,6 +13250,10 @@ function buildExternalIdTable(docIds) {
|
|
|
13270
13250
|
ordinalOf(docId) {
|
|
13271
13251
|
return ordinals.get(docId) ?? -1;
|
|
13272
13252
|
},
|
|
13253
|
+
sortedOrdinalAt(position) {
|
|
13254
|
+
if (sortedOrdinals === null) sortedOrdinals = codePointOrder(docIds);
|
|
13255
|
+
return sortedOrdinals[position];
|
|
13256
|
+
},
|
|
13273
13257
|
collectSortedIds(excluded) {
|
|
13274
13258
|
if (sortedIds === null) {
|
|
13275
13259
|
const ids = [...docIds];
|
|
@@ -13841,6 +13825,10 @@ function createFrozenRemovalPass(documents, surfaces) {
|
|
|
13841
13825
|
});
|
|
13842
13826
|
return spellings;
|
|
13843
13827
|
}
|
|
13828
|
+
function subtract(ordinal, analysis) {
|
|
13829
|
+
if (surfaces.tableStaysOnDisk()) surfaces.subtractOnceLoaded(ordinal, analysis);
|
|
13830
|
+
else surfaces.subtractSpellings(ordinal, heldSpellingsOf(ordinal, analysis));
|
|
13831
|
+
}
|
|
13844
13832
|
return {
|
|
13845
13833
|
get settled() {
|
|
13846
13834
|
return pending.length === 0;
|
|
@@ -13849,17 +13837,17 @@ function createFrozenRemovalPass(documents, surfaces) {
|
|
|
13849
13837
|
if (bound !== null) return;
|
|
13850
13838
|
const scoped = { ...analysis, fields: documents.textFields };
|
|
13851
13839
|
bound = scoped;
|
|
13852
|
-
for (const ordinal of pending)
|
|
13840
|
+
for (const ordinal of pending) subtract(ordinal, scoped);
|
|
13853
13841
|
pending = [];
|
|
13854
13842
|
},
|
|
13855
13843
|
spellingsOf(ordinal) {
|
|
13856
|
-
return bound === null ? void 0 : heldSpellingsOf(ordinal, bound);
|
|
13844
|
+
return bound === null || surfaces.tableStaysOnDisk() ? void 0 : heldSpellingsOf(ordinal, bound);
|
|
13857
13845
|
},
|
|
13858
13846
|
afterRemoval(ordinal, spellings) {
|
|
13859
13847
|
if (spellings !== void 0) {
|
|
13860
13848
|
surfaces.subtractSpellings(ordinal, spellings);
|
|
13861
13849
|
} else if (bound !== null) {
|
|
13862
|
-
|
|
13850
|
+
subtract(ordinal, bound);
|
|
13863
13851
|
} else {
|
|
13864
13852
|
pending.push(ordinal);
|
|
13865
13853
|
}
|
|
@@ -13883,15 +13871,32 @@ function addForms(target, forms) {
|
|
|
13883
13871
|
if (Array.isArray(value)) target.add(surface, value[1], value[0]);
|
|
13884
13872
|
}
|
|
13885
13873
|
}
|
|
13886
|
-
function createFrozenSurfaceSlot(load, documents) {
|
|
13874
|
+
function createFrozenSurfaceSlot(load, documents, tableOnDisk) {
|
|
13887
13875
|
let registry = null;
|
|
13888
13876
|
let uncounted = null;
|
|
13877
|
+
let table = tableOnDisk;
|
|
13878
|
+
let waitingRemovals = [];
|
|
13879
|
+
let waitingAnalysis = null;
|
|
13880
|
+
function subtractWaitingRemovals(target) {
|
|
13881
|
+
if (waitingAnalysis === null || target.size() === 0) return;
|
|
13882
|
+
for (const ordinal of waitingRemovals) {
|
|
13883
|
+
const spellings = [];
|
|
13884
|
+
visitDocumentSurfaceSpellings(documents.documentAt(ordinal), waitingAnalysis, (spelling) => {
|
|
13885
|
+
if (target.holds(spelling)) spellings.push(spelling);
|
|
13886
|
+
});
|
|
13887
|
+
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
13888
|
+
}
|
|
13889
|
+
}
|
|
13889
13890
|
function loaded3() {
|
|
13890
13891
|
if (registry === null) {
|
|
13891
13892
|
const built = createSurfaceRegistry();
|
|
13892
13893
|
const forms = load();
|
|
13893
13894
|
if (forms !== null) built.deserialize(forms);
|
|
13895
|
+
subtractWaitingRemovals(built);
|
|
13894
13896
|
registry = built;
|
|
13897
|
+
table = void 0;
|
|
13898
|
+
waitingRemovals = [];
|
|
13899
|
+
waitingAnalysis = null;
|
|
13895
13900
|
}
|
|
13896
13901
|
return registry;
|
|
13897
13902
|
}
|
|
@@ -13899,12 +13904,22 @@ function createFrozenSurfaceSlot(load, documents) {
|
|
|
13899
13904
|
return uncounted === null || uncounted[ordinal] === 0;
|
|
13900
13905
|
}
|
|
13901
13906
|
return {
|
|
13907
|
+
heldBytes: () => table?.heldBytes() ?? 0,
|
|
13902
13908
|
reader: {
|
|
13903
|
-
candidatesForPrefix: (prefix) => loaded3().candidatesForPrefix(prefix),
|
|
13909
|
+
candidatesForPrefix: (prefix) => table !== void 0 && waitingRemovals.length === 0 ? table.candidatesForPrefix(prefix) : loaded3().candidatesForPrefix(prefix),
|
|
13904
13910
|
stemChangedTotalFor: (token) => loaded3().stemChangedTotalFor(token),
|
|
13905
13911
|
size: () => loaded3().size(),
|
|
13906
13912
|
serialize: () => loaded3().serialize()
|
|
13907
13913
|
},
|
|
13914
|
+
tableStaysOnDisk() {
|
|
13915
|
+
return registry === null && tableOnDisk !== void 0;
|
|
13916
|
+
},
|
|
13917
|
+
subtractOnceLoaded(ordinal, analysis) {
|
|
13918
|
+
uncounted ??= new Uint8Array(documents.count);
|
|
13919
|
+
uncounted[ordinal] = 1;
|
|
13920
|
+
waitingAnalysis = analysis;
|
|
13921
|
+
waitingRemovals.push(ordinal);
|
|
13922
|
+
},
|
|
13908
13923
|
holdsAny() {
|
|
13909
13924
|
return loaded3().size() > 0;
|
|
13910
13925
|
},
|
|
@@ -14052,6 +14067,9 @@ function wrapFrozenTokenTable(data) {
|
|
|
14052
14067
|
}
|
|
14053
14068
|
return token;
|
|
14054
14069
|
},
|
|
14070
|
+
tokenBytesAt(sortedIndex) {
|
|
14071
|
+
return blob.subarray(offsets[sortedIndex], offsets[sortedIndex + 1]);
|
|
14072
|
+
},
|
|
14055
14073
|
byteLengthAt(sortedIndex) {
|
|
14056
14074
|
return offsets[sortedIndex + 1] - offsets[sortedIndex];
|
|
14057
14075
|
},
|
|
@@ -14326,10 +14344,10 @@ function buildGeoReaders(entries) {
|
|
|
14326
14344
|
}
|
|
14327
14345
|
return readers;
|
|
14328
14346
|
}
|
|
14329
|
-
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms) {
|
|
14347
|
+
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable, survivingSurfaceForms, pagedParts) {
|
|
14330
14348
|
const tombstones = createFrozenTombstones();
|
|
14331
|
-
const counts = createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
|
|
14332
|
-
const postingViews = createFrozenPostingViews(source, tombstones, counts);
|
|
14349
|
+
const counts = pagedParts?.createCounts(tombstones) ?? createFrozenLiveCounts(source, source.tokenTable, tombstones, storedFrequenciesOf(source));
|
|
14350
|
+
const postingViews = pagedParts?.createViews(tombstones, counts) ?? createFrozenPostingViews(source, tombstones, counts);
|
|
14333
14351
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
14334
14352
|
for (let i = 0; i < source.fieldLengthNames.length; i++) {
|
|
14335
14353
|
lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
|
|
@@ -14346,7 +14364,11 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
14346
14364
|
documentAt: (ordinal) => documentSource.readAt(ordinal)
|
|
14347
14365
|
};
|
|
14348
14366
|
const loadSurfaceForms = survivingSurfaceForms === void 0 ? source.readSurfaceForms : () => survivingSurfaceForms;
|
|
14349
|
-
const surfaces = createFrozenSurfaceSlot(
|
|
14367
|
+
const surfaces = createFrozenSurfaceSlot(
|
|
14368
|
+
loadSurfaceForms,
|
|
14369
|
+
textDocuments,
|
|
14370
|
+
survivingSurfaceForms === void 0 ? source.surfaceTableOnDisk : void 0
|
|
14371
|
+
);
|
|
14350
14372
|
const removals = createFrozenRemovalPass(textDocuments, surfaces);
|
|
14351
14373
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
14352
14374
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
@@ -14368,6 +14390,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
14368
14390
|
segmentId,
|
|
14369
14391
|
documentSource,
|
|
14370
14392
|
sharedSnapshot,
|
|
14393
|
+
paged: pagedParts?.access ?? null,
|
|
14371
14394
|
arrays: source,
|
|
14372
14395
|
invertedIdx: createFrozenInvertedReader(source.tokenTable, postingViews, counts),
|
|
14373
14396
|
docStore,
|
|
@@ -14383,7 +14406,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
14383
14406
|
sizeIndexes,
|
|
14384
14407
|
fieldNameTable,
|
|
14385
14408
|
partialWords,
|
|
14386
|
-
trackPositions: source.positionOffsets !== null,
|
|
14409
|
+
trackPositions: pagedParts?.access.snapshot.tracksPositions ?? source.positionOffsets !== null,
|
|
14387
14410
|
flatSchemaCache: null,
|
|
14388
14411
|
lastSchemaRef: null,
|
|
14389
14412
|
sortColumns: null,
|
|
@@ -14427,6 +14450,9 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
14427
14450
|
};
|
|
14428
14451
|
return segment;
|
|
14429
14452
|
}
|
|
14453
|
+
function assemblePagedFrozenSegment(segmentId, source, documentSource, partialWordTable, parts) {
|
|
14454
|
+
return assembleFrozenSegment(segmentId, source, documentSource, null, partialWordTable, void 0, parts);
|
|
14455
|
+
}
|
|
14430
14456
|
function createFrozenSegment(payload, documents, segmentId) {
|
|
14431
14457
|
const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
|
|
14432
14458
|
return assembleFrozenSegment(
|
|
@@ -15290,7 +15316,20 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
15290
15316
|
|
|
15291
15317
|
// src/workers/segment-actions.ts
|
|
15292
15318
|
function isSegmentAction(action) {
|
|
15293
|
-
return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail";
|
|
15319
|
+
return action.type === "buildSegment" || action.type === "mergeSegments" || action.type === "attachSegments" || action.type === "compactSegments" || action.type === "swapSegments" || action.type === "freezeLiveTail" || action.type === "adoptDiskSegments";
|
|
15320
|
+
}
|
|
15321
|
+
function isPagedSnapshot(snapshot) {
|
|
15322
|
+
return "path" in snapshot;
|
|
15323
|
+
}
|
|
15324
|
+
async function attachSegments(entry, action) {
|
|
15325
|
+
for (const segment of action.segments) {
|
|
15326
|
+
growPartitionsTo(entry.manager, segment.partitionId);
|
|
15327
|
+
const surfaceForms = segment.survivingSurfaceForms;
|
|
15328
|
+
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
15329
|
+
const frozen = isPagedSnapshot(segment.snapshot) ? (await import('./module-BUFNZMEU.mjs')).createPagedFrozenSegment(segment.snapshot) : createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
15330
|
+
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
15331
|
+
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
15332
|
+
}
|
|
15294
15333
|
}
|
|
15295
15334
|
function growPartitionsTo(manager, partitionId) {
|
|
15296
15335
|
while (manager.partitionCount <= partitionId) {
|
|
@@ -15339,17 +15378,10 @@ function runSegmentAction(entry, action) {
|
|
|
15339
15378
|
}
|
|
15340
15379
|
return void 0;
|
|
15341
15380
|
}
|
|
15342
|
-
case "attachSegments":
|
|
15343
|
-
|
|
15344
|
-
|
|
15345
|
-
|
|
15346
|
-
const frozen = createSharedFrozenSegment(segment.snapshot, surfaceForms);
|
|
15347
|
-
const spellings = surfaceForms === void 0 ? void 0 : NO_REMOVED_SPELLINGS;
|
|
15348
|
-
for (const docId of segment.tombstonedDocIds ?? []) frozen.tombstoneDocument(docId, spellings);
|
|
15349
|
-
entry.manager.attachFrozenSegment(segment.partitionId, frozen);
|
|
15350
|
-
}
|
|
15351
|
-
return void 0;
|
|
15352
|
-
}
|
|
15381
|
+
case "attachSegments":
|
|
15382
|
+
return attachSegments(entry, action);
|
|
15383
|
+
case "adoptDiskSegments":
|
|
15384
|
+
return import('./module-BUFNZMEU.mjs').then((code) => code.adoptSegmentsOnWorker(entry.manager, action));
|
|
15353
15385
|
case "compactSegments": {
|
|
15354
15386
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
15355
15387
|
return mergeFrozenSegments(
|
|
@@ -15397,7 +15429,7 @@ function requireLoadedVectorCode() {
|
|
|
15397
15429
|
function loadVectorCode() {
|
|
15398
15430
|
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
15399
15431
|
if (loading2 === null) {
|
|
15400
|
-
loading2 = import('./module-
|
|
15432
|
+
loading2 = import('./module-BVP4TKZU.mjs').then(
|
|
15401
15433
|
(code) => {
|
|
15402
15434
|
loaded2 = code;
|
|
15403
15435
|
return code;
|
|
@@ -15473,7 +15505,7 @@ function closeHeldVectorCopies(held) {
|
|
|
15473
15505
|
held.fields.clear();
|
|
15474
15506
|
held.searchers.clear();
|
|
15475
15507
|
}
|
|
15476
|
-
function
|
|
15508
|
+
function yieldToEventLoop() {
|
|
15477
15509
|
return new Promise((resolve) => {
|
|
15478
15510
|
if (typeof setImmediate === "function") setImmediate(resolve);
|
|
15479
15511
|
else setTimeout(resolve, 0);
|
|
@@ -15484,7 +15516,7 @@ async function insertIntoHeldGraph(held, fieldName, handle, ordinals) {
|
|
|
15484
15516
|
if (field === void 0) return null;
|
|
15485
15517
|
for (const ordinal of ordinals) {
|
|
15486
15518
|
field.view.insertOrdinal(ordinal);
|
|
15487
|
-
await
|
|
15519
|
+
await yieldToEventLoop();
|
|
15488
15520
|
}
|
|
15489
15521
|
return field.view.takeOutcome();
|
|
15490
15522
|
}
|
|
@@ -15492,6 +15524,34 @@ function heldVectorOf(held, fieldName, docId) {
|
|
|
15492
15524
|
return held.fields.get(fieldName)?.view.vectorOf(docId);
|
|
15493
15525
|
}
|
|
15494
15526
|
|
|
15527
|
+
// src/workers/direct-executor/entry.ts
|
|
15528
|
+
function indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad) {
|
|
15529
|
+
return {
|
|
15530
|
+
manager,
|
|
15531
|
+
get config() {
|
|
15532
|
+
return manager.config;
|
|
15533
|
+
},
|
|
15534
|
+
get language() {
|
|
15535
|
+
return manager.language;
|
|
15536
|
+
},
|
|
15537
|
+
searchOptions: {
|
|
15538
|
+
get stopWords() {
|
|
15539
|
+
return manager.analysis.stopWords;
|
|
15540
|
+
},
|
|
15541
|
+
get customTokenizer() {
|
|
15542
|
+
return manager.analysis.customTokenizer;
|
|
15543
|
+
},
|
|
15544
|
+
get synonyms() {
|
|
15545
|
+
return manager.synonyms;
|
|
15546
|
+
}
|
|
15547
|
+
},
|
|
15548
|
+
vectorIndexes,
|
|
15549
|
+
vectorCopies: createHeldVectorCopies(),
|
|
15550
|
+
analysisStale,
|
|
15551
|
+
awaitedCopyLoad
|
|
15552
|
+
};
|
|
15553
|
+
}
|
|
15554
|
+
|
|
15495
15555
|
// src/workers/direct-executor/index.ts
|
|
15496
15556
|
var NO_VECTOR_WORKER_COPIES = { enabled: false };
|
|
15497
15557
|
function createDirectExecutor(options) {
|
|
@@ -15543,22 +15603,7 @@ function createDirectExecutor(options) {
|
|
|
15543
15603
|
const partitionCount = config.partitions?.maxPartitions ?? 1;
|
|
15544
15604
|
const vectorIndexes = vectorIndexesFor(indexName, config, copies);
|
|
15545
15605
|
const manager = createPartitionManager(indexName, config, language, router, partitionCount, vectorIndexes);
|
|
15546
|
-
return
|
|
15547
|
-
manager,
|
|
15548
|
-
config,
|
|
15549
|
-
language,
|
|
15550
|
-
searchOptions: {
|
|
15551
|
-
stopWords: manager.analysis.stopWords,
|
|
15552
|
-
customTokenizer: manager.analysis.customTokenizer,
|
|
15553
|
-
get synonyms() {
|
|
15554
|
-
return manager.synonyms;
|
|
15555
|
-
}
|
|
15556
|
-
},
|
|
15557
|
-
vectorIndexes,
|
|
15558
|
-
vectorCopies: createHeldVectorCopies(),
|
|
15559
|
-
analysisStale,
|
|
15560
|
-
awaitedCopyLoad
|
|
15561
|
-
};
|
|
15606
|
+
return indexEntryOf(manager, vectorIndexes, analysisStale, awaitedCopyLoad);
|
|
15562
15607
|
}
|
|
15563
15608
|
function disposeEntry(entry) {
|
|
15564
15609
|
for (const vectorIndex of entry.vectorIndexes.values()) {
|
|
@@ -15779,40 +15824,73 @@ function createDirectExecutor(options) {
|
|
|
15779
15824
|
};
|
|
15780
15825
|
}
|
|
15781
15826
|
|
|
15827
|
+
// src/engine/orchestration/turn.ts
|
|
15828
|
+
function afterCurrentTurn(callback) {
|
|
15829
|
+
if (typeof setImmediate === "function") {
|
|
15830
|
+
setImmediate(callback);
|
|
15831
|
+
return;
|
|
15832
|
+
}
|
|
15833
|
+
setTimeout(callback, 0);
|
|
15834
|
+
}
|
|
15835
|
+
function yieldToEventLoop2() {
|
|
15836
|
+
return new Promise((resolve) => afterCurrentTurn(resolve));
|
|
15837
|
+
}
|
|
15838
|
+
|
|
15839
|
+
// src/partitioning/constants.ts
|
|
15840
|
+
var REBALANCE_CHUNK_SIZE = 1e3;
|
|
15841
|
+
var DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE = 1e4;
|
|
15842
|
+
var TEXT_REBUILD_CHUNK_DOCUMENTS = 1e3;
|
|
15843
|
+
var TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS = 1e3;
|
|
15844
|
+
|
|
15845
|
+
// src/partitioning/rewrite-spill-of.ts
|
|
15846
|
+
async function rewriteSpillOf(storage, manager, target) {
|
|
15847
|
+
if (storage === void 0 || (target?.config ?? manager.config).textStorage !== "disk") return null;
|
|
15848
|
+
const { openRewriteSpill } = await import('./module-BUFNZMEU.mjs');
|
|
15849
|
+
return openRewriteSpill(storage, manager, target);
|
|
15850
|
+
}
|
|
15851
|
+
|
|
15782
15852
|
// src/partitioning/text-rebuild.ts
|
|
15853
|
+
function textRebuildTargetOf(config, language, analysis) {
|
|
15854
|
+
return { config, language, analysis, partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis) };
|
|
15855
|
+
}
|
|
15783
15856
|
function interrupted(pass) {
|
|
15784
15857
|
return pass.log.partitionsReplaced || !pass.hooks.isCurrent();
|
|
15785
15858
|
}
|
|
15786
15859
|
async function buildPartition(pass, source, partitionId) {
|
|
15787
|
-
const { manager, options } = pass;
|
|
15788
|
-
const target = createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
15860
|
+
const { manager, language, options } = pass;
|
|
15861
|
+
const target = pass.spill?.newPartition(partitionId) ?? createPartitionIndex(partitionId, manager.config.trackPositions ?? true);
|
|
15789
15862
|
const docIds = [...source.docIds()];
|
|
15790
15863
|
target.beginBatch();
|
|
15791
15864
|
for (let start = 0; start < docIds.length; start += TEXT_REBUILD_CHUNK_DOCUMENTS) {
|
|
15792
15865
|
const end = Math.min(start + TEXT_REBUILD_CHUNK_DOCUMENTS, docIds.length);
|
|
15793
15866
|
for (let i = start; i < end; i++) {
|
|
15794
15867
|
const document = manager.getRef(docIds[i]);
|
|
15795
|
-
if (document !== void 0) target.insert(docIds[i], document, manager.schema,
|
|
15868
|
+
if (document !== void 0) target.insert(docIds[i], document, manager.schema, language, options);
|
|
15796
15869
|
}
|
|
15797
|
-
|
|
15870
|
+
if (pass.spill !== null) {
|
|
15871
|
+
target.endBatch();
|
|
15872
|
+
await pass.spill.spillIfFull(target, partitionId);
|
|
15873
|
+
target.beginBatch();
|
|
15874
|
+
}
|
|
15875
|
+
await yieldToEventLoop2();
|
|
15798
15876
|
if (interrupted(pass)) return null;
|
|
15799
15877
|
}
|
|
15800
15878
|
target.endBatch();
|
|
15801
|
-
if (
|
|
15879
|
+
if (pass.partialWordAnalysis !== null) target.preparePartialWords(pass.partialWordAnalysis);
|
|
15802
15880
|
return target;
|
|
15803
15881
|
}
|
|
15804
15882
|
function applyWrittenDocuments(pass, built) {
|
|
15805
|
-
const { manager, options } = pass;
|
|
15883
|
+
const { manager, language, options } = pass;
|
|
15806
15884
|
for (const docId of pass.log.takeWrittenDocIds()) {
|
|
15807
15885
|
for (const partition of built) {
|
|
15808
15886
|
if (!partition.has(docId)) continue;
|
|
15809
|
-
partition.remove(docId, manager.schema,
|
|
15887
|
+
partition.remove(docId, manager.schema, language, options);
|
|
15810
15888
|
break;
|
|
15811
15889
|
}
|
|
15812
15890
|
const partitionId = manager.partitionIdOf(docId);
|
|
15813
15891
|
const document = manager.getRef(docId);
|
|
15814
15892
|
if (partitionId === void 0 || document === void 0) continue;
|
|
15815
|
-
built[partitionId].insert(docId, document, manager.schema,
|
|
15893
|
+
built[partitionId].insert(docId, document, manager.schema, language, options);
|
|
15816
15894
|
}
|
|
15817
15895
|
}
|
|
15818
15896
|
async function rebuildOnce(pass) {
|
|
@@ -15826,22 +15904,48 @@ async function rebuildOnce(pass) {
|
|
|
15826
15904
|
}
|
|
15827
15905
|
while (pass.log.pendingDocuments > TEXT_REBUILD_SWITCH_BACKLOG_DOCUMENTS) {
|
|
15828
15906
|
applyWrittenDocuments(pass, built);
|
|
15829
|
-
await
|
|
15907
|
+
await yieldToEventLoop2();
|
|
15830
15908
|
if (interrupted(pass)) return false;
|
|
15831
15909
|
}
|
|
15832
|
-
|
|
15833
|
-
|
|
15834
|
-
|
|
15835
|
-
|
|
15910
|
+
const keepCurrent = () => {
|
|
15911
|
+
if (interrupted(pass)) return false;
|
|
15912
|
+
applyWrittenDocuments(pass, built);
|
|
15913
|
+
return true;
|
|
15914
|
+
};
|
|
15915
|
+
if (pass.hooks.holdBeforeSwitch !== void 0 && !await pass.hooks.holdBeforeSwitch(keepCurrent)) return false;
|
|
15916
|
+
const swap = () => {
|
|
15917
|
+
if (interrupted(pass)) return false;
|
|
15918
|
+
applyWrittenDocuments(pass, built);
|
|
15919
|
+
pass.manager.adoptRebuiltPartitions(built);
|
|
15920
|
+
pass.hooks.onSwitch();
|
|
15921
|
+
return true;
|
|
15922
|
+
};
|
|
15923
|
+
return pass.hooks.runSwitch === void 0 ? swap() : pass.hooks.runSwitch(swap);
|
|
15836
15924
|
}
|
|
15837
|
-
async function rebuildTextIndexes(manager, hooks) {
|
|
15838
|
-
const options = resolvePartitionInsertOptions(
|
|
15925
|
+
async function rebuildTextIndexes(manager, hooks, target) {
|
|
15926
|
+
const options = resolvePartitionInsertOptions(
|
|
15927
|
+
target?.config ?? manager.config,
|
|
15928
|
+
target?.analysis ?? manager.analysis,
|
|
15929
|
+
{
|
|
15930
|
+
validate: false,
|
|
15931
|
+
skipClone: true
|
|
15932
|
+
}
|
|
15933
|
+
);
|
|
15934
|
+
const language = target?.language ?? manager.language;
|
|
15935
|
+
const partialWordAnalysis = target === void 0 ? manager.partialWordAnalysis : target.partialWordAnalysis;
|
|
15936
|
+
const spill = await rewriteSpillOf(hooks.rewriteStorage?.(), manager, target);
|
|
15839
15937
|
while (hooks.isCurrent()) {
|
|
15840
15938
|
const log = manager.openWriteLog();
|
|
15939
|
+
let swapped = false;
|
|
15841
15940
|
try {
|
|
15842
|
-
|
|
15941
|
+
swapped = await rebuildOnce({ manager, log, hooks, language, partialWordAnalysis, options, spill });
|
|
15942
|
+
if (swapped) {
|
|
15943
|
+
hooks.onRewriteSpill?.(spill);
|
|
15944
|
+
return true;
|
|
15945
|
+
}
|
|
15843
15946
|
} finally {
|
|
15844
15947
|
log.close();
|
|
15948
|
+
if (!swapped) await spill?.abandon();
|
|
15845
15949
|
}
|
|
15846
15950
|
}
|
|
15847
15951
|
return false;
|
|
@@ -15875,8 +15979,10 @@ function toError(value) {
|
|
|
15875
15979
|
function wireAnalysisRebuild(wiring) {
|
|
15876
15980
|
return createAnalysisRebuildCoordinator(wiring.config, {
|
|
15877
15981
|
getManager: wiring.getManager,
|
|
15982
|
+
rewriteStorage: () => wiring.durabilityManager?.rewriteStorage?.(),
|
|
15878
15983
|
desyncIndex: wiring.desyncIndex,
|
|
15879
15984
|
resyncIndex: wiring.resyncIndex,
|
|
15985
|
+
hasPendingChange: wiring.hasPendingChange,
|
|
15880
15986
|
async persistAnalysisRevision(indexName) {
|
|
15881
15987
|
const durability = wiring.durabilityManager;
|
|
15882
15988
|
if (durability === null) {
|
|
@@ -15894,6 +16000,7 @@ function wireAnalysisRebuild(wiring) {
|
|
|
15894
16000
|
}
|
|
15895
16001
|
function createAnalysisRebuildCoordinator(config, deps) {
|
|
15896
16002
|
const stale = /* @__PURE__ */ new Map();
|
|
16003
|
+
const exclusiveRuns = /* @__PURE__ */ new Map();
|
|
15897
16004
|
const rebuildMode = config?.rebuild ?? "auto";
|
|
15898
16005
|
let queue = Promise.resolve();
|
|
15899
16006
|
let active = null;
|
|
@@ -15906,8 +16013,13 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
15906
16013
|
const partitionCount = manager.partitionCount;
|
|
15907
16014
|
deps.emit({ indexName, status: "started", partitionsRebuilt: 0, partitionCount });
|
|
15908
16015
|
const wasPromoted = deps.desyncIndex(indexName);
|
|
16016
|
+
let spill = null;
|
|
15909
16017
|
try {
|
|
15910
16018
|
const rebuilt = await rebuildTextIndexes(manager, {
|
|
16019
|
+
rewriteStorage: () => deps.rewriteStorage?.(),
|
|
16020
|
+
onRewriteSpill(kept) {
|
|
16021
|
+
spill = kept;
|
|
16022
|
+
},
|
|
15911
16023
|
isCurrent: () => deps.getManager(indexName) === manager && stale.get(indexName) === entry && !entry.abandoned,
|
|
15912
16024
|
onPartitionBuilt(partitionsBuilt) {
|
|
15913
16025
|
entry.partitionsRebuilt = partitionsBuilt;
|
|
@@ -15922,6 +16034,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
15922
16034
|
return;
|
|
15923
16035
|
}
|
|
15924
16036
|
await deps.persistAnalysisRevision(indexName);
|
|
16037
|
+
await spill?.removeUnread(manager);
|
|
15925
16038
|
await deps.resyncIndex(indexName, wasPromoted);
|
|
15926
16039
|
deps.emit({
|
|
15927
16040
|
indexName,
|
|
@@ -15969,7 +16082,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
15969
16082
|
await config?.onStaleAnalysis?.({ ...entry.index, documentCount }, () => enqueue(indexName, entry));
|
|
15970
16083
|
}
|
|
15971
16084
|
async function review(indexName, entry) {
|
|
15972
|
-
if (entry.run !== null || deps.getManager(indexName) === void 0) return;
|
|
16085
|
+
if (entry.run !== null || deps.getManager(indexName) === void 0 || deps.hasPendingChange(indexName)) return;
|
|
15973
16086
|
try {
|
|
15974
16087
|
await announce(indexName, entry);
|
|
15975
16088
|
} catch (err) {
|
|
@@ -16001,8 +16114,22 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
16001
16114
|
return stale.has(indexName);
|
|
16002
16115
|
},
|
|
16003
16116
|
isRunning(indexName) {
|
|
16117
|
+
if ((exclusiveRuns.get(indexName) ?? 0) > 0) return true;
|
|
16004
16118
|
return stale.get(indexName)?.run !== null && stale.get(indexName)?.run !== void 0;
|
|
16005
16119
|
},
|
|
16120
|
+
runExclusive(indexName, work, turnEnds) {
|
|
16121
|
+
exclusiveRuns.set(indexName, (exclusiveRuns.get(indexName) ?? 0) + 1);
|
|
16122
|
+
const run = queue.then(work).finally(() => {
|
|
16123
|
+
const remaining = (exclusiveRuns.get(indexName) ?? 1) - 1;
|
|
16124
|
+
if (remaining === 0) exclusiveRuns.delete(indexName);
|
|
16125
|
+
else exclusiveRuns.set(indexName, remaining);
|
|
16126
|
+
});
|
|
16127
|
+
queue = Promise.race([run, turnEnds ?? run]).then(
|
|
16128
|
+
() => void 0,
|
|
16129
|
+
() => void 0
|
|
16130
|
+
);
|
|
16131
|
+
return run;
|
|
16132
|
+
},
|
|
16006
16133
|
async abandon(indexName) {
|
|
16007
16134
|
const entry = stale.get(indexName);
|
|
16008
16135
|
if (entry !== void 0 && entry.run !== null) entry.abandoned = true;
|
|
@@ -16181,6 +16308,9 @@ function reconstructSchemaFromMetadata(metadata) {
|
|
|
16181
16308
|
if (metadata.trackPositions !== void 0) {
|
|
16182
16309
|
config.trackPositions = metadata.trackPositions;
|
|
16183
16310
|
}
|
|
16311
|
+
if (metadata.textStorage !== void 0) {
|
|
16312
|
+
config.textStorage = metadata.textStorage;
|
|
16313
|
+
}
|
|
16184
16314
|
if (metadata.strict !== void 0) {
|
|
16185
16315
|
config.strict = metadata.strict;
|
|
16186
16316
|
}
|
|
@@ -16401,6 +16531,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
16401
16531
|
if (loadingVectorCode !== null) await loadingVectorCode;
|
|
16402
16532
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
16403
16533
|
if (existing !== void 0) {
|
|
16534
|
+
if ((metadata.analysisGeneration ?? 0) > existing.analysisGeneration) {
|
|
16535
|
+
const { adoptSavedAnalysis } = await import('./analysis-change-JPV775LR.mjs');
|
|
16536
|
+
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
16537
|
+
}
|
|
16404
16538
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
16405
16539
|
deps.executor.createIndex(
|
|
16406
16540
|
metadata.indexName,
|
|
@@ -16431,6 +16565,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
16431
16565
|
const synonyms = prepared?.encoded ?? null;
|
|
16432
16566
|
try {
|
|
16433
16567
|
validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
|
|
16568
|
+
validateTextStorage(indexConfig.textStorage, deps.filesystemDurability);
|
|
16434
16569
|
if (loadData) {
|
|
16435
16570
|
deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
|
|
16436
16571
|
}
|
|
@@ -16449,7 +16584,10 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
16449
16584
|
partitionCount: metadata.partitionCount,
|
|
16450
16585
|
analysisRevision: metadata.analysisRevision ?? null,
|
|
16451
16586
|
arrayPositionGap: metadata.arrayPositionGap ?? null,
|
|
16452
|
-
synonyms
|
|
16587
|
+
synonyms,
|
|
16588
|
+
analysisGeneration: metadata.analysisGeneration ?? 0,
|
|
16589
|
+
pendingAnalysis: metadata.pendingAnalysis ?? null,
|
|
16590
|
+
durableAnalysis: null
|
|
16453
16591
|
});
|
|
16454
16592
|
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
16455
16593
|
else deps.indexState().registerClosed(metadata.indexName);
|
|
@@ -16849,6 +16987,7 @@ function wireIndexState(wiring) {
|
|
|
16849
16987
|
}
|
|
16850
16988
|
try {
|
|
16851
16989
|
await durability.manager.recoverIndex(indexName);
|
|
16990
|
+
await wiring.resumeChange(indexName);
|
|
16852
16991
|
await analysisRebuild.review(indexName);
|
|
16853
16992
|
await orchestrator.openIndex(indexName);
|
|
16854
16993
|
await wiring.onOpen?.(indexName);
|
|
@@ -17178,6 +17317,9 @@ function checkHeapAfterRecovery(core) {
|
|
|
17178
17317
|
|
|
17179
17318
|
// src/core/partition/frozen/share.ts
|
|
17180
17319
|
function shareFrozenSegment(segment, analysis) {
|
|
17320
|
+
if (segment.paged !== null) {
|
|
17321
|
+
return { snapshot: segment.paged.snapshot, tombstonedDocIds: segment.tombstonedDocIds() };
|
|
17322
|
+
}
|
|
17181
17323
|
if (segment.sharedSnapshot !== null) {
|
|
17182
17324
|
const tombstonedDocIds = segment.tombstonedDocIds();
|
|
17183
17325
|
if (tombstonedDocIds.length === 0 || !segment.removals.settled) {
|
|
@@ -17569,14 +17711,18 @@ function assertConfigReachesWorker(indexName, config, bootstrapModule) {
|
|
|
17569
17711
|
{ indexName }
|
|
17570
17712
|
);
|
|
17571
17713
|
}
|
|
17714
|
+
if (bootstrapModule !== void 0) return;
|
|
17572
17715
|
const language = config.language ?? "english";
|
|
17573
|
-
|
|
17716
|
+
const requireRegistered = (setting, name) => {
|
|
17574
17717
|
throw new NarsilError(
|
|
17575
17718
|
ErrorCodes.CONFIG_INVALID,
|
|
17576
|
-
`Index "${indexName}" uses
|
|
17577
|
-
{ indexName,
|
|
17719
|
+
`Index "${indexName}" uses ${setting} "${name}", which a worker thread registers only from a bootstrap module. Set workers.bootstrapModule to a module that registers it`,
|
|
17720
|
+
{ indexName, [setting]: name }
|
|
17578
17721
|
);
|
|
17579
|
-
}
|
|
17722
|
+
};
|
|
17723
|
+
if (language !== "english") requireRegistered("language", language);
|
|
17724
|
+
if (typeof config.tokenizer === "string") requireRegistered("tokenizer", config.tokenizer);
|
|
17725
|
+
if (typeof config.stopWords === "string") requireRegistered("stopWords", config.stopWords);
|
|
17580
17726
|
}
|
|
17581
17727
|
function workerIneligibility(indexName, config, bootstrapModule) {
|
|
17582
17728
|
try {
|
|
@@ -17772,7 +17918,7 @@ async function replicateToWorkers(state, action) {
|
|
|
17772
17918
|
const loading3 = state.copyLoadBuffers.get(action.indexName);
|
|
17773
17919
|
if (loading3 !== void 0) {
|
|
17774
17920
|
queueForCopies(state, detached);
|
|
17775
|
-
if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await
|
|
17921
|
+
if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop2();
|
|
17776
17922
|
return;
|
|
17777
17923
|
}
|
|
17778
17924
|
const queue = state.replicationQueues.get(action.indexName);
|
|
@@ -17980,7 +18126,7 @@ async function compactPartitionSegments(state, indexName, manager, partitionId,
|
|
|
17980
18126
|
}
|
|
17981
18127
|
async function compactIndexSegments(state, indexName, policy) {
|
|
17982
18128
|
const manager = state.executor.getManager(indexName);
|
|
17983
|
-
if (!manager) return;
|
|
18129
|
+
if (!manager || manager.config.textStorage === "disk") return;
|
|
17984
18130
|
const partitionIds = /* @__PURE__ */ new Set();
|
|
17985
18131
|
for (let partitionId = 0; partitionId < manager.partitionCount; partitionId++) partitionIds.add(partitionId);
|
|
17986
18132
|
for (const partitionId of state.segmentLedger.get(indexName)?.keys() ?? []) partitionIds.add(partitionId);
|
|
@@ -19431,6 +19577,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
19431
19577
|
scaleOutBeforeBatch: (indexName, incomingCount) => scaleOutBeforeBatch(state, indexName, incomingCount),
|
|
19432
19578
|
replicateToWorkers: replicate,
|
|
19433
19579
|
awaitReplication: (indexName) => awaitReplicationIdle(state, indexName),
|
|
19580
|
+
queueDiskAdoption: (action) => queueForCopies(state, action),
|
|
19434
19581
|
awaitWrites: (indexName) => awaitWritesApplied(state, indexName),
|
|
19435
19582
|
awaitCompactions: () => awaitCompactions(state),
|
|
19436
19583
|
openIndex,
|
|
@@ -19504,7 +19651,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
19504
19651
|
// src/engine/core.ts
|
|
19505
19652
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
19506
19653
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
19507
|
-
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-
|
|
19654
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-VEUH674B.mjs')).resolveRerankSettings(config);
|
|
19508
19655
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
19509
19656
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
19510
19657
|
const vectorCopyPolicy = {
|
|
@@ -19515,7 +19662,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19515
19662
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
19516
19663
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
19517
19664
|
}
|
|
19518
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
19665
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-SNOWIT6Z.mjs');
|
|
19519
19666
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
19520
19667
|
const executor = createDirectExecutor({
|
|
19521
19668
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -19575,7 +19722,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19575
19722
|
},
|
|
19576
19723
|
vectorCopyPolicy
|
|
19577
19724
|
);
|
|
19578
|
-
const rebalancer = createRebalancer();
|
|
19579
19725
|
const rebalanceRouter = createPartitionRouter();
|
|
19580
19726
|
const waqMap = /* @__PURE__ */ new Map();
|
|
19581
19727
|
const rebalanceTargets = /* @__PURE__ */ new Map();
|
|
@@ -19635,6 +19781,8 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19635
19781
|
emitCheckpointHeld(report) {
|
|
19636
19782
|
emitEngineEvent(eventHandlers, "checkpointHeld", report);
|
|
19637
19783
|
},
|
|
19784
|
+
queueDiskAdoption: orchestrator.queueDiskAdoption,
|
|
19785
|
+
awaitCopiesOfIndex: (indexName) => orchestrator.awaitReplication(indexName),
|
|
19638
19786
|
publishCheckpointedPartitions: (indexName, partitions) => invalidation?.publishPartitions(indexName, partitions) ?? Promise.resolve(),
|
|
19639
19787
|
recordCheckpoint(indexName, documentCount, partitionCount) {
|
|
19640
19788
|
const entry = indexRegistry.get(indexName);
|
|
@@ -19685,6 +19833,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19685
19833
|
entry.analysisRevision = entry.language.revision;
|
|
19686
19834
|
entry.arrayPositionGap = ARRAY_POSITION_GAP;
|
|
19687
19835
|
},
|
|
19836
|
+
hasPendingChange: (indexName) => (indexRegistry.get(indexName)?.pendingAnalysis ?? null) !== null,
|
|
19688
19837
|
durabilityManager: durability?.manager ?? null
|
|
19689
19838
|
});
|
|
19690
19839
|
const indexState = wireIndexState({
|
|
@@ -19696,6 +19845,10 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19696
19845
|
indexRegistry,
|
|
19697
19846
|
rebalancingIndexes,
|
|
19698
19847
|
requireManager,
|
|
19848
|
+
resumeChange: async (indexName) => {
|
|
19849
|
+
if ((indexRegistry.get(indexName)?.pendingAnalysis ?? null) === null) return;
|
|
19850
|
+
await (await import('./analysis-change-entry-7HR4LWL3.mjs')).resumeChangeOf(engineCore, indexName);
|
|
19851
|
+
},
|
|
19699
19852
|
onClose: hooks?.onIndexClose,
|
|
19700
19853
|
onOpen: (indexName) => {
|
|
19701
19854
|
heapPressureNotifier.check(indexName);
|
|
@@ -19724,7 +19877,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19724
19877
|
checkHeapPressure: heapPressureNotifier.check
|
|
19725
19878
|
};
|
|
19726
19879
|
const rebalanceCtx = {
|
|
19727
|
-
rebalancer,
|
|
19728
19880
|
router: rebalanceRouter,
|
|
19729
19881
|
waqMap,
|
|
19730
19882
|
rebalancingIndexes,
|
|
@@ -19736,7 +19888,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19736
19888
|
checkWatermark: watermarkNotifier.check,
|
|
19737
19889
|
requireIndex
|
|
19738
19890
|
};
|
|
19739
|
-
|
|
19891
|
+
const engineCore = {
|
|
19740
19892
|
executor,
|
|
19741
19893
|
pluginRegistry,
|
|
19742
19894
|
durability,
|
|
@@ -19751,7 +19903,6 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19751
19903
|
shutdownState,
|
|
19752
19904
|
abortController,
|
|
19753
19905
|
orchestrator,
|
|
19754
|
-
rebalancer,
|
|
19755
19906
|
rebalanceRouter,
|
|
19756
19907
|
rebalancingIndexes,
|
|
19757
19908
|
waqMap,
|
|
@@ -19767,6 +19918,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19767
19918
|
rebalanceCtx,
|
|
19768
19919
|
patternWorkCap: config?.patternWorkCap ?? DEFAULT_PATTERN_WORK_CAP
|
|
19769
19920
|
};
|
|
19921
|
+
return engineCore;
|
|
19770
19922
|
}
|
|
19771
19923
|
|
|
19772
19924
|
// src/engine/lifecycle.ts
|
|
@@ -19777,6 +19929,7 @@ async function startEngineCore(core, config) {
|
|
|
19777
19929
|
checkHeapAfterRecovery(core);
|
|
19778
19930
|
}
|
|
19779
19931
|
if (core.invalidation) await core.invalidation.start();
|
|
19932
|
+
if (core.durability) await (await import('./analysis-change-entry-7HR4LWL3.mjs')).resumeEveryChange(core);
|
|
19780
19933
|
if (config?.lifecycle === void 0) await core.analysisRebuild.reviewStaleIndexes();
|
|
19781
19934
|
} catch (err) {
|
|
19782
19935
|
await core.durability?.manager.shutdown().catch(() => void 0);
|
|
@@ -19814,7 +19967,7 @@ async function shutdownEngine(core) {
|
|
|
19814
19967
|
}
|
|
19815
19968
|
}
|
|
19816
19969
|
if (core.rerankAdapters.size > 0) {
|
|
19817
|
-
await (await import('./runtime-
|
|
19970
|
+
await (await import('./runtime-VEUH674B.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
19818
19971
|
}
|
|
19819
19972
|
await orchestrator.shutdown();
|
|
19820
19973
|
await executor.shutdown();
|
|
@@ -20124,9 +20277,12 @@ function requireWithinFileLimit(byteLength, what) {
|
|
|
20124
20277
|
);
|
|
20125
20278
|
}
|
|
20126
20279
|
async function packChecksummedEnvelope(payloadChunks, envelopeFormatVersion, what) {
|
|
20280
|
+
requireWithinFileLimit(HEADER_SIZE + payloadByteLength(payloadChunks), what);
|
|
20281
|
+
return packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, await checksumOfChunks(payloadChunks));
|
|
20282
|
+
}
|
|
20283
|
+
function packEnvelopeWithChecksum(payloadChunks, envelopeFormatVersion, what, checksum) {
|
|
20127
20284
|
const payloadLength = payloadByteLength(payloadChunks);
|
|
20128
20285
|
requireWithinFileLimit(HEADER_SIZE + payloadLength, what);
|
|
20129
|
-
const checksum = await checksumOfChunks(payloadChunks);
|
|
20130
20286
|
const [major, minor, patch] = engineVersionParts();
|
|
20131
20287
|
const header = writeHeader({
|
|
20132
20288
|
envelopeFormatVersion,
|
|
@@ -20244,7 +20400,8 @@ function indexConfigurationOf(entry) {
|
|
|
20244
20400
|
entry.embeddingAdapterName,
|
|
20245
20401
|
entry.analysisRevision,
|
|
20246
20402
|
entry.arrayPositionGap,
|
|
20247
|
-
synonyms
|
|
20403
|
+
synonyms,
|
|
20404
|
+
entry.analysisGeneration
|
|
20248
20405
|
);
|
|
20249
20406
|
}
|
|
20250
20407
|
function requireConfigurationWithinSizeLimit(config, language) {
|
|
@@ -20265,7 +20422,7 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
20265
20422
|
{ bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
|
|
20266
20423
|
);
|
|
20267
20424
|
}
|
|
20268
|
-
function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms) {
|
|
20425
|
+
function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms, analysisGeneration = 0) {
|
|
20269
20426
|
const bm25 = config.bm25;
|
|
20270
20427
|
return {
|
|
20271
20428
|
schema: config.schema,
|
|
@@ -20275,6 +20432,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
|
|
|
20275
20432
|
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
20276
20433
|
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
20277
20434
|
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
|
|
20435
|
+
...analysisGeneration > 0 ? { analysisGeneration } : {},
|
|
20278
20436
|
...bm25 !== void 0 ? {
|
|
20279
20437
|
bm25: {
|
|
20280
20438
|
...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
|
|
@@ -20286,6 +20444,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
|
|
|
20286
20444
|
...config.partitions !== void 0 ? { partitionConfig: config.partitions } : {},
|
|
20287
20445
|
...config.defaultScoring !== void 0 ? { defaultScoring: config.defaultScoring } : {},
|
|
20288
20446
|
...config.trackPositions !== void 0 ? { trackPositions: config.trackPositions } : {},
|
|
20447
|
+
...config.textStorage !== void 0 ? { textStorage: config.textStorage } : {},
|
|
20289
20448
|
...config.strict !== void 0 ? { strict: config.strict } : {},
|
|
20290
20449
|
...config.required !== void 0 ? { required: config.required } : {},
|
|
20291
20450
|
...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
|
|
@@ -20374,6 +20533,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
20374
20533
|
requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
|
|
20375
20534
|
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
20376
20535
|
validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
|
|
20536
|
+
validateTextStorage(indexConfig.textStorage, core.filesystemDurability);
|
|
20377
20537
|
if (indexConfig.bm25 !== void 0) {
|
|
20378
20538
|
validateBM25Params(
|
|
20379
20539
|
indexConfig.bm25,
|
|
@@ -20449,7 +20609,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
20449
20609
|
partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
|
|
20450
20610
|
analysisRevision: language.revision,
|
|
20451
20611
|
arrayPositionGap: ARRAY_POSITION_GAP,
|
|
20452
|
-
synonyms: synonyms?.encoded ?? null
|
|
20612
|
+
synonyms: synonyms?.encoded ?? null,
|
|
20613
|
+
analysisGeneration: 0,
|
|
20614
|
+
pendingAnalysis: null,
|
|
20615
|
+
durableAnalysis: null
|
|
20453
20616
|
});
|
|
20454
20617
|
await core.indexState.registerOpen(name);
|
|
20455
20618
|
if (core.durability) {
|
|
@@ -20460,8 +20623,14 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
20460
20623
|
async function dropEngineIndex(core, name) {
|
|
20461
20624
|
core.guardShutdown();
|
|
20462
20625
|
const entry = core.requireIndex(name);
|
|
20626
|
+
const changeRecorded = () => entry.pendingAnalysis !== null || entry.durableAnalysis !== null;
|
|
20627
|
+
const abandonChange = async () => {
|
|
20628
|
+
await (await import('./analysis-change-entry-7HR4LWL3.mjs')).abandonChangeOf(core, name);
|
|
20629
|
+
};
|
|
20630
|
+
if (changeRecorded()) await abandonChange();
|
|
20463
20631
|
await core.indexState.drop(name, async () => {
|
|
20464
20632
|
await core.analysisRebuild.abandon(name);
|
|
20633
|
+
if (changeRecorded()) await abandonChange();
|
|
20465
20634
|
await core.orchestrator.closeIndex(name);
|
|
20466
20635
|
if (core.durability) {
|
|
20467
20636
|
try {
|
|
@@ -21293,7 +21462,7 @@ function snapshotSegments(segments) {
|
|
|
21293
21462
|
}
|
|
21294
21463
|
return shared2;
|
|
21295
21464
|
}
|
|
21296
|
-
function
|
|
21465
|
+
function attachSegments2(orchestrator, indexName, segments) {
|
|
21297
21466
|
return orchestrator.replicateToWorkers({
|
|
21298
21467
|
type: "attachSegments",
|
|
21299
21468
|
indexName,
|
|
@@ -21304,11 +21473,11 @@ function attachSegments(orchestrator, indexName, segments) {
|
|
|
21304
21473
|
async function broadcastBuiltSegments(orchestrator, indexName, segments, skipClone) {
|
|
21305
21474
|
const frozen = tryFreezeSegmentsForAttach(segments);
|
|
21306
21475
|
if (frozen !== null) {
|
|
21307
|
-
await
|
|
21476
|
+
await attachSegments2(orchestrator, indexName, frozen);
|
|
21308
21477
|
return;
|
|
21309
21478
|
}
|
|
21310
21479
|
const shared2 = snapshotSegments(segments);
|
|
21311
|
-
if (shared2.length > 0) await
|
|
21480
|
+
if (shared2.length > 0) await attachSegments2(orchestrator, indexName, shared2);
|
|
21312
21481
|
const plain = payloadSegments(segments);
|
|
21313
21482
|
if (plain.length === 0) return;
|
|
21314
21483
|
await orchestrator.replicateToWorkers({
|
|
@@ -22035,303 +22204,33 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
22035
22204
|
for (let i = 0; i < prepared.length; i++) {
|
|
22036
22205
|
const update = prepared[i];
|
|
22037
22206
|
const failure = failures[i];
|
|
22038
|
-
if (failure !== null) {
|
|
22039
|
-
let error = failure.error;
|
|
22040
|
-
try {
|
|
22041
|
-
await rollbackUpdatedDocument(ctx, indexName, update.docId, update.rollbackDoc, error);
|
|
22042
|
-
} catch (rollbackError) {
|
|
22043
|
-
error = rollbackError;
|
|
22044
|
-
}
|
|
22045
|
-
failed.push({ docId: update.docId, error: asUpdateError(error) });
|
|
22046
|
-
continue;
|
|
22047
|
-
}
|
|
22048
|
-
if (!update.buffered) await settleUpdate(ctx, indexName, update);
|
|
22049
|
-
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
22050
|
-
if (vec !== null) touchedVectorFields.add(fieldPath);
|
|
22051
|
-
}
|
|
22052
|
-
succeeded.push(update.docId);
|
|
22053
|
-
}
|
|
22054
|
-
if (chunkEnd < updates.length) {
|
|
22055
|
-
ctx.checkHeapPressure(indexName);
|
|
22056
|
-
await new Promise((r) => setTimeout(r, 0));
|
|
22057
|
-
}
|
|
22058
|
-
}
|
|
22059
|
-
for (const fieldPath of touchedVectorFields) {
|
|
22060
|
-
updateBatchVecIndexes.get(fieldPath)?.scheduleBuild();
|
|
22061
|
-
}
|
|
22062
|
-
ctx.checkHeapPressure(indexName);
|
|
22063
|
-
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
22064
|
-
return { succeeded, failed };
|
|
22065
|
-
}
|
|
22066
|
-
|
|
22067
|
-
// src/partitioning/write-ahead-queue.ts
|
|
22068
|
-
function createWriteAheadQueue(maxSize = DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE) {
|
|
22069
|
-
const entries = [];
|
|
22070
|
-
const docStates = /* @__PURE__ */ new Map();
|
|
22071
|
-
let nextSequence = 1;
|
|
22072
|
-
let resolveReplayed = () => void 0;
|
|
22073
|
-
const replayed = new Promise((resolve) => {
|
|
22074
|
-
resolveReplayed = resolve;
|
|
22075
|
-
});
|
|
22076
|
-
return {
|
|
22077
|
-
push(entry) {
|
|
22078
|
-
if (entries.length >= maxSize) {
|
|
22079
|
-
throw new NarsilError(
|
|
22080
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
22081
|
-
`Write-ahead queue is full (${maxSize} entries). Retry after rebalancing completes.`,
|
|
22082
|
-
{ maxSize, currentSize: entries.length }
|
|
22083
|
-
);
|
|
22084
|
-
}
|
|
22085
|
-
const seq = nextSequence++;
|
|
22086
|
-
entries.push({ ...entry, sequenceNumber: seq });
|
|
22087
|
-
docStates.set(entry.docId, entry.action === "remove" ? "absent" : "present");
|
|
22088
|
-
return seq;
|
|
22089
|
-
},
|
|
22090
|
-
drain() {
|
|
22091
|
-
const result = entries.splice(0);
|
|
22092
|
-
result.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
|
|
22093
|
-
return result;
|
|
22094
|
-
},
|
|
22095
|
-
clear() {
|
|
22096
|
-
entries.length = 0;
|
|
22097
|
-
docStates.clear();
|
|
22098
|
-
},
|
|
22099
|
-
bufferedDocState(docId) {
|
|
22100
|
-
return docStates.get(docId);
|
|
22101
|
-
},
|
|
22102
|
-
whenReplayed() {
|
|
22103
|
-
return replayed;
|
|
22104
|
-
},
|
|
22105
|
-
markReplayed() {
|
|
22106
|
-
resolveReplayed();
|
|
22107
|
-
},
|
|
22108
|
-
get size() {
|
|
22109
|
-
return entries.length;
|
|
22110
|
-
},
|
|
22111
|
-
get isFull() {
|
|
22112
|
-
return entries.length >= maxSize;
|
|
22113
|
-
}
|
|
22114
|
-
};
|
|
22115
|
-
}
|
|
22116
|
-
|
|
22117
|
-
// src/engine/rebalance-executor.ts
|
|
22118
|
-
function warnHookError(hook, err) {
|
|
22119
|
-
console.warn(`${hook} plugin hook error:`, err instanceof Error ? err.message : String(err));
|
|
22120
|
-
}
|
|
22121
|
-
function applyReplayUpdateVectors(entry, vectorFieldPaths2) {
|
|
22122
|
-
const updateVectors = /* @__PURE__ */ new Map();
|
|
22123
|
-
for (const fieldPath of vectorFieldPaths2) {
|
|
22124
|
-
updateVectors.set(fieldPath, extractVectorFromDoc(entry.document, fieldPath));
|
|
22125
|
-
}
|
|
22126
|
-
let updateDoc = entry.document;
|
|
22127
|
-
if (updateVectors.size > 0) {
|
|
22128
|
-
updateDoc = structuredClone(entry.document);
|
|
22129
|
-
for (const fieldPath of updateVectors.keys()) {
|
|
22130
|
-
deleteNestedValue(updateDoc, fieldPath);
|
|
22131
|
-
}
|
|
22132
|
-
}
|
|
22133
|
-
return { updateDoc, updateVectors };
|
|
22134
|
-
}
|
|
22135
|
-
async function replayEntry(manager, indexName, entry, ctx) {
|
|
22136
|
-
const vecIndexes = manager.getVectorIndexes();
|
|
22137
|
-
const vectorFieldPaths2 = ctx.requireIndex(indexName).vectorFieldPaths;
|
|
22138
|
-
if (entry.action === "insert" && entry.document) {
|
|
22139
|
-
const { partitionDoc, extractedVectors } = prepareDocumentVectors(
|
|
22140
|
-
entry.document,
|
|
22141
|
-
vectorFieldPaths2
|
|
22142
|
-
);
|
|
22143
|
-
const overwrote = manager.has(entry.docId);
|
|
22144
|
-
const removedSpellings = overwrote ? manager.removedSpellingsOf(entry.docId) : void 0;
|
|
22145
|
-
if (overwrote) {
|
|
22146
|
-
removeDocumentVectors(entry.docId, vecIndexes);
|
|
22147
|
-
manager.remove(entry.docId, removedSpellings);
|
|
22148
|
-
}
|
|
22149
|
-
manager.insert(entry.docId, partitionDoc);
|
|
22150
|
-
if (extractedVectors.size > 0) {
|
|
22151
|
-
insertDocumentVectors(entry.docId, extractedVectors, vecIndexes, manager.partitionIdOf(entry.docId));
|
|
22152
|
-
for (const fieldPath of extractedVectors.keys()) {
|
|
22153
|
-
vecIndexes.get(fieldPath)?.scheduleBuild();
|
|
22154
|
-
}
|
|
22155
|
-
}
|
|
22156
|
-
try {
|
|
22157
|
-
await ctx.pluginRegistry.runHook("afterInsert", { indexName, docId: entry.docId, document: entry.document });
|
|
22158
|
-
} catch (err) {
|
|
22159
|
-
warnHookError("afterInsert", err);
|
|
22160
|
-
}
|
|
22161
|
-
const replicated = {
|
|
22162
|
-
indexName,
|
|
22163
|
-
docId: entry.docId,
|
|
22164
|
-
document: entry.document,
|
|
22165
|
-
requestId: `replicate-insert-${entry.docId}`
|
|
22166
|
-
};
|
|
22167
|
-
await ctx.orchestrator.replicateToWorkers(
|
|
22168
|
-
overwrote ? { type: "update", ...replicated, removedSpellings } : { type: "insert", ...replicated }
|
|
22169
|
-
);
|
|
22170
|
-
return;
|
|
22171
|
-
}
|
|
22172
|
-
if (entry.action === "remove") {
|
|
22173
|
-
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
22174
|
-
manager.remove(entry.docId, removedSpellings);
|
|
22175
|
-
removeDocumentVectors(entry.docId, vecIndexes);
|
|
22176
|
-
try {
|
|
22177
|
-
await ctx.pluginRegistry.runHook("afterRemove", { indexName, docId: entry.docId });
|
|
22178
|
-
} catch (err) {
|
|
22179
|
-
warnHookError("afterRemove", err);
|
|
22180
|
-
}
|
|
22181
|
-
await ctx.orchestrator.replicateToWorkers({
|
|
22182
|
-
type: "remove",
|
|
22183
|
-
indexName,
|
|
22184
|
-
docId: entry.docId,
|
|
22185
|
-
requestId: `replicate-remove-${entry.docId}`,
|
|
22186
|
-
removedSpellings
|
|
22187
|
-
});
|
|
22188
|
-
return;
|
|
22189
|
-
}
|
|
22190
|
-
if (entry.action === "update" && entry.document) {
|
|
22191
|
-
const oldDocument = manager.get(entry.docId);
|
|
22192
|
-
const { updateDoc, updateVectors } = applyReplayUpdateVectors(entry, vectorFieldPaths2);
|
|
22193
|
-
const removedSpellings = manager.removedSpellingsOf(entry.docId);
|
|
22194
|
-
if (manager.has(entry.docId)) {
|
|
22195
|
-
manager.update(entry.docId, updateDoc, void 0, removedSpellings);
|
|
22196
|
-
} else {
|
|
22197
|
-
manager.insert(entry.docId, updateDoc);
|
|
22198
|
-
}
|
|
22199
|
-
for (const [fieldPath, newVec] of updateVectors) {
|
|
22200
|
-
const vecIndex = vecIndexes.get(fieldPath);
|
|
22201
|
-
if (!vecIndex) continue;
|
|
22202
|
-
if (newVec === null) {
|
|
22203
|
-
if (vecIndex.has(entry.docId)) {
|
|
22204
|
-
vecIndex.remove(entry.docId);
|
|
22205
|
-
}
|
|
22206
|
-
} else {
|
|
22207
|
-
const oldVec = vecIndex.getVector(entry.docId);
|
|
22208
|
-
if (!vectorsEqual(oldVec, newVec)) {
|
|
22209
|
-
if (vecIndex.has(entry.docId)) {
|
|
22210
|
-
vecIndex.remove(entry.docId);
|
|
22211
|
-
}
|
|
22212
|
-
vecIndex.insert(entry.docId, newVec);
|
|
22213
|
-
vecIndex.scheduleBuild();
|
|
22214
|
-
}
|
|
22215
|
-
}
|
|
22216
|
-
}
|
|
22217
|
-
try {
|
|
22218
|
-
await ctx.pluginRegistry.runHook("afterUpdate", {
|
|
22219
|
-
indexName,
|
|
22220
|
-
docId: entry.docId,
|
|
22221
|
-
oldDocument: oldDocument ?? {},
|
|
22222
|
-
newDocument: entry.document
|
|
22223
|
-
});
|
|
22224
|
-
} catch (err) {
|
|
22225
|
-
warnHookError("afterUpdate", err);
|
|
22226
|
-
}
|
|
22227
|
-
await ctx.orchestrator.replicateToWorkers({
|
|
22228
|
-
type: "update",
|
|
22229
|
-
indexName,
|
|
22230
|
-
docId: entry.docId,
|
|
22231
|
-
document: entry.document,
|
|
22232
|
-
requestId: `replicate-update-${entry.docId}`,
|
|
22233
|
-
removedSpellings
|
|
22234
|
-
});
|
|
22235
|
-
}
|
|
22236
|
-
}
|
|
22237
|
-
async function replayQueued(manager, indexName, waq, ctx) {
|
|
22238
|
-
while (true) {
|
|
22239
|
-
const entries = waq.drain();
|
|
22240
|
-
if (entries.length === 0) break;
|
|
22241
|
-
for (const entry of entries) {
|
|
22242
|
-
try {
|
|
22243
|
-
await replayEntry(manager, indexName, entry, ctx);
|
|
22244
|
-
} catch (replayErr) {
|
|
22245
|
-
const isMissing = replayErr instanceof NarsilError && replayErr.code === ErrorCodes.DOC_NOT_FOUND;
|
|
22246
|
-
if (!isMissing) {
|
|
22247
|
-
console.warn(`WAQ replay failed for ${entry.action} on doc "${entry.docId}":`, replayErr);
|
|
22248
|
-
}
|
|
22249
|
-
}
|
|
22250
|
-
}
|
|
22251
|
-
}
|
|
22252
|
-
}
|
|
22253
|
-
async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
22254
|
-
if (targetPartitionCount <= 0 || !Number.isInteger(targetPartitionCount)) {
|
|
22255
|
-
throw new NarsilError(
|
|
22256
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
22257
|
-
`Target partition count must be a positive integer, got ${targetPartitionCount}`,
|
|
22258
|
-
{ targetPartitionCount }
|
|
22259
|
-
);
|
|
22260
|
-
}
|
|
22261
|
-
const entry = ctx.requireIndex(indexName);
|
|
22262
|
-
const maxPartitions = entry.config.partitions?.maxPartitions;
|
|
22263
|
-
if (maxPartitions !== void 0 && targetPartitionCount > maxPartitions) {
|
|
22264
|
-
throw new NarsilError(
|
|
22265
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
22266
|
-
`Target partition count (${targetPartitionCount}) is above the maximum of ${maxPartitions} partitions`,
|
|
22267
|
-
{ targetPartitionCount, maxPartitions }
|
|
22268
|
-
);
|
|
22269
|
-
}
|
|
22270
|
-
if (ctx.rebalancingIndexes.has(indexName)) {
|
|
22271
|
-
throw new NarsilError(
|
|
22272
|
-
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
22273
|
-
`Index "${indexName}" is already being rebalanced`
|
|
22274
|
-
);
|
|
22275
|
-
}
|
|
22276
|
-
if (targetPartitionCount === manager.partitionCount) {
|
|
22277
|
-
return;
|
|
22278
|
-
}
|
|
22279
|
-
const waq = createWriteAheadQueue();
|
|
22280
|
-
ctx.waqMap.set(indexName, waq);
|
|
22281
|
-
ctx.rebalancingIndexes.add(indexName);
|
|
22282
|
-
ctx.rebalanceTargets.set(indexName, targetPartitionCount);
|
|
22283
|
-
const wasPromoted = ctx.orchestrator.desyncIndex(indexName);
|
|
22284
|
-
try {
|
|
22285
|
-
const oldCount = manager.partitionCount;
|
|
22286
|
-
await ctx.rebalancer.rebalance(manager, targetPartitionCount, ctx.router, (progress) => {
|
|
22287
|
-
if (progress.phase === "complete") {
|
|
22288
|
-
emitEngineEvent(ctx.eventHandlers, "partitionRebalance", {
|
|
22289
|
-
indexName,
|
|
22290
|
-
oldCount,
|
|
22291
|
-
newCount: targetPartitionCount
|
|
22292
|
-
});
|
|
22207
|
+
if (failure !== null) {
|
|
22208
|
+
let error = failure.error;
|
|
22209
|
+
try {
|
|
22210
|
+
await rollbackUpdatedDocument(ctx, indexName, update.docId, update.rollbackDoc, error);
|
|
22211
|
+
} catch (rollbackError) {
|
|
22212
|
+
error = rollbackError;
|
|
22213
|
+
}
|
|
22214
|
+
failed.push({ docId: update.docId, error: asUpdateError(error) });
|
|
22215
|
+
continue;
|
|
22293
22216
|
}
|
|
22294
|
-
|
|
22295
|
-
|
|
22296
|
-
|
|
22297
|
-
|
|
22298
|
-
|
|
22299
|
-
newPartitionCount: targetPartitionCount
|
|
22300
|
-
});
|
|
22301
|
-
} catch (hookErr) {
|
|
22302
|
-
console.warn("onPartitionSplit plugin hook failed:", hookErr instanceof Error ? hookErr.message : String(hookErr));
|
|
22303
|
-
}
|
|
22304
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
22305
|
-
if (ctx.durabilityManager) {
|
|
22306
|
-
await ctx.durabilityManager.persistMetadata(indexName);
|
|
22307
|
-
await (ctx.durabilityManager.checkpointFromMemory?.(indexName) ?? ctx.durabilityManager.checkpoint(indexName));
|
|
22308
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
22309
|
-
}
|
|
22310
|
-
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
22311
|
-
do {
|
|
22312
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
22313
|
-
} while (waq.size > 0);
|
|
22314
|
-
} catch (err) {
|
|
22315
|
-
try {
|
|
22316
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
22317
|
-
await ctx.orchestrator.resyncIndex(indexName, wasPromoted);
|
|
22318
|
-
do {
|
|
22319
|
-
await replayQueued(manager, indexName, waq, ctx);
|
|
22320
|
-
} while (waq.size > 0);
|
|
22321
|
-
} catch (recoveryErr) {
|
|
22322
|
-
console.warn(
|
|
22323
|
-
`Rebalance failure recovery for index "${indexName}" did not complete:`,
|
|
22324
|
-
recoveryErr instanceof Error ? recoveryErr.message : String(recoveryErr)
|
|
22325
|
-
);
|
|
22217
|
+
if (!update.buffered) await settleUpdate(ctx, indexName, update);
|
|
22218
|
+
for (const [fieldPath, vec] of update.extractedVectors) {
|
|
22219
|
+
if (vec !== null) touchedVectorFields.add(fieldPath);
|
|
22220
|
+
}
|
|
22221
|
+
succeeded.push(update.docId);
|
|
22326
22222
|
}
|
|
22327
|
-
|
|
22328
|
-
|
|
22329
|
-
|
|
22330
|
-
|
|
22331
|
-
|
|
22332
|
-
|
|
22223
|
+
if (chunkEnd < updates.length) {
|
|
22224
|
+
ctx.checkHeapPressure(indexName);
|
|
22225
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
22226
|
+
}
|
|
22227
|
+
}
|
|
22228
|
+
for (const fieldPath of touchedVectorFields) {
|
|
22229
|
+
updateBatchVecIndexes.get(fieldPath)?.scheduleBuild();
|
|
22333
22230
|
}
|
|
22334
|
-
ctx.
|
|
22231
|
+
ctx.checkHeapPressure(indexName);
|
|
22232
|
+
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
22233
|
+
return { succeeded, failed };
|
|
22335
22234
|
}
|
|
22336
22235
|
|
|
22337
22236
|
// src/engine/vector-maintenance.ts
|
|
@@ -22373,6 +22272,7 @@ function getVectorMaintenanceStatus(manager) {
|
|
|
22373
22272
|
bufferSize: status.bufferSize,
|
|
22374
22273
|
building: status.building,
|
|
22375
22274
|
waitingForMemory: status.waitingForMemory,
|
|
22275
|
+
loadingFromFiles: status.loadingFromFiles === true,
|
|
22376
22276
|
estimatedCompactMs: status.estimatedCompactMs,
|
|
22377
22277
|
estimatedOptimizeMs: status.estimatedOptimizeMs
|
|
22378
22278
|
});
|
|
@@ -22479,646 +22379,6 @@ function bindEngineCore(engine, core) {
|
|
|
22479
22379
|
engineCores.set(engine, core);
|
|
22480
22380
|
}
|
|
22481
22381
|
|
|
22482
|
-
// src/engine/snapshot/restore-config.ts
|
|
22483
|
-
function rejectSnapshot(message) {
|
|
22484
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Invalid snapshot: ${message}`);
|
|
22485
|
-
}
|
|
22486
|
-
function isPlainObject3(value) {
|
|
22487
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
22488
|
-
}
|
|
22489
|
-
function restoredBm25(raw, reject2) {
|
|
22490
|
-
if (raw === void 0) {
|
|
22491
|
-
return void 0;
|
|
22492
|
-
}
|
|
22493
|
-
if (!isPlainObject3(raw)) {
|
|
22494
|
-
reject2("bm25 must be an object");
|
|
22495
|
-
}
|
|
22496
|
-
const fields = restoredBm25Fields(raw.fields, reject2);
|
|
22497
|
-
return {
|
|
22498
|
-
...restoredBm25Pair(raw, "bm25", reject2),
|
|
22499
|
-
...fields !== void 0 ? { fields } : {}
|
|
22500
|
-
};
|
|
22501
|
-
}
|
|
22502
|
-
function restoredBm25Pair(raw, label, reject2) {
|
|
22503
|
-
const { k1, b } = raw;
|
|
22504
|
-
if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
|
|
22505
|
-
reject2(`${label} k1 must be a finite number`);
|
|
22506
|
-
}
|
|
22507
|
-
if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
|
|
22508
|
-
reject2(`${label} b must be a finite number`);
|
|
22509
|
-
}
|
|
22510
|
-
return {
|
|
22511
|
-
...k1 !== void 0 ? { k1 } : {},
|
|
22512
|
-
...b !== void 0 ? { b } : {}
|
|
22513
|
-
};
|
|
22514
|
-
}
|
|
22515
|
-
function restoredBm25Fields(raw, reject2) {
|
|
22516
|
-
if (raw === void 0) return void 0;
|
|
22517
|
-
if (!isPlainObject3(raw)) reject2("bm25 fields must be an object");
|
|
22518
|
-
const source = raw;
|
|
22519
|
-
const fields = /* @__PURE__ */ Object.create(null);
|
|
22520
|
-
for (const name of Object.keys(source)) {
|
|
22521
|
-
const pair = source[name];
|
|
22522
|
-
if (!isPlainObject3(pair)) reject2("bm25 fields must map each field to an object");
|
|
22523
|
-
fields[name] = restoredBm25Pair(pair, "bm25 field", reject2);
|
|
22524
|
-
}
|
|
22525
|
-
return fields;
|
|
22526
|
-
}
|
|
22527
|
-
function restoredStopWords(name, list, reject2) {
|
|
22528
|
-
if (name !== void 0 && typeof name !== "string") {
|
|
22529
|
-
reject2("stopWords must be a name");
|
|
22530
|
-
}
|
|
22531
|
-
if (list !== void 0 && (!Array.isArray(list) || !list.every((word) => typeof word === "string"))) {
|
|
22532
|
-
reject2("stopWordList must be a list of words");
|
|
22533
|
-
}
|
|
22534
|
-
if (typeof name === "string" && Array.isArray(list)) {
|
|
22535
|
-
reject2("stopWords and stopWordList cannot both be present");
|
|
22536
|
-
}
|
|
22537
|
-
if (typeof name === "string") {
|
|
22538
|
-
return name;
|
|
22539
|
-
}
|
|
22540
|
-
if (Array.isArray(list)) {
|
|
22541
|
-
return new Set(list);
|
|
22542
|
-
}
|
|
22543
|
-
return void 0;
|
|
22544
|
-
}
|
|
22545
|
-
function restoredPartitionConfig(raw, reject2 = rejectSnapshot) {
|
|
22546
|
-
if (raw === void 0) {
|
|
22547
|
-
return void 0;
|
|
22548
|
-
}
|
|
22549
|
-
if (!isPlainObject3(raw)) {
|
|
22550
|
-
reject2("partitionConfig must be an object");
|
|
22551
|
-
}
|
|
22552
|
-
const { maxDocsPerPartition, maxPartitions, watermark } = raw;
|
|
22553
|
-
if (maxDocsPerPartition !== void 0 && (typeof maxDocsPerPartition !== "number" || !Number.isInteger(maxDocsPerPartition) || maxDocsPerPartition < 1)) {
|
|
22554
|
-
reject2("partitionConfig maxDocsPerPartition must be a positive integer");
|
|
22555
|
-
}
|
|
22556
|
-
if (maxPartitions !== void 0 && (typeof maxPartitions !== "number" || !Number.isInteger(maxPartitions) || maxPartitions < 1)) {
|
|
22557
|
-
reject2("partitionConfig maxPartitions must be a positive integer");
|
|
22558
|
-
}
|
|
22559
|
-
if (watermark !== void 0 && (typeof watermark !== "number" || !(watermark > 0) || watermark > 1)) {
|
|
22560
|
-
reject2("partitionConfig watermark must be above 0 and at most 1");
|
|
22561
|
-
}
|
|
22562
|
-
return {
|
|
22563
|
-
...typeof maxDocsPerPartition === "number" ? { maxDocsPerPartition } : {},
|
|
22564
|
-
...typeof maxPartitions === "number" ? { maxPartitions } : {},
|
|
22565
|
-
...typeof watermark === "number" ? { watermark } : {}
|
|
22566
|
-
};
|
|
22567
|
-
}
|
|
22568
|
-
function restoredScoring(raw, reject2) {
|
|
22569
|
-
if (raw === void 0) {
|
|
22570
|
-
return void 0;
|
|
22571
|
-
}
|
|
22572
|
-
if (raw !== "local" && raw !== "dfs" && raw !== "broadcast") {
|
|
22573
|
-
reject2("defaultScoring must be 'local', 'dfs', or 'broadcast'");
|
|
22574
|
-
}
|
|
22575
|
-
return raw;
|
|
22576
|
-
}
|
|
22577
|
-
function restoredBoolean(raw, field, reject2) {
|
|
22578
|
-
if (raw === void 0) {
|
|
22579
|
-
return void 0;
|
|
22580
|
-
}
|
|
22581
|
-
if (typeof raw !== "boolean") {
|
|
22582
|
-
reject2(`${field} must be a boolean`);
|
|
22583
|
-
}
|
|
22584
|
-
return raw;
|
|
22585
|
-
}
|
|
22586
|
-
function restoredRequired(raw, reject2) {
|
|
22587
|
-
if (raw === void 0) {
|
|
22588
|
-
return void 0;
|
|
22589
|
-
}
|
|
22590
|
-
if (!Array.isArray(raw) || !raw.every((field) => typeof field === "string")) {
|
|
22591
|
-
reject2("required must be a list of field paths");
|
|
22592
|
-
}
|
|
22593
|
-
return raw;
|
|
22594
|
-
}
|
|
22595
|
-
function restoredVectorPromotion(raw, reject2) {
|
|
22596
|
-
if (raw === void 0) {
|
|
22597
|
-
return void 0;
|
|
22598
|
-
}
|
|
22599
|
-
if (!isPlainObject3(raw) || raw.hnswConfig !== void 0 && !isPlainObject3(raw.hnswConfig)) {
|
|
22600
|
-
reject2("vectorPromotion must be an object");
|
|
22601
|
-
}
|
|
22602
|
-
return raw;
|
|
22603
|
-
}
|
|
22604
|
-
function restoredEmbedding(raw, reject2 = rejectSnapshot) {
|
|
22605
|
-
if (raw === void 0) {
|
|
22606
|
-
return void 0;
|
|
22607
|
-
}
|
|
22608
|
-
if (!isPlainObject3(raw)) {
|
|
22609
|
-
reject2("embedding must be an object");
|
|
22610
|
-
}
|
|
22611
|
-
const { fields, adapter } = raw;
|
|
22612
|
-
if (adapter !== void 0 && typeof adapter !== "string") {
|
|
22613
|
-
reject2("embedding adapter must be a name");
|
|
22614
|
-
}
|
|
22615
|
-
if (!isPlainObject3(fields)) {
|
|
22616
|
-
reject2("embedding fields must be an object");
|
|
22617
|
-
}
|
|
22618
|
-
for (const value of Object.values(fields)) {
|
|
22619
|
-
const isPath = typeof value === "string";
|
|
22620
|
-
const isPathList = Array.isArray(value) && value.every((path) => typeof path === "string");
|
|
22621
|
-
if (!isPath && !isPathList) {
|
|
22622
|
-
reject2("embedding fields must map vector fields to source paths");
|
|
22623
|
-
}
|
|
22624
|
-
}
|
|
22625
|
-
return {
|
|
22626
|
-
fields,
|
|
22627
|
-
...adapter !== void 0 ? { adapter } : {}
|
|
22628
|
-
};
|
|
22629
|
-
}
|
|
22630
|
-
function readSharedConfigFields(source, reject2) {
|
|
22631
|
-
if (source.tokenizer !== void 0 && typeof source.tokenizer !== "string") {
|
|
22632
|
-
reject2("tokenizer must be a name");
|
|
22633
|
-
}
|
|
22634
|
-
const stopWords2 = restoredStopWords(source.stopWords, source.stopWordList, reject2);
|
|
22635
|
-
const bm25 = restoredBm25(source.bm25, reject2);
|
|
22636
|
-
const surfaceForms = restoredBoolean(source.surfaceForms, "surfaceForms", reject2);
|
|
22637
|
-
const defaultScoring = restoredScoring(source.defaultScoring, reject2);
|
|
22638
|
-
const trackPositions = restoredBoolean(source.trackPositions, "trackPositions", reject2);
|
|
22639
|
-
const strict = restoredBoolean(source.strict, "strict", reject2);
|
|
22640
|
-
const required = restoredRequired(source.required, reject2);
|
|
22641
|
-
const vectorPromotion = restoredVectorPromotion(source.vectorPromotion, reject2);
|
|
22642
|
-
if (source.patternValueLimit !== void 0 && typeof source.patternValueLimit !== "number") {
|
|
22643
|
-
reject2("patternValueLimit must be a number");
|
|
22644
|
-
}
|
|
22645
|
-
return {
|
|
22646
|
-
...typeof source.tokenizer === "string" ? { tokenizer: source.tokenizer } : {},
|
|
22647
|
-
...stopWords2 !== void 0 ? { stopWords: stopWords2 } : {},
|
|
22648
|
-
...bm25 !== void 0 ? { bm25 } : {},
|
|
22649
|
-
...surfaceForms !== void 0 ? { surfaceForms } : {},
|
|
22650
|
-
...defaultScoring !== void 0 ? { defaultScoring } : {},
|
|
22651
|
-
...trackPositions !== void 0 ? { trackPositions } : {},
|
|
22652
|
-
...strict !== void 0 ? { strict } : {},
|
|
22653
|
-
...required !== void 0 ? { required } : {},
|
|
22654
|
-
...vectorPromotion !== void 0 ? { vectorPromotion } : {},
|
|
22655
|
-
...typeof source.patternValueLimit === "number" ? { patternValueLimit: source.patternValueLimit } : {}
|
|
22656
|
-
};
|
|
22657
|
-
}
|
|
22658
|
-
function restoredConfigFields(envelope, reject2 = rejectSnapshot) {
|
|
22659
|
-
const shared2 = readSharedConfigFields(envelope, reject2);
|
|
22660
|
-
const partitions = restoredPartitionConfig(envelope.partitionConfig, reject2);
|
|
22661
|
-
return {
|
|
22662
|
-
...shared2,
|
|
22663
|
-
...partitions !== void 0 ? { partitions } : {}
|
|
22664
|
-
};
|
|
22665
|
-
}
|
|
22666
|
-
|
|
22667
|
-
// src/persistence/archive/byte-reader.ts
|
|
22668
|
-
async function* chunksOf(source) {
|
|
22669
|
-
if (source instanceof Uint8Array) {
|
|
22670
|
-
yield source;
|
|
22671
|
-
return;
|
|
22672
|
-
}
|
|
22673
|
-
if (typeof source.getReader === "function") {
|
|
22674
|
-
const reader = source.getReader();
|
|
22675
|
-
try {
|
|
22676
|
-
for (; ; ) {
|
|
22677
|
-
const step = await reader.read();
|
|
22678
|
-
if (step.done) return;
|
|
22679
|
-
yield step.value;
|
|
22680
|
-
}
|
|
22681
|
-
} finally {
|
|
22682
|
-
reader.releaseLock();
|
|
22683
|
-
}
|
|
22684
|
-
}
|
|
22685
|
-
yield* source;
|
|
22686
|
-
}
|
|
22687
|
-
function createByteReader(source) {
|
|
22688
|
-
const iterator = chunksOf(source);
|
|
22689
|
-
const pending = [];
|
|
22690
|
-
let pendingBytes = 0;
|
|
22691
|
-
let ended = false;
|
|
22692
|
-
async function fill(length) {
|
|
22693
|
-
while (pendingBytes < length && !ended) {
|
|
22694
|
-
const step = await iterator.next();
|
|
22695
|
-
if (step.done) {
|
|
22696
|
-
ended = true;
|
|
22697
|
-
break;
|
|
22698
|
-
}
|
|
22699
|
-
if (step.value.byteLength === 0) continue;
|
|
22700
|
-
pending.push(step.value);
|
|
22701
|
-
pendingBytes += step.value.byteLength;
|
|
22702
|
-
}
|
|
22703
|
-
}
|
|
22704
|
-
function take(length) {
|
|
22705
|
-
const out = new Uint8Array(length);
|
|
22706
|
-
let filled = 0;
|
|
22707
|
-
while (filled < length) {
|
|
22708
|
-
const head = pending[0];
|
|
22709
|
-
const needed = length - filled;
|
|
22710
|
-
if (head.byteLength <= needed) {
|
|
22711
|
-
out.set(head, filled);
|
|
22712
|
-
filled += head.byteLength;
|
|
22713
|
-
pending.shift();
|
|
22714
|
-
} else {
|
|
22715
|
-
out.set(head.subarray(0, needed), filled);
|
|
22716
|
-
pending[0] = head.subarray(needed);
|
|
22717
|
-
filled += needed;
|
|
22718
|
-
}
|
|
22719
|
-
}
|
|
22720
|
-
pendingBytes -= length;
|
|
22721
|
-
return out;
|
|
22722
|
-
}
|
|
22723
|
-
return {
|
|
22724
|
-
async readExactly(length) {
|
|
22725
|
-
await fill(length);
|
|
22726
|
-
if (pendingBytes === 0 && length > 0) return null;
|
|
22727
|
-
if (pendingBytes < length) return take(pendingBytes);
|
|
22728
|
-
return take(length);
|
|
22729
|
-
},
|
|
22730
|
-
async peek(length) {
|
|
22731
|
-
await fill(length);
|
|
22732
|
-
const available = Math.min(length, pendingBytes);
|
|
22733
|
-
const copy = take(available);
|
|
22734
|
-
pending.unshift(copy);
|
|
22735
|
-
pendingBytes += copy.byteLength;
|
|
22736
|
-
return copy;
|
|
22737
|
-
},
|
|
22738
|
-
async close() {
|
|
22739
|
-
await iterator.return(void 0);
|
|
22740
|
-
}
|
|
22741
|
-
};
|
|
22742
|
-
}
|
|
22743
|
-
|
|
22744
|
-
// src/engine/export/lines.ts
|
|
22745
|
-
var NEWLINE = 10;
|
|
22746
|
-
function joined(parts, total) {
|
|
22747
|
-
if (parts.length === 1) return parts[0];
|
|
22748
|
-
const bytes = new Uint8Array(total);
|
|
22749
|
-
let offset = 0;
|
|
22750
|
-
for (const part of parts) {
|
|
22751
|
-
bytes.set(part, offset);
|
|
22752
|
-
offset += part.byteLength;
|
|
22753
|
-
}
|
|
22754
|
-
return bytes;
|
|
22755
|
-
}
|
|
22756
|
-
async function* exportLines(source, maxLineBytes) {
|
|
22757
|
-
const decoder4 = new TextDecoder("utf-8", { fatal: true });
|
|
22758
|
-
let parts = [];
|
|
22759
|
-
let pending = 0;
|
|
22760
|
-
let overflowing = false;
|
|
22761
|
-
let line = 0;
|
|
22762
|
-
const finishLine = () => {
|
|
22763
|
-
line += 1;
|
|
22764
|
-
if (overflowing) {
|
|
22765
|
-
overflowing = false;
|
|
22766
|
-
parts = [];
|
|
22767
|
-
pending = 0;
|
|
22768
|
-
return { line, refused: "too-long" };
|
|
22769
|
-
}
|
|
22770
|
-
const bytes = joined(parts, pending);
|
|
22771
|
-
parts = [];
|
|
22772
|
-
pending = 0;
|
|
22773
|
-
try {
|
|
22774
|
-
return { line, text: decoder4.decode(bytes) };
|
|
22775
|
-
} catch {
|
|
22776
|
-
return { line, refused: "not-utf8" };
|
|
22777
|
-
}
|
|
22778
|
-
};
|
|
22779
|
-
for await (const chunk of chunksOf(source)) {
|
|
22780
|
-
let start = 0;
|
|
22781
|
-
for (; ; ) {
|
|
22782
|
-
const newline = chunk.indexOf(NEWLINE, start);
|
|
22783
|
-
const end = newline < 0 ? chunk.byteLength : newline;
|
|
22784
|
-
const piece = chunk.subarray(start, end);
|
|
22785
|
-
if (!overflowing && pending + piece.byteLength <= maxLineBytes) {
|
|
22786
|
-
if (piece.byteLength > 0) parts.push(piece.slice());
|
|
22787
|
-
pending += piece.byteLength;
|
|
22788
|
-
} else {
|
|
22789
|
-
overflowing = true;
|
|
22790
|
-
parts = [];
|
|
22791
|
-
}
|
|
22792
|
-
if (newline < 0) break;
|
|
22793
|
-
yield finishLine();
|
|
22794
|
-
start = newline + 1;
|
|
22795
|
-
}
|
|
22796
|
-
}
|
|
22797
|
-
if (pending > 0 || overflowing) yield finishLine();
|
|
22798
|
-
}
|
|
22799
|
-
|
|
22800
|
-
// src/serialization/base64.ts
|
|
22801
|
-
var ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
|
22802
|
-
var PAD = 61;
|
|
22803
|
-
var BYTES_PER_GROUP = 3;
|
|
22804
|
-
var CHARACTERS_PER_GROUP = 4;
|
|
22805
|
-
var ASCII_LIMIT = 128;
|
|
22806
|
-
var ENCODE = new Uint8Array(ALPHABET.length);
|
|
22807
|
-
var DECODE = new Int8Array(ASCII_LIMIT).fill(-1);
|
|
22808
|
-
for (let at = 0; at < ALPHABET.length; at++) {
|
|
22809
|
-
ENCODE[at] = ALPHABET.charCodeAt(at);
|
|
22810
|
-
DECODE[ALPHABET.charCodeAt(at)] = at;
|
|
22811
|
-
}
|
|
22812
|
-
var asciiDecoder = new TextDecoder();
|
|
22813
|
-
function encodeBase64(bytes) {
|
|
22814
|
-
const out = new Uint8Array(Math.ceil(bytes.byteLength / BYTES_PER_GROUP) * CHARACTERS_PER_GROUP);
|
|
22815
|
-
let write = 0;
|
|
22816
|
-
let read = 0;
|
|
22817
|
-
for (; read + BYTES_PER_GROUP <= bytes.byteLength; read += BYTES_PER_GROUP) {
|
|
22818
|
-
const group = bytes[read] << 16 | bytes[read + 1] << 8 | bytes[read + 2];
|
|
22819
|
-
out[write++] = ENCODE[group >>> 18];
|
|
22820
|
-
out[write++] = ENCODE[group >>> 12 & 63];
|
|
22821
|
-
out[write++] = ENCODE[group >>> 6 & 63];
|
|
22822
|
-
out[write++] = ENCODE[group & 63];
|
|
22823
|
-
}
|
|
22824
|
-
const left = bytes.byteLength - read;
|
|
22825
|
-
if (left > 0) {
|
|
22826
|
-
const group = bytes[read] << 16 | (left === 2 ? bytes[read + 1] << 8 : 0);
|
|
22827
|
-
out[write++] = ENCODE[group >>> 18];
|
|
22828
|
-
out[write++] = ENCODE[group >>> 12 & 63];
|
|
22829
|
-
out[write++] = left === 2 ? ENCODE[group >>> 6 & 63] : PAD;
|
|
22830
|
-
out[write++] = PAD;
|
|
22831
|
-
}
|
|
22832
|
-
return asciiDecoder.decode(out);
|
|
22833
|
-
}
|
|
22834
|
-
function sextetAt(text, at) {
|
|
22835
|
-
const code = text.charCodeAt(at);
|
|
22836
|
-
return code < ASCII_LIMIT ? DECODE[code] : -1;
|
|
22837
|
-
}
|
|
22838
|
-
function decodeExactBase64(text) {
|
|
22839
|
-
if (text.length % CHARACTERS_PER_GROUP !== 0) return null;
|
|
22840
|
-
const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0;
|
|
22841
|
-
const out = new Uint8Array(text.length / CHARACTERS_PER_GROUP * BYTES_PER_GROUP - padding);
|
|
22842
|
-
const fullEnd = padding === 0 ? text.length : text.length - CHARACTERS_PER_GROUP;
|
|
22843
|
-
let write = 0;
|
|
22844
|
-
for (let at = 0; at < fullEnd; at += CHARACTERS_PER_GROUP) {
|
|
22845
|
-
const a2 = sextetAt(text, at);
|
|
22846
|
-
const b2 = sextetAt(text, at + 1);
|
|
22847
|
-
const c2 = sextetAt(text, at + 2);
|
|
22848
|
-
const d = sextetAt(text, at + 3);
|
|
22849
|
-
if ((a2 | b2 | c2 | d) < 0) return null;
|
|
22850
|
-
const group = a2 << 18 | b2 << 12 | c2 << 6 | d;
|
|
22851
|
-
out[write++] = group >>> 16;
|
|
22852
|
-
out[write++] = group >>> 8 & 255;
|
|
22853
|
-
out[write++] = group & 255;
|
|
22854
|
-
}
|
|
22855
|
-
if (padding === 0) return out;
|
|
22856
|
-
const a = sextetAt(text, fullEnd);
|
|
22857
|
-
const b = sextetAt(text, fullEnd + 1);
|
|
22858
|
-
const c = padding === 1 ? sextetAt(text, fullEnd + 2) : 0;
|
|
22859
|
-
if ((a | b | c) < 0) return null;
|
|
22860
|
-
if (padding === 2 && (b & 15) !== 0) return null;
|
|
22861
|
-
if (padding === 1 && (c & 3) !== 0) return null;
|
|
22862
|
-
out[write++] = a << 2 | b >>> 4;
|
|
22863
|
-
if (padding === 1) out[write] = (b & 15) << 4 | c >>> 2;
|
|
22864
|
-
return out;
|
|
22865
|
-
}
|
|
22866
|
-
|
|
22867
|
-
// src/engine/export/tags.ts
|
|
22868
|
-
var TAG_PREFIX = "$";
|
|
22869
|
-
var DATE_TAG = "$date";
|
|
22870
|
-
var BYTES_TAG = "$bytes";
|
|
22871
|
-
var NUMBER_TAG = "$number";
|
|
22872
|
-
var OBJECT_TAG = "$object";
|
|
22873
|
-
var MILLISECONDS_PER_SECOND = 1e3;
|
|
22874
|
-
var NANOSECONDS_PER_MILLISECOND = 1e6;
|
|
22875
|
-
var MAX_NANOSECONDS = 999999999;
|
|
22876
|
-
var MAX_DATE_MILLISECONDS = 864e13;
|
|
22877
|
-
var ESCAPED_TAG_PREFIX = "\\u0024";
|
|
22878
|
-
var NON_FINITE_NUMBERS = /* @__PURE__ */ new Map([
|
|
22879
|
-
["NaN", Number.NaN],
|
|
22880
|
-
["Infinity", Number.POSITIVE_INFINITY],
|
|
22881
|
-
["-Infinity", Number.NEGATIVE_INFINITY]
|
|
22882
|
-
]);
|
|
22883
|
-
function refuse(path, reason) {
|
|
22884
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
22885
|
-
field: path
|
|
22886
|
-
});
|
|
22887
|
-
}
|
|
22888
|
-
function isRecord(value) {
|
|
22889
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
22890
|
-
}
|
|
22891
|
-
function startsWithTagPrefix(key) {
|
|
22892
|
-
return key.startsWith(TAG_PREFIX);
|
|
22893
|
-
}
|
|
22894
|
-
function soleTagKey(record) {
|
|
22895
|
-
let sole = null;
|
|
22896
|
-
for (const key of Object.keys(record)) {
|
|
22897
|
-
if (sole !== null) return null;
|
|
22898
|
-
sole = key;
|
|
22899
|
-
}
|
|
22900
|
-
return sole !== null && startsWithTagPrefix(sole) ? sole : null;
|
|
22901
|
-
}
|
|
22902
|
-
function dateOf(value, path) {
|
|
22903
|
-
if (!Array.isArray(value) || value.length !== 2)
|
|
22904
|
-
refuse(path, `holds a "${DATE_TAG}" tag other than [seconds, nanoseconds]`);
|
|
22905
|
-
const seconds = value[0];
|
|
22906
|
-
const nanoseconds = value[1];
|
|
22907
|
-
if (typeof seconds !== "number" || !Number.isSafeInteger(seconds)) {
|
|
22908
|
-
refuse(
|
|
22909
|
-
path,
|
|
22910
|
-
`holds a "${DATE_TAG}" tag whose seconds are not an integer from -${Number.MAX_SAFE_INTEGER} to ${Number.MAX_SAFE_INTEGER}`
|
|
22911
|
-
);
|
|
22912
|
-
}
|
|
22913
|
-
if (typeof nanoseconds !== "number" || !Number.isInteger(nanoseconds) || nanoseconds < 0 || nanoseconds > MAX_NANOSECONDS) {
|
|
22914
|
-
refuse(path, `holds a "${DATE_TAG}" tag whose nanoseconds are not an integer from 0 to ${MAX_NANOSECONDS}`);
|
|
22915
|
-
}
|
|
22916
|
-
const milliseconds = seconds * MILLISECONDS_PER_SECOND + nanoseconds / NANOSECONDS_PER_MILLISECOND;
|
|
22917
|
-
if (Math.abs(milliseconds) > MAX_DATE_MILLISECONDS)
|
|
22918
|
-
refuse(path, `holds a "${DATE_TAG}" tag outside the range of a date`);
|
|
22919
|
-
return new Date(milliseconds);
|
|
22920
|
-
}
|
|
22921
|
-
function bytesOf(value, path) {
|
|
22922
|
-
const bytes = typeof value === "string" ? decodeExactBase64(value) : null;
|
|
22923
|
-
if (bytes === null) refuse(path, `holds a "${BYTES_TAG}" tag whose text is not padded base64`);
|
|
22924
|
-
return bytes;
|
|
22925
|
-
}
|
|
22926
|
-
function numberOf(value, path) {
|
|
22927
|
-
const number = typeof value === "string" ? NON_FINITE_NUMBERS.get(value) : void 0;
|
|
22928
|
-
if (number === void 0) refuse(path, `holds a "${NUMBER_TAG}" tag other than "NaN", "Infinity", or "-Infinity"`);
|
|
22929
|
-
return number;
|
|
22930
|
-
}
|
|
22931
|
-
function requireDepth(path, depth) {
|
|
22932
|
-
if (depth >= MAX_DOCUMENT_NESTING_DEPTH) refuse(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
|
|
22933
|
-
}
|
|
22934
|
-
function untagMap(record, path, depth) {
|
|
22935
|
-
requireDepth(path, depth);
|
|
22936
|
-
for (const key of Object.keys(record)) {
|
|
22937
|
-
record[key] = untag(record[key], path === "" ? key : `${path}.${key}`, depth + 1);
|
|
22938
|
-
}
|
|
22939
|
-
return record;
|
|
22940
|
-
}
|
|
22941
|
-
function untagList(list, path, depth) {
|
|
22942
|
-
requireDepth(path, depth);
|
|
22943
|
-
for (let at = 0; at < list.length; at++) list[at] = untag(list[at], `${path}[${at}]`, depth + 1);
|
|
22944
|
-
return list;
|
|
22945
|
-
}
|
|
22946
|
-
function untagTag(tag, value, path, depth) {
|
|
22947
|
-
switch (tag) {
|
|
22948
|
-
case DATE_TAG:
|
|
22949
|
-
return dateOf(value, path);
|
|
22950
|
-
case BYTES_TAG:
|
|
22951
|
-
return bytesOf(value, path);
|
|
22952
|
-
case NUMBER_TAG:
|
|
22953
|
-
return numberOf(value, path);
|
|
22954
|
-
case OBJECT_TAG:
|
|
22955
|
-
if (!isRecord(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
22956
|
-
return untagMap(value, path, depth);
|
|
22957
|
-
default:
|
|
22958
|
-
return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
22959
|
-
}
|
|
22960
|
-
}
|
|
22961
|
-
function untag(value, path, depth) {
|
|
22962
|
-
if (typeof value !== "object" || value === null) return value;
|
|
22963
|
-
if (Array.isArray(value)) return untagList(value, path, depth);
|
|
22964
|
-
const record = value;
|
|
22965
|
-
const tag = soleTagKey(record);
|
|
22966
|
-
return tag === null ? untagMap(record, path, depth) : untagTag(tag, record[tag], path, depth);
|
|
22967
|
-
}
|
|
22968
|
-
function untagDocumentLine(text, parsed) {
|
|
22969
|
-
if (!text.includes(`"${TAG_PREFIX}`) && !text.includes(ESCAPED_TAG_PREFIX)) return parsed;
|
|
22970
|
-
return untagMap(parsed, "", 0);
|
|
22971
|
-
}
|
|
22972
|
-
|
|
22973
|
-
// src/engine/export/import.ts
|
|
22974
|
-
var DOCUMENT_EXPORT_FORMAT = "narsil-export";
|
|
22975
|
-
var DOCUMENT_EXPORT_VERSION = 1;
|
|
22976
|
-
function invalidHeader(message) {
|
|
22977
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
22978
|
-
}
|
|
22979
|
-
function isRecord2(value) {
|
|
22980
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
22981
|
-
}
|
|
22982
|
-
function configOfHeader(line) {
|
|
22983
|
-
if (line === void 0) invalidHeader("is missing");
|
|
22984
|
-
if (!("text" in line)) {
|
|
22985
|
-
invalidHeader(
|
|
22986
|
-
line.refused === "too-long" ? `passes the line limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "is not UTF-8"
|
|
22987
|
-
);
|
|
22988
|
-
}
|
|
22989
|
-
let raw;
|
|
22990
|
-
try {
|
|
22991
|
-
raw = JSON.parse(line.text);
|
|
22992
|
-
} catch {
|
|
22993
|
-
invalidHeader("is not JSON");
|
|
22994
|
-
}
|
|
22995
|
-
if (!isRecord2(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
22996
|
-
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
22997
|
-
}
|
|
22998
|
-
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
22999
|
-
const fields = restoredConfigFields(configuration, (message) => invalidHeader(message));
|
|
23000
|
-
const embedding = restoredEmbedding(configuration.embedding, (message) => invalidHeader(message));
|
|
23001
|
-
return {
|
|
23002
|
-
schema: configuration.schema,
|
|
23003
|
-
language: configuration.language,
|
|
23004
|
-
...fields,
|
|
23005
|
-
...embedding === void 0 ? {} : {
|
|
23006
|
-
embedding: {
|
|
23007
|
-
fields: embedding.fields,
|
|
23008
|
-
...embedding.adapter === void 0 ? {} : { adapter: embedding.adapter }
|
|
23009
|
-
}
|
|
23010
|
-
}
|
|
23011
|
-
};
|
|
23012
|
-
}
|
|
23013
|
-
async function prepareIndex(engine, indexName, config) {
|
|
23014
|
-
if (!engine.listIndexes().some((index) => index.name === indexName)) {
|
|
23015
|
-
await engine.createIndex(indexName, config);
|
|
23016
|
-
return;
|
|
23017
|
-
}
|
|
23018
|
-
const existing = flattenSchema(engine.getStats(indexName).schema);
|
|
23019
|
-
const exported = flattenSchema(config.schema);
|
|
23020
|
-
const fields = /* @__PURE__ */ new Set([...Object.keys(existing), ...Object.keys(exported)]);
|
|
23021
|
-
for (const field of fields) {
|
|
23022
|
-
if (existing[field] === exported[field]) continue;
|
|
23023
|
-
throw new NarsilError(
|
|
23024
|
-
ErrorCodes.CONFIG_INVALID,
|
|
23025
|
-
`Index "${indexName}" gives field "${field}" another type than the export does, so the import cannot write into it`,
|
|
23026
|
-
{ indexName, field }
|
|
23027
|
-
);
|
|
23028
|
-
}
|
|
23029
|
-
}
|
|
23030
|
-
function highestReportedAt(refused) {
|
|
23031
|
-
let highest = 0;
|
|
23032
|
-
for (let at = 1; at < refused.length; at++) {
|
|
23033
|
-
if (refused[at].line > refused[highest].line) highest = at;
|
|
23034
|
-
}
|
|
23035
|
-
return highest;
|
|
23036
|
-
}
|
|
23037
|
-
function refuse2(state, line, code, message) {
|
|
23038
|
-
state.result.refusedCount += 1;
|
|
23039
|
-
const refused = state.result.refused;
|
|
23040
|
-
if (refused.length < MAX_IMPORT_REFUSALS_REPORTED) {
|
|
23041
|
-
refused.push({ line, code, message });
|
|
23042
|
-
if (refused.length === MAX_IMPORT_REFUSALS_REPORTED) state.highestReported = highestReportedAt(refused);
|
|
23043
|
-
return;
|
|
23044
|
-
}
|
|
23045
|
-
if (line >= refused[state.highestReported].line) return;
|
|
23046
|
-
refused[state.highestReported] = { line, code, message };
|
|
23047
|
-
state.highestReported = highestReportedAt(refused);
|
|
23048
|
-
}
|
|
23049
|
-
async function flush(state) {
|
|
23050
|
-
if (state.batch.length === 0) return;
|
|
23051
|
-
const batch = state.batch;
|
|
23052
|
-
state.batch = [];
|
|
23053
|
-
state.batchIds = /* @__PURE__ */ new Set();
|
|
23054
|
-
const lineOf = new Map(batch.map((entry) => [entry.docId, entry.line]));
|
|
23055
|
-
const inserts = [];
|
|
23056
|
-
const updates = [];
|
|
23057
|
-
for (const entry of batch) {
|
|
23058
|
-
if (await state.engine.has(state.indexName, entry.docId))
|
|
23059
|
-
updates.push({ docId: entry.docId, document: entry.document });
|
|
23060
|
-
else inserts.push(entry.document);
|
|
23061
|
-
}
|
|
23062
|
-
for (const outcome of [
|
|
23063
|
-
inserts.length === 0 ? null : await state.engine.insertBatch(state.indexName, inserts, { skipClone: true }),
|
|
23064
|
-
updates.length === 0 ? null : await state.engine.updateBatch(state.indexName, updates)
|
|
23065
|
-
]) {
|
|
23066
|
-
if (outcome === null) continue;
|
|
23067
|
-
state.result.imported += outcome.succeeded.length;
|
|
23068
|
-
for (const failure of outcome.failed) {
|
|
23069
|
-
const code = failure.error instanceof NarsilError ? failure.error.code : ErrorCodes.DOC_VALIDATION_FAILED;
|
|
23070
|
-
refuse2(state, lineOf.get(failure.docId) ?? 0, code, failure.error.message);
|
|
23071
|
-
}
|
|
23072
|
-
}
|
|
23073
|
-
}
|
|
23074
|
-
async function takeLine(state, entry) {
|
|
23075
|
-
if (!("text" in entry)) {
|
|
23076
|
-
const message = entry.refused === "too-long" ? `The line passes the limit of ${MAX_EXPORT_LINE_BYTES} bytes` : "The line is not UTF-8";
|
|
23077
|
-
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, message);
|
|
23078
|
-
return;
|
|
23079
|
-
}
|
|
23080
|
-
if (entry.text.length === 0) return;
|
|
23081
|
-
let document;
|
|
23082
|
-
try {
|
|
23083
|
-
document = JSON.parse(entry.text);
|
|
23084
|
-
} catch {
|
|
23085
|
-
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
23086
|
-
return;
|
|
23087
|
-
}
|
|
23088
|
-
if (!isRecord2(document) || typeof document.id !== "string" || document.id.length === 0) {
|
|
23089
|
-
refuse2(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
23090
|
-
return;
|
|
23091
|
-
}
|
|
23092
|
-
try {
|
|
23093
|
-
untagDocumentLine(entry.text, document);
|
|
23094
|
-
} catch (error) {
|
|
23095
|
-
if (!(error instanceof NarsilError)) throw error;
|
|
23096
|
-
refuse2(state, entry.line, error.code, error.message);
|
|
23097
|
-
return;
|
|
23098
|
-
}
|
|
23099
|
-
if (state.batchIds.has(document.id)) await flush(state);
|
|
23100
|
-
state.batch.push({ line: entry.line, docId: document.id, document });
|
|
23101
|
-
state.batchIds.add(document.id);
|
|
23102
|
-
if (state.batch.length >= IMPORT_BATCH_DOCUMENTS) await flush(state);
|
|
23103
|
-
}
|
|
23104
|
-
async function readDocumentExport(engine, indexName, source) {
|
|
23105
|
-
const lines = exportLines(source, MAX_EXPORT_LINE_BYTES);
|
|
23106
|
-
const header = await lines.next();
|
|
23107
|
-
await prepareIndex(engine, indexName, configOfHeader(header.done === true ? void 0 : header.value));
|
|
23108
|
-
const state = {
|
|
23109
|
-
engine,
|
|
23110
|
-
indexName,
|
|
23111
|
-
batch: [],
|
|
23112
|
-
batchIds: /* @__PURE__ */ new Set(),
|
|
23113
|
-
result: { imported: 0, refusedCount: 0, refused: [] },
|
|
23114
|
-
highestReported: 0
|
|
23115
|
-
};
|
|
23116
|
-
for await (const entry of lines) await takeLine(state, entry);
|
|
23117
|
-
await flush(state);
|
|
23118
|
-
state.result.refused.sort((a, b) => a.line - b.line);
|
|
23119
|
-
return state.result;
|
|
23120
|
-
}
|
|
23121
|
-
|
|
23122
22382
|
// src/narsil/portability.ts
|
|
23123
22383
|
function portabilityOperations(core, config, engine) {
|
|
23124
22384
|
async function withOpenIndex2(indexName, action) {
|
|
@@ -23150,14 +22410,14 @@ function portabilityOperations(core, config, engine) {
|
|
|
23150
22410
|
return {
|
|
23151
22411
|
snapshot(indexName) {
|
|
23152
22412
|
return withOpenIndex2(indexName, async () => {
|
|
23153
|
-
const { createSnapshot } = await import('./snapshot-
|
|
22413
|
+
const { createSnapshot } = await import('./snapshot-QXBKHJAG.mjs');
|
|
23154
22414
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
23155
22415
|
});
|
|
23156
22416
|
},
|
|
23157
22417
|
async restore(indexName, data) {
|
|
23158
22418
|
core.guardShutdown();
|
|
23159
22419
|
validateIndexName(indexName);
|
|
23160
|
-
const restored = import('./restore-queue-
|
|
22420
|
+
const restored = import('./restore-queue-ONYNYUDE.mjs').then(
|
|
23161
22421
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
23162
22422
|
);
|
|
23163
22423
|
await emitWriteEventAfter(core, indexName, restored);
|
|
@@ -23166,8 +22426,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
23166
22426
|
exportDocuments(indexName) {
|
|
23167
22427
|
return withOpenIndex2(indexName, async () => {
|
|
23168
22428
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
23169
|
-
import('./snapshot-
|
|
23170
|
-
import('./write-
|
|
22429
|
+
import('./snapshot-QXBKHJAG.mjs'),
|
|
22430
|
+
import('./write-WHFBDWQS.mjs')
|
|
23171
22431
|
]);
|
|
23172
22432
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
23173
22433
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -23175,7 +22435,10 @@ function portabilityOperations(core, config, engine) {
|
|
|
23175
22435
|
},
|
|
23176
22436
|
importDocumentExport(indexName, data) {
|
|
23177
22437
|
core.guardShutdown();
|
|
23178
|
-
|
|
22438
|
+
const imported = import('./import-TLB2A5LJ.mjs').then(
|
|
22439
|
+
({ readDocumentExport }) => readDocumentExport(engine(), indexName, data)
|
|
22440
|
+
);
|
|
22441
|
+
return emitWriteEventAfter(core, indexName, imported);
|
|
23179
22442
|
}
|
|
23180
22443
|
};
|
|
23181
22444
|
}
|
|
@@ -23185,14 +22448,14 @@ var ABSENT_TAG = 0;
|
|
|
23185
22448
|
var NULL_TAG = 1;
|
|
23186
22449
|
var FALSE_TAG = 2;
|
|
23187
22450
|
var TRUE_TAG = 3;
|
|
23188
|
-
var
|
|
22451
|
+
var NUMBER_TAG = 4;
|
|
23189
22452
|
var STRING_TAG = 5;
|
|
23190
22453
|
var LIST_TAG = 6;
|
|
23191
22454
|
var MAP_TAG = 7;
|
|
23192
22455
|
var LENGTH_BYTES = 4;
|
|
23193
22456
|
var LAST_ASCII_UNIT = 127;
|
|
23194
22457
|
var encoder5 = new TextEncoder();
|
|
23195
|
-
function
|
|
22458
|
+
function flush(stream) {
|
|
23196
22459
|
stream.hash = fnv1aContinue(stream.hash, stream.bytes, stream.length);
|
|
23197
22460
|
stream.length = 0;
|
|
23198
22461
|
}
|
|
@@ -23256,8 +22519,8 @@ function writeNumberList(stream, list) {
|
|
|
23256
22519
|
if (stream.clock !== void 0) stream.clock.tick(list.members.length);
|
|
23257
22520
|
writeByte(stream, LIST_TAG);
|
|
23258
22521
|
writeUint32(stream, list.members.length);
|
|
23259
|
-
|
|
23260
|
-
stream.hash = fnv1aContinueTaggedNumbers(stream.hash,
|
|
22522
|
+
flush(stream);
|
|
22523
|
+
stream.hash = fnv1aContinueTaggedNumbers(stream.hash, NUMBER_TAG, list.members);
|
|
23261
22524
|
}
|
|
23262
22525
|
function writeValue(stream, value) {
|
|
23263
22526
|
if (value instanceof NumberList) {
|
|
@@ -23277,7 +22540,7 @@ function writeValue(stream, value) {
|
|
|
23277
22540
|
return;
|
|
23278
22541
|
}
|
|
23279
22542
|
if (typeof value === "number") {
|
|
23280
|
-
writeByte(stream,
|
|
22543
|
+
writeByte(stream, NUMBER_TAG);
|
|
23281
22544
|
writeNumber(stream, value);
|
|
23282
22545
|
return;
|
|
23283
22546
|
}
|
|
@@ -23351,7 +22614,7 @@ function bindingOf(values, clock) {
|
|
|
23351
22614
|
for (const value of values) {
|
|
23352
22615
|
writeValue(stream, value);
|
|
23353
22616
|
}
|
|
23354
|
-
|
|
22617
|
+
flush(stream);
|
|
23355
22618
|
return (stream.hash >>> 0).toString(16).padStart(8, "0");
|
|
23356
22619
|
}
|
|
23357
22620
|
function queryBindingOf(params, clock) {
|
|
@@ -24472,7 +23735,7 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
24472
23735
|
}
|
|
24473
23736
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
24474
23737
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
24475
|
-
return import('./runtime-
|
|
23738
|
+
return import('./runtime-VEUH674B.mjs').then(
|
|
24476
23739
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
24477
23740
|
);
|
|
24478
23741
|
}
|
|
@@ -24482,7 +23745,7 @@ function executeQuery(request, callerContext) {
|
|
|
24482
23745
|
if (loadingPhraseCode === null) return runQuery(request, callerContext);
|
|
24483
23746
|
return loadingPhraseCode.then(() => runQuery(request, callerContext));
|
|
24484
23747
|
}
|
|
24485
|
-
return import('./runtime-
|
|
23748
|
+
return import('./runtime-VEUH674B.mjs').then(
|
|
24486
23749
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
24487
23750
|
);
|
|
24488
23751
|
}
|
|
@@ -24838,16 +24101,22 @@ function pickDisplaySurface(surfaceOccurrences) {
|
|
|
24838
24101
|
// src/narsil/reads.ts
|
|
24839
24102
|
async function enterRead(core, indexName) {
|
|
24840
24103
|
core.guardShutdown();
|
|
24841
|
-
if (core.rebalancingIndexes.has(indexName)) await
|
|
24104
|
+
if (core.rebalancingIndexes.has(indexName)) await yieldToEventLoop2();
|
|
24842
24105
|
}
|
|
24843
24106
|
async function readUnderOneAnalysis(core, indexName, read) {
|
|
24844
|
-
|
|
24845
|
-
|
|
24846
|
-
|
|
24847
|
-
analysisStale = core.analysisRebuild.isStale(indexName);
|
|
24848
|
-
|
|
24107
|
+
const manager = core.requireManager(indexName);
|
|
24108
|
+
for (; ; ) {
|
|
24109
|
+
const analysis = manager.analysis;
|
|
24110
|
+
const analysisStale = core.analysisRebuild.isStale(indexName);
|
|
24111
|
+
try {
|
|
24112
|
+
const result = await read();
|
|
24113
|
+
if (manager.analysis === analysis && core.analysisRebuild.isStale(indexName) === analysisStale) {
|
|
24114
|
+
return { result, analysisStale };
|
|
24115
|
+
}
|
|
24116
|
+
} catch (error) {
|
|
24117
|
+
if (manager.analysis === analysis) throw error;
|
|
24118
|
+
}
|
|
24849
24119
|
}
|
|
24850
|
-
return { result, analysisStale };
|
|
24851
24120
|
}
|
|
24852
24121
|
function phraseReadinessOf(core, indexName, config) {
|
|
24853
24122
|
return () => {
|
|
@@ -24875,7 +24144,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24875
24144
|
const manager = core.requireManager(indexName);
|
|
24876
24145
|
const scoped = scopedParams(params, options);
|
|
24877
24146
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
24878
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-
|
|
24147
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-VEUH674B.mjs');
|
|
24879
24148
|
const resolvedParams = await resolveVectorText(
|
|
24880
24149
|
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
24881
24150
|
entry.embeddingAdapter,
|
|
@@ -24883,25 +24152,24 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24883
24152
|
entry.embeddingAdapterName
|
|
24884
24153
|
);
|
|
24885
24154
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
24886
|
-
const context = {
|
|
24887
|
-
manager,
|
|
24888
|
-
language: entry.language,
|
|
24889
|
-
config: entry.config,
|
|
24890
|
-
workerSearch,
|
|
24891
|
-
indexName,
|
|
24892
|
-
broadcastStats: broadcastStatsFor(core, options),
|
|
24893
|
-
partitionIds: options?.partitionIds,
|
|
24894
|
-
rescoreWindow: options?.rescoreWindow,
|
|
24895
|
-
indexBm25: options?.indexBm25,
|
|
24896
|
-
cursorBinding: lazyQueryBinding(params),
|
|
24897
|
-
patternWorkCap: core.patternWorkCap,
|
|
24898
|
-
rerank: rerank?.rerankRunnerFor(core),
|
|
24899
|
-
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24900
|
-
};
|
|
24901
24155
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
24902
24156
|
core,
|
|
24903
24157
|
indexName,
|
|
24904
|
-
() => executeQuery(resolvedParams,
|
|
24158
|
+
() => executeQuery(resolvedParams, {
|
|
24159
|
+
manager,
|
|
24160
|
+
language: manager.language,
|
|
24161
|
+
config: manager.config,
|
|
24162
|
+
workerSearch,
|
|
24163
|
+
indexName,
|
|
24164
|
+
broadcastStats: broadcastStatsFor(core, options),
|
|
24165
|
+
partitionIds: options?.partitionIds,
|
|
24166
|
+
rescoreWindow: options?.rescoreWindow,
|
|
24167
|
+
indexBm25: options?.indexBm25,
|
|
24168
|
+
cursorBinding: lazyQueryBinding(params),
|
|
24169
|
+
patternWorkCap: core.patternWorkCap,
|
|
24170
|
+
rerank: rerank?.rerankRunnerFor(core),
|
|
24171
|
+
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24172
|
+
})
|
|
24905
24173
|
);
|
|
24906
24174
|
if (analysisStale) {
|
|
24907
24175
|
result.analysisStale = true;
|
|
@@ -24935,23 +24203,22 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
24935
24203
|
entry.embeddingAdapterName
|
|
24936
24204
|
);
|
|
24937
24205
|
const workerSearch = core.orchestrator.hasWorkerPool() ? core.orchestrator.searchViaWorker.bind(core.orchestrator) : void 0;
|
|
24938
|
-
const context = {
|
|
24939
|
-
manager,
|
|
24940
|
-
language: entry.language,
|
|
24941
|
-
config: entry.config,
|
|
24942
|
-
workerSearch,
|
|
24943
|
-
indexName,
|
|
24944
|
-
broadcastStats: broadcastStatsFor(core, options),
|
|
24945
|
-
partitionIds: options?.partitionIds,
|
|
24946
|
-
indexBm25: options?.indexBm25,
|
|
24947
|
-
cursorBinding: lazyQueryBinding(params),
|
|
24948
|
-
patternWorkCap: core.patternWorkCap,
|
|
24949
|
-
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24950
|
-
};
|
|
24951
24206
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
24952
24207
|
core,
|
|
24953
24208
|
indexName,
|
|
24954
|
-
() => executePreflight(resolvedParams,
|
|
24209
|
+
() => executePreflight(resolvedParams, {
|
|
24210
|
+
manager,
|
|
24211
|
+
language: manager.language,
|
|
24212
|
+
config: manager.config,
|
|
24213
|
+
workerSearch,
|
|
24214
|
+
indexName,
|
|
24215
|
+
broadcastStats: broadcastStatsFor(core, options),
|
|
24216
|
+
partitionIds: options?.partitionIds,
|
|
24217
|
+
indexBm25: options?.indexBm25,
|
|
24218
|
+
cursorBinding: lazyQueryBinding(params),
|
|
24219
|
+
patternWorkCap: core.patternWorkCap,
|
|
24220
|
+
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24221
|
+
})
|
|
24955
24222
|
);
|
|
24956
24223
|
if (analysisStale) {
|
|
24957
24224
|
result.analysisStale = true;
|
|
@@ -25039,8 +24306,11 @@ function createNarsilFromCore(core, config) {
|
|
|
25039
24306
|
release();
|
|
25040
24307
|
}
|
|
25041
24308
|
}
|
|
24309
|
+
const rebalancesStarting = /* @__PURE__ */ new Map();
|
|
25042
24310
|
function withOpenIndexWrite(indexName, action) {
|
|
25043
|
-
|
|
24311
|
+
const starting = rebalancesStarting.get(indexName);
|
|
24312
|
+
const written = starting === void 0 ? withOpenIndex2(indexName, action) : starting.then(() => withOpenIndex2(indexName, action));
|
|
24313
|
+
return emitWriteEventAfter(core, indexName, written);
|
|
25044
24314
|
}
|
|
25045
24315
|
const narsil = {
|
|
25046
24316
|
createIndex(name, indexConfig) {
|
|
@@ -25164,9 +24434,21 @@ function createNarsilFromCore(core, config) {
|
|
|
25164
24434
|
return runEngineSuggest(core, indexName, params);
|
|
25165
24435
|
},
|
|
25166
24436
|
async rebuildAnalysis(indexName) {
|
|
24437
|
+
guardShutdown();
|
|
24438
|
+
const entry = requireIndex(indexName);
|
|
24439
|
+
await withOpenIndexWrite(indexName, async () => {
|
|
24440
|
+
if (entry.pendingAnalysis === null) return core.analysisRebuild.rebuild(indexName);
|
|
24441
|
+
return (await import('./analysis-change-JPV775LR.mjs')).resumeAnalysisChange(core, indexName);
|
|
24442
|
+
});
|
|
24443
|
+
},
|
|
24444
|
+
async changeAnalysis(indexName, change) {
|
|
25167
24445
|
guardShutdown();
|
|
25168
24446
|
requireIndex(indexName);
|
|
25169
|
-
|
|
24447
|
+
const changed = withOpenIndex2(
|
|
24448
|
+
indexName,
|
|
24449
|
+
async () => (await import('./analysis-change-JPV775LR.mjs')).changeIndexAnalysis(core, indexName, change)
|
|
24450
|
+
);
|
|
24451
|
+
await emitWriteEventAfter(core, indexName, changed);
|
|
25170
24452
|
},
|
|
25171
24453
|
...portabilityOperations(core, config, () => narsil),
|
|
25172
24454
|
...synonymOperations(core),
|
|
@@ -25185,15 +24467,29 @@ function createNarsilFromCore(core, config) {
|
|
|
25185
24467
|
await executor.execute({ type: "clear", indexName, requestId: indexName });
|
|
25186
24468
|
await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
|
|
25187
24469
|
core.watermarkNotifier.forget(indexName);
|
|
24470
|
+
const durable = core.durability?.manager;
|
|
24471
|
+
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
25188
24472
|
});
|
|
25189
24473
|
},
|
|
25190
24474
|
async rebalance(indexName, targetPartitionCount) {
|
|
25191
24475
|
guardShutdown();
|
|
25192
24476
|
requireIndex(indexName);
|
|
25193
|
-
|
|
25194
|
-
|
|
25195
|
-
|
|
25196
|
-
);
|
|
24477
|
+
let writesMayFollow = () => void 0;
|
|
24478
|
+
const started = new Promise((resolve) => {
|
|
24479
|
+
writesMayFollow = resolve;
|
|
24480
|
+
});
|
|
24481
|
+
function releaseWrites() {
|
|
24482
|
+
writesMayFollow();
|
|
24483
|
+
if (rebalancesStarting.get(indexName) === started) rebalancesStarting.delete(indexName);
|
|
24484
|
+
}
|
|
24485
|
+
rebalancesStarting.set(indexName, started);
|
|
24486
|
+
const rebalanced = withOpenIndex2(indexName, async () => {
|
|
24487
|
+
const { executeRebalance } = await import('./analysis-change-JPV775LR.mjs');
|
|
24488
|
+
const running = executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx);
|
|
24489
|
+
releaseWrites();
|
|
24490
|
+
return running;
|
|
24491
|
+
});
|
|
24492
|
+
return emitWriteEventAfter(core, indexName, rebalanced).finally(releaseWrites);
|
|
25197
24493
|
},
|
|
25198
24494
|
async updatePartitionConfig(indexName, partitionConfig) {
|
|
25199
24495
|
guardShutdown();
|
|
@@ -25312,4 +24608,4 @@ async function createNarsil(config) {
|
|
|
25312
24608
|
// src/index.ts
|
|
25313
24609
|
var VERSION = "0.1.0";
|
|
25314
24610
|
|
|
25315
|
-
export { ARRAY_POSITION_GAP,
|
|
24611
|
+
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, DEFAULT_WRITE_AHEAD_QUEUE_MAX_SIZE, HEADER_SIZE, MAX_DOCUMENT_NESTING_DEPTH, REBALANCE_CHUNK_SIZE, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, assemblePagedFrozenSegment, buildPartialWordTableFrom, buildSegmentPayload, checksumOfChunks, clearNormalizationCache, configureNormalizationCache, crc32, crc32Final, crc32Init, crc32Update, createCompositePartition, createEncodedDocumentSource, createFrozenSegment, createNarsil, createPartitionIndex, createPartitionManager, createPartitionRouter, createRequestId, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, decodeSurfaceTable, deleteNestedValue, emitEngineEvent, encodeDocument, encodeIndexConfiguration, encodeSurfaceTable, endOfValue, engineVersionParts, extractVectorFieldsFromSchema, extractVectorFromDoc, fieldIndexRemapOf, flattenSchema, fnv1a, freezeSegmentShared, generateId, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, holdsADiscardedField, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeDocFrequencies, mergeFieldIndexes2 as mergeFieldIndexes, mergeFieldLengths, mergePartialWords, mergeSegmentContents, mergeSurfaceForms2 as mergeSurfaceForms, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, packEnvelopeWithChecksum, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareConfiguredSynonyms, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, rebuildTextIndexes, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, removeDocumentVectors, requireConfigurationShape, requireConfigurationWithinSizeLimit, requireSurfaceFormsForPartialFields, requireValidFieldTypes, requireWithinFileLimit, resetNormalizationCache, resolveIndexAnalysis, resolvePartialWordAnalysis, resolvePartitionInsertOptions, rewriteSpillOf, rulesOfStoredSynonyms, scanPostingBlocks, segmentArraysOf, selectionKeeps, shareSegmentArrays, storedArrayPositionGap, survivorDocumentsOf, textRebuildTargetOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateTextStorage, validateVectorPromotion, validateVectorStorage, vectorsEqual, wrapFrozenTokenTable, writeHeader, yieldToEventLoop2 as yieldToEventLoop };
|