@delali/narsil 0.3.1-next.90 → 0.3.1-next.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-66KYAOIK.mjs} +1 -1
- package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-B6XE7KXY.mjs} +11 -3
- package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-NOUZ6RH3.mjs} +2219 -2005
- package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-F4OGQUSU.mjs} +3 -3
- package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-5EJ7I7QY.mjs} +12 -6
- package/dist/browser/{write-6SBYTS62.mjs → write-K5V4XQ72.mjs} +2 -2
- package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-CvExBThq.d.ts} +11 -3
- package/dist/{chunk-MWJIAREY.mjs → chunk-3RDS4UHC.mjs} +2 -2
- package/dist/{chunk-QAMUC2BJ.mjs → chunk-3TTYK5A7.mjs} +7 -6
- package/dist/{chunk-URGLSNKR.mjs → chunk-43ORORN6.mjs} +201 -15
- package/dist/{chunk-A3UB2MXL.mjs → chunk-5ER245FI.mjs} +2 -505
- package/dist/{chunk-VY4DXZ5P.mjs → chunk-67Z2V6TC.mjs} +1 -1
- package/dist/{chunk-EMXAYPTQ.mjs → chunk-6VZ3Q73N.mjs} +2265 -3461
- package/dist/{chunk-S4LQDSUL.mjs → chunk-7OQHSDK6.mjs} +3 -3
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/{chunk-R6BMMQXH.mjs → chunk-A4G7JSYO.mjs} +106 -41
- package/dist/{chunk-DCK4P5BI.mjs → chunk-BJ2YKRNX.mjs} +16 -7
- package/dist/{chunk-J2BPXTD7.mjs → chunk-CO4MRKST.mjs} +1 -1
- package/dist/{chunk-CL4OM7GB.mjs → chunk-EONIHG3V.mjs} +8 -4
- package/dist/chunk-MKWUXWQH.mjs +35 -0
- package/dist/chunk-NVL4ZR3K.mjs +426 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
- package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
- package/dist/{chunk-V7YUSKKZ.mjs → chunk-T4DB5YFB.mjs} +7 -3
- package/dist/{chunk-XSZVS4S3.mjs → chunk-UXQE2ROB.mjs} +6 -2
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XLJ2S2BH.mjs +3572 -0
- package/dist/chunk-ZIJ63TBT.mjs +507 -0
- package/dist/{client-BMK53IR2.d.ts → client-Cc42b42j.d.ts} +27 -3
- package/dist/client.d.ts +9 -8
- package/dist/client.mjs +15 -2
- package/dist/{config-BUR-7eVQ.d.ts → config-BYne-yCI.d.ts} +2 -2
- package/dist/{config-UJx1dyX1.d.ts → config-DyRowvVP.d.ts} +2 -2
- package/dist/{direct-executor-EGESD26S.mjs → direct-executor-CHXVCBN7.mjs} +9 -5
- package/dist/distribution/coordinator/etcd.d.ts +3 -2
- package/dist/distribution/coordinator/etcd.mjs +4 -0
- package/dist/distribution/coordinator/in-memory.d.ts +3 -2
- package/dist/distribution/coordinator/in-memory.mjs +4 -0
- package/dist/distribution/transport/grpc.mjs +2 -2
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +2 -2
- package/dist/distribution.d.ts +43 -9
- package/dist/distribution.mjs +4590 -4263
- package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-6AN4QTT3.mjs} +20 -16
- package/dist/{engine-9X6W5naj.d.ts → engine-96_kLtSY.d.ts} +43 -7
- package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
- package/dist/index.browser.mjs +1 -1
- package/dist/index.d.ts +44 -25
- package/dist/index.mjs +13 -9
- package/dist/{memory-CV9NAxLY.d.ts → memory-dd3GESdc.d.ts} +3 -3
- package/dist/partial-words.mjs +2 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
- package/dist/persistence/durability/checkpoint-worker.mjs +18 -14
- package/dist/react.d.ts +8 -7
- package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
- package/dist/server.d.ts +11 -10
- package/dist/server.mjs +33 -6
- package/dist/{snapshot-22CUEYYW.mjs → snapshot-JFZVOGZV.mjs} +28 -18
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-BGaZJyoY.d.ts → tasks-DA1Ff0G4.d.ts} +4 -1
- package/dist/{thread-ZP3UMIZT.mjs → thread-VTDLLYXC.mjs} +7 -5
- package/dist/vector/search-worker.d.ts +2 -1
- package/dist/workers/entry.mjs +4 -4
- package/dist/{write-PXDKPG2R.mjs → write-OJR3H4BI.mjs} +14 -10
- package/package.json +15 -7
- package/dist/chunk-PR4O7CAP.mjs +0 -18
- package/dist/chunk-X4SEXL4U.mjs +0 -2310
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ErrorCodes, singleFoldTable, appendFoldedCodePoint, NarsilError, compareCodePoints, createNarsilError, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, toReducedSortValue, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, resolveRequestThreadCount, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, defaultSortMode, holdsUnpairedSurrogate, MIN_SWEEP_INTERVAL_MS, MAX_SUGGEST_LIMIT, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_INDEX_CONFIGURATION_BYTES, compareComparableValues, readSortField, exceedsCodePoints, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, compareComparableKeys, sortModeOf, toComparableSortValue, DEFAULT_FUZZY_PREFIX_LENGTH, SORT_VALUE_MAX_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, truncateSortString, MAX_IMPORT_REFUSALS_REPORTED, MIN_OVERSAMPLE, DEFAULT_MAX_PER_GROUP, compareSortValues, isSortMode, SORT_MODES, MIN_DOCUMENTS_FOR_SEGMENTS, SHUTDOWN_TIMEOUT_MS,
|
|
1
|
+
import { ErrorCodes, singleFoldTable, appendFoldedCodePoint, NarsilError, compareCodePoints, createNarsilError, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, toReducedSortValue, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, resolveRequestThreadCount, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, defaultSortMode, holdsUnpairedSurrogate, MIN_SWEEP_INTERVAL_MS, MAX_SUGGEST_LIMIT, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_INDEX_CONFIGURATION_BYTES, compareComparableValues, readSortField, exceedsCodePoints, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, RESULT_WINDOW, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, compareComparableKeys, sortModeOf, toComparableSortValue, DEFAULT_FUZZY_PREFIX_LENGTH, SORT_VALUE_MAX_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, truncateSortString, MAX_IMPORT_REFUSALS_REPORTED, MIN_OVERSAMPLE, DEFAULT_MAX_PER_GROUP, compareSortValues, isSortMode, SORT_MODES, MIN_DOCUMENTS_FOR_SEGMENTS, SHUTDOWN_TIMEOUT_MS, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-FHFFWLEK.mjs';
|
|
2
2
|
import { Decoder, encode, decode } from '@msgpack/msgpack';
|
|
3
3
|
|
|
4
4
|
// src/analysis/registry.ts
|
|
@@ -503,6 +503,42 @@ function requirePatternSearch(fieldPath) {
|
|
|
503
503
|
);
|
|
504
504
|
}
|
|
505
505
|
|
|
506
|
+
// src/core/synonyms/platform.browser.ts
|
|
507
|
+
function platformSynonymSearch() {
|
|
508
|
+
return null;
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
// src/core/synonyms/registry.ts
|
|
512
|
+
var registered3 = null;
|
|
513
|
+
var MODULE_FUNCTIONS3 = ["prepare", "storedLookup", "decodeRules", "replace", "applyLoaded"];
|
|
514
|
+
function isSynonymSearchModule(value) {
|
|
515
|
+
if (value === null || typeof value !== "object" || value.name !== "synonyms") return false;
|
|
516
|
+
return MODULE_FUNCTIONS3.every((name) => typeof Reflect.get(value, name) === "function");
|
|
517
|
+
}
|
|
518
|
+
function registerSynonyms(module) {
|
|
519
|
+
if (!isSynonymSearchModule(module)) {
|
|
520
|
+
throw new NarsilError(
|
|
521
|
+
ErrorCodes.CONFIG_INVALID,
|
|
522
|
+
'Pass registerSynonyms the synonyms object that "@delali/narsil/synonyms" exports'
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
registered3 = module;
|
|
526
|
+
}
|
|
527
|
+
function availableSynonymModule() {
|
|
528
|
+
return registered3 ?? platformSynonymSearch();
|
|
529
|
+
}
|
|
530
|
+
function synonymModule() {
|
|
531
|
+
const module = availableSynonymModule();
|
|
532
|
+
if (module !== null) return module;
|
|
533
|
+
throw new NarsilError(
|
|
534
|
+
ErrorCodes.CONFIG_INVALID,
|
|
535
|
+
"This bundle holds no synonym code, so import { synonyms } from '@delali/narsil/synonyms' and pass it to registerSynonyms before you use a synonym set"
|
|
536
|
+
);
|
|
537
|
+
}
|
|
538
|
+
function isEmptySynonymSet(value) {
|
|
539
|
+
return value === void 0 || Array.isArray(value) && value.length === 0;
|
|
540
|
+
}
|
|
541
|
+
|
|
506
542
|
// src/core/tokenizer/constants.ts
|
|
507
543
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
508
544
|
var TOKEN_CACHE_SIZE_CEILING = 2e6;
|
|
@@ -7467,19 +7503,22 @@ function termGroupOf(groups, token) {
|
|
|
7467
7503
|
function typoMatchesFor(index, token, group, exact, tolerance, prefixLength) {
|
|
7468
7504
|
if (group !== void 0) {
|
|
7469
7505
|
if (!group.typoMatches || exact || tolerance === 0) return [];
|
|
7470
|
-
return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token);
|
|
7506
|
+
return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token && group.synonyms?.has(match.token) !== true);
|
|
7471
7507
|
}
|
|
7472
7508
|
if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
|
|
7473
7509
|
const postingList = index.lookup(token);
|
|
7474
7510
|
return postingList ? [{ token, postingList }] : [];
|
|
7475
7511
|
}
|
|
7476
7512
|
function resolveGroupMatches(context, group) {
|
|
7477
|
-
|
|
7513
|
+
const frequencyOf = (token) => context.globalFrequency(token) ?? context.index.documentFrequency(token);
|
|
7514
|
+
const lookup = (token) => context.index.lookup(token);
|
|
7515
|
+
const synonyms = group.synonyms;
|
|
7516
|
+
let groupDocumentFrequency = synonyms?.documentFrequency(frequencyOf) ?? 0;
|
|
7478
7517
|
const found = [];
|
|
7479
7518
|
for (const member of group.members) {
|
|
7480
|
-
const documents =
|
|
7519
|
+
const documents = frequencyOf(member.token);
|
|
7481
7520
|
if (documents > groupDocumentFrequency) groupDocumentFrequency = documents;
|
|
7482
|
-
const postingList =
|
|
7521
|
+
const postingList = synonyms !== void 0 && member.token === group.token ? synonyms.postings(lookup, member.token) : lookup(member.token);
|
|
7483
7522
|
if (postingList === void 0) continue;
|
|
7484
7523
|
found.push({ token: member.token, factor: member.factor, partialFactor: member.partialFactor, postingList });
|
|
7485
7524
|
}
|
|
@@ -7586,6 +7625,10 @@ function searchFulltextMatches(state, params) {
|
|
|
7586
7625
|
if (postingList === void 0) continue;
|
|
7587
7626
|
markPostingList(matched, postingList, member.factor > 0 ? searched : searchedPartial, filterBitset);
|
|
7588
7627
|
}
|
|
7628
|
+
for (const synonym of group.synonyms?.tokens ?? []) {
|
|
7629
|
+
const postingList = state.invertedIdx.lookup(synonym);
|
|
7630
|
+
if (postingList !== void 0) markPostingList(matched, postingList, searched, filterBitset);
|
|
7631
|
+
}
|
|
7589
7632
|
}
|
|
7590
7633
|
const docStore = state.docStore;
|
|
7591
7634
|
const resolver = docStore.resolver();
|
|
@@ -9730,205 +9773,253 @@ function readHeapStatistics() {
|
|
|
9730
9773
|
return null;
|
|
9731
9774
|
}
|
|
9732
9775
|
|
|
9733
|
-
// src/
|
|
9734
|
-
|
|
9735
|
-
|
|
9736
|
-
return new NarsilError(
|
|
9737
|
-
ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
|
|
9738
|
-
`The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
|
|
9739
|
-
{ cap }
|
|
9740
|
-
);
|
|
9741
|
-
}
|
|
9742
|
-
function localMeter(cap, spent, matcherStateBytes, flushUnits) {
|
|
9743
|
-
let total = spent;
|
|
9744
|
-
let sharedCount = null;
|
|
9745
|
-
const spentSoFar = () => sharedCount === null ? total : Number(Atomics.load(sharedCount, 0));
|
|
9746
|
-
const meter = {
|
|
9747
|
-
cap,
|
|
9748
|
-
matcherStateBytes,
|
|
9749
|
-
flushUnits,
|
|
9750
|
-
add(units) {
|
|
9751
|
-
if (units <= 0) return;
|
|
9752
|
-
if (sharedCount === null) total += units;
|
|
9753
|
-
else total = Number(Atomics.add(sharedCount, 0, BigInt(units))) + units;
|
|
9754
|
-
if (total > cap) throw capExceeded(cap);
|
|
9755
|
-
},
|
|
9756
|
-
remaining() {
|
|
9757
|
-
return cap - spentSoFar();
|
|
9758
|
-
},
|
|
9759
|
-
fork(workerCount) {
|
|
9760
|
-
const copies = Math.max(1, workerCount);
|
|
9761
|
-
const seed = spentSoFar();
|
|
9762
|
-
let counter = null;
|
|
9763
|
-
if (typeof SharedArrayBuffer === "function") {
|
|
9764
|
-
counter = new SharedArrayBuffer(COUNTER_BYTES);
|
|
9765
|
-
new BigInt64Array(counter)[0] = BigInt(seed);
|
|
9766
|
-
}
|
|
9767
|
-
const shared2 = {
|
|
9768
|
-
cap,
|
|
9769
|
-
spent: seed,
|
|
9770
|
-
counter,
|
|
9771
|
-
matcherStateBytes: Math.floor(matcherStateBytes / copies),
|
|
9772
|
-
flushUnits: Math.max(1, Math.floor(flushUnits / copies))
|
|
9773
|
-
};
|
|
9774
|
-
return {
|
|
9775
|
-
shared: shared2,
|
|
9776
|
-
settle(reportedUnits) {
|
|
9777
|
-
if (counter === null) {
|
|
9778
|
-
meter.add(reportedUnits);
|
|
9779
|
-
return;
|
|
9780
|
-
}
|
|
9781
|
-
const unitsSinceFork = spentSoFar() - seed;
|
|
9782
|
-
const view = new BigInt64Array(counter);
|
|
9783
|
-
if (unitsSinceFork > 0) Atomics.add(view, 0, BigInt(unitsSinceFork));
|
|
9784
|
-
sharedCount = view;
|
|
9785
|
-
total = spentSoFar();
|
|
9786
|
-
if (total > cap) throw capExceeded(cap);
|
|
9787
|
-
}
|
|
9788
|
-
};
|
|
9789
|
-
}
|
|
9790
|
-
};
|
|
9791
|
-
return meter;
|
|
9792
|
-
}
|
|
9793
|
-
function createPatternWorkMeter(cap = DEFAULT_PATTERN_WORK_CAP) {
|
|
9794
|
-
return localMeter(cap, 0, MATCHER_STATE_BUDGET_BYTES, PATTERN_WORK_FLUSH_UNITS);
|
|
9795
|
-
}
|
|
9796
|
-
function sharedMeter(work, counter) {
|
|
9797
|
-
const view = new BigInt64Array(counter);
|
|
9798
|
-
return {
|
|
9799
|
-
matcherStateBytes: work.matcherStateBytes,
|
|
9800
|
-
flushUnits: work.flushUnits,
|
|
9801
|
-
add(units) {
|
|
9802
|
-
if (units <= 0) return;
|
|
9803
|
-
const total = Number(Atomics.add(view, 0, BigInt(units))) + units;
|
|
9804
|
-
if (total > work.cap) throw capExceeded(work.cap);
|
|
9805
|
-
},
|
|
9806
|
-
remaining() {
|
|
9807
|
-
return work.cap - Number(Atomics.load(view, 0));
|
|
9808
|
-
}
|
|
9809
|
-
};
|
|
9810
|
-
}
|
|
9811
|
-
function joinPatternWorkMeter(work) {
|
|
9812
|
-
const meter = work.counter === null ? localMeter(work.cap, work.spent, work.matcherStateBytes, work.flushUnits) : sharedMeter(work, work.counter);
|
|
9813
|
-
let spent = 0;
|
|
9814
|
-
return {
|
|
9815
|
-
matcherStateBytes: meter.matcherStateBytes,
|
|
9816
|
-
flushUnits: meter.flushUnits,
|
|
9817
|
-
add(units) {
|
|
9818
|
-
if (units <= 0) return;
|
|
9819
|
-
spent += units;
|
|
9820
|
-
meter.add(units);
|
|
9821
|
-
},
|
|
9822
|
-
remaining: () => meter.remaining(),
|
|
9823
|
-
spent: () => spent
|
|
9824
|
-
};
|
|
9776
|
+
// src/search/oversample.ts
|
|
9777
|
+
function oversampledShardSize(size) {
|
|
9778
|
+
return Math.ceil(size * 1.5) + 10;
|
|
9825
9779
|
}
|
|
9826
9780
|
|
|
9827
|
-
// src/
|
|
9828
|
-
function
|
|
9829
|
-
|
|
9830
|
-
stem: true,
|
|
9831
|
-
removeStopWords: true,
|
|
9832
|
-
customTokenizer: analysis.customTokenizer,
|
|
9833
|
-
stopWordOverride: analysis.stopWords
|
|
9834
|
-
});
|
|
9835
|
-
const unique = /* @__PURE__ */ new Set();
|
|
9836
|
-
for (const { token } of tokens) {
|
|
9837
|
-
unique.add(token);
|
|
9838
|
-
}
|
|
9839
|
-
return unique;
|
|
9781
|
+
// src/search/facets.ts
|
|
9782
|
+
function keptValueCount(fieldConfig) {
|
|
9783
|
+
return keptFacetValues(fieldConfig?.limit);
|
|
9840
9784
|
}
|
|
9841
|
-
function
|
|
9842
|
-
const
|
|
9843
|
-
for (const [
|
|
9844
|
-
|
|
9845
|
-
copy[key] = value;
|
|
9846
|
-
}
|
|
9785
|
+
function everyValueFacetConfig(config) {
|
|
9786
|
+
const widened = {};
|
|
9787
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
9788
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
9847
9789
|
}
|
|
9848
|
-
return
|
|
9849
|
-
}
|
|
9850
|
-
function sanitizeGlobalStats(stats) {
|
|
9851
|
-
return {
|
|
9852
|
-
totalDocuments: typeof stats.totalDocuments === "number" && Number.isFinite(stats.totalDocuments) ? stats.totalDocuments : 0,
|
|
9853
|
-
docFrequencies: copyNumericRecord(stats.docFrequencies),
|
|
9854
|
-
totalFieldLengths: copyNumericRecord(stats.totalFieldLengths),
|
|
9855
|
-
averageFieldLengths: copyNumericRecord(stats.averageFieldLengths)
|
|
9856
|
-
};
|
|
9790
|
+
return widened;
|
|
9857
9791
|
}
|
|
9858
|
-
function
|
|
9859
|
-
const
|
|
9860
|
-
for (const
|
|
9861
|
-
if (
|
|
9862
|
-
|
|
9792
|
+
function oversampledFacetConfig(config) {
|
|
9793
|
+
const widened = {};
|
|
9794
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
9795
|
+
if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
|
|
9796
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
9797
|
+
continue;
|
|
9863
9798
|
}
|
|
9799
|
+
const kept = keptValueCount(fieldConfig);
|
|
9800
|
+
widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
|
|
9864
9801
|
}
|
|
9865
|
-
return
|
|
9866
|
-
totalDocuments: stats.totalDocuments,
|
|
9867
|
-
docFrequencies,
|
|
9868
|
-
totalFieldLengths: stats.totalFieldLengths,
|
|
9869
|
-
averageFieldLengths: stats.averageFieldLengths
|
|
9870
|
-
};
|
|
9802
|
+
return widened;
|
|
9871
9803
|
}
|
|
9872
|
-
function
|
|
9873
|
-
const
|
|
9874
|
-
|
|
9875
|
-
const
|
|
9876
|
-
|
|
9877
|
-
|
|
9878
|
-
|
|
9879
|
-
|
|
9880
|
-
|
|
9881
|
-
|
|
9882
|
-
|
|
9883
|
-
|
|
9884
|
-
|
|
9885
|
-
if (Object.hasOwn(stats.docFrequencies, token)) {
|
|
9886
|
-
docFrequencies[token] = (docFrequencies[token] ?? 0) + stats.docFrequencies[token];
|
|
9804
|
+
function mergeFacets(partitionFacets, config) {
|
|
9805
|
+
const merged = /* @__PURE__ */ new Map();
|
|
9806
|
+
const bounds = /* @__PURE__ */ new Map();
|
|
9807
|
+
for (const partition of partitionFacets) {
|
|
9808
|
+
for (const [field, facetResult] of Object.entries(partition)) {
|
|
9809
|
+
let fieldMap = merged.get(field);
|
|
9810
|
+
if (!fieldMap) {
|
|
9811
|
+
fieldMap = /* @__PURE__ */ new Map();
|
|
9812
|
+
merged.set(field, fieldMap);
|
|
9813
|
+
}
|
|
9814
|
+
bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
|
|
9815
|
+
for (const [value, count] of Object.entries(facetResult.values)) {
|
|
9816
|
+
fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
|
|
9887
9817
|
}
|
|
9888
9818
|
}
|
|
9889
9819
|
}
|
|
9890
|
-
const
|
|
9891
|
-
|
|
9892
|
-
|
|
9893
|
-
|
|
9894
|
-
|
|
9895
|
-
|
|
9896
|
-
|
|
9897
|
-
|
|
9898
|
-
|
|
9899
|
-
|
|
9900
|
-
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
9901
|
-
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
9902
|
-
for (const stats of statsArray) {
|
|
9903
|
-
totalDocuments += stats.totalDocuments;
|
|
9904
|
-
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
9905
|
-
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
9906
|
-
}
|
|
9907
|
-
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
9908
|
-
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
9820
|
+
const result = {};
|
|
9821
|
+
for (const [field, valueMap] of merged) {
|
|
9822
|
+
const fieldConfig = config[field];
|
|
9823
|
+
const ascending = fieldConfig?.sort === "asc";
|
|
9824
|
+
const ordered = Array.from(valueMap.entries());
|
|
9825
|
+
ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
9826
|
+
const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
|
|
9827
|
+
let largestLeftOut = 0;
|
|
9828
|
+
for (let index = kept; index < ordered.length; index++) {
|
|
9829
|
+
if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
|
|
9909
9830
|
}
|
|
9910
|
-
|
|
9911
|
-
|
|
9912
|
-
|
|
9913
|
-
|
|
9914
|
-
averageFieldLengths[field] = totalLength / totalDocuments;
|
|
9831
|
+
const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
|
|
9832
|
+
const values = {};
|
|
9833
|
+
for (let index = 0; index < kept; index++) {
|
|
9834
|
+
values[ordered[index][0]] = ordered[index][1];
|
|
9915
9835
|
}
|
|
9836
|
+
result[field] = { values, count: kept, errorBound };
|
|
9916
9837
|
}
|
|
9917
|
-
return
|
|
9918
|
-
totalDocuments,
|
|
9919
|
-
docFrequencies,
|
|
9920
|
-
totalFieldLengths,
|
|
9921
|
-
averageFieldLengths
|
|
9922
|
-
};
|
|
9838
|
+
return result;
|
|
9923
9839
|
}
|
|
9924
9840
|
|
|
9925
|
-
// src/core/partition/
|
|
9926
|
-
function
|
|
9927
|
-
const
|
|
9928
|
-
|
|
9929
|
-
|
|
9930
|
-
|
|
9931
|
-
|
|
9841
|
+
// src/core/partition/frozen/memory.ts
|
|
9842
|
+
function frozenSegmentBytes(segment) {
|
|
9843
|
+
const arrays = segment.arrays;
|
|
9844
|
+
let bytes = arrays.tokenTable.bytes + arrays.idTable.bytes;
|
|
9845
|
+
bytes += arrays.postingOffsets.byteLength;
|
|
9846
|
+
bytes += arrays.postingDocIds.byteLength;
|
|
9847
|
+
bytes += arrays.postingFrequencies.byteLength;
|
|
9848
|
+
bytes += arrays.postingFieldIndices.byteLength;
|
|
9849
|
+
if (arrays.positionOffsets !== null) bytes += arrays.positionOffsets.byteLength;
|
|
9850
|
+
if (arrays.positionValues !== null) bytes += arrays.positionValues.byteLength;
|
|
9851
|
+
if (arrays.documentTable !== null) {
|
|
9852
|
+
bytes += arrays.documentTable.blob.byteLength + arrays.documentTable.offsets.byteLength;
|
|
9853
|
+
}
|
|
9854
|
+
for (const column of arrays.fieldLengthColumns) bytes += column.byteLength;
|
|
9855
|
+
for (const entry of [...arrays.numeric, ...arrays.sizes]) bytes += entry.docIds.byteLength + entry.values.byteLength;
|
|
9856
|
+
for (const entry of arrays.boolean) bytes += entry.trueDocs.byteLength + entry.falseDocs.byteLength;
|
|
9857
|
+
for (const entry of arrays.enums) bytes += entry.offsets.byteLength + entry.docIds.byteLength;
|
|
9858
|
+
for (const entry of arrays.geo) {
|
|
9859
|
+
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
9860
|
+
}
|
|
9861
|
+
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
9862
|
+
return bytes + segment.partialWords.bytes() + segment.removalBytes();
|
|
9863
|
+
}
|
|
9864
|
+
|
|
9865
|
+
// src/core/partition/composite/compaction.ts
|
|
9866
|
+
function resolveFrozenSegments(frozen, segmentIds, partitionId) {
|
|
9867
|
+
const byId = new Map(frozen.map((segment) => [segment.segmentId, segment]));
|
|
9868
|
+
return segmentIds.map((segmentId) => {
|
|
9869
|
+
const segment = byId.get(segmentId);
|
|
9870
|
+
if (segment === void 0) {
|
|
9871
|
+
throw new NarsilError(
|
|
9872
|
+
ErrorCodes.PARTITION_CORRUPTED,
|
|
9873
|
+
`Partition ${partitionId} does not hold frozen segment "${segmentId}"`,
|
|
9874
|
+
{ partitionId, segmentId }
|
|
9875
|
+
);
|
|
9876
|
+
}
|
|
9877
|
+
return segment;
|
|
9878
|
+
});
|
|
9879
|
+
}
|
|
9880
|
+
function survivorDocumentsOf(sub, payload) {
|
|
9881
|
+
return payload.docIds.map((docId) => {
|
|
9882
|
+
const stored = sub.docStore.get(docId);
|
|
9883
|
+
if (stored === void 0) {
|
|
9884
|
+
throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while compacting a segment`, {
|
|
9885
|
+
docId
|
|
9886
|
+
});
|
|
9887
|
+
}
|
|
9888
|
+
return stored.fields;
|
|
9889
|
+
});
|
|
9890
|
+
}
|
|
9891
|
+
function freezeLiveTailInto(live, liveState, frozen, freeze) {
|
|
9892
|
+
if (live.count() === 0) return null;
|
|
9893
|
+
const payload = live.encodeSegment();
|
|
9894
|
+
const segment = freeze(payload, survivorDocumentsOf(liveState, payload));
|
|
9895
|
+
if (segment === null) return null;
|
|
9896
|
+
live.clear();
|
|
9897
|
+
frozen.push(segment);
|
|
9898
|
+
return segment;
|
|
9899
|
+
}
|
|
9900
|
+
function swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings) {
|
|
9901
|
+
const dropSet = new Set(dropSegmentIds);
|
|
9902
|
+
const dropped = frozen.filter((segment) => dropSet.has(segment.segmentId));
|
|
9903
|
+
const applied = /* @__PURE__ */ new Map();
|
|
9904
|
+
for (const segment of dropped) {
|
|
9905
|
+
for (const docId of segment.tombstonedDocIds()) {
|
|
9906
|
+
if (!replacement.hasDocument(docId)) continue;
|
|
9907
|
+
if (dropped.some((other) => other.hasDocument(docId))) continue;
|
|
9908
|
+
const spellings = removedSpellings?.get(docId) ?? replacement.removedSpellingsOf(docId);
|
|
9909
|
+
if (spellings !== void 0) applied.set(docId, spellings);
|
|
9910
|
+
replacement.tombstoneDocument(docId, spellings);
|
|
9911
|
+
}
|
|
9912
|
+
}
|
|
9913
|
+
const kept = frozen.filter((segment) => !dropSet.has(segment.segmentId));
|
|
9914
|
+
frozen.length = 0;
|
|
9915
|
+
frozen.push(...kept, replacement);
|
|
9916
|
+
return applied;
|
|
9917
|
+
}
|
|
9918
|
+
|
|
9919
|
+
// src/core/partition/composite/documents.ts
|
|
9920
|
+
function compositeDocument(frozen, live, docId, projection) {
|
|
9921
|
+
for (const segment of frozen) {
|
|
9922
|
+
const projected = segment.docStore.projectedDocument(docId, projection);
|
|
9923
|
+
if (projected !== void 0) return projected;
|
|
9924
|
+
}
|
|
9925
|
+
return live.get(docId, projection);
|
|
9926
|
+
}
|
|
9927
|
+
function compositeFieldValues(frozen, live, docId, fieldPaths) {
|
|
9928
|
+
for (const segment of frozen) {
|
|
9929
|
+
const internalId = segment.docStore.getInternalId(docId);
|
|
9930
|
+
if (internalId === void 0) continue;
|
|
9931
|
+
return fieldPaths.map((fieldPath) => segment.docStore.fieldReader(fieldPath)(internalId));
|
|
9932
|
+
}
|
|
9933
|
+
return live.fieldValues(docId, fieldPaths);
|
|
9934
|
+
}
|
|
9935
|
+
|
|
9936
|
+
// src/core/partition/composite/filters.ts
|
|
9937
|
+
var WORD_BITS = 32;
|
|
9938
|
+
function alignToWord(value) {
|
|
9939
|
+
return value + WORD_BITS - 1 & -32;
|
|
9940
|
+
}
|
|
9941
|
+
function computeOrdinalLayout(subs) {
|
|
9942
|
+
const bases = new Array(subs.length);
|
|
9943
|
+
let cursor = 0;
|
|
9944
|
+
for (let i = 0; i < subs.length; i++) {
|
|
9945
|
+
bases[i] = cursor;
|
|
9946
|
+
cursor += alignToWord(subs[i].docStore.internalIdCapacity());
|
|
9947
|
+
}
|
|
9948
|
+
return { bases, totalCapacity: cursor };
|
|
9949
|
+
}
|
|
9950
|
+
function subBitsetView(composite, layout, subIndex) {
|
|
9951
|
+
const startWord = layout.bases[subIndex] >> 5;
|
|
9952
|
+
const endWord = subIndex + 1 < layout.bases.length ? layout.bases[subIndex + 1] >> 5 : Math.ceil(layout.totalCapacity / WORD_BITS);
|
|
9953
|
+
return composite.subarray(startWord, endWord);
|
|
9954
|
+
}
|
|
9955
|
+
function placeSubBitset(composite, layout, subIndex, subBits) {
|
|
9956
|
+
const words = subBitsetView(composite, layout, subIndex).length;
|
|
9957
|
+
if (words === 0) return;
|
|
9958
|
+
composite.set(subBits.length > words ? subBits.subarray(0, words) : subBits, layout.bases[subIndex] >> 5);
|
|
9959
|
+
}
|
|
9960
|
+
function compositeFiltersBitset(subs, layout, filters, schema, meter) {
|
|
9961
|
+
const composite = new Uint32Array(Math.ceil(layout.totalCapacity / WORD_BITS));
|
|
9962
|
+
for (let i = 0; i < subs.length; i++) {
|
|
9963
|
+
placeSubBitset(composite, layout, i, applyPartitionFiltersBitset(subs[i], filters, schema, meter));
|
|
9964
|
+
}
|
|
9965
|
+
return composite;
|
|
9966
|
+
}
|
|
9967
|
+
function compositeFilters(subs, filters, schema, meter) {
|
|
9968
|
+
const combined = /* @__PURE__ */ new Set();
|
|
9969
|
+
for (const sub of subs) {
|
|
9970
|
+
for (const docId of applyPartitionFilters(sub, filters, schema, meter)) {
|
|
9971
|
+
combined.add(docId);
|
|
9972
|
+
}
|
|
9973
|
+
}
|
|
9974
|
+
return combined;
|
|
9975
|
+
}
|
|
9976
|
+
function compositeFilterMatches(subs, layout, filters, schema, meter) {
|
|
9977
|
+
const bits = compositeFiltersBitset(subs, layout, filters, schema, meter);
|
|
9978
|
+
let count = 0;
|
|
9979
|
+
for (let i = 0; i < subs.length; i++) {
|
|
9980
|
+
const resolver = subs[i].docStore.resolver();
|
|
9981
|
+
const view = subBitsetView(bits, layout, i);
|
|
9982
|
+
for (let wordIndex = 0; wordIndex < view.length; wordIndex++) {
|
|
9983
|
+
let word = view[wordIndex];
|
|
9984
|
+
if (word === 0) continue;
|
|
9985
|
+
const base = wordIndex << 5;
|
|
9986
|
+
while (word !== 0) {
|
|
9987
|
+
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
9988
|
+
if (resolver.isLive(base + trailingZeros)) count++;
|
|
9989
|
+
word &= word - 1;
|
|
9990
|
+
}
|
|
9991
|
+
}
|
|
9992
|
+
}
|
|
9993
|
+
return {
|
|
9994
|
+
count,
|
|
9995
|
+
hasExternal(docId) {
|
|
9996
|
+
for (let i = 0; i < subs.length; i++) {
|
|
9997
|
+
const internalId = subs[i].docStore.getInternalId(docId);
|
|
9998
|
+
if (internalId !== void 0) return bitsetHas(bits, layout.bases[i] + internalId);
|
|
9999
|
+
}
|
|
10000
|
+
return false;
|
|
10001
|
+
},
|
|
10002
|
+
hasInternal(internalId) {
|
|
10003
|
+
return bitsetHas(bits, internalId);
|
|
10004
|
+
}
|
|
10005
|
+
};
|
|
10006
|
+
}
|
|
10007
|
+
function subMatchesView(matches, layout, subIndex) {
|
|
10008
|
+
const base = layout.bases[subIndex];
|
|
10009
|
+
return {
|
|
10010
|
+
count: matches.count,
|
|
10011
|
+
hasExternal: (docId) => matches.hasExternal(docId),
|
|
10012
|
+
hasInternal: (internalId) => matches.hasInternal(base + internalId)
|
|
10013
|
+
};
|
|
10014
|
+
}
|
|
10015
|
+
|
|
10016
|
+
// src/core/partition/scored-merge.ts
|
|
10017
|
+
function kWayMerge(arrays) {
|
|
10018
|
+
const nonEmpty = arrays.filter((a) => a.length > 0);
|
|
10019
|
+
if (nonEmpty.length === 0) return [];
|
|
10020
|
+
if (nonEmpty.length === 1) return nonEmpty[0];
|
|
10021
|
+
if (nonEmpty.length <= 4) {
|
|
10022
|
+
return sequentialMerge(nonEmpty);
|
|
9932
10023
|
}
|
|
9933
10024
|
return heapMerge(nonEmpty);
|
|
9934
10025
|
}
|
|
@@ -10039,1921 +10130,1944 @@ function heapPop(heap) {
|
|
|
10039
10130
|
return top;
|
|
10040
10131
|
}
|
|
10041
10132
|
|
|
10042
|
-
// src/
|
|
10043
|
-
function
|
|
10044
|
-
|
|
10045
|
-
|
|
10046
|
-
|
|
10047
|
-
|
|
10048
|
-
|
|
10049
|
-
|
|
10133
|
+
// src/core/partition/composite/stats.ts
|
|
10134
|
+
function collectCandidateTerms(subs, params) {
|
|
10135
|
+
const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
|
|
10136
|
+
const terms = /* @__PURE__ */ new Set();
|
|
10137
|
+
for (const qt of queryTokens) {
|
|
10138
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
10139
|
+
if (group !== void 0) {
|
|
10140
|
+
for (const member of group.members) terms.add(member.token);
|
|
10141
|
+
for (const synonym of group.synonyms?.tokens ?? []) terms.add(synonym);
|
|
10142
|
+
}
|
|
10143
|
+
if (group === void 0 && (exact || tolerance === 0)) {
|
|
10144
|
+
terms.add(qt.token);
|
|
10145
|
+
continue;
|
|
10146
|
+
}
|
|
10147
|
+
for (const sub of subs) {
|
|
10148
|
+
for (const match of typoMatchesFor(sub.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
10149
|
+
terms.add(match.token);
|
|
10150
|
+
}
|
|
10151
|
+
}
|
|
10152
|
+
}
|
|
10153
|
+
return terms;
|
|
10050
10154
|
}
|
|
10051
|
-
function
|
|
10052
|
-
const
|
|
10053
|
-
for (const
|
|
10054
|
-
|
|
10155
|
+
function resolveQueryTermFrequencies(subs, params) {
|
|
10156
|
+
const frequencies = /* @__PURE__ */ Object.create(null);
|
|
10157
|
+
for (const term of collectCandidateTerms(subs, params)) {
|
|
10158
|
+
const total = partitionDocumentFrequency(subs, term);
|
|
10159
|
+
if (total > 0) frequencies[term] = total;
|
|
10055
10160
|
}
|
|
10056
|
-
return
|
|
10161
|
+
return frequencies;
|
|
10057
10162
|
}
|
|
10058
|
-
function
|
|
10059
|
-
|
|
10060
|
-
|
|
10061
|
-
|
|
10062
|
-
|
|
10063
|
-
|
|
10163
|
+
function aggregateFieldStats(subs) {
|
|
10164
|
+
let totalDocuments = 0;
|
|
10165
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
10166
|
+
for (const sub of subs) {
|
|
10167
|
+
totalDocuments += sub.stats.totalDocuments;
|
|
10168
|
+
for (const [field, length] of Object.entries(sub.stats.totalFieldLengths)) {
|
|
10169
|
+
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
10064
10170
|
}
|
|
10065
|
-
const kept = keptValueCount(fieldConfig);
|
|
10066
|
-
widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
|
|
10067
10171
|
}
|
|
10068
|
-
|
|
10172
|
+
const averageFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
10173
|
+
if (totalDocuments > 0) {
|
|
10174
|
+
for (const [field, length] of Object.entries(totalFieldLengths)) {
|
|
10175
|
+
averageFieldLengths[field] = length / totalDocuments;
|
|
10176
|
+
}
|
|
10177
|
+
}
|
|
10178
|
+
return { totalDocuments, totalFieldLengths, averageFieldLengths };
|
|
10069
10179
|
}
|
|
10070
|
-
function
|
|
10071
|
-
const merged = /* @__PURE__ */
|
|
10072
|
-
const
|
|
10073
|
-
|
|
10074
|
-
|
|
10075
|
-
let fieldMap = merged.get(field);
|
|
10076
|
-
if (!fieldMap) {
|
|
10077
|
-
fieldMap = /* @__PURE__ */ new Map();
|
|
10078
|
-
merged.set(field, fieldMap);
|
|
10079
|
-
}
|
|
10080
|
-
bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
|
|
10081
|
-
for (const [value, count] of Object.entries(facetResult.values)) {
|
|
10082
|
-
fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
|
|
10083
|
-
}
|
|
10180
|
+
function mergeDocFrequencies(subs) {
|
|
10181
|
+
const merged = /* @__PURE__ */ Object.create(null);
|
|
10182
|
+
for (const sub of subs) {
|
|
10183
|
+
for (const [term, frequency] of Object.entries(sub.stats.docFrequencies)) {
|
|
10184
|
+
merged[term] = (merged[term] ?? 0) + frequency;
|
|
10084
10185
|
}
|
|
10085
10186
|
}
|
|
10086
|
-
|
|
10087
|
-
for (const [field, valueMap] of merged) {
|
|
10088
|
-
const fieldConfig = config[field];
|
|
10089
|
-
const ascending = fieldConfig?.sort === "asc";
|
|
10090
|
-
const ordered = Array.from(valueMap.entries());
|
|
10091
|
-
ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
10092
|
-
const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
|
|
10093
|
-
let largestLeftOut = 0;
|
|
10094
|
-
for (let index = kept; index < ordered.length; index++) {
|
|
10095
|
-
if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
|
|
10096
|
-
}
|
|
10097
|
-
const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
|
|
10098
|
-
const values = {};
|
|
10099
|
-
for (let index = 0; index < kept; index++) {
|
|
10100
|
-
values[ordered[index][0]] = ordered[index][1];
|
|
10101
|
-
}
|
|
10102
|
-
result[field] = { values, count: kept, errorBound };
|
|
10103
|
-
}
|
|
10104
|
-
return result;
|
|
10187
|
+
return merged;
|
|
10105
10188
|
}
|
|
10106
|
-
|
|
10107
|
-
|
|
10108
|
-
|
|
10109
|
-
|
|
10110
|
-
|
|
10111
|
-
|
|
10112
|
-
|
|
10113
|
-
|
|
10189
|
+
function buildAggregateStatsView(subs, docFrequencies) {
|
|
10190
|
+
const fields = aggregateFieldStats(subs);
|
|
10191
|
+
return {
|
|
10192
|
+
...fields,
|
|
10193
|
+
get docFrequencies() {
|
|
10194
|
+
return docFrequencies();
|
|
10195
|
+
}
|
|
10196
|
+
};
|
|
10114
10197
|
}
|
|
10115
|
-
function
|
|
10116
|
-
|
|
10117
|
-
const
|
|
10118
|
-
|
|
10119
|
-
|
|
10120
|
-
|
|
10198
|
+
function compositeQueryStats(subs, params) {
|
|
10199
|
+
const resolved = resolveQueryTermFrequencies(subs, params);
|
|
10200
|
+
const caller = params.globalStats;
|
|
10201
|
+
if (caller === void 0) {
|
|
10202
|
+
const fields = aggregateFieldStats(subs);
|
|
10203
|
+
return { ...fields, docFrequencies: resolved };
|
|
10204
|
+
}
|
|
10205
|
+
return {
|
|
10206
|
+
totalDocuments: caller.totalDocuments,
|
|
10207
|
+
totalFieldLengths: caller.totalFieldLengths,
|
|
10208
|
+
averageFieldLengths: caller.averageFieldLengths,
|
|
10209
|
+
docFrequencies: { ...resolved, ...caller.docFrequencies }
|
|
10210
|
+
};
|
|
10121
10211
|
}
|
|
10122
10212
|
|
|
10123
|
-
// src/search
|
|
10124
|
-
function
|
|
10125
|
-
return
|
|
10213
|
+
// src/core/partition/composite/search.ts
|
|
10214
|
+
function subParams(params, layout, subIndex, globalStats) {
|
|
10215
|
+
return {
|
|
10216
|
+
...params,
|
|
10217
|
+
globalStats,
|
|
10218
|
+
filterBitset: params.filterBitset === void 0 ? void 0 : subBitsetView(params.filterBitset, layout, subIndex)
|
|
10219
|
+
};
|
|
10126
10220
|
}
|
|
10127
|
-
function
|
|
10128
|
-
|
|
10129
|
-
|
|
10130
|
-
|
|
10131
|
-
|
|
10132
|
-
|
|
10221
|
+
function onlyPopulatedSub(subs) {
|
|
10222
|
+
let found = -1;
|
|
10223
|
+
for (let i = 0; i < subs.length; i++) {
|
|
10224
|
+
if (subs[i].stats.totalDocuments === 0) continue;
|
|
10225
|
+
if (found !== -1) return -1;
|
|
10226
|
+
found = i;
|
|
10133
10227
|
}
|
|
10134
|
-
return
|
|
10135
|
-
}
|
|
10136
|
-
function sortModesOf(fields) {
|
|
10137
|
-
return fields.map(sortModeOf);
|
|
10228
|
+
return found;
|
|
10138
10229
|
}
|
|
10139
|
-
function
|
|
10140
|
-
const
|
|
10141
|
-
|
|
10230
|
+
function searchSingleSub(subs, layout, params, index) {
|
|
10231
|
+
const result = searchFulltext(subs[index], subParams(params, layout, index, params.globalStats));
|
|
10232
|
+
if (result.matchedOrdinalBitset === void 0) return result;
|
|
10233
|
+
const matchedOrdinalBitset = createBitSet(layout.totalCapacity);
|
|
10234
|
+
placeSubBitset(matchedOrdinalBitset, layout, index, result.matchedOrdinalBitset);
|
|
10235
|
+
return { ...result, matchedOrdinalBitset };
|
|
10142
10236
|
}
|
|
10143
|
-
function
|
|
10144
|
-
if (
|
|
10145
|
-
|
|
10146
|
-
throw new NarsilError(
|
|
10147
|
-
ErrorCodes.SEARCH_INVALID_MODE,
|
|
10148
|
-
`A sort mode is one of ${SORT_MODES.map((mode) => `"${mode}"`).join(", ")}, and the mode on "${entry.field}" is "${String(entry.mode)}"`,
|
|
10149
|
-
{ field: entry.field, mode: String(entry.mode) }
|
|
10150
|
-
);
|
|
10237
|
+
function compositeSearchFulltext(subs, layout, params) {
|
|
10238
|
+
if (params.queryTokens.length === 0) {
|
|
10239
|
+
return { scored: [], totalMatched: 0 };
|
|
10151
10240
|
}
|
|
10152
|
-
const
|
|
10153
|
-
if (
|
|
10154
|
-
|
|
10155
|
-
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
10156
|
-
`The engine applies the "${entry.mode}" mode to a number field alone, and "${entry.field}" is a ${fieldType} field`,
|
|
10157
|
-
{ field: entry.field, fieldType, mode: entry.mode }
|
|
10158
|
-
);
|
|
10241
|
+
const single = onlyPopulatedSub(subs);
|
|
10242
|
+
if (single !== -1) {
|
|
10243
|
+
return searchSingleSub(subs, layout, params, single);
|
|
10159
10244
|
}
|
|
10160
|
-
|
|
10161
|
-
|
|
10162
|
-
const
|
|
10163
|
-
if (
|
|
10164
|
-
|
|
10165
|
-
|
|
10166
|
-
|
|
10167
|
-
|
|
10168
|
-
|
|
10169
|
-
|
|
10170
|
-
|
|
10171
|
-
|
|
10172
|
-
|
|
10173
|
-
|
|
10174
|
-
|
|
10175
|
-
|
|
10176
|
-
|
|
10177
|
-
|
|
10178
|
-
throw new NarsilError(
|
|
10179
|
-
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
10180
|
-
`The engine sorts by text field "${entry.field}" only where its type includes "sortable", as in "${withSortableOption(parsed)}", because it keeps up to the first ${SORT_VALUE_MAX_CODE_POINTS} code points of every document's value in memory to sort by a text field`,
|
|
10181
|
-
{ field: entry.field, fieldType }
|
|
10182
|
-
);
|
|
10183
|
-
}
|
|
10184
|
-
if (isGeopointOrVectorType(fieldType)) {
|
|
10185
|
-
throw new NarsilError(
|
|
10186
|
-
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
10187
|
-
`The engine sorts by a number, boolean, enum, or sortable text field, and "${entry.field}" is a ${fieldType} field`,
|
|
10188
|
-
{ field: entry.field, fieldType }
|
|
10189
|
-
);
|
|
10245
|
+
const globalStats = compositeQueryStats(subs, params);
|
|
10246
|
+
const results = subs.map((sub, index) => searchFulltext(sub, subParams(params, layout, index, globalStats)));
|
|
10247
|
+
const scored = kWayMerge(results.map((result) => result.scored));
|
|
10248
|
+
if (params.maxResults !== void 0 && scored.length > params.maxResults) {
|
|
10249
|
+
scored.length = params.maxResults;
|
|
10250
|
+
}
|
|
10251
|
+
let totalMatched = 0;
|
|
10252
|
+
for (const result of results) {
|
|
10253
|
+
totalMatched += result.totalMatched;
|
|
10254
|
+
}
|
|
10255
|
+
if (params.collectMatchedSet === void 0) {
|
|
10256
|
+
return { scored, totalMatched };
|
|
10257
|
+
}
|
|
10258
|
+
if (params.collectMatchedSet === "ordinals") {
|
|
10259
|
+
const matchedOrdinalBitset = createBitSet(layout.totalCapacity);
|
|
10260
|
+
for (let i = 0; i < results.length; i++) {
|
|
10261
|
+
const subBitset = results[i].matchedOrdinalBitset;
|
|
10262
|
+
if (subBitset !== void 0) placeSubBitset(matchedOrdinalBitset, layout, i, subBitset);
|
|
10190
10263
|
}
|
|
10264
|
+
return { scored, totalMatched, matchedOrdinalBitset };
|
|
10191
10265
|
}
|
|
10192
|
-
|
|
10193
|
-
|
|
10194
|
-
|
|
10195
|
-
if (normalized.length === 0) return hits;
|
|
10196
|
-
const directions = normalized.map((entry) => entry.direction);
|
|
10197
|
-
const keyCache = /* @__PURE__ */ new Map();
|
|
10198
|
-
for (const hit of hits) {
|
|
10199
|
-
keyCache.set(hit.id, sortKeyOf(hit.id));
|
|
10266
|
+
const matchedIds = [];
|
|
10267
|
+
for (const result of results) {
|
|
10268
|
+
if (result.matchedIds !== void 0) matchedIds.push(...result.matchedIds);
|
|
10200
10269
|
}
|
|
10201
|
-
|
|
10202
|
-
sorted.sort(
|
|
10203
|
-
(a, b) => compareComparableKeys(keyCache.get(a.id) ?? [], keyCache.get(b.id) ?? [], directions) || compareCodePoints(a.id, b.id)
|
|
10204
|
-
);
|
|
10205
|
-
return sorted;
|
|
10206
|
-
}
|
|
10207
|
-
|
|
10208
|
-
// src/search/cursor.ts
|
|
10209
|
-
var CURSOR_VERSION = 4;
|
|
10210
|
-
function invalidCursor(cursor, reason) {
|
|
10211
|
-
const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
|
|
10212
|
-
return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
|
|
10270
|
+
return { scored, totalMatched, matchedIds };
|
|
10213
10271
|
}
|
|
10214
|
-
function
|
|
10215
|
-
const
|
|
10216
|
-
|
|
10217
|
-
|
|
10218
|
-
|
|
10219
|
-
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
10220
|
-
`A sort names at most ${MAX_SORT_FIELDS} fields, because the paging cursor carries one value per field`,
|
|
10221
|
-
{ count: fields.length, limit: MAX_SORT_FIELDS }
|
|
10222
|
-
);
|
|
10272
|
+
function compositeSearchMatches(subs, layout, params) {
|
|
10273
|
+
const subMatches = subs.map((sub, index) => searchFulltextMatches(sub, subParams(params, layout, index, void 0)));
|
|
10274
|
+
let count = 0;
|
|
10275
|
+
for (const matches of subMatches) {
|
|
10276
|
+
count += matches.count;
|
|
10223
10277
|
}
|
|
10224
|
-
|
|
10225
|
-
|
|
10226
|
-
|
|
10227
|
-
|
|
10228
|
-
|
|
10229
|
-
|
|
10230
|
-
|
|
10278
|
+
return {
|
|
10279
|
+
count,
|
|
10280
|
+
hasExternal(docId) {
|
|
10281
|
+
for (const matches of subMatches) {
|
|
10282
|
+
if (matches.hasExternal(docId)) return true;
|
|
10283
|
+
}
|
|
10284
|
+
return false;
|
|
10285
|
+
},
|
|
10286
|
+
hasInternal(internalId) {
|
|
10287
|
+
for (let i = subMatches.length - 1; i >= 0; i--) {
|
|
10288
|
+
if (internalId >= layout.bases[i]) {
|
|
10289
|
+
return subMatches[i].hasInternal(internalId - layout.bases[i]);
|
|
10290
|
+
}
|
|
10291
|
+
}
|
|
10292
|
+
return false;
|
|
10293
|
+
},
|
|
10294
|
+
matchedDocIds() {
|
|
10295
|
+
const combined = /* @__PURE__ */ new Set();
|
|
10296
|
+
for (const matches of subMatches) {
|
|
10297
|
+
for (const docId of matches.matchedDocIds()) {
|
|
10298
|
+
combined.add(docId);
|
|
10299
|
+
}
|
|
10300
|
+
}
|
|
10301
|
+
return combined;
|
|
10302
|
+
},
|
|
10303
|
+
ordinalBitset() {
|
|
10304
|
+
const combined = createBitSet(layout.totalCapacity);
|
|
10305
|
+
for (let i = 0; i < subMatches.length; i++) {
|
|
10306
|
+
placeSubBitset(combined, layout, i, subMatches[i].ordinalBitset());
|
|
10307
|
+
}
|
|
10308
|
+
return combined;
|
|
10231
10309
|
}
|
|
10232
|
-
}
|
|
10233
|
-
return JSON.stringify(fields.map(sortSignatureEntry));
|
|
10310
|
+
};
|
|
10234
10311
|
}
|
|
10235
|
-
|
|
10236
|
-
|
|
10237
|
-
|
|
10238
|
-
|
|
10239
|
-
|
|
10240
|
-
|
|
10241
|
-
payload.s = cursor.score;
|
|
10312
|
+
|
|
10313
|
+
// src/core/partition/composite/sorting.ts
|
|
10314
|
+
function compareEntries(a, b, directions) {
|
|
10315
|
+
for (let i = 0; i < directions.length; i++) {
|
|
10316
|
+
const comparison = compareComparableValues(a.key[i] ?? null, b.key[i] ?? null, directions[i]);
|
|
10317
|
+
if (comparison !== 0) return comparison;
|
|
10242
10318
|
}
|
|
10243
|
-
|
|
10244
|
-
payload.q = cursor.binding;
|
|
10245
|
-
return encodeCursorText(JSON.stringify(payload));
|
|
10319
|
+
return compareCodePoints(a.id, b.id);
|
|
10246
10320
|
}
|
|
10247
|
-
function
|
|
10248
|
-
|
|
10249
|
-
|
|
10250
|
-
|
|
10251
|
-
|
|
10252
|
-
|
|
10253
|
-
|
|
10254
|
-
|
|
10255
|
-
|
|
10256
|
-
|
|
10257
|
-
|
|
10258
|
-
key.push(entry);
|
|
10259
|
-
continue;
|
|
10260
|
-
}
|
|
10261
|
-
if (typeof entry === "string") {
|
|
10262
|
-
if (exceedsCodePoints(entry, SORT_VALUE_MAX_CODE_POINTS)) {
|
|
10263
|
-
throw invalidCursor(cursor, '"k" holds an oversized sort value');
|
|
10264
|
-
}
|
|
10265
|
-
key.push(entry);
|
|
10266
|
-
continue;
|
|
10267
|
-
}
|
|
10268
|
-
throw invalidCursor(cursor, '"k" accepts a string, a finite number, a boolean, or null');
|
|
10269
|
-
}
|
|
10270
|
-
return key;
|
|
10321
|
+
function compositeSortedPage(subs, layout, request) {
|
|
10322
|
+
const pages = subs.map(
|
|
10323
|
+
(sub, index) => sortedPageOf(sub, {
|
|
10324
|
+
...request,
|
|
10325
|
+
matches: request.matches === null ? null : subMatchesView(request.matches, layout, index)
|
|
10326
|
+
})
|
|
10327
|
+
);
|
|
10328
|
+
const merged = pages.flat();
|
|
10329
|
+
merged.sort((a, b) => compareEntries(a, b, request.directions));
|
|
10330
|
+
if (merged.length > request.limit) merged.length = request.limit;
|
|
10331
|
+
return merged;
|
|
10271
10332
|
}
|
|
10272
|
-
function
|
|
10273
|
-
|
|
10274
|
-
|
|
10275
|
-
|
|
10276
|
-
|
|
10277
|
-
} catch {
|
|
10278
|
-
throw invalidCursor(cursor, "invalid base64 encoding");
|
|
10279
|
-
}
|
|
10280
|
-
let parsed;
|
|
10281
|
-
try {
|
|
10282
|
-
parsed = JSON.parse(json);
|
|
10283
|
-
} catch {
|
|
10284
|
-
throw invalidCursor(cursor, "invalid JSON");
|
|
10285
|
-
}
|
|
10286
|
-
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
10287
|
-
throw invalidCursor(cursor, "expected an object");
|
|
10288
|
-
}
|
|
10289
|
-
const { v, a, s, k, o, d, q } = parsed;
|
|
10290
|
-
if (v !== CURSOR_VERSION) throw invalidCursor(cursor, `unsupported cursor version ${String(v)}`);
|
|
10291
|
-
if (typeof q !== "string" || !/^[0-9a-f]{8}$/.test(q)) {
|
|
10292
|
-
throw invalidCursor(cursor, '"q" must be 8 lowercase hex digits');
|
|
10293
|
-
}
|
|
10294
|
-
if (typeof a !== "string" || a.length === 0) throw invalidCursor(cursor, '"a" must be a non-empty string');
|
|
10295
|
-
if (exceedsCodePoints(a, MAX_CURSOR_ANCHOR_CODE_POINTS)) {
|
|
10296
|
-
throw invalidCursor(cursor, '"a" is too long to be a document id');
|
|
10297
|
-
}
|
|
10298
|
-
if (s !== void 0 && (typeof s !== "number" || !Number.isFinite(s))) {
|
|
10299
|
-
throw invalidCursor(cursor, '"s" must be a finite number');
|
|
10300
|
-
}
|
|
10301
|
-
if (s !== void 0 && k !== void 0) throw invalidCursor(cursor, 'a cursor carries "s" or "k", never both');
|
|
10302
|
-
if (d !== void 0 && (typeof d !== "number" || !Number.isSafeInteger(d) || d < 1)) {
|
|
10303
|
-
throw invalidCursor(cursor, '"d" must be a positive integer');
|
|
10304
|
-
}
|
|
10305
|
-
const sortKey = k === void 0 ? null : decodeSortKey(k, cursor);
|
|
10306
|
-
if (o !== void 0 && typeof o !== "string") throw invalidCursor(cursor, '"o" must be a string');
|
|
10307
|
-
if (sortKey === null !== (o === void 0)) {
|
|
10308
|
-
throw invalidCursor(cursor, '"k" and "o" have to arrive together');
|
|
10333
|
+
function compositeSortValues(subs, docId, fields, fieldTypes, modes) {
|
|
10334
|
+
for (const sub of subs) {
|
|
10335
|
+
if (sub.docStore.has(docId)) {
|
|
10336
|
+
return sortValuesOf(sub, docId, fields, fieldTypes, modes);
|
|
10337
|
+
}
|
|
10309
10338
|
}
|
|
10310
|
-
return
|
|
10311
|
-
anchor: a,
|
|
10312
|
-
score: typeof s === "number" ? s : null,
|
|
10313
|
-
sortKey,
|
|
10314
|
-
sortSignature: typeof o === "string" ? o : null,
|
|
10315
|
-
depth: typeof d === "number" ? d : 0,
|
|
10316
|
-
binding: q
|
|
10317
|
-
};
|
|
10339
|
+
return fields.map(() => null);
|
|
10318
10340
|
}
|
|
10319
|
-
|
|
10320
|
-
|
|
10321
|
-
|
|
10322
|
-
|
|
10323
|
-
|
|
10324
|
-
|
|
10325
|
-
|
|
10341
|
+
|
|
10342
|
+
// src/core/partition/composite/suggest.ts
|
|
10343
|
+
function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
10344
|
+
if (limit <= 0) return [];
|
|
10345
|
+
const merged = /* @__PURE__ */ new Map();
|
|
10346
|
+
for (const sub of subs) {
|
|
10347
|
+
for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix, Number.MAX_SAFE_INTEGER)) {
|
|
10348
|
+
let entry = merged.get(suggestion.token);
|
|
10349
|
+
if (entry === void 0) {
|
|
10350
|
+
entry = { documentFrequency: 0, surfaceOccurrences: /* @__PURE__ */ new Map() };
|
|
10351
|
+
merged.set(suggestion.token, entry);
|
|
10352
|
+
}
|
|
10353
|
+
entry.documentFrequency += suggestion.documentFrequency;
|
|
10354
|
+
for (const surface of suggestion.surfaces) {
|
|
10355
|
+
entry.surfaceOccurrences.set(
|
|
10356
|
+
surface.surface,
|
|
10357
|
+
(entry.surfaceOccurrences.get(surface.surface) ?? 0) + surface.occurrences
|
|
10358
|
+
);
|
|
10359
|
+
}
|
|
10326
10360
|
}
|
|
10327
|
-
return;
|
|
10328
|
-
}
|
|
10329
|
-
if (cursor.sortSignature !== null) {
|
|
10330
|
-
throw invalidCursor(encoded, "the cursor belongs to a sorted request");
|
|
10331
10361
|
}
|
|
10332
|
-
|
|
10333
|
-
|
|
10334
|
-
|
|
10335
|
-
|
|
10336
|
-
|
|
10362
|
+
const suggestions = [];
|
|
10363
|
+
for (const [token, entry] of merged) {
|
|
10364
|
+
const surfaces = Array.from(entry.surfaceOccurrences, ([surface, occurrences]) => ({ surface, occurrences }));
|
|
10365
|
+
surfaces.sort((a, b) => b.occurrences - a.occurrences || compareCodePoints(a.surface, b.surface));
|
|
10366
|
+
suggestions.push({ token, documentFrequency: entry.documentFrequency, surfaces });
|
|
10337
10367
|
}
|
|
10368
|
+
suggestions.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.token, b.token));
|
|
10369
|
+
if (suggestions.length > limit) suggestions.length = limit;
|
|
10370
|
+
return suggestions;
|
|
10338
10371
|
}
|
|
10339
10372
|
|
|
10340
|
-
// src/
|
|
10341
|
-
function
|
|
10342
|
-
|
|
10343
|
-
return Math.max(0, Math.floor(value));
|
|
10344
|
-
}
|
|
10345
|
-
function requireWithinResultWindow(limit, offset) {
|
|
10346
|
-
const depth = offset + limit;
|
|
10347
|
-
if (depth <= RESULT_WINDOW) return;
|
|
10348
|
-
throw new NarsilError(
|
|
10349
|
-
ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
|
|
10350
|
-
`A request reaches the first ${RESULT_WINDOW} results, and offset + limit is ${depth}. Page past that with the cursor each result carries`,
|
|
10351
|
-
{ limit, offset, window: RESULT_WINDOW }
|
|
10352
|
-
);
|
|
10373
|
+
// src/core/partition/composite/index.ts
|
|
10374
|
+
function isCompositePartition(partition) {
|
|
10375
|
+
return "attachFrozenSegment" in partition;
|
|
10353
10376
|
}
|
|
10354
|
-
function
|
|
10355
|
-
|
|
10356
|
-
|
|
10377
|
+
function createCompositePartition(partitionId, trackPositions = true, existingLive, textAnalysis) {
|
|
10378
|
+
const live = existingLive ?? createPartitionIndex(partitionId, trackPositions);
|
|
10379
|
+
const liveState = partitionStateOf(live);
|
|
10380
|
+
const frozen = [];
|
|
10381
|
+
let docFrequenciesCache = null;
|
|
10382
|
+
function invalidateDocFrequencies() {
|
|
10383
|
+
docFrequenciesCache = null;
|
|
10357
10384
|
}
|
|
10358
|
-
|
|
10359
|
-
|
|
10360
|
-
|
|
10361
|
-
|
|
10362
|
-
const comparison = compareSortValues(sort.sortKeyOf(result.id), anchor.sortKey, sort.directions) || compareCodePoints(result.id, anchor.anchor);
|
|
10363
|
-
return comparison > 0;
|
|
10385
|
+
function bindSegments() {
|
|
10386
|
+
invalidateDocFrequencies();
|
|
10387
|
+
if (textAnalysis === void 0) return;
|
|
10388
|
+
for (const segment of frozen) segment.removals.bind(textAnalysis);
|
|
10364
10389
|
}
|
|
10365
|
-
|
|
10366
|
-
|
|
10367
|
-
|
|
10368
|
-
|
|
10369
|
-
|
|
10370
|
-
function applyPagination(results, limit, offset, binding, cursor, sort, pinnedIds) {
|
|
10371
|
-
const decoded = cursor ? decodePageCursor(cursor) : null;
|
|
10372
|
-
if (decoded !== null && cursor !== void 0) {
|
|
10373
|
-
requireMatchingCursor(decoded, cursor, sort?.signature ?? null, true, binding);
|
|
10390
|
+
function cachedDocFrequencies() {
|
|
10391
|
+
if (docFrequenciesCache === null) {
|
|
10392
|
+
docFrequenciesCache = mergeDocFrequencies(subs());
|
|
10393
|
+
}
|
|
10394
|
+
return docFrequenciesCache;
|
|
10374
10395
|
}
|
|
10375
|
-
|
|
10376
|
-
return
|
|
10396
|
+
function subs() {
|
|
10397
|
+
return [...frozen, liveState];
|
|
10377
10398
|
}
|
|
10378
|
-
|
|
10379
|
-
|
|
10380
|
-
|
|
10381
|
-
|
|
10382
|
-
if (
|
|
10383
|
-
|
|
10384
|
-
break;
|
|
10385
|
-
}
|
|
10399
|
+
function composedScratch() {
|
|
10400
|
+
const scratch = createPartitionIndex(partitionId, trackPositions);
|
|
10401
|
+
for (const sub of subs()) {
|
|
10402
|
+
const payload = encodeSegmentState(sub);
|
|
10403
|
+
if (payload.documentCount === 0) continue;
|
|
10404
|
+
scratch.mergeSegmentPayload(payload, survivorDocumentsOf(sub, payload));
|
|
10386
10405
|
}
|
|
10406
|
+
return scratch;
|
|
10387
10407
|
}
|
|
10388
|
-
|
|
10389
|
-
|
|
10390
|
-
let nextCursor;
|
|
10391
|
-
const hasMore = afterOffset + limit < results.length;
|
|
10392
|
-
const lastResult = lastOrganicResult(sliced, pinnedIds);
|
|
10393
|
-
const depth = afterOffset + sliced.length;
|
|
10394
|
-
if (hasMore && lastResult !== void 0) {
|
|
10395
|
-
nextCursor = sort !== void 0 ? encodePageCursor({
|
|
10396
|
-
anchor: lastResult.id,
|
|
10397
|
-
score: null,
|
|
10398
|
-
sortKey: sort.sortKeyOf(lastResult.id).map(toComparableSortValue),
|
|
10399
|
-
sortSignature: sort.signature,
|
|
10400
|
-
depth,
|
|
10401
|
-
binding
|
|
10402
|
-
}) : encodePageCursor({
|
|
10403
|
-
anchor: lastResult.id,
|
|
10404
|
-
score: lastResult.score ?? null,
|
|
10405
|
-
sortKey: null,
|
|
10406
|
-
sortSignature: null,
|
|
10407
|
-
depth,
|
|
10408
|
-
binding
|
|
10409
|
-
});
|
|
10408
|
+
function layout() {
|
|
10409
|
+
return computeOrdinalLayout(subs());
|
|
10410
10410
|
}
|
|
10411
|
-
|
|
10412
|
-
|
|
10413
|
-
|
|
10414
|
-
|
|
10415
|
-
|
|
10416
|
-
const unstemmed = tokenize(term, language, {
|
|
10417
|
-
stem: false,
|
|
10418
|
-
removeStopWords: true,
|
|
10419
|
-
customTokenizer: analysis?.customTokenizer,
|
|
10420
|
-
stopWordOverride: analysis?.stopWords
|
|
10421
|
-
});
|
|
10422
|
-
const written = /* @__PURE__ */ new Map();
|
|
10423
|
-
for (const token of unstemmed.tokens) {
|
|
10424
|
-
if (!written.has(token.position)) written.set(token.position, token.token);
|
|
10411
|
+
function frozenOwner(docId) {
|
|
10412
|
+
for (const segment of frozen) {
|
|
10413
|
+
if (segment.hasDocument(docId)) return segment;
|
|
10414
|
+
}
|
|
10415
|
+
return void 0;
|
|
10425
10416
|
}
|
|
10426
|
-
|
|
10427
|
-
|
|
10428
|
-
|
|
10429
|
-
|
|
10430
|
-
|
|
10431
|
-
|
|
10432
|
-
groups.set(token, group);
|
|
10417
|
+
function frozenStored(docId) {
|
|
10418
|
+
for (const segment of frozen) {
|
|
10419
|
+
const stored = segment.docStore.get(docId);
|
|
10420
|
+
if (stored !== void 0) return stored;
|
|
10421
|
+
}
|
|
10422
|
+
return void 0;
|
|
10433
10423
|
}
|
|
10434
|
-
return group;
|
|
10435
|
-
}
|
|
10436
|
-
function addCompletions(groups, partition, lastToken, typedPrefix) {
|
|
10437
|
-
if (typedPrefix === void 0 || typedPrefix.length === 0) return;
|
|
10438
|
-
const group = groupFor(groups, lastToken.token, false);
|
|
10439
|
-
const ownLength = codePointCount(lastToken.token);
|
|
10440
|
-
for (const completion of partition.expandTermPrefix(typedPrefix, lastToken.token, PREFIX_MAX_EXPANSIONS)) {
|
|
10441
|
-
if (completion === lastToken.token) continue;
|
|
10442
|
-
const factor = Math.min(1, ownLength / codePointCount(completion));
|
|
10443
|
-
group.members.push({ token: completion, factor, partialFactor: factor });
|
|
10444
|
-
}
|
|
10445
|
-
}
|
|
10446
|
-
function partialWordAnalysisOf(fields, language, analysis) {
|
|
10447
10424
|
return {
|
|
10448
|
-
|
|
10449
|
-
|
|
10450
|
-
|
|
10451
|
-
|
|
10452
|
-
|
|
10453
|
-
|
|
10425
|
+
get partitionId() {
|
|
10426
|
+
return partitionId;
|
|
10427
|
+
},
|
|
10428
|
+
get live() {
|
|
10429
|
+
return live;
|
|
10430
|
+
},
|
|
10431
|
+
get stats() {
|
|
10432
|
+
return buildAggregateStatsView([...frozen, liveState], cachedDocFrequencies);
|
|
10433
|
+
},
|
|
10434
|
+
frozenSegmentCount() {
|
|
10435
|
+
return frozen.length;
|
|
10436
|
+
},
|
|
10437
|
+
frozenSegmentSizes() {
|
|
10438
|
+
return frozen.map((segment) => ({ segmentId: segment.segmentId, liveDocumentCount: segment.liveDocumentCount() }));
|
|
10439
|
+
},
|
|
10440
|
+
frozenSegmentsById(segmentIds) {
|
|
10441
|
+
return resolveFrozenSegments(frozen, segmentIds, partitionId);
|
|
10442
|
+
},
|
|
10443
|
+
swapFrozenSegments(dropSegmentIds, replacement, removedSpellings) {
|
|
10444
|
+
if (textAnalysis !== void 0) replacement.removals.bind(textAnalysis);
|
|
10445
|
+
const applied = swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings);
|
|
10446
|
+
bindSegments();
|
|
10447
|
+
return applied;
|
|
10448
|
+
},
|
|
10449
|
+
freezeLiveTail(freeze) {
|
|
10450
|
+
const segment = freezeLiveTailInto(live, liveState, frozen, freeze);
|
|
10451
|
+
if (segment !== null) bindSegments();
|
|
10452
|
+
return segment;
|
|
10453
|
+
},
|
|
10454
|
+
replaceLiveTail(segment) {
|
|
10455
|
+
live.clear();
|
|
10456
|
+
frozen.push(segment);
|
|
10457
|
+
bindSegments();
|
|
10458
|
+
},
|
|
10459
|
+
attachFrozenSegment(segment) {
|
|
10460
|
+
frozen.push(segment);
|
|
10461
|
+
bindSegments();
|
|
10462
|
+
},
|
|
10463
|
+
mergeSegment(segment) {
|
|
10464
|
+
invalidateDocFrequencies();
|
|
10465
|
+
live.mergeSegment(segment);
|
|
10466
|
+
},
|
|
10467
|
+
mergeSegmentPayload(payload, documents) {
|
|
10468
|
+
invalidateDocFrequencies();
|
|
10469
|
+
live.mergeSegmentPayload(payload, documents);
|
|
10470
|
+
},
|
|
10471
|
+
encodeSegment() {
|
|
10472
|
+
if (frozen.length === 0) return live.encodeSegment();
|
|
10473
|
+
return composedScratch().encodeSegment();
|
|
10474
|
+
},
|
|
10475
|
+
serialize(indexName, totalPartitions, language, schema) {
|
|
10476
|
+
if (frozen.length === 0) return live.serialize(indexName, totalPartitions, language, schema);
|
|
10477
|
+
return composedScratch().serialize(indexName, totalPartitions, language, schema);
|
|
10478
|
+
},
|
|
10479
|
+
deserialize(data, schema) {
|
|
10480
|
+
frozen.length = 0;
|
|
10481
|
+
invalidateDocFrequencies();
|
|
10482
|
+
live.deserialize(data, schema);
|
|
10483
|
+
},
|
|
10484
|
+
clear() {
|
|
10485
|
+
frozen.length = 0;
|
|
10486
|
+
invalidateDocFrequencies();
|
|
10487
|
+
live.clear();
|
|
10488
|
+
},
|
|
10489
|
+
insert(docId, document, schema, language, options) {
|
|
10490
|
+
if (frozenOwner(docId) !== void 0) {
|
|
10491
|
+
throw new NarsilError(
|
|
10492
|
+
ErrorCodes.DOC_ALREADY_EXISTS,
|
|
10493
|
+
`Document "${docId}" already exists in partition ${partitionId}`,
|
|
10494
|
+
{ docId, partitionId }
|
|
10495
|
+
);
|
|
10496
|
+
}
|
|
10497
|
+
invalidateDocFrequencies();
|
|
10498
|
+
live.insert(docId, document, schema, language, options);
|
|
10499
|
+
},
|
|
10500
|
+
update(docId, document, schema, language, options, removedSpellings) {
|
|
10501
|
+
invalidateDocFrequencies();
|
|
10502
|
+
const owner = frozenOwner(docId);
|
|
10503
|
+
if (owner === void 0) {
|
|
10504
|
+
live.update(docId, document, schema, language, options);
|
|
10505
|
+
return;
|
|
10506
|
+
}
|
|
10507
|
+
live.insert(docId, document, schema, language, options);
|
|
10508
|
+
owner.tombstoneDocument(docId, removedSpellings);
|
|
10509
|
+
},
|
|
10510
|
+
remove(docId, schema, language, options, removedSpellings) {
|
|
10511
|
+
if (live.has(docId)) {
|
|
10512
|
+
invalidateDocFrequencies();
|
|
10513
|
+
live.remove(docId, schema, language, options);
|
|
10514
|
+
return;
|
|
10515
|
+
}
|
|
10516
|
+
const owner = frozenOwner(docId);
|
|
10517
|
+
if (owner === void 0) {
|
|
10518
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in partition ${partitionId}`, {
|
|
10519
|
+
docId,
|
|
10520
|
+
partitionId
|
|
10521
|
+
});
|
|
10522
|
+
}
|
|
10523
|
+
invalidateDocFrequencies();
|
|
10524
|
+
owner.tombstoneDocument(docId, removedSpellings);
|
|
10525
|
+
},
|
|
10526
|
+
removedSpellingsOf(docId) {
|
|
10527
|
+
return live.has(docId) ? void 0 : frozenOwner(docId)?.removedSpellingsOf(docId);
|
|
10528
|
+
},
|
|
10529
|
+
beginBatch() {
|
|
10530
|
+
live.beginBatch();
|
|
10531
|
+
},
|
|
10532
|
+
endBatch() {
|
|
10533
|
+
live.endBatch();
|
|
10534
|
+
},
|
|
10535
|
+
searchFulltext(params) {
|
|
10536
|
+
return compositeSearchFulltext(subs(), layout(), params);
|
|
10537
|
+
},
|
|
10538
|
+
searchFulltextMatches(params) {
|
|
10539
|
+
return compositeSearchMatches(subs(), layout(), params);
|
|
10540
|
+
},
|
|
10541
|
+
applyFilters(filters, schema, meter) {
|
|
10542
|
+
return compositeFilters(subs(), filters, schema, meter);
|
|
10543
|
+
},
|
|
10544
|
+
applyFiltersBitset(filters, schema, meter) {
|
|
10545
|
+
return compositeFiltersBitset(subs(), layout(), filters, schema, meter);
|
|
10546
|
+
},
|
|
10547
|
+
filterMatches(filters, schema, meter) {
|
|
10548
|
+
return compositeFilterMatches(subs(), layout(), filters, schema, meter);
|
|
10549
|
+
},
|
|
10550
|
+
computeFacets(matched, config, schema) {
|
|
10551
|
+
const everyValue = everyValueFacetConfig(config);
|
|
10552
|
+
if ("ordinalBitset" in matched) {
|
|
10553
|
+
const ordinalLayout = layout();
|
|
10554
|
+
return mergeFacets(
|
|
10555
|
+
subs().map(
|
|
10556
|
+
(sub, index) => computeFacets(
|
|
10557
|
+
sub,
|
|
10558
|
+
{ ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
|
|
10559
|
+
everyValue,
|
|
10560
|
+
schema
|
|
10561
|
+
)
|
|
10562
|
+
),
|
|
10563
|
+
config
|
|
10564
|
+
);
|
|
10565
|
+
}
|
|
10566
|
+
return mergeFacets(
|
|
10567
|
+
subs().map((sub) => computeFacets(sub, matched, everyValue, schema)),
|
|
10568
|
+
config
|
|
10569
|
+
);
|
|
10570
|
+
},
|
|
10571
|
+
sortedPage(request) {
|
|
10572
|
+
return compositeSortedPage(subs(), layout(), request);
|
|
10573
|
+
},
|
|
10574
|
+
sortValues(docId, fields, fieldTypes, modes) {
|
|
10575
|
+
return compositeSortValues(subs(), docId, fields, fieldTypes, modes);
|
|
10576
|
+
},
|
|
10577
|
+
suggestTerms(surfacePrefix, stemmedPrefix, limit) {
|
|
10578
|
+
return compositeSuggestTerms(subs(), surfacePrefix, stemmedPrefix, limit);
|
|
10579
|
+
},
|
|
10580
|
+
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
10581
|
+
return expandTermPrefix(subs(), surfacePrefix, stemmedToken, maxExpansions);
|
|
10582
|
+
},
|
|
10583
|
+
expandPartialWords(analysis, requests) {
|
|
10584
|
+
return expandPartialWords(subs(), analysis, requests);
|
|
10585
|
+
},
|
|
10586
|
+
preparePartialWords(analysis) {
|
|
10587
|
+
expandPartialWords(subs(), analysis, []);
|
|
10588
|
+
},
|
|
10589
|
+
get(docId, projection) {
|
|
10590
|
+
return compositeDocument(frozen, live, docId, projection);
|
|
10591
|
+
},
|
|
10592
|
+
fieldValues(docId, fieldPaths) {
|
|
10593
|
+
return compositeFieldValues(frozen, live, docId, fieldPaths);
|
|
10594
|
+
},
|
|
10595
|
+
getRef(docId) {
|
|
10596
|
+
const stored = frozenStored(docId);
|
|
10597
|
+
if (stored !== void 0) return stored.fields;
|
|
10598
|
+
return live.getRef(docId);
|
|
10599
|
+
},
|
|
10600
|
+
has(docId) {
|
|
10601
|
+
return live.has(docId) || frozenOwner(docId) !== void 0;
|
|
10602
|
+
},
|
|
10603
|
+
count() {
|
|
10604
|
+
let total = live.count();
|
|
10605
|
+
for (const segment of frozen) {
|
|
10606
|
+
total += segment.liveDocumentCount();
|
|
10607
|
+
}
|
|
10608
|
+
return total;
|
|
10609
|
+
},
|
|
10610
|
+
*docIds() {
|
|
10611
|
+
for (const segment of frozen) {
|
|
10612
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
10613
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
10614
|
+
if (docId !== void 0) yield docId;
|
|
10615
|
+
}
|
|
10616
|
+
}
|
|
10617
|
+
yield* live.docIds();
|
|
10618
|
+
},
|
|
10619
|
+
sortedDocIds() {
|
|
10620
|
+
const merged = [];
|
|
10621
|
+
for (const segment of frozen) {
|
|
10622
|
+
merged.push(...segment.docStore.sortedDocIds());
|
|
10623
|
+
}
|
|
10624
|
+
merged.push(...live.sortedDocIds());
|
|
10625
|
+
merged.sort(compareCodePoints);
|
|
10626
|
+
return merged;
|
|
10627
|
+
},
|
|
10628
|
+
releaseSortedDocIds() {
|
|
10629
|
+
for (const segment of frozen) {
|
|
10630
|
+
segment.docStore.releaseSortedDocIds();
|
|
10631
|
+
}
|
|
10632
|
+
live.releaseSortedDocIds();
|
|
10633
|
+
},
|
|
10634
|
+
estimateMemoryBytes() {
|
|
10635
|
+
let total = live.estimateMemoryBytes();
|
|
10636
|
+
for (const segment of frozen) {
|
|
10637
|
+
total += frozenSegmentBytes(segment);
|
|
10638
|
+
}
|
|
10639
|
+
return total;
|
|
10454
10640
|
}
|
|
10455
10641
|
};
|
|
10456
10642
|
}
|
|
10457
|
-
|
|
10458
|
-
|
|
10459
|
-
|
|
10460
|
-
|
|
10643
|
+
|
|
10644
|
+
// src/partitioning/insert-options.ts
|
|
10645
|
+
function resolvePartialWordAnalysis(config, language, analysis) {
|
|
10646
|
+
const fields = partialFieldsOf(config.schema);
|
|
10647
|
+
if (fields.length === 0) return null;
|
|
10648
|
+
return { fields, language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
10649
|
+
}
|
|
10650
|
+
function resolvePartitionInsertOptions(config, analysis, options) {
|
|
10651
|
+
const applyStrict = config.strict === true;
|
|
10652
|
+
const applyAnalyzer = analysis.stopWords !== void 0 || analysis.customTokenizer !== void 0;
|
|
10653
|
+
const applySurfaces = config.surfaceForms !== false;
|
|
10654
|
+
const applyPatternLimit = config.patternValueLimit !== void 0;
|
|
10655
|
+
if (!applyStrict && !applyAnalyzer && !applySurfaces && !applyPatternLimit) return options;
|
|
10656
|
+
const resolved = { ...options };
|
|
10657
|
+
if (applyStrict) resolved.strict = true;
|
|
10658
|
+
if (applyPatternLimit) resolved.patternValueLimit = config.patternValueLimit;
|
|
10659
|
+
if (applyAnalyzer) {
|
|
10660
|
+
resolved.stopWordOverride = options?.stopWordOverride ?? analysis.stopWords;
|
|
10661
|
+
resolved.customTokenizer = options?.customTokenizer ?? analysis.customTokenizer;
|
|
10461
10662
|
}
|
|
10462
|
-
|
|
10663
|
+
if (applySurfaces) resolved.collectSurfaces = true;
|
|
10664
|
+
return resolved;
|
|
10463
10665
|
}
|
|
10464
|
-
|
|
10465
|
-
|
|
10466
|
-
|
|
10467
|
-
const
|
|
10468
|
-
|
|
10469
|
-
|
|
10470
|
-
|
|
10471
|
-
|
|
10472
|
-
|
|
10473
|
-
|
|
10474
|
-
|
|
10475
|
-
|
|
10476
|
-
|
|
10477
|
-
|
|
10478
|
-
|
|
10479
|
-
|
|
10480
|
-
|
|
10481
|
-
|
|
10482
|
-
|
|
10483
|
-
|
|
10484
|
-
|
|
10485
|
-
|
|
10486
|
-
|
|
10487
|
-
|
|
10488
|
-
|
|
10489
|
-
|
|
10490
|
-
|
|
10491
|
-
|
|
10666
|
+
|
|
10667
|
+
// src/partitioning/manager/capacity.ts
|
|
10668
|
+
function createCapacityChecks(source) {
|
|
10669
|
+
const { indexName, config } = source;
|
|
10670
|
+
return {
|
|
10671
|
+
assertCapacity(pendingWrites = 0, partitionCountCap) {
|
|
10672
|
+
const maxDocsPerPartition = config.partitions?.maxDocsPerPartition;
|
|
10673
|
+
if (maxDocsPerPartition === void 0) return;
|
|
10674
|
+
const partitionCount = source.partitionCount();
|
|
10675
|
+
const effectivePartitionCount = partitionCountCap === void 0 ? partitionCount : Math.min(partitionCount, partitionCountCap);
|
|
10676
|
+
const totalCapacity = maxDocsPerPartition * effectivePartitionCount;
|
|
10677
|
+
const currentCount = source.documentCount() + pendingWrites;
|
|
10678
|
+
if (currentCount < totalCapacity) return;
|
|
10679
|
+
throw new NarsilError(
|
|
10680
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
10681
|
+
`Index "${indexName}" has reached its capacity of ${totalCapacity} documents (${maxDocsPerPartition} per partition \xD7 ${effectivePartitionCount} partitions)`,
|
|
10682
|
+
{
|
|
10683
|
+
indexName,
|
|
10684
|
+
currentCount,
|
|
10685
|
+
totalCapacity,
|
|
10686
|
+
maxDocsPerPartition,
|
|
10687
|
+
partitionCount: effectivePartitionCount
|
|
10688
|
+
}
|
|
10689
|
+
);
|
|
10690
|
+
},
|
|
10691
|
+
assertPartitionCapacity(partitionId, pendingWrites = 0) {
|
|
10692
|
+
const maxDocsPerPartition = config.partitions?.maxDocsPerPartition;
|
|
10693
|
+
if (maxDocsPerPartition === void 0) return;
|
|
10694
|
+
const currentCount = source.getPartition(partitionId).count() + pendingWrites;
|
|
10695
|
+
if (currentCount < maxDocsPerPartition) return;
|
|
10696
|
+
throw new NarsilError(
|
|
10697
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
10698
|
+
`Partition ${partitionId} of index "${indexName}" has reached its capacity of ${maxDocsPerPartition} documents`,
|
|
10699
|
+
{ indexName, partitionId, currentCount, maxDocsPerPartition }
|
|
10700
|
+
);
|
|
10701
|
+
}
|
|
10702
|
+
};
|
|
10492
10703
|
}
|
|
10493
10704
|
|
|
10494
|
-
// src/
|
|
10495
|
-
function
|
|
10496
|
-
if (!
|
|
10497
|
-
|
|
10498
|
-
|
|
10499
|
-
if (params.fields) {
|
|
10500
|
-
validateSearchFields(params.fields, flatSchema);
|
|
10705
|
+
// src/partitioning/manager/nested-values.ts
|
|
10706
|
+
function setNestedValue(obj, path, value) {
|
|
10707
|
+
if (!path.includes(".")) {
|
|
10708
|
+
obj[path] = value;
|
|
10709
|
+
return;
|
|
10501
10710
|
}
|
|
10502
|
-
const
|
|
10503
|
-
|
|
10504
|
-
|
|
10505
|
-
|
|
10506
|
-
|
|
10507
|
-
|
|
10508
|
-
|
|
10509
|
-
|
|
10510
|
-
|
|
10511
|
-
if (params.filters) {
|
|
10512
|
-
filterBitset = partition.applyFiltersBitset(
|
|
10513
|
-
params.filters,
|
|
10514
|
-
schema,
|
|
10515
|
-
options?.patternWork ?? createPatternWorkMeter()
|
|
10516
|
-
);
|
|
10517
|
-
if (bitsetIsEmpty(filterBitset)) return null;
|
|
10711
|
+
const segments = path.split(".");
|
|
10712
|
+
let current = obj;
|
|
10713
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
10714
|
+
let next = current[segments[i]];
|
|
10715
|
+
if (next === null || next === void 0 || typeof next !== "object") {
|
|
10716
|
+
next = {};
|
|
10717
|
+
current[segments[i]] = next;
|
|
10718
|
+
}
|
|
10719
|
+
current = next;
|
|
10518
10720
|
}
|
|
10519
|
-
|
|
10520
|
-
const groups = resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, options);
|
|
10521
|
-
return { queryTokens, ...groups, filterBitset };
|
|
10522
|
-
}
|
|
10523
|
-
function fulltextMatches(partition, params, language, schema, options) {
|
|
10524
|
-
const prepared = prepareFulltextQuery(partition, params, language, schema, options);
|
|
10525
|
-
if (prepared === null) return null;
|
|
10526
|
-
return partition.searchFulltextMatches({
|
|
10527
|
-
queryTokens: prepared.queryTokens,
|
|
10528
|
-
termGroups: prepared.termGroups,
|
|
10529
|
-
partialFields: prepared.partialFields,
|
|
10530
|
-
fields: params.fields,
|
|
10531
|
-
tolerance: params.tolerance ?? 0,
|
|
10532
|
-
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
10533
|
-
exact: params.exact ?? false,
|
|
10534
|
-
filterBitset: prepared.filterBitset
|
|
10535
|
-
});
|
|
10721
|
+
current[segments[segments.length - 1]] = value;
|
|
10536
10722
|
}
|
|
10537
|
-
function
|
|
10538
|
-
|
|
10539
|
-
|
|
10540
|
-
|
|
10541
|
-
|
|
10542
|
-
const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
|
|
10543
|
-
const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
10544
|
-
const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
|
|
10545
|
-
const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
|
|
10546
|
-
const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
|
|
10547
|
-
const rawResult = partition.searchFulltext({
|
|
10548
|
-
queryTokens,
|
|
10549
|
-
termGroups,
|
|
10550
|
-
partialFields,
|
|
10551
|
-
fields: params.fields,
|
|
10552
|
-
boost: params.boost,
|
|
10553
|
-
tolerance: params.tolerance ?? 0,
|
|
10554
|
-
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
10555
|
-
exact: params.exact ?? false,
|
|
10556
|
-
bm25Params: options?.bm25Params,
|
|
10557
|
-
globalStats: options?.globalStats,
|
|
10558
|
-
maxResults,
|
|
10559
|
-
termMatch: params.termMatch,
|
|
10560
|
-
filterBitset,
|
|
10561
|
-
collectComponents,
|
|
10562
|
-
collectMatchedSet
|
|
10563
|
-
});
|
|
10564
|
-
let scored = rawResult.scored;
|
|
10565
|
-
const termMatch = params.termMatch ?? "any";
|
|
10566
|
-
if (termMatch !== "any") {
|
|
10567
|
-
scored = filterByTermCoverage(
|
|
10568
|
-
scored,
|
|
10569
|
-
queryTokens,
|
|
10570
|
-
termMatch,
|
|
10571
|
-
params.tolerance ?? 0,
|
|
10572
|
-
params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
10573
|
-
params.exact ?? false,
|
|
10574
|
-
termGroups,
|
|
10575
|
-
partialFields ?? []
|
|
10576
|
-
);
|
|
10577
|
-
}
|
|
10578
|
-
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
10579
|
-
const threshold = params.minScore;
|
|
10580
|
-
scored = scored.filter((doc) => doc.score >= threshold);
|
|
10581
|
-
}
|
|
10582
|
-
const totalMatched = needsAllResults ? scored.length : rawResult.totalMatched;
|
|
10583
|
-
if (params.facets === void 0) {
|
|
10584
|
-
return { scored, totalMatched };
|
|
10585
|
-
}
|
|
10586
|
-
if (needsAllResults) {
|
|
10587
|
-
return { scored, totalMatched, matchedIds: scored.map((doc) => doc.docId) };
|
|
10723
|
+
function copyVectorsInto(document, docId, vectorIndexes, projection) {
|
|
10724
|
+
for (const [fieldPath, vectorIndex] of vectorIndexes) {
|
|
10725
|
+
if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
|
|
10726
|
+
const vector = vectorIndex.getVector(docId);
|
|
10727
|
+
if (vector) setNestedValue(document, fieldPath, vector);
|
|
10588
10728
|
}
|
|
10589
|
-
return { scored, totalMatched, matchedOrdinalBitset: rawResult.matchedOrdinalBitset };
|
|
10590
10729
|
}
|
|
10591
|
-
|
|
10592
|
-
|
|
10593
|
-
|
|
10594
|
-
|
|
10595
|
-
|
|
10596
|
-
|
|
10597
|
-
|
|
10598
|
-
|
|
10599
|
-
|
|
10600
|
-
|
|
10601
|
-
{ field, fieldType }
|
|
10602
|
-
);
|
|
10603
|
-
}
|
|
10730
|
+
|
|
10731
|
+
// src/partitioning/manager/segments.ts
|
|
10732
|
+
function requireNewSegmentDocuments(segment, docPartitionMap) {
|
|
10733
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
10734
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
10735
|
+
if (docId === void 0 || !docPartitionMap.has(docId)) continue;
|
|
10736
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists`, {
|
|
10737
|
+
docId,
|
|
10738
|
+
partitionId: docPartitionMap.get(docId)
|
|
10739
|
+
});
|
|
10604
10740
|
}
|
|
10605
10741
|
}
|
|
10606
|
-
|
|
10607
|
-
|
|
10608
|
-
|
|
10609
|
-
|
|
10610
|
-
|
|
10611
|
-
|
|
10612
|
-
|
|
10742
|
+
|
|
10743
|
+
// src/partitioning/manager/stats.ts
|
|
10744
|
+
function aggregateStatsOf(partitions) {
|
|
10745
|
+
let totalDocuments = 0;
|
|
10746
|
+
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
10747
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
10748
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
10749
|
+
const stats = partitions[i].stats;
|
|
10750
|
+
totalDocuments += stats.totalDocuments;
|
|
10751
|
+
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
10752
|
+
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
10613
10753
|
}
|
|
10614
|
-
|
|
10615
|
-
|
|
10616
|
-
}
|
|
10617
|
-
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
|
|
10618
|
-
const requiredCount = policy === "all" ? queryTokens.length : policy;
|
|
10619
|
-
if (requiredCount <= 0) return scored;
|
|
10620
|
-
if (requiredCount > queryTokens.length) return [];
|
|
10621
|
-
const groups = /* @__PURE__ */ new Map();
|
|
10622
|
-
for (const group of termGroups ?? []) {
|
|
10623
|
-
const partialOnly = /* @__PURE__ */ new Set();
|
|
10624
|
-
for (const member of group.members) {
|
|
10625
|
-
if (member.factor === 0) partialOnly.add(member.token);
|
|
10754
|
+
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
10755
|
+
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
10626
10756
|
}
|
|
10627
|
-
groups.set(group.token, {
|
|
10628
|
-
members: new Set(group.members.map((member) => member.token)),
|
|
10629
|
-
partialOnly,
|
|
10630
|
-
typoMatches: group.typoMatches
|
|
10631
|
-
});
|
|
10632
10757
|
}
|
|
10633
|
-
return
|
|
10634
|
-
const matched = countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields);
|
|
10635
|
-
return matched >= requiredCount;
|
|
10636
|
-
});
|
|
10758
|
+
return { totalDocuments, docFrequencies, totalFieldLengths };
|
|
10637
10759
|
}
|
|
10638
|
-
function
|
|
10639
|
-
|
|
10640
|
-
|
|
10641
|
-
|
|
10642
|
-
for (const qt of queryTokens) {
|
|
10643
|
-
const group = groups.get(qt.token);
|
|
10644
|
-
const satisfied = group === void 0 ? queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact) : groupSatisfied(group, doc, indexTokens, partialFields) || group.typoMatches && queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact);
|
|
10645
|
-
if (satisfied) count++;
|
|
10760
|
+
function estimateBytesOf(partitions, vecIndexes) {
|
|
10761
|
+
let total = 0;
|
|
10762
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
10763
|
+
total += partitions[i].estimateMemoryBytes();
|
|
10646
10764
|
}
|
|
10647
|
-
|
|
10648
|
-
|
|
10649
|
-
function matchedInPartialField(doc, token, partialFields) {
|
|
10650
|
-
for (const field of partialFields) {
|
|
10651
|
-
if (doc.termFrequencies[`${field}:${token}`] !== void 0) return true;
|
|
10765
|
+
for (const [, vecIndex] of vecIndexes) {
|
|
10766
|
+
total += vecIndex.estimateMemoryBytes();
|
|
10652
10767
|
}
|
|
10653
|
-
return
|
|
10768
|
+
return total;
|
|
10654
10769
|
}
|
|
10655
|
-
function
|
|
10656
|
-
|
|
10657
|
-
|
|
10658
|
-
|
|
10659
|
-
|
|
10660
|
-
|
|
10661
|
-
}
|
|
10662
|
-
function queryTermSatisfied(queryToken, indexTokens, tolerance, prefixLength, exact) {
|
|
10663
|
-
if (exact || tolerance === 0) {
|
|
10664
|
-
return indexTokens.includes(queryToken);
|
|
10665
|
-
}
|
|
10666
|
-
for (const indexToken of indexTokens) {
|
|
10667
|
-
if (fuzzyTermMatches(queryToken, indexToken, tolerance, prefixLength)) {
|
|
10668
|
-
return true;
|
|
10669
|
-
}
|
|
10670
|
-
}
|
|
10671
|
-
return false;
|
|
10770
|
+
function partitionStatsOf(partitions) {
|
|
10771
|
+
return partitions.map((p, i) => ({
|
|
10772
|
+
partitionId: i,
|
|
10773
|
+
documentCount: p.count(),
|
|
10774
|
+
estimatedMemoryBytes: p.estimateMemoryBytes()
|
|
10775
|
+
}));
|
|
10672
10776
|
}
|
|
10673
10777
|
|
|
10674
|
-
// src/partitioning/
|
|
10675
|
-
function
|
|
10676
|
-
|
|
10677
|
-
|
|
10678
|
-
|
|
10679
|
-
|
|
10680
|
-
|
|
10681
|
-
|
|
10682
|
-
|
|
10683
|
-
|
|
10778
|
+
// src/partitioning/manager/write-log.ts
|
|
10779
|
+
function createWriteLogRecorder() {
|
|
10780
|
+
let current = null;
|
|
10781
|
+
return {
|
|
10782
|
+
open() {
|
|
10783
|
+
const log = { docIds: /* @__PURE__ */ new Set(), replaced: false };
|
|
10784
|
+
current = log;
|
|
10785
|
+
return {
|
|
10786
|
+
get partitionsReplaced() {
|
|
10787
|
+
return log.replaced;
|
|
10788
|
+
},
|
|
10789
|
+
get pendingDocuments() {
|
|
10790
|
+
return log.docIds.size;
|
|
10791
|
+
},
|
|
10792
|
+
takeWrittenDocIds() {
|
|
10793
|
+
const taken = log.docIds;
|
|
10794
|
+
log.docIds = /* @__PURE__ */ new Set();
|
|
10795
|
+
return taken;
|
|
10796
|
+
},
|
|
10797
|
+
close() {
|
|
10798
|
+
if (current === log) current = null;
|
|
10799
|
+
}
|
|
10800
|
+
};
|
|
10801
|
+
},
|
|
10802
|
+
documentWritten(docId) {
|
|
10803
|
+
if (current !== null) current.docIds.add(docId);
|
|
10804
|
+
},
|
|
10805
|
+
segmentWritten(segment) {
|
|
10806
|
+
if (current === null) return;
|
|
10807
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
10808
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
10809
|
+
if (docId !== void 0) current.docIds.add(docId);
|
|
10810
|
+
}
|
|
10811
|
+
},
|
|
10812
|
+
partitionsReplaced() {
|
|
10813
|
+
if (current !== null) current.replaced = true;
|
|
10684
10814
|
}
|
|
10685
|
-
}
|
|
10686
|
-
return partitions;
|
|
10815
|
+
};
|
|
10687
10816
|
}
|
|
10688
10817
|
|
|
10689
|
-
// src/partitioning/
|
|
10690
|
-
|
|
10691
|
-
|
|
10692
|
-
|
|
10693
|
-
|
|
10818
|
+
// src/partitioning/manager/index.ts
|
|
10819
|
+
function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
|
|
10820
|
+
requirePatternSearchFor(config.schema);
|
|
10821
|
+
requirePartialWordSearchFor(config.schema);
|
|
10822
|
+
const count = initialPartitionCount ?? 1;
|
|
10823
|
+
const analysis = resolveIndexAnalysis(config);
|
|
10824
|
+
const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
10825
|
+
const textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
10826
|
+
const trackPositions = config.trackPositions ?? true;
|
|
10827
|
+
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
10828
|
+
let partitions = [];
|
|
10829
|
+
const docPartitionMap = /* @__PURE__ */ new Map();
|
|
10830
|
+
const writeLog = createWriteLogRecorder();
|
|
10831
|
+
for (let i = 0; i < count; i++) {
|
|
10832
|
+
partitions.push(createPartitionIndex(i, trackPositions));
|
|
10694
10833
|
}
|
|
10695
|
-
|
|
10696
|
-
|
|
10697
|
-
|
|
10698
|
-
|
|
10699
|
-
|
|
10700
|
-
|
|
10701
|
-
params.term,
|
|
10702
|
-
language,
|
|
10703
|
-
{
|
|
10704
|
-
stopWords: searchOptions?.stopWords,
|
|
10705
|
-
customTokenizer: searchOptions?.customTokenizer
|
|
10706
|
-
},
|
|
10707
|
-
config.partitionIds
|
|
10834
|
+
function validatePartitionId(partitionId) {
|
|
10835
|
+
if (partitionId < 0 || partitionId >= partitions.length) {
|
|
10836
|
+
throw new NarsilError(
|
|
10837
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
10838
|
+
`Partition ${partitionId} is out of bounds (0..${partitions.length - 1})`,
|
|
10839
|
+
{ partitionId, partitionCount: partitions.length }
|
|
10708
10840
|
);
|
|
10709
10841
|
}
|
|
10710
|
-
} else if (effectiveMode === "broadcast") {
|
|
10711
|
-
globalStats = config.globalStats;
|
|
10712
|
-
}
|
|
10713
|
-
const options = buildSearchOptions(searchOptions, globalStats);
|
|
10714
|
-
if (partitions.length === 1 && !config.dispatcher) {
|
|
10715
|
-
const result = dispatchSinglePartition(partitions[0], params, language, schema, options);
|
|
10716
|
-
let facets2;
|
|
10717
|
-
if (params.facets) {
|
|
10718
|
-
facets2 = partitions[0].computeFacets(facetMatchSetOf(result), params.facets, schema);
|
|
10719
|
-
}
|
|
10720
|
-
return { scored: result.scored, totalMatched: result.totalMatched, facets: facets2 };
|
|
10721
|
-
}
|
|
10722
|
-
let outcomes;
|
|
10723
|
-
let facets;
|
|
10724
|
-
if (config.dispatcher) {
|
|
10725
|
-
outcomes = await dispatchWithDispatcher(partitions, params, language, schema, options, config.dispatcher);
|
|
10726
|
-
if (params.facets) {
|
|
10727
|
-
facets = collectAndMergeFacets(outcomes, params.facets, schema);
|
|
10728
|
-
}
|
|
10729
|
-
} else {
|
|
10730
|
-
outcomes = [];
|
|
10731
|
-
const partitionFacets = [];
|
|
10732
|
-
const everyValue = params.facets ? everyValueFacetConfig(params.facets) : void 0;
|
|
10733
|
-
for (const partition of partitions) {
|
|
10734
|
-
const result = dispatchSinglePartition(partition, params, language, schema, options);
|
|
10735
|
-
if (everyValue) {
|
|
10736
|
-
partitionFacets.push(partition.computeFacets(facetMatchSetOf(result), everyValue, schema));
|
|
10737
|
-
}
|
|
10738
|
-
outcomes.push({ result, partition });
|
|
10739
|
-
}
|
|
10740
|
-
if (params.facets) {
|
|
10741
|
-
facets = mergeFacets(partitionFacets, params.facets);
|
|
10742
|
-
}
|
|
10743
10842
|
}
|
|
10744
|
-
|
|
10745
|
-
|
|
10746
|
-
let totalMatched = 0;
|
|
10747
|
-
for (const outcome of outcomes) {
|
|
10748
|
-
totalMatched += outcome.result.totalMatched;
|
|
10843
|
+
function resolveInsertOptions(options) {
|
|
10844
|
+
return resolvePartitionInsertOptions(config, analysis, options);
|
|
10749
10845
|
}
|
|
10750
|
-
|
|
10751
|
-
|
|
10752
|
-
function facetMatchSetOf(result) {
|
|
10753
|
-
if (result.matchedOrdinalBitset !== void 0) {
|
|
10754
|
-
return { ordinalBitset: result.matchedOrdinalBitset };
|
|
10846
|
+
function preparePartialWords(partitionId) {
|
|
10847
|
+
if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
|
|
10755
10848
|
}
|
|
10756
|
-
|
|
10757
|
-
|
|
10758
|
-
function resolveEffectiveMode(config) {
|
|
10759
|
-
if (config.scoringMode === "broadcast" && !config.globalStats) {
|
|
10760
|
-
return "dfs";
|
|
10849
|
+
function locateDocument(docId) {
|
|
10850
|
+
return docPartitionMap.get(docId);
|
|
10761
10851
|
}
|
|
10762
|
-
|
|
10763
|
-
|
|
10764
|
-
|
|
10765
|
-
|
|
10766
|
-
|
|
10767
|
-
|
|
10768
|
-
patternWork: base?.patternWork ?? createPatternWorkMeter()
|
|
10769
|
-
};
|
|
10770
|
-
}
|
|
10771
|
-
function dispatchWithDispatcher(partitions, params, language, schema, options, dispatcher) {
|
|
10772
|
-
const promises = partitions.map(
|
|
10773
|
-
(partition) => dispatcher.dispatch(partition, params, language, schema, options).then((result) => ({ result, partition }))
|
|
10774
|
-
);
|
|
10775
|
-
return Promise.all(promises);
|
|
10776
|
-
}
|
|
10777
|
-
function dispatchSinglePartition(partition, params, language, schema, options) {
|
|
10778
|
-
return fulltextSearch(partition, params, language, schema, options);
|
|
10779
|
-
}
|
|
10780
|
-
function collectAndMergeFacets(outcomes, facetConfig, schema) {
|
|
10781
|
-
const everyValue = everyValueFacetConfig(facetConfig);
|
|
10782
|
-
const partitionFacets = [];
|
|
10783
|
-
for (const outcome of outcomes) {
|
|
10784
|
-
partitionFacets.push(outcome.partition.computeFacets(facetMatchSetOf(outcome.result), everyValue, schema));
|
|
10852
|
+
function asCompositePartition(partitionId) {
|
|
10853
|
+
const existing = partitions[partitionId];
|
|
10854
|
+
if (isCompositePartition(existing)) return existing;
|
|
10855
|
+
const composite = createCompositePartition(partitionId, trackPositions, existing, textAnalysis);
|
|
10856
|
+
partitions[partitionId] = composite;
|
|
10857
|
+
return composite;
|
|
10785
10858
|
}
|
|
10786
|
-
|
|
10787
|
-
|
|
10788
|
-
|
|
10789
|
-
|
|
10790
|
-
|
|
10791
|
-
|
|
10792
|
-
|
|
10793
|
-
bytes += arrays.postingOffsets.byteLength;
|
|
10794
|
-
bytes += arrays.postingDocIds.byteLength;
|
|
10795
|
-
bytes += arrays.postingFrequencies.byteLength;
|
|
10796
|
-
bytes += arrays.postingFieldIndices.byteLength;
|
|
10797
|
-
if (arrays.positionOffsets !== null) bytes += arrays.positionOffsets.byteLength;
|
|
10798
|
-
if (arrays.positionValues !== null) bytes += arrays.positionValues.byteLength;
|
|
10799
|
-
if (arrays.documentTable !== null) {
|
|
10800
|
-
bytes += arrays.documentTable.blob.byteLength + arrays.documentTable.offsets.byteLength;
|
|
10859
|
+
function rebuildDocPartitionMap() {
|
|
10860
|
+
docPartitionMap.clear();
|
|
10861
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
10862
|
+
for (const docId of partitions[i].docIds()) {
|
|
10863
|
+
docPartitionMap.set(docId, i);
|
|
10864
|
+
}
|
|
10865
|
+
}
|
|
10801
10866
|
}
|
|
10802
|
-
|
|
10803
|
-
|
|
10804
|
-
|
|
10805
|
-
|
|
10806
|
-
|
|
10807
|
-
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
10867
|
+
function registerSegmentDocuments(partitionId, segment) {
|
|
10868
|
+
for (const internalId of segment.docStore.allInternalIds()) {
|
|
10869
|
+
const docId = segment.docStore.getExternalId(internalId);
|
|
10870
|
+
if (docId !== void 0) docPartitionMap.set(docId, partitionId);
|
|
10871
|
+
}
|
|
10808
10872
|
}
|
|
10809
|
-
|
|
10810
|
-
|
|
10811
|
-
|
|
10812
|
-
|
|
10813
|
-
|
|
10814
|
-
function resolveFrozenSegments(frozen, segmentIds, partitionId) {
|
|
10815
|
-
const byId = new Map(frozen.map((segment) => [segment.segmentId, segment]));
|
|
10816
|
-
return segmentIds.map((segmentId) => {
|
|
10817
|
-
const segment = byId.get(segmentId);
|
|
10818
|
-
if (segment === void 0) {
|
|
10819
|
-
throw new NarsilError(
|
|
10820
|
-
ErrorCodes.PARTITION_CORRUPTED,
|
|
10821
|
-
`Partition ${partitionId} does not hold frozen segment "${segmentId}"`,
|
|
10822
|
-
{ partitionId, segmentId }
|
|
10823
|
-
);
|
|
10873
|
+
function rebuildDocPartitionMapForPartition(partitionId) {
|
|
10874
|
+
for (const [docId, pid] of docPartitionMap) {
|
|
10875
|
+
if (pid === partitionId) {
|
|
10876
|
+
docPartitionMap.delete(docId);
|
|
10877
|
+
}
|
|
10824
10878
|
}
|
|
10825
|
-
|
|
10826
|
-
|
|
10827
|
-
}
|
|
10828
|
-
function survivorDocumentsOf(sub, payload) {
|
|
10829
|
-
return payload.docIds.map((docId) => {
|
|
10830
|
-
const stored = sub.docStore.get(docId);
|
|
10831
|
-
if (stored === void 0) {
|
|
10832
|
-
throw new NarsilError(ErrorCodes.PARTITION_CORRUPTED, `Document "${docId}" vanished while compacting a segment`, {
|
|
10833
|
-
docId
|
|
10834
|
-
});
|
|
10835
|
-
}
|
|
10836
|
-
return stored.fields;
|
|
10837
|
-
});
|
|
10838
|
-
}
|
|
10839
|
-
function freezeLiveTailInto(live, liveState, frozen, freeze) {
|
|
10840
|
-
if (live.count() === 0) return null;
|
|
10841
|
-
const payload = live.encodeSegment();
|
|
10842
|
-
const segment = freeze(payload, survivorDocumentsOf(liveState, payload));
|
|
10843
|
-
if (segment === null) return null;
|
|
10844
|
-
live.clear();
|
|
10845
|
-
frozen.push(segment);
|
|
10846
|
-
return segment;
|
|
10847
|
-
}
|
|
10848
|
-
function swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings) {
|
|
10849
|
-
const dropSet = new Set(dropSegmentIds);
|
|
10850
|
-
const dropped = frozen.filter((segment) => dropSet.has(segment.segmentId));
|
|
10851
|
-
const applied = /* @__PURE__ */ new Map();
|
|
10852
|
-
for (const segment of dropped) {
|
|
10853
|
-
for (const docId of segment.tombstonedDocIds()) {
|
|
10854
|
-
if (!replacement.hasDocument(docId)) continue;
|
|
10855
|
-
if (dropped.some((other) => other.hasDocument(docId))) continue;
|
|
10856
|
-
const spellings = removedSpellings?.get(docId) ?? replacement.removedSpellingsOf(docId);
|
|
10857
|
-
if (spellings !== void 0) applied.set(docId, spellings);
|
|
10858
|
-
replacement.tombstoneDocument(docId, spellings);
|
|
10859
|
-
}
|
|
10860
|
-
}
|
|
10861
|
-
const kept = frozen.filter((segment) => !dropSet.has(segment.segmentId));
|
|
10862
|
-
frozen.length = 0;
|
|
10863
|
-
frozen.push(...kept, replacement);
|
|
10864
|
-
return applied;
|
|
10865
|
-
}
|
|
10866
|
-
|
|
10867
|
-
// src/core/partition/composite/documents.ts
|
|
10868
|
-
function compositeDocument(frozen, live, docId, projection) {
|
|
10869
|
-
for (const segment of frozen) {
|
|
10870
|
-
const projected = segment.docStore.projectedDocument(docId, projection);
|
|
10871
|
-
if (projected !== void 0) return projected;
|
|
10872
|
-
}
|
|
10873
|
-
return live.get(docId, projection);
|
|
10874
|
-
}
|
|
10875
|
-
function compositeFieldValues(frozen, live, docId, fieldPaths) {
|
|
10876
|
-
for (const segment of frozen) {
|
|
10877
|
-
const internalId = segment.docStore.getInternalId(docId);
|
|
10878
|
-
if (internalId === void 0) continue;
|
|
10879
|
-
return fieldPaths.map((fieldPath) => segment.docStore.fieldReader(fieldPath)(internalId));
|
|
10880
|
-
}
|
|
10881
|
-
return live.fieldValues(docId, fieldPaths);
|
|
10882
|
-
}
|
|
10883
|
-
|
|
10884
|
-
// src/core/partition/composite/filters.ts
|
|
10885
|
-
var WORD_BITS = 32;
|
|
10886
|
-
function alignToWord(value) {
|
|
10887
|
-
return value + WORD_BITS - 1 & -32;
|
|
10888
|
-
}
|
|
10889
|
-
function computeOrdinalLayout(subs) {
|
|
10890
|
-
const bases = new Array(subs.length);
|
|
10891
|
-
let cursor = 0;
|
|
10892
|
-
for (let i = 0; i < subs.length; i++) {
|
|
10893
|
-
bases[i] = cursor;
|
|
10894
|
-
cursor += alignToWord(subs[i].docStore.internalIdCapacity());
|
|
10895
|
-
}
|
|
10896
|
-
return { bases, totalCapacity: cursor };
|
|
10897
|
-
}
|
|
10898
|
-
function subBitsetView(composite, layout, subIndex) {
|
|
10899
|
-
const startWord = layout.bases[subIndex] >> 5;
|
|
10900
|
-
const endWord = subIndex + 1 < layout.bases.length ? layout.bases[subIndex + 1] >> 5 : Math.ceil(layout.totalCapacity / WORD_BITS);
|
|
10901
|
-
return composite.subarray(startWord, endWord);
|
|
10902
|
-
}
|
|
10903
|
-
function placeSubBitset(composite, layout, subIndex, subBits) {
|
|
10904
|
-
const words = subBitsetView(composite, layout, subIndex).length;
|
|
10905
|
-
if (words === 0) return;
|
|
10906
|
-
composite.set(subBits.length > words ? subBits.subarray(0, words) : subBits, layout.bases[subIndex] >> 5);
|
|
10907
|
-
}
|
|
10908
|
-
function compositeFiltersBitset(subs, layout, filters, schema, meter) {
|
|
10909
|
-
const composite = new Uint32Array(Math.ceil(layout.totalCapacity / WORD_BITS));
|
|
10910
|
-
for (let i = 0; i < subs.length; i++) {
|
|
10911
|
-
placeSubBitset(composite, layout, i, applyPartitionFiltersBitset(subs[i], filters, schema, meter));
|
|
10912
|
-
}
|
|
10913
|
-
return composite;
|
|
10914
|
-
}
|
|
10915
|
-
function compositeFilters(subs, filters, schema, meter) {
|
|
10916
|
-
const combined = /* @__PURE__ */ new Set();
|
|
10917
|
-
for (const sub of subs) {
|
|
10918
|
-
for (const docId of applyPartitionFilters(sub, filters, schema, meter)) {
|
|
10919
|
-
combined.add(docId);
|
|
10879
|
+
for (const docId of partitions[partitionId].docIds()) {
|
|
10880
|
+
docPartitionMap.set(docId, partitionId);
|
|
10920
10881
|
}
|
|
10921
10882
|
}
|
|
10922
|
-
|
|
10923
|
-
|
|
10924
|
-
|
|
10925
|
-
|
|
10926
|
-
|
|
10927
|
-
|
|
10928
|
-
|
|
10929
|
-
|
|
10930
|
-
|
|
10931
|
-
|
|
10932
|
-
|
|
10933
|
-
|
|
10934
|
-
|
|
10935
|
-
|
|
10936
|
-
|
|
10937
|
-
|
|
10883
|
+
const manager = {
|
|
10884
|
+
get partitionCount() {
|
|
10885
|
+
return partitions.length;
|
|
10886
|
+
},
|
|
10887
|
+
get indexName() {
|
|
10888
|
+
return indexName;
|
|
10889
|
+
},
|
|
10890
|
+
get schema() {
|
|
10891
|
+
return config.schema;
|
|
10892
|
+
},
|
|
10893
|
+
get language() {
|
|
10894
|
+
return language;
|
|
10895
|
+
},
|
|
10896
|
+
get config() {
|
|
10897
|
+
return config;
|
|
10898
|
+
},
|
|
10899
|
+
get analysis() {
|
|
10900
|
+
return analysis;
|
|
10901
|
+
},
|
|
10902
|
+
get partialWordAnalysis() {
|
|
10903
|
+
return partialWordAnalysis;
|
|
10904
|
+
},
|
|
10905
|
+
synonyms: null,
|
|
10906
|
+
get textAnalysis() {
|
|
10907
|
+
return textAnalysis;
|
|
10908
|
+
},
|
|
10909
|
+
getPartition(partitionId) {
|
|
10910
|
+
validatePartitionId(partitionId);
|
|
10911
|
+
return partitions[partitionId];
|
|
10912
|
+
},
|
|
10913
|
+
partitionAt(index) {
|
|
10914
|
+
return partitions[index];
|
|
10915
|
+
},
|
|
10916
|
+
getAllPartitions() {
|
|
10917
|
+
return [...partitions];
|
|
10918
|
+
},
|
|
10919
|
+
setPartitions(newPartitions) {
|
|
10920
|
+
writeLog.partitionsReplaced();
|
|
10921
|
+
partitions = [...newPartitions];
|
|
10922
|
+
rebuildDocPartitionMap();
|
|
10923
|
+
},
|
|
10924
|
+
openWriteLog() {
|
|
10925
|
+
return writeLog.open();
|
|
10926
|
+
},
|
|
10927
|
+
adoptRebuiltPartitions(rebuilt) {
|
|
10928
|
+
if (rebuilt.length !== partitions.length) {
|
|
10929
|
+
throw new NarsilError(
|
|
10930
|
+
ErrorCodes.PARTITION_CORRUPTED,
|
|
10931
|
+
`Index "${indexName}" holds ${partitions.length} partitions, but the rebuild produced ${rebuilt.length}`,
|
|
10932
|
+
{ indexName, partitionCount: partitions.length, rebuiltCount: rebuilt.length }
|
|
10933
|
+
);
|
|
10938
10934
|
}
|
|
10939
|
-
|
|
10940
|
-
|
|
10941
|
-
|
|
10942
|
-
|
|
10943
|
-
|
|
10944
|
-
|
|
10945
|
-
|
|
10946
|
-
|
|
10935
|
+
partitions = [...rebuilt];
|
|
10936
|
+
},
|
|
10937
|
+
addPartition() {
|
|
10938
|
+
const currentMaxPartitions = config.partitions?.maxPartitions;
|
|
10939
|
+
if (currentMaxPartitions !== void 0 && partitions.length >= currentMaxPartitions) {
|
|
10940
|
+
throw new NarsilError(
|
|
10941
|
+
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
10942
|
+
`Cannot add partition: maximum of ${currentMaxPartitions} partitions reached`,
|
|
10943
|
+
{ maxPartitions: currentMaxPartitions, currentCount: partitions.length }
|
|
10944
|
+
);
|
|
10947
10945
|
}
|
|
10948
|
-
|
|
10946
|
+
const partition = createPartitionIndex(partitions.length, trackPositions);
|
|
10947
|
+
writeLog.partitionsReplaced();
|
|
10948
|
+
partitions.push(partition);
|
|
10949
|
+
return partition;
|
|
10949
10950
|
},
|
|
10950
|
-
|
|
10951
|
-
|
|
10952
|
-
|
|
10953
|
-
|
|
10954
|
-
|
|
10955
|
-
|
|
10956
|
-
|
|
10957
|
-
|
|
10958
|
-
|
|
10959
|
-
|
|
10960
|
-
|
|
10961
|
-
|
|
10962
|
-
}
|
|
10963
|
-
|
|
10964
|
-
// src/core/partition/composite/stats.ts
|
|
10965
|
-
function collectCandidateTerms(subs, params) {
|
|
10966
|
-
const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
|
|
10967
|
-
const terms = /* @__PURE__ */ new Set();
|
|
10968
|
-
for (const qt of queryTokens) {
|
|
10969
|
-
const group = termGroupOf(termGroups, qt.token);
|
|
10970
|
-
if (group !== void 0) {
|
|
10971
|
-
for (const member of group.members) terms.add(member.token);
|
|
10972
|
-
}
|
|
10973
|
-
if (group === void 0 && (exact || tolerance === 0)) {
|
|
10974
|
-
terms.add(qt.token);
|
|
10975
|
-
continue;
|
|
10976
|
-
}
|
|
10977
|
-
for (const sub of subs) {
|
|
10978
|
-
for (const match of typoMatchesFor(sub.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
10979
|
-
terms.add(match.token);
|
|
10951
|
+
removePartition(partitionId) {
|
|
10952
|
+
validatePartitionId(partitionId);
|
|
10953
|
+
writeLog.partitionsReplaced();
|
|
10954
|
+
partitions.splice(partitionId, 1);
|
|
10955
|
+
rebuildDocPartitionMap();
|
|
10956
|
+
},
|
|
10957
|
+
trimPartitions(newCount) {
|
|
10958
|
+
if (newCount < 1) {
|
|
10959
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Cannot trim index "${indexName}" below one partition`, {
|
|
10960
|
+
indexName,
|
|
10961
|
+
requestedCount: newCount
|
|
10962
|
+
});
|
|
10980
10963
|
}
|
|
10981
|
-
|
|
10982
|
-
|
|
10983
|
-
return terms;
|
|
10984
|
-
}
|
|
10985
|
-
function resolveQueryTermFrequencies(subs, params) {
|
|
10986
|
-
const frequencies = /* @__PURE__ */ Object.create(null);
|
|
10987
|
-
for (const term of collectCandidateTerms(subs, params)) {
|
|
10988
|
-
const total = partitionDocumentFrequency(subs, term);
|
|
10989
|
-
if (total > 0) frequencies[term] = total;
|
|
10990
|
-
}
|
|
10991
|
-
return frequencies;
|
|
10992
|
-
}
|
|
10993
|
-
function aggregateFieldStats(subs) {
|
|
10994
|
-
let totalDocuments = 0;
|
|
10995
|
-
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
10996
|
-
for (const sub of subs) {
|
|
10997
|
-
totalDocuments += sub.stats.totalDocuments;
|
|
10998
|
-
for (const [field, length] of Object.entries(sub.stats.totalFieldLengths)) {
|
|
10999
|
-
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
11000
|
-
}
|
|
11001
|
-
}
|
|
11002
|
-
const averageFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
11003
|
-
if (totalDocuments > 0) {
|
|
11004
|
-
for (const [field, length] of Object.entries(totalFieldLengths)) {
|
|
11005
|
-
averageFieldLengths[field] = length / totalDocuments;
|
|
11006
|
-
}
|
|
11007
|
-
}
|
|
11008
|
-
return { totalDocuments, totalFieldLengths, averageFieldLengths };
|
|
11009
|
-
}
|
|
11010
|
-
function mergeDocFrequencies(subs) {
|
|
11011
|
-
const merged = /* @__PURE__ */ Object.create(null);
|
|
11012
|
-
for (const sub of subs) {
|
|
11013
|
-
for (const [term, frequency] of Object.entries(sub.stats.docFrequencies)) {
|
|
11014
|
-
merged[term] = (merged[term] ?? 0) + frequency;
|
|
11015
|
-
}
|
|
11016
|
-
}
|
|
11017
|
-
return merged;
|
|
11018
|
-
}
|
|
11019
|
-
function buildAggregateStatsView(subs, docFrequencies) {
|
|
11020
|
-
const fields = aggregateFieldStats(subs);
|
|
11021
|
-
return {
|
|
11022
|
-
...fields,
|
|
11023
|
-
get docFrequencies() {
|
|
11024
|
-
return docFrequencies();
|
|
11025
|
-
}
|
|
11026
|
-
};
|
|
11027
|
-
}
|
|
11028
|
-
function compositeQueryStats(subs, params) {
|
|
11029
|
-
const resolved = resolveQueryTermFrequencies(subs, params);
|
|
11030
|
-
const caller = params.globalStats;
|
|
11031
|
-
if (caller === void 0) {
|
|
11032
|
-
const fields = aggregateFieldStats(subs);
|
|
11033
|
-
return { ...fields, docFrequencies: resolved };
|
|
11034
|
-
}
|
|
11035
|
-
return {
|
|
11036
|
-
totalDocuments: caller.totalDocuments,
|
|
11037
|
-
totalFieldLengths: caller.totalFieldLengths,
|
|
11038
|
-
averageFieldLengths: caller.averageFieldLengths,
|
|
11039
|
-
docFrequencies: { ...resolved, ...caller.docFrequencies }
|
|
11040
|
-
};
|
|
11041
|
-
}
|
|
11042
|
-
|
|
11043
|
-
// src/core/partition/composite/search.ts
|
|
11044
|
-
function subParams(params, layout, subIndex, globalStats) {
|
|
11045
|
-
return {
|
|
11046
|
-
...params,
|
|
11047
|
-
globalStats,
|
|
11048
|
-
filterBitset: params.filterBitset === void 0 ? void 0 : subBitsetView(params.filterBitset, layout, subIndex)
|
|
11049
|
-
};
|
|
11050
|
-
}
|
|
11051
|
-
function onlyPopulatedSub(subs) {
|
|
11052
|
-
let found = -1;
|
|
11053
|
-
for (let i = 0; i < subs.length; i++) {
|
|
11054
|
-
if (subs[i].stats.totalDocuments === 0) continue;
|
|
11055
|
-
if (found !== -1) return -1;
|
|
11056
|
-
found = i;
|
|
11057
|
-
}
|
|
11058
|
-
return found;
|
|
11059
|
-
}
|
|
11060
|
-
function searchSingleSub(subs, layout, params, index) {
|
|
11061
|
-
const result = searchFulltext(subs[index], subParams(params, layout, index, params.globalStats));
|
|
11062
|
-
if (result.matchedOrdinalBitset === void 0) return result;
|
|
11063
|
-
const matchedOrdinalBitset = createBitSet(layout.totalCapacity);
|
|
11064
|
-
placeSubBitset(matchedOrdinalBitset, layout, index, result.matchedOrdinalBitset);
|
|
11065
|
-
return { ...result, matchedOrdinalBitset };
|
|
11066
|
-
}
|
|
11067
|
-
function compositeSearchFulltext(subs, layout, params) {
|
|
11068
|
-
if (params.queryTokens.length === 0) {
|
|
11069
|
-
return { scored: [], totalMatched: 0 };
|
|
11070
|
-
}
|
|
11071
|
-
const single = onlyPopulatedSub(subs);
|
|
11072
|
-
if (single !== -1) {
|
|
11073
|
-
return searchSingleSub(subs, layout, params, single);
|
|
11074
|
-
}
|
|
11075
|
-
const globalStats = compositeQueryStats(subs, params);
|
|
11076
|
-
const results = subs.map((sub, index) => searchFulltext(sub, subParams(params, layout, index, globalStats)));
|
|
11077
|
-
const scored = kWayMerge(results.map((result) => result.scored));
|
|
11078
|
-
if (params.maxResults !== void 0 && scored.length > params.maxResults) {
|
|
11079
|
-
scored.length = params.maxResults;
|
|
11080
|
-
}
|
|
11081
|
-
let totalMatched = 0;
|
|
11082
|
-
for (const result of results) {
|
|
11083
|
-
totalMatched += result.totalMatched;
|
|
11084
|
-
}
|
|
11085
|
-
if (params.collectMatchedSet === void 0) {
|
|
11086
|
-
return { scored, totalMatched };
|
|
11087
|
-
}
|
|
11088
|
-
if (params.collectMatchedSet === "ordinals") {
|
|
11089
|
-
const matchedOrdinalBitset = createBitSet(layout.totalCapacity);
|
|
11090
|
-
for (let i = 0; i < results.length; i++) {
|
|
11091
|
-
const subBitset = results[i].matchedOrdinalBitset;
|
|
11092
|
-
if (subBitset !== void 0) placeSubBitset(matchedOrdinalBitset, layout, i, subBitset);
|
|
11093
|
-
}
|
|
11094
|
-
return { scored, totalMatched, matchedOrdinalBitset };
|
|
11095
|
-
}
|
|
11096
|
-
const matchedIds = [];
|
|
11097
|
-
for (const result of results) {
|
|
11098
|
-
if (result.matchedIds !== void 0) matchedIds.push(...result.matchedIds);
|
|
11099
|
-
}
|
|
11100
|
-
return { scored, totalMatched, matchedIds };
|
|
11101
|
-
}
|
|
11102
|
-
function compositeSearchMatches(subs, layout, params) {
|
|
11103
|
-
const subMatches = subs.map((sub, index) => searchFulltextMatches(sub, subParams(params, layout, index, void 0)));
|
|
11104
|
-
let count = 0;
|
|
11105
|
-
for (const matches of subMatches) {
|
|
11106
|
-
count += matches.count;
|
|
11107
|
-
}
|
|
11108
|
-
return {
|
|
11109
|
-
count,
|
|
11110
|
-
hasExternal(docId) {
|
|
11111
|
-
for (const matches of subMatches) {
|
|
11112
|
-
if (matches.hasExternal(docId)) return true;
|
|
10964
|
+
if (newCount >= partitions.length) {
|
|
10965
|
+
return;
|
|
11113
10966
|
}
|
|
11114
|
-
|
|
11115
|
-
|
|
11116
|
-
|
|
11117
|
-
|
|
11118
|
-
if (internalId >= layout.bases[i]) {
|
|
11119
|
-
return subMatches[i].hasInternal(internalId - layout.bases[i]);
|
|
10967
|
+
writeLog.partitionsReplaced();
|
|
10968
|
+
for (const [docId, pid] of docPartitionMap) {
|
|
10969
|
+
if (pid >= newCount) {
|
|
10970
|
+
docPartitionMap.delete(docId);
|
|
11120
10971
|
}
|
|
11121
10972
|
}
|
|
11122
|
-
|
|
10973
|
+
partitions.length = newCount;
|
|
11123
10974
|
},
|
|
11124
|
-
|
|
11125
|
-
|
|
11126
|
-
|
|
11127
|
-
|
|
11128
|
-
|
|
11129
|
-
|
|
10975
|
+
...createCapacityChecks({
|
|
10976
|
+
indexName,
|
|
10977
|
+
config,
|
|
10978
|
+
partitionCount: () => partitions.length,
|
|
10979
|
+
documentCount: () => manager.countDocuments(),
|
|
10980
|
+
getPartition: (partitionId) => manager.getPartition(partitionId)
|
|
10981
|
+
}),
|
|
10982
|
+
routePartition(docId) {
|
|
10983
|
+
return router.route(docId, partitions.length);
|
|
10984
|
+
},
|
|
10985
|
+
insert(docId, document, options) {
|
|
10986
|
+
const pid = router.route(docId, partitions.length);
|
|
10987
|
+
const insertOpts = resolveInsertOptions(options);
|
|
10988
|
+
partitions[pid].insert(docId, document, config.schema, language, insertOpts);
|
|
10989
|
+
docPartitionMap.set(docId, pid);
|
|
10990
|
+
writeLog.documentWritten(docId);
|
|
10991
|
+
},
|
|
10992
|
+
removedSpellingsOf(docId) {
|
|
10993
|
+
const pid = locateDocument(docId);
|
|
10994
|
+
const partition = pid === void 0 ? void 0 : partitions[pid];
|
|
10995
|
+
return partition !== void 0 && isCompositePartition(partition) ? partition.removedSpellingsOf(docId) : void 0;
|
|
10996
|
+
},
|
|
10997
|
+
remove(docId, removedSpellings) {
|
|
10998
|
+
const pid = locateDocument(docId);
|
|
10999
|
+
if (pid === void 0) {
|
|
11000
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
11130
11001
|
}
|
|
11131
|
-
|
|
11002
|
+
partitions[pid].remove(docId, config.schema, language, resolveInsertOptions(), removedSpellings);
|
|
11003
|
+
docPartitionMap.delete(docId);
|
|
11004
|
+
writeLog.documentWritten(docId);
|
|
11132
11005
|
},
|
|
11133
|
-
|
|
11134
|
-
|
|
11135
|
-
|
|
11136
|
-
placeSubBitset(combined, layout, i, subMatches[i].ordinalBitset());
|
|
11006
|
+
beginBatchRemove() {
|
|
11007
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
11008
|
+
partitions[i].beginBatch();
|
|
11137
11009
|
}
|
|
11138
|
-
|
|
11139
|
-
|
|
11140
|
-
|
|
11141
|
-
|
|
11142
|
-
|
|
11143
|
-
// src/core/partition/composite/sorting.ts
|
|
11144
|
-
function compareEntries(a, b, directions) {
|
|
11145
|
-
for (let i = 0; i < directions.length; i++) {
|
|
11146
|
-
const comparison = compareComparableValues(a.key[i] ?? null, b.key[i] ?? null, directions[i]);
|
|
11147
|
-
if (comparison !== 0) return comparison;
|
|
11148
|
-
}
|
|
11149
|
-
return compareCodePoints(a.id, b.id);
|
|
11150
|
-
}
|
|
11151
|
-
function compositeSortedPage(subs, layout, request) {
|
|
11152
|
-
const pages = subs.map(
|
|
11153
|
-
(sub, index) => sortedPageOf(sub, {
|
|
11154
|
-
...request,
|
|
11155
|
-
matches: request.matches === null ? null : subMatchesView(request.matches, layout, index)
|
|
11156
|
-
})
|
|
11157
|
-
);
|
|
11158
|
-
const merged = pages.flat();
|
|
11159
|
-
merged.sort((a, b) => compareEntries(a, b, request.directions));
|
|
11160
|
-
if (merged.length > request.limit) merged.length = request.limit;
|
|
11161
|
-
return merged;
|
|
11162
|
-
}
|
|
11163
|
-
function compositeSortValues(subs, docId, fields, fieldTypes, modes) {
|
|
11164
|
-
for (const sub of subs) {
|
|
11165
|
-
if (sub.docStore.has(docId)) {
|
|
11166
|
-
return sortValuesOf(sub, docId, fields, fieldTypes, modes);
|
|
11167
|
-
}
|
|
11168
|
-
}
|
|
11169
|
-
return fields.map(() => null);
|
|
11170
|
-
}
|
|
11171
|
-
|
|
11172
|
-
// src/core/partition/composite/suggest.ts
|
|
11173
|
-
function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
11174
|
-
if (limit <= 0) return [];
|
|
11175
|
-
const merged = /* @__PURE__ */ new Map();
|
|
11176
|
-
for (const sub of subs) {
|
|
11177
|
-
for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix, Number.MAX_SAFE_INTEGER)) {
|
|
11178
|
-
let entry = merged.get(suggestion.token);
|
|
11179
|
-
if (entry === void 0) {
|
|
11180
|
-
entry = { documentFrequency: 0, surfaceOccurrences: /* @__PURE__ */ new Map() };
|
|
11181
|
-
merged.set(suggestion.token, entry);
|
|
11010
|
+
},
|
|
11011
|
+
endBatchRemove() {
|
|
11012
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
11013
|
+
partitions[i].endBatch();
|
|
11182
11014
|
}
|
|
11183
|
-
|
|
11184
|
-
|
|
11185
|
-
|
|
11186
|
-
|
|
11187
|
-
|
|
11188
|
-
);
|
|
11015
|
+
},
|
|
11016
|
+
update(docId, document, options, removedSpellings) {
|
|
11017
|
+
const pid = locateDocument(docId);
|
|
11018
|
+
if (pid === void 0) {
|
|
11019
|
+
throw new NarsilError(ErrorCodes.DOC_NOT_FOUND, `Document "${docId}" not found in any partition`, { docId });
|
|
11189
11020
|
}
|
|
11190
|
-
|
|
11191
|
-
|
|
11192
|
-
|
|
11193
|
-
|
|
11194
|
-
const surfaces = Array.from(entry.surfaceOccurrences, ([surface, occurrences]) => ({ surface, occurrences }));
|
|
11195
|
-
surfaces.sort((a, b) => b.occurrences - a.occurrences || compareCodePoints(a.surface, b.surface));
|
|
11196
|
-
suggestions.push({ token, documentFrequency: entry.documentFrequency, surfaces });
|
|
11197
|
-
}
|
|
11198
|
-
suggestions.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.token, b.token));
|
|
11199
|
-
if (suggestions.length > limit) suggestions.length = limit;
|
|
11200
|
-
return suggestions;
|
|
11201
|
-
}
|
|
11202
|
-
|
|
11203
|
-
// src/core/partition/composite/index.ts
|
|
11204
|
-
function isCompositePartition(partition) {
|
|
11205
|
-
return "attachFrozenSegment" in partition;
|
|
11206
|
-
}
|
|
11207
|
-
function createCompositePartition(partitionId, trackPositions = true, existingLive, textAnalysis) {
|
|
11208
|
-
const live = existingLive ?? createPartitionIndex(partitionId, trackPositions);
|
|
11209
|
-
const liveState = partitionStateOf(live);
|
|
11210
|
-
const frozen = [];
|
|
11211
|
-
let docFrequenciesCache = null;
|
|
11212
|
-
function invalidateDocFrequencies() {
|
|
11213
|
-
docFrequenciesCache = null;
|
|
11214
|
-
}
|
|
11215
|
-
function bindSegments() {
|
|
11216
|
-
invalidateDocFrequencies();
|
|
11217
|
-
if (textAnalysis === void 0) return;
|
|
11218
|
-
for (const segment of frozen) segment.removals.bind(textAnalysis);
|
|
11219
|
-
}
|
|
11220
|
-
function cachedDocFrequencies() {
|
|
11221
|
-
if (docFrequenciesCache === null) {
|
|
11222
|
-
docFrequenciesCache = mergeDocFrequencies(subs());
|
|
11223
|
-
}
|
|
11224
|
-
return docFrequenciesCache;
|
|
11225
|
-
}
|
|
11226
|
-
function subs() {
|
|
11227
|
-
return [...frozen, liveState];
|
|
11228
|
-
}
|
|
11229
|
-
function composedScratch() {
|
|
11230
|
-
const scratch = createPartitionIndex(partitionId, trackPositions);
|
|
11231
|
-
for (const sub of subs()) {
|
|
11232
|
-
const payload = encodeSegmentState(sub);
|
|
11233
|
-
if (payload.documentCount === 0) continue;
|
|
11234
|
-
scratch.mergeSegmentPayload(payload, survivorDocumentsOf(sub, payload));
|
|
11235
|
-
}
|
|
11236
|
-
return scratch;
|
|
11237
|
-
}
|
|
11238
|
-
function layout() {
|
|
11239
|
-
return computeOrdinalLayout(subs());
|
|
11240
|
-
}
|
|
11241
|
-
function frozenOwner(docId) {
|
|
11242
|
-
for (const segment of frozen) {
|
|
11243
|
-
if (segment.hasDocument(docId)) return segment;
|
|
11244
|
-
}
|
|
11245
|
-
return void 0;
|
|
11246
|
-
}
|
|
11247
|
-
function frozenStored(docId) {
|
|
11248
|
-
for (const segment of frozen) {
|
|
11249
|
-
const stored = segment.docStore.get(docId);
|
|
11250
|
-
if (stored !== void 0) return stored;
|
|
11251
|
-
}
|
|
11252
|
-
return void 0;
|
|
11253
|
-
}
|
|
11254
|
-
return {
|
|
11255
|
-
get partitionId() {
|
|
11256
|
-
return partitionId;
|
|
11257
|
-
},
|
|
11258
|
-
get live() {
|
|
11259
|
-
return live;
|
|
11260
|
-
},
|
|
11261
|
-
get stats() {
|
|
11262
|
-
return buildAggregateStatsView([...frozen, liveState], cachedDocFrequencies);
|
|
11263
|
-
},
|
|
11264
|
-
frozenSegmentCount() {
|
|
11265
|
-
return frozen.length;
|
|
11266
|
-
},
|
|
11267
|
-
frozenSegmentSizes() {
|
|
11268
|
-
return frozen.map((segment) => ({ segmentId: segment.segmentId, liveDocumentCount: segment.liveDocumentCount() }));
|
|
11269
|
-
},
|
|
11270
|
-
frozenSegmentsById(segmentIds) {
|
|
11271
|
-
return resolveFrozenSegments(frozen, segmentIds, partitionId);
|
|
11272
|
-
},
|
|
11273
|
-
swapFrozenSegments(dropSegmentIds, replacement, removedSpellings) {
|
|
11274
|
-
if (textAnalysis !== void 0) replacement.removals.bind(textAnalysis);
|
|
11275
|
-
const applied = swapFrozenSegmentList(frozen, dropSegmentIds, replacement, removedSpellings);
|
|
11276
|
-
bindSegments();
|
|
11277
|
-
return applied;
|
|
11278
|
-
},
|
|
11279
|
-
freezeLiveTail(freeze) {
|
|
11280
|
-
const segment = freezeLiveTailInto(live, liveState, frozen, freeze);
|
|
11281
|
-
if (segment !== null) bindSegments();
|
|
11282
|
-
return segment;
|
|
11283
|
-
},
|
|
11284
|
-
replaceLiveTail(segment) {
|
|
11285
|
-
live.clear();
|
|
11286
|
-
frozen.push(segment);
|
|
11287
|
-
bindSegments();
|
|
11288
|
-
},
|
|
11289
|
-
attachFrozenSegment(segment) {
|
|
11290
|
-
frozen.push(segment);
|
|
11291
|
-
bindSegments();
|
|
11021
|
+
const updateOpts = resolveInsertOptions(options);
|
|
11022
|
+
partitions[pid].update(docId, document, config.schema, language, updateOpts, removedSpellings);
|
|
11023
|
+
docPartitionMap.set(docId, pid);
|
|
11024
|
+
writeLog.documentWritten(docId);
|
|
11292
11025
|
},
|
|
11293
|
-
|
|
11294
|
-
|
|
11295
|
-
|
|
11026
|
+
get(docId, projection) {
|
|
11027
|
+
const pid = locateDocument(docId);
|
|
11028
|
+
if (pid === void 0) return void 0;
|
|
11029
|
+
const doc = partitions[pid].get(docId, projection);
|
|
11030
|
+
if (doc) copyVectorsInto(doc, docId, vecIndexes, projection);
|
|
11031
|
+
return doc;
|
|
11296
11032
|
},
|
|
11297
|
-
|
|
11298
|
-
|
|
11299
|
-
|
|
11033
|
+
getRef(docId) {
|
|
11034
|
+
const pid = locateDocument(docId);
|
|
11035
|
+
if (pid === void 0) return void 0;
|
|
11036
|
+
return partitions[pid].getRef(docId);
|
|
11300
11037
|
},
|
|
11301
|
-
|
|
11302
|
-
|
|
11303
|
-
return
|
|
11038
|
+
fieldValues(docId, fieldPaths) {
|
|
11039
|
+
const pid = locateDocument(docId);
|
|
11040
|
+
return pid === void 0 ? void 0 : partitions[pid].fieldValues(docId, fieldPaths);
|
|
11304
11041
|
},
|
|
11305
|
-
|
|
11306
|
-
|
|
11307
|
-
|
|
11042
|
+
sortValues(docId, fields, fieldTypes, modes) {
|
|
11043
|
+
const pid = locateDocument(docId);
|
|
11044
|
+
if (pid === void 0) return fields.map(() => null);
|
|
11045
|
+
return partitions[pid].sortValues(docId, fields, fieldTypes, modes);
|
|
11308
11046
|
},
|
|
11309
|
-
|
|
11310
|
-
|
|
11311
|
-
invalidateDocFrequencies();
|
|
11312
|
-
live.deserialize(data, schema);
|
|
11047
|
+
has(docId) {
|
|
11048
|
+
return locateDocument(docId) !== void 0;
|
|
11313
11049
|
},
|
|
11314
|
-
|
|
11315
|
-
|
|
11316
|
-
invalidateDocFrequencies();
|
|
11317
|
-
live.clear();
|
|
11050
|
+
partitionIdOf(docId) {
|
|
11051
|
+
return locateDocument(docId);
|
|
11318
11052
|
},
|
|
11319
|
-
|
|
11320
|
-
|
|
11321
|
-
|
|
11322
|
-
|
|
11323
|
-
`Document "${docId}" already exists in partition ${partitionId}`,
|
|
11324
|
-
{ docId, partitionId }
|
|
11325
|
-
);
|
|
11053
|
+
countDocuments() {
|
|
11054
|
+
let total = 0;
|
|
11055
|
+
for (let i = 0; i < partitions.length; i++) {
|
|
11056
|
+
total += partitions[i].count();
|
|
11326
11057
|
}
|
|
11327
|
-
|
|
11328
|
-
live.insert(docId, document, schema, language, options);
|
|
11058
|
+
return total;
|
|
11329
11059
|
},
|
|
11330
|
-
|
|
11331
|
-
|
|
11332
|
-
|
|
11333
|
-
if (owner === void 0) {
|
|
11334
|
-
live.update(docId, document, schema, language, options);
|
|
11335
|
-
return;
|
|
11336
|
-
}
|
|
11337
|
-
live.insert(docId, document, schema, language, options);
|
|
11338
|
-
owner.tombstoneDocument(docId, removedSpellings);
|
|
11060
|
+
serializePartition(partitionId) {
|
|
11061
|
+
validatePartitionId(partitionId);
|
|
11062
|
+
return partitions[partitionId].serialize(indexName, partitions.length, language.name, config.schema);
|
|
11339
11063
|
},
|
|
11340
|
-
|
|
11341
|
-
|
|
11342
|
-
|
|
11343
|
-
|
|
11344
|
-
|
|
11345
|
-
|
|
11346
|
-
const
|
|
11347
|
-
|
|
11348
|
-
|
|
11349
|
-
docId,
|
|
11350
|
-
partitionId
|
|
11351
|
-
});
|
|
11064
|
+
mergeSegment(partitionId, payload, documents) {
|
|
11065
|
+
validatePartitionId(partitionId);
|
|
11066
|
+
const partition = partitions[partitionId];
|
|
11067
|
+
partition.beginBatch();
|
|
11068
|
+
partition.mergeSegmentPayload(payload, documents);
|
|
11069
|
+
partition.endBatch();
|
|
11070
|
+
for (const docId of payload.docIds) {
|
|
11071
|
+
docPartitionMap.set(docId, partitionId);
|
|
11072
|
+
writeLog.documentWritten(docId);
|
|
11352
11073
|
}
|
|
11353
|
-
invalidateDocFrequencies();
|
|
11354
|
-
owner.tombstoneDocument(docId, removedSpellings);
|
|
11355
|
-
},
|
|
11356
|
-
removedSpellingsOf(docId) {
|
|
11357
|
-
return live.has(docId) ? void 0 : frozenOwner(docId)?.removedSpellingsOf(docId);
|
|
11358
|
-
},
|
|
11359
|
-
beginBatch() {
|
|
11360
|
-
live.beginBatch();
|
|
11361
|
-
},
|
|
11362
|
-
endBatch() {
|
|
11363
|
-
live.endBatch();
|
|
11364
|
-
},
|
|
11365
|
-
searchFulltext(params) {
|
|
11366
|
-
return compositeSearchFulltext(subs(), layout(), params);
|
|
11367
|
-
},
|
|
11368
|
-
searchFulltextMatches(params) {
|
|
11369
|
-
return compositeSearchMatches(subs(), layout(), params);
|
|
11370
|
-
},
|
|
11371
|
-
applyFilters(filters, schema, meter) {
|
|
11372
|
-
return compositeFilters(subs(), filters, schema, meter);
|
|
11373
11074
|
},
|
|
11374
|
-
|
|
11375
|
-
|
|
11376
|
-
|
|
11377
|
-
|
|
11378
|
-
|
|
11075
|
+
attachFrozenSegment(partitionId, segment) {
|
|
11076
|
+
validatePartitionId(partitionId);
|
|
11077
|
+
requireNewSegmentDocuments(segment, docPartitionMap);
|
|
11078
|
+
asCompositePartition(partitionId).attachFrozenSegment(segment);
|
|
11079
|
+
registerSegmentDocuments(partitionId, segment);
|
|
11080
|
+
writeLog.segmentWritten(segment);
|
|
11379
11081
|
},
|
|
11380
|
-
|
|
11381
|
-
|
|
11382
|
-
|
|
11383
|
-
|
|
11384
|
-
|
|
11385
|
-
|
|
11386
|
-
(sub, index) => computeFacets(
|
|
11387
|
-
sub,
|
|
11388
|
-
{ ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
|
|
11389
|
-
everyValue,
|
|
11390
|
-
schema
|
|
11391
|
-
)
|
|
11392
|
-
),
|
|
11393
|
-
config
|
|
11394
|
-
);
|
|
11395
|
-
}
|
|
11396
|
-
return mergeFacets(
|
|
11397
|
-
subs().map((sub) => computeFacets(sub, matched, everyValue, schema)),
|
|
11398
|
-
config
|
|
11082
|
+
swapFrozenSegments(partitionId, dropSegmentIds, replacement, removedSpellings) {
|
|
11083
|
+
validatePartitionId(partitionId);
|
|
11084
|
+
const applied = asCompositePartition(partitionId).swapFrozenSegments(
|
|
11085
|
+
dropSegmentIds,
|
|
11086
|
+
replacement,
|
|
11087
|
+
removedSpellings
|
|
11399
11088
|
);
|
|
11089
|
+
registerSegmentDocuments(partitionId, replacement);
|
|
11090
|
+
return applied;
|
|
11400
11091
|
},
|
|
11401
|
-
|
|
11402
|
-
|
|
11403
|
-
|
|
11404
|
-
sortValues(docId, fields, fieldTypes, modes) {
|
|
11405
|
-
return compositeSortValues(subs(), docId, fields, fieldTypes, modes);
|
|
11406
|
-
},
|
|
11407
|
-
suggestTerms(surfacePrefix, stemmedPrefix, limit) {
|
|
11408
|
-
return compositeSuggestTerms(subs(), surfacePrefix, stemmedPrefix, limit);
|
|
11409
|
-
},
|
|
11410
|
-
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
11411
|
-
return expandTermPrefix(subs(), surfacePrefix, stemmedToken, maxExpansions);
|
|
11092
|
+
freezeLiveTail(partitionId, freeze) {
|
|
11093
|
+
validatePartitionId(partitionId);
|
|
11094
|
+
return asCompositePartition(partitionId).freezeLiveTail(freeze);
|
|
11412
11095
|
},
|
|
11413
|
-
|
|
11414
|
-
|
|
11096
|
+
replaceLiveTail(partitionId, segment) {
|
|
11097
|
+
validatePartitionId(partitionId);
|
|
11098
|
+
asCompositePartition(partitionId).replaceLiveTail(segment);
|
|
11099
|
+
registerSegmentDocuments(partitionId, segment);
|
|
11415
11100
|
},
|
|
11416
|
-
|
|
11417
|
-
|
|
11101
|
+
deserializePartition(partitionId, data) {
|
|
11102
|
+
validatePartitionId(partitionId);
|
|
11103
|
+
writeLog.partitionsReplaced();
|
|
11104
|
+
partitions[partitionId].deserialize(data, config.schema);
|
|
11105
|
+
preparePartialWords(partitionId);
|
|
11106
|
+
rebuildDocPartitionMapForPartition(partitionId);
|
|
11418
11107
|
},
|
|
11419
|
-
|
|
11420
|
-
return
|
|
11108
|
+
getAggregateStats() {
|
|
11109
|
+
return aggregateStatsOf(partitions);
|
|
11421
11110
|
},
|
|
11422
|
-
|
|
11423
|
-
return
|
|
11111
|
+
estimateMemoryBytes() {
|
|
11112
|
+
return estimateBytesOf(partitions, vecIndexes);
|
|
11424
11113
|
},
|
|
11425
|
-
|
|
11426
|
-
|
|
11427
|
-
if (stored !== void 0) return stored.fields;
|
|
11428
|
-
return live.getRef(docId);
|
|
11114
|
+
getPartitionStats() {
|
|
11115
|
+
return partitionStatsOf(partitions);
|
|
11429
11116
|
},
|
|
11430
|
-
|
|
11431
|
-
return
|
|
11117
|
+
getVectorIndexes() {
|
|
11118
|
+
return vecIndexes;
|
|
11432
11119
|
},
|
|
11433
|
-
|
|
11434
|
-
|
|
11435
|
-
for (const
|
|
11436
|
-
|
|
11120
|
+
resetVectorIndexes(newIndexes) {
|
|
11121
|
+
vecIndexes.clear();
|
|
11122
|
+
for (const [key, value] of newIndexes) {
|
|
11123
|
+
vecIndexes.set(key, value);
|
|
11437
11124
|
}
|
|
11438
|
-
|
|
11439
|
-
|
|
11440
|
-
|
|
11441
|
-
|
|
11442
|
-
|
|
11443
|
-
|
|
11444
|
-
|
|
11445
|
-
|
|
11446
|
-
|
|
11447
|
-
|
|
11448
|
-
},
|
|
11449
|
-
|
|
11450
|
-
|
|
11451
|
-
|
|
11452
|
-
|
|
11453
|
-
|
|
11454
|
-
|
|
11455
|
-
|
|
11456
|
-
|
|
11125
|
+
}
|
|
11126
|
+
};
|
|
11127
|
+
return manager;
|
|
11128
|
+
}
|
|
11129
|
+
|
|
11130
|
+
// src/core/pattern-index/work-meter.ts
|
|
11131
|
+
var COUNTER_BYTES = 8;
|
|
11132
|
+
function capExceeded(cap) {
|
|
11133
|
+
return new NarsilError(
|
|
11134
|
+
ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
|
|
11135
|
+
`The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
|
|
11136
|
+
{ cap }
|
|
11137
|
+
);
|
|
11138
|
+
}
|
|
11139
|
+
function localMeter(cap, spent, matcherStateBytes, flushUnits) {
|
|
11140
|
+
let total = spent;
|
|
11141
|
+
let sharedCount = null;
|
|
11142
|
+
const spentSoFar = () => sharedCount === null ? total : Number(Atomics.load(sharedCount, 0));
|
|
11143
|
+
const meter = {
|
|
11144
|
+
cap,
|
|
11145
|
+
matcherStateBytes,
|
|
11146
|
+
flushUnits,
|
|
11147
|
+
add(units) {
|
|
11148
|
+
if (units <= 0) return;
|
|
11149
|
+
if (sharedCount === null) total += units;
|
|
11150
|
+
else total = Number(Atomics.add(sharedCount, 0, BigInt(units))) + units;
|
|
11151
|
+
if (total > cap) throw capExceeded(cap);
|
|
11457
11152
|
},
|
|
11458
|
-
|
|
11459
|
-
|
|
11460
|
-
segment.docStore.releaseSortedDocIds();
|
|
11461
|
-
}
|
|
11462
|
-
live.releaseSortedDocIds();
|
|
11153
|
+
remaining() {
|
|
11154
|
+
return cap - spentSoFar();
|
|
11463
11155
|
},
|
|
11464
|
-
|
|
11465
|
-
|
|
11466
|
-
|
|
11467
|
-
|
|
11156
|
+
fork(workerCount) {
|
|
11157
|
+
const copies = Math.max(1, workerCount);
|
|
11158
|
+
const seed = spentSoFar();
|
|
11159
|
+
let counter = null;
|
|
11160
|
+
if (typeof SharedArrayBuffer === "function") {
|
|
11161
|
+
counter = new SharedArrayBuffer(COUNTER_BYTES);
|
|
11162
|
+
new BigInt64Array(counter)[0] = BigInt(seed);
|
|
11468
11163
|
}
|
|
11469
|
-
|
|
11164
|
+
const shared2 = {
|
|
11165
|
+
cap,
|
|
11166
|
+
spent: seed,
|
|
11167
|
+
counter,
|
|
11168
|
+
matcherStateBytes: Math.floor(matcherStateBytes / copies),
|
|
11169
|
+
flushUnits: Math.max(1, Math.floor(flushUnits / copies))
|
|
11170
|
+
};
|
|
11171
|
+
return {
|
|
11172
|
+
shared: shared2,
|
|
11173
|
+
settle(reportedUnits) {
|
|
11174
|
+
if (counter === null) {
|
|
11175
|
+
meter.add(reportedUnits);
|
|
11176
|
+
return;
|
|
11177
|
+
}
|
|
11178
|
+
const unitsSinceFork = spentSoFar() - seed;
|
|
11179
|
+
const view = new BigInt64Array(counter);
|
|
11180
|
+
if (unitsSinceFork > 0) Atomics.add(view, 0, BigInt(unitsSinceFork));
|
|
11181
|
+
sharedCount = view;
|
|
11182
|
+
total = spentSoFar();
|
|
11183
|
+
if (total > cap) throw capExceeded(cap);
|
|
11184
|
+
}
|
|
11185
|
+
};
|
|
11470
11186
|
}
|
|
11471
11187
|
};
|
|
11188
|
+
return meter;
|
|
11472
11189
|
}
|
|
11473
|
-
|
|
11474
|
-
|
|
11475
|
-
function resolvePartialWordAnalysis(config, language, analysis) {
|
|
11476
|
-
const fields = partialFieldsOf(config.schema);
|
|
11477
|
-
if (fields.length === 0) return null;
|
|
11478
|
-
return { fields, language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
11479
|
-
}
|
|
11480
|
-
function resolvePartitionInsertOptions(config, analysis, options) {
|
|
11481
|
-
const applyStrict = config.strict === true;
|
|
11482
|
-
const applyAnalyzer = analysis.stopWords !== void 0 || analysis.customTokenizer !== void 0;
|
|
11483
|
-
const applySurfaces = config.surfaceForms !== false;
|
|
11484
|
-
const applyPatternLimit = config.patternValueLimit !== void 0;
|
|
11485
|
-
if (!applyStrict && !applyAnalyzer && !applySurfaces && !applyPatternLimit) return options;
|
|
11486
|
-
const resolved = { ...options };
|
|
11487
|
-
if (applyStrict) resolved.strict = true;
|
|
11488
|
-
if (applyPatternLimit) resolved.patternValueLimit = config.patternValueLimit;
|
|
11489
|
-
if (applyAnalyzer) {
|
|
11490
|
-
resolved.stopWordOverride = options?.stopWordOverride ?? analysis.stopWords;
|
|
11491
|
-
resolved.customTokenizer = options?.customTokenizer ?? analysis.customTokenizer;
|
|
11492
|
-
}
|
|
11493
|
-
if (applySurfaces) resolved.collectSurfaces = true;
|
|
11494
|
-
return resolved;
|
|
11190
|
+
function createPatternWorkMeter(cap = DEFAULT_PATTERN_WORK_CAP) {
|
|
11191
|
+
return localMeter(cap, 0, MATCHER_STATE_BUDGET_BYTES, PATTERN_WORK_FLUSH_UNITS);
|
|
11495
11192
|
}
|
|
11496
|
-
|
|
11497
|
-
|
|
11498
|
-
function createCapacityChecks(source) {
|
|
11499
|
-
const { indexName, config } = source;
|
|
11193
|
+
function sharedMeter(work, counter) {
|
|
11194
|
+
const view = new BigInt64Array(counter);
|
|
11500
11195
|
return {
|
|
11501
|
-
|
|
11502
|
-
|
|
11503
|
-
|
|
11504
|
-
|
|
11505
|
-
const
|
|
11506
|
-
|
|
11507
|
-
const currentCount = source.documentCount() + pendingWrites;
|
|
11508
|
-
if (currentCount < totalCapacity) return;
|
|
11509
|
-
throw new NarsilError(
|
|
11510
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
11511
|
-
`Index "${indexName}" has reached its capacity of ${totalCapacity} documents (${maxDocsPerPartition} per partition \xD7 ${effectivePartitionCount} partitions)`,
|
|
11512
|
-
{
|
|
11513
|
-
indexName,
|
|
11514
|
-
currentCount,
|
|
11515
|
-
totalCapacity,
|
|
11516
|
-
maxDocsPerPartition,
|
|
11517
|
-
partitionCount: effectivePartitionCount
|
|
11518
|
-
}
|
|
11519
|
-
);
|
|
11196
|
+
matcherStateBytes: work.matcherStateBytes,
|
|
11197
|
+
flushUnits: work.flushUnits,
|
|
11198
|
+
add(units) {
|
|
11199
|
+
if (units <= 0) return;
|
|
11200
|
+
const total = Number(Atomics.add(view, 0, BigInt(units))) + units;
|
|
11201
|
+
if (total > work.cap) throw capExceeded(work.cap);
|
|
11520
11202
|
},
|
|
11521
|
-
|
|
11522
|
-
|
|
11523
|
-
if (maxDocsPerPartition === void 0) return;
|
|
11524
|
-
const currentCount = source.getPartition(partitionId).count() + pendingWrites;
|
|
11525
|
-
if (currentCount < maxDocsPerPartition) return;
|
|
11526
|
-
throw new NarsilError(
|
|
11527
|
-
ErrorCodes.PARTITION_CAPACITY_EXCEEDED,
|
|
11528
|
-
`Partition ${partitionId} of index "${indexName}" has reached its capacity of ${maxDocsPerPartition} documents`,
|
|
11529
|
-
{ indexName, partitionId, currentCount, maxDocsPerPartition }
|
|
11530
|
-
);
|
|
11203
|
+
remaining() {
|
|
11204
|
+
return work.cap - Number(Atomics.load(view, 0));
|
|
11531
11205
|
}
|
|
11532
11206
|
};
|
|
11533
11207
|
}
|
|
11208
|
+
function joinPatternWorkMeter(work) {
|
|
11209
|
+
const meter = work.counter === null ? localMeter(work.cap, work.spent, work.matcherStateBytes, work.flushUnits) : sharedMeter(work, work.counter);
|
|
11210
|
+
let spent = 0;
|
|
11211
|
+
return {
|
|
11212
|
+
matcherStateBytes: meter.matcherStateBytes,
|
|
11213
|
+
flushUnits: meter.flushUnits,
|
|
11214
|
+
add(units) {
|
|
11215
|
+
if (units <= 0) return;
|
|
11216
|
+
spent += units;
|
|
11217
|
+
meter.add(units);
|
|
11218
|
+
},
|
|
11219
|
+
remaining: () => meter.remaining(),
|
|
11220
|
+
spent: () => spent
|
|
11221
|
+
};
|
|
11222
|
+
}
|
|
11534
11223
|
|
|
11535
|
-
// src/
|
|
11536
|
-
function
|
|
11537
|
-
|
|
11538
|
-
|
|
11539
|
-
|
|
11540
|
-
|
|
11541
|
-
|
|
11542
|
-
|
|
11543
|
-
|
|
11544
|
-
|
|
11545
|
-
if (
|
|
11546
|
-
next = {};
|
|
11547
|
-
current[segments[i]] = next;
|
|
11548
|
-
}
|
|
11549
|
-
current = next;
|
|
11224
|
+
// src/search/term-groups.ts
|
|
11225
|
+
function writtenWordsByPosition(term, language, analysis) {
|
|
11226
|
+
const unstemmed = tokenize(term, language, {
|
|
11227
|
+
stem: false,
|
|
11228
|
+
removeStopWords: true,
|
|
11229
|
+
customTokenizer: analysis?.customTokenizer,
|
|
11230
|
+
stopWordOverride: analysis?.stopWords
|
|
11231
|
+
});
|
|
11232
|
+
const written = /* @__PURE__ */ new Map();
|
|
11233
|
+
for (const token of unstemmed.tokens) {
|
|
11234
|
+
if (!written.has(token.position)) written.set(token.position, token.token);
|
|
11550
11235
|
}
|
|
11551
|
-
|
|
11236
|
+
return written;
|
|
11552
11237
|
}
|
|
11553
|
-
function
|
|
11554
|
-
|
|
11555
|
-
|
|
11556
|
-
|
|
11557
|
-
|
|
11238
|
+
function groupFor(groups, token, typoMatches) {
|
|
11239
|
+
let group = groups.get(token);
|
|
11240
|
+
if (group === void 0) {
|
|
11241
|
+
group = { token, members: [{ token, factor: 1, partialFactor: 1 }], typoMatches };
|
|
11242
|
+
groups.set(token, group);
|
|
11558
11243
|
}
|
|
11244
|
+
return group;
|
|
11559
11245
|
}
|
|
11560
|
-
|
|
11561
|
-
|
|
11562
|
-
|
|
11563
|
-
|
|
11564
|
-
|
|
11565
|
-
if (
|
|
11566
|
-
|
|
11567
|
-
|
|
11568
|
-
partitionId: docPartitionMap.get(docId)
|
|
11569
|
-
});
|
|
11246
|
+
function addCompletions(groups, partition, lastToken, typedPrefix) {
|
|
11247
|
+
if (typedPrefix === void 0 || typedPrefix.length === 0) return;
|
|
11248
|
+
const group = groupFor(groups, lastToken.token, false);
|
|
11249
|
+
const ownLength = codePointCount(lastToken.token);
|
|
11250
|
+
for (const completion of partition.expandTermPrefix(typedPrefix, lastToken.token, PREFIX_MAX_EXPANSIONS)) {
|
|
11251
|
+
if (completion === lastToken.token) continue;
|
|
11252
|
+
const factor = Math.min(1, ownLength / codePointCount(completion));
|
|
11253
|
+
group.members.push({ token: completion, factor, partialFactor: factor });
|
|
11570
11254
|
}
|
|
11571
11255
|
}
|
|
11572
|
-
|
|
11573
|
-
|
|
11574
|
-
|
|
11575
|
-
|
|
11576
|
-
|
|
11577
|
-
|
|
11578
|
-
|
|
11579
|
-
|
|
11580
|
-
totalDocuments += stats.totalDocuments;
|
|
11581
|
-
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
11582
|
-
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
11583
|
-
}
|
|
11584
|
-
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
11585
|
-
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
11256
|
+
function partialWordAnalysisOf(fields, language, analysis) {
|
|
11257
|
+
return {
|
|
11258
|
+
fields,
|
|
11259
|
+
language,
|
|
11260
|
+
options: {
|
|
11261
|
+
stopWordOverride: analysis?.stopWords,
|
|
11262
|
+
customTokenizer: analysis?.customTokenizer,
|
|
11263
|
+
collectSurfaces: true
|
|
11586
11264
|
}
|
|
11587
|
-
}
|
|
11588
|
-
return { totalDocuments, docFrequencies, totalFieldLengths };
|
|
11265
|
+
};
|
|
11589
11266
|
}
|
|
11590
|
-
function
|
|
11591
|
-
|
|
11592
|
-
for (
|
|
11593
|
-
|
|
11594
|
-
}
|
|
11595
|
-
for (const [, vecIndex] of vecIndexes) {
|
|
11596
|
-
total += vecIndex.estimateMemoryBytes();
|
|
11267
|
+
function searchesAPartialField(params, partialFields) {
|
|
11268
|
+
if (params.fields === void 0) return true;
|
|
11269
|
+
for (const field of params.fields) {
|
|
11270
|
+
if (partialFields.includes(field)) return true;
|
|
11597
11271
|
}
|
|
11598
|
-
return
|
|
11272
|
+
return false;
|
|
11599
11273
|
}
|
|
11600
|
-
function
|
|
11601
|
-
|
|
11602
|
-
|
|
11603
|
-
|
|
11604
|
-
|
|
11605
|
-
|
|
11274
|
+
function resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, analysis, synonyms) {
|
|
11275
|
+
const term = params.term;
|
|
11276
|
+
if (term === void 0 || params.exact === true) return {};
|
|
11277
|
+
const usesPrefix = params.prefix === true;
|
|
11278
|
+
const partialFields = params.partialWords === true ? partialFieldsOf(schema) : [];
|
|
11279
|
+
const usesPartialWords = partialFields.length > 0 && searchesAPartialField(params, partialFields);
|
|
11280
|
+
if (!usesPrefix && !usesPartialWords && synonyms === void 0) return {};
|
|
11281
|
+
const written = usesPrefix || usesPartialWords ? writtenWordsByPosition(term, language, analysis) : /* @__PURE__ */ new Map();
|
|
11282
|
+
const groups = /* @__PURE__ */ new Map();
|
|
11283
|
+
if (usesPrefix) addCompletions(groups, partition, lastToken, written.get(lastToken.position));
|
|
11284
|
+
const prefixToken = groups.has(lastToken.token) ? lastToken.token : void 0;
|
|
11285
|
+
if (usesPartialWords) {
|
|
11286
|
+
addPartialWordMembers(
|
|
11287
|
+
{
|
|
11288
|
+
queryTokens,
|
|
11289
|
+
written,
|
|
11290
|
+
prefixToken,
|
|
11291
|
+
tolerance: params.tolerance ?? 0,
|
|
11292
|
+
prefixLength: params.prefixLength,
|
|
11293
|
+
analysis: partialWordAnalysisOf(partialFields, language, analysis)
|
|
11294
|
+
},
|
|
11295
|
+
(partialAnalysis, requests) => partition.expandPartialWords(partialAnalysis, requests),
|
|
11296
|
+
(token, typoMatches) => groupFor(groups, token, typoMatches)
|
|
11297
|
+
);
|
|
11298
|
+
}
|
|
11299
|
+
synonyms?.addTo((token) => groupFor(groups, token, true));
|
|
11300
|
+
const termGroups = [...groups.values()];
|
|
11301
|
+
if (termGroups.length === 0) return {};
|
|
11302
|
+
return usesPartialWords ? { termGroups, partialFields: [...partialFields] } : { termGroups };
|
|
11606
11303
|
}
|
|
11607
11304
|
|
|
11608
|
-
// src/partitioning/
|
|
11609
|
-
function
|
|
11610
|
-
|
|
11611
|
-
|
|
11612
|
-
|
|
11613
|
-
|
|
11614
|
-
|
|
11615
|
-
|
|
11616
|
-
|
|
11617
|
-
|
|
11618
|
-
|
|
11619
|
-
|
|
11620
|
-
|
|
11621
|
-
|
|
11622
|
-
|
|
11623
|
-
|
|
11624
|
-
|
|
11625
|
-
|
|
11626
|
-
|
|
11627
|
-
|
|
11628
|
-
|
|
11629
|
-
|
|
11630
|
-
|
|
11631
|
-
|
|
11632
|
-
|
|
11633
|
-
|
|
11634
|
-
|
|
11635
|
-
|
|
11636
|
-
|
|
11637
|
-
|
|
11638
|
-
|
|
11639
|
-
|
|
11640
|
-
}
|
|
11641
|
-
}
|
|
11642
|
-
|
|
11643
|
-
|
|
11644
|
-
|
|
11305
|
+
// src/partitioning/expanded-tokens.ts
|
|
11306
|
+
function queryTokensOf(term, language, analysis) {
|
|
11307
|
+
const { tokens } = tokenize(term, language, {
|
|
11308
|
+
stem: true,
|
|
11309
|
+
removeStopWords: true,
|
|
11310
|
+
customTokenizer: analysis.customTokenizer,
|
|
11311
|
+
stopWordOverride: analysis.stopWords
|
|
11312
|
+
});
|
|
11313
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11314
|
+
const unique = [];
|
|
11315
|
+
for (const token of tokens) {
|
|
11316
|
+
if (seen.has(token.token)) continue;
|
|
11317
|
+
seen.add(token.token);
|
|
11318
|
+
unique.push(token);
|
|
11319
|
+
}
|
|
11320
|
+
return tokens.length === 0 ? null : { unique, last: tokens[tokens.length - 1] };
|
|
11321
|
+
}
|
|
11322
|
+
function readStatesOf(partition) {
|
|
11323
|
+
if (!isCompositePartition(partition)) return [partitionStateOf(partition)];
|
|
11324
|
+
const segmentIds = partition.frozenSegmentSizes().map((segment) => segment.segmentId);
|
|
11325
|
+
return [...partition.frozenSegmentsById(segmentIds), partitionStateOf(partition.live)];
|
|
11326
|
+
}
|
|
11327
|
+
function addPartitionTokens(tokens, addedMembers, partition, term, analysed, expansion, language, analysis) {
|
|
11328
|
+
const { options, schema } = expansion;
|
|
11329
|
+
const { unique: queryTokens, last: lastToken } = analysed;
|
|
11330
|
+
const tolerance = options.tolerance ?? 0;
|
|
11331
|
+
const prefixLength = options.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH;
|
|
11332
|
+
if (tolerance > 0) {
|
|
11333
|
+
const states = readStatesOf(partition);
|
|
11334
|
+
for (const { token } of queryTokens) {
|
|
11335
|
+
for (const state of states) {
|
|
11336
|
+
for (const match of state.invertedIdx.fuzzyLookup(token, tolerance, prefixLength)) tokens.add(match.token);
|
|
11337
|
+
}
|
|
11338
|
+
}
|
|
11339
|
+
}
|
|
11340
|
+
const params = {
|
|
11341
|
+
term,
|
|
11342
|
+
tolerance,
|
|
11343
|
+
prefixLength,
|
|
11344
|
+
prefix: options.prefix ?? void 0,
|
|
11345
|
+
partialWords: options.partialWords ?? void 0,
|
|
11346
|
+
fields: options.fields ?? void 0
|
|
11645
11347
|
};
|
|
11348
|
+
const { termGroups } = resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, analysis);
|
|
11349
|
+
for (const group of termGroups ?? []) {
|
|
11350
|
+
for (const member of group.members) {
|
|
11351
|
+
if (member.token !== group.token) addedMembers.add(member.token);
|
|
11352
|
+
}
|
|
11353
|
+
}
|
|
11354
|
+
}
|
|
11355
|
+
function addReachedTokens(tokens, partitions, term, expansion, language, analysis) {
|
|
11356
|
+
const { options } = expansion;
|
|
11357
|
+
if (options.exact === true) return [];
|
|
11358
|
+
if (!((options.tolerance ?? 0) > 0 || options.prefix === true || options.partialWords === true)) return [];
|
|
11359
|
+
const analysed = queryTokensOf(term, language, analysis);
|
|
11360
|
+
if (analysed === null) return [];
|
|
11361
|
+
const addedMembers = /* @__PURE__ */ new Set();
|
|
11362
|
+
for (const partition of partitions) {
|
|
11363
|
+
addPartitionTokens(tokens, addedMembers, partition, term, analysed, expansion, language, analysis);
|
|
11364
|
+
}
|
|
11365
|
+
for (const token of addedMembers) tokens.add(token);
|
|
11366
|
+
return [...addedMembers];
|
|
11646
11367
|
}
|
|
11647
11368
|
|
|
11648
|
-
// src/partitioning/
|
|
11649
|
-
function
|
|
11650
|
-
|
|
11651
|
-
|
|
11652
|
-
const count = initialPartitionCount ?? 1;
|
|
11653
|
-
const analysis = resolveIndexAnalysis(config);
|
|
11654
|
-
const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
11655
|
-
const textAnalysis = { language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
11656
|
-
const trackPositions = config.trackPositions ?? true;
|
|
11657
|
-
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
11658
|
-
let partitions = [];
|
|
11659
|
-
const docPartitionMap = /* @__PURE__ */ new Map();
|
|
11660
|
-
const writeLog = createWriteLogRecorder();
|
|
11661
|
-
for (let i = 0; i < count; i++) {
|
|
11662
|
-
partitions.push(createPartitionIndex(i, trackPositions));
|
|
11369
|
+
// src/partitioning/partition-selection.ts
|
|
11370
|
+
function partitionsIn(manager, partitionIds) {
|
|
11371
|
+
if (partitionIds === void 0) {
|
|
11372
|
+
return manager.getAllPartitions();
|
|
11663
11373
|
}
|
|
11664
|
-
|
|
11665
|
-
|
|
11666
|
-
|
|
11667
|
-
|
|
11668
|
-
|
|
11669
|
-
{ partitionId, partitionCount: partitions.length }
|
|
11670
|
-
);
|
|
11374
|
+
const partitions = [];
|
|
11375
|
+
for (const partitionId of partitionIds) {
|
|
11376
|
+
const partition = manager.partitionAt(partitionId);
|
|
11377
|
+
if (partition !== void 0) {
|
|
11378
|
+
partitions.push(partition);
|
|
11671
11379
|
}
|
|
11672
11380
|
}
|
|
11673
|
-
|
|
11674
|
-
|
|
11675
|
-
|
|
11676
|
-
|
|
11677
|
-
|
|
11381
|
+
return partitions;
|
|
11382
|
+
}
|
|
11383
|
+
|
|
11384
|
+
// src/partitioning/distributed-scoring.ts
|
|
11385
|
+
function analysedQueryTokens(term, language, analysis) {
|
|
11386
|
+
const { tokens } = tokenize(term, language, {
|
|
11387
|
+
stem: true,
|
|
11388
|
+
removeStopWords: true,
|
|
11389
|
+
customTokenizer: analysis.customTokenizer,
|
|
11390
|
+
stopWordOverride: analysis.stopWords
|
|
11391
|
+
});
|
|
11392
|
+
const unique = /* @__PURE__ */ new Set();
|
|
11393
|
+
for (const { token } of tokens) {
|
|
11394
|
+
unique.add(token);
|
|
11678
11395
|
}
|
|
11679
|
-
|
|
11680
|
-
|
|
11396
|
+
for (const synonym of analysis.synonyms?.expand(unique)?.tokens ?? []) unique.add(synonym);
|
|
11397
|
+
return unique;
|
|
11398
|
+
}
|
|
11399
|
+
function copyNumericRecord(source) {
|
|
11400
|
+
const copy = /* @__PURE__ */ Object.create(null);
|
|
11401
|
+
for (const [key, value] of Object.entries(source)) {
|
|
11402
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
11403
|
+
copy[key] = value;
|
|
11404
|
+
}
|
|
11681
11405
|
}
|
|
11682
|
-
|
|
11683
|
-
|
|
11684
|
-
|
|
11685
|
-
|
|
11686
|
-
|
|
11687
|
-
|
|
11406
|
+
return copy;
|
|
11407
|
+
}
|
|
11408
|
+
function sanitizeGlobalStats(stats) {
|
|
11409
|
+
return {
|
|
11410
|
+
totalDocuments: typeof stats.totalDocuments === "number" && Number.isFinite(stats.totalDocuments) ? stats.totalDocuments : 0,
|
|
11411
|
+
docFrequencies: copyNumericRecord(stats.docFrequencies),
|
|
11412
|
+
totalFieldLengths: copyNumericRecord(stats.totalFieldLengths),
|
|
11413
|
+
averageFieldLengths: copyNumericRecord(stats.averageFieldLengths)
|
|
11414
|
+
};
|
|
11415
|
+
}
|
|
11416
|
+
function pruneStatsToQueryTerms(stats, term, language, analysis, reached) {
|
|
11417
|
+
const tokens = analysedQueryTokens(term, language, analysis);
|
|
11418
|
+
if (reached !== void 0) addReachedTokens(tokens, reached.partitions, term, reached.expansion, language, analysis);
|
|
11419
|
+
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
11420
|
+
for (const token of tokens) {
|
|
11421
|
+
if (Object.hasOwn(stats.docFrequencies, token)) {
|
|
11422
|
+
docFrequencies[token] = stats.docFrequencies[token];
|
|
11423
|
+
}
|
|
11688
11424
|
}
|
|
11689
|
-
|
|
11690
|
-
|
|
11691
|
-
|
|
11692
|
-
|
|
11693
|
-
|
|
11425
|
+
return {
|
|
11426
|
+
totalDocuments: stats.totalDocuments,
|
|
11427
|
+
docFrequencies,
|
|
11428
|
+
totalFieldLengths: stats.totalFieldLengths,
|
|
11429
|
+
averageFieldLengths: stats.averageFieldLengths
|
|
11430
|
+
};
|
|
11431
|
+
}
|
|
11432
|
+
function collectQueryTermStats(manager, term, language, analysis, partitionIds, scope) {
|
|
11433
|
+
const queryTokens = analysedQueryTokens(term, language, analysis);
|
|
11434
|
+
let totalDocuments = 0;
|
|
11435
|
+
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
11436
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
11437
|
+
const statsPartitions = partitionsIn(manager, partitionIds);
|
|
11438
|
+
for (const token of scope?.tokens ?? []) queryTokens.add(token);
|
|
11439
|
+
const expanded = scope?.expansion === void 0 ? [] : addReachedTokens(queryTokens, statsPartitions, term, scope.expansion, language, analysis);
|
|
11440
|
+
for (const partition of statsPartitions) {
|
|
11441
|
+
const stats = partition.stats;
|
|
11442
|
+
totalDocuments += stats.totalDocuments;
|
|
11443
|
+
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
11444
|
+
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
11445
|
+
}
|
|
11446
|
+
for (const token of queryTokens) {
|
|
11447
|
+
if (Object.hasOwn(stats.docFrequencies, token)) {
|
|
11448
|
+
docFrequencies[token] = (docFrequencies[token] ?? 0) + stats.docFrequencies[token];
|
|
11694
11449
|
}
|
|
11695
11450
|
}
|
|
11696
11451
|
}
|
|
11697
|
-
|
|
11698
|
-
|
|
11699
|
-
|
|
11700
|
-
|
|
11452
|
+
const averageFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
11453
|
+
if (totalDocuments > 0) {
|
|
11454
|
+
for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
|
|
11455
|
+
averageFieldLengths[field] = totalLength / totalDocuments;
|
|
11701
11456
|
}
|
|
11702
11457
|
}
|
|
11703
|
-
|
|
11704
|
-
|
|
11705
|
-
|
|
11706
|
-
|
|
11707
|
-
|
|
11458
|
+
return { totalDocuments, docFrequencies, totalFieldLengths, averageFieldLengths, expanded };
|
|
11459
|
+
}
|
|
11460
|
+
function mergePartitionStats(statsArray) {
|
|
11461
|
+
let totalDocuments = 0;
|
|
11462
|
+
const docFrequencies = /* @__PURE__ */ Object.create(null);
|
|
11463
|
+
const totalFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
11464
|
+
for (const stats of statsArray) {
|
|
11465
|
+
totalDocuments += stats.totalDocuments;
|
|
11466
|
+
for (const [term, freq] of Object.entries(stats.docFrequencies)) {
|
|
11467
|
+
docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
|
|
11708
11468
|
}
|
|
11709
|
-
for (const
|
|
11710
|
-
|
|
11469
|
+
for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
|
|
11470
|
+
totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
|
|
11711
11471
|
}
|
|
11712
11472
|
}
|
|
11713
|
-
const
|
|
11714
|
-
|
|
11715
|
-
|
|
11716
|
-
|
|
11717
|
-
|
|
11718
|
-
|
|
11719
|
-
|
|
11720
|
-
|
|
11721
|
-
|
|
11722
|
-
|
|
11723
|
-
|
|
11724
|
-
|
|
11725
|
-
|
|
11726
|
-
|
|
11727
|
-
|
|
11728
|
-
|
|
11729
|
-
|
|
11730
|
-
|
|
11731
|
-
|
|
11732
|
-
|
|
11733
|
-
|
|
11734
|
-
|
|
11735
|
-
|
|
11736
|
-
|
|
11737
|
-
|
|
11738
|
-
|
|
11739
|
-
|
|
11740
|
-
|
|
11741
|
-
|
|
11742
|
-
|
|
11743
|
-
|
|
11744
|
-
|
|
11745
|
-
|
|
11746
|
-
|
|
11747
|
-
|
|
11748
|
-
|
|
11749
|
-
|
|
11750
|
-
|
|
11751
|
-
|
|
11752
|
-
|
|
11753
|
-
|
|
11754
|
-
|
|
11755
|
-
|
|
11756
|
-
|
|
11757
|
-
|
|
11758
|
-
|
|
11759
|
-
|
|
11760
|
-
|
|
11761
|
-
|
|
11762
|
-
|
|
11763
|
-
|
|
11764
|
-
|
|
11765
|
-
|
|
11766
|
-
|
|
11767
|
-
|
|
11768
|
-
|
|
11769
|
-
|
|
11770
|
-
|
|
11771
|
-
|
|
11772
|
-
|
|
11773
|
-
|
|
11774
|
-
|
|
11775
|
-
|
|
11776
|
-
|
|
11777
|
-
|
|
11778
|
-
|
|
11779
|
-
|
|
11780
|
-
|
|
11781
|
-
|
|
11782
|
-
|
|
11783
|
-
|
|
11784
|
-
|
|
11785
|
-
|
|
11786
|
-
|
|
11787
|
-
|
|
11788
|
-
|
|
11789
|
-
|
|
11790
|
-
|
|
11791
|
-
})
|
|
11792
|
-
|
|
11793
|
-
|
|
11794
|
-
|
|
11795
|
-
|
|
11796
|
-
|
|
11797
|
-
|
|
11798
|
-
|
|
11799
|
-
|
|
11800
|
-
}
|
|
11801
|
-
|
|
11802
|
-
|
|
11803
|
-
}
|
|
11804
|
-
|
|
11805
|
-
|
|
11806
|
-
|
|
11807
|
-
|
|
11808
|
-
|
|
11809
|
-
|
|
11810
|
-
}
|
|
11811
|
-
|
|
11812
|
-
|
|
11813
|
-
|
|
11814
|
-
|
|
11815
|
-
|
|
11816
|
-
|
|
11817
|
-
|
|
11818
|
-
|
|
11819
|
-
|
|
11820
|
-
|
|
11821
|
-
|
|
11822
|
-
|
|
11823
|
-
|
|
11824
|
-
|
|
11825
|
-
|
|
11826
|
-
|
|
11827
|
-
|
|
11828
|
-
|
|
11829
|
-
|
|
11830
|
-
|
|
11831
|
-
|
|
11832
|
-
|
|
11833
|
-
|
|
11834
|
-
|
|
11835
|
-
|
|
11836
|
-
|
|
11837
|
-
|
|
11838
|
-
|
|
11839
|
-
|
|
11840
|
-
|
|
11841
|
-
|
|
11842
|
-
|
|
11843
|
-
|
|
11844
|
-
|
|
11845
|
-
|
|
11846
|
-
|
|
11847
|
-
|
|
11848
|
-
|
|
11849
|
-
|
|
11850
|
-
|
|
11851
|
-
|
|
11852
|
-
|
|
11853
|
-
|
|
11854
|
-
|
|
11855
|
-
|
|
11856
|
-
|
|
11857
|
-
|
|
11858
|
-
|
|
11859
|
-
|
|
11860
|
-
|
|
11861
|
-
|
|
11862
|
-
|
|
11863
|
-
|
|
11864
|
-
|
|
11865
|
-
|
|
11866
|
-
|
|
11867
|
-
|
|
11868
|
-
|
|
11869
|
-
|
|
11870
|
-
|
|
11871
|
-
|
|
11872
|
-
|
|
11873
|
-
|
|
11874
|
-
|
|
11875
|
-
}
|
|
11876
|
-
|
|
11877
|
-
|
|
11878
|
-
|
|
11879
|
-
|
|
11880
|
-
|
|
11881
|
-
|
|
11882
|
-
|
|
11883
|
-
|
|
11884
|
-
for (let i = 0; i < partitions.length; i++) {
|
|
11885
|
-
total += partitions[i].count();
|
|
11473
|
+
const averageFieldLengths = /* @__PURE__ */ Object.create(null);
|
|
11474
|
+
if (totalDocuments > 0) {
|
|
11475
|
+
for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
|
|
11476
|
+
averageFieldLengths[field] = totalLength / totalDocuments;
|
|
11477
|
+
}
|
|
11478
|
+
}
|
|
11479
|
+
return {
|
|
11480
|
+
totalDocuments,
|
|
11481
|
+
docFrequencies,
|
|
11482
|
+
totalFieldLengths,
|
|
11483
|
+
averageFieldLengths
|
|
11484
|
+
};
|
|
11485
|
+
}
|
|
11486
|
+
|
|
11487
|
+
// src/search/cursor-codec.ts
|
|
11488
|
+
function encodeCursorText(text) {
|
|
11489
|
+
if (typeof Buffer !== "undefined") return Buffer.from(text, "utf-8").toString("base64");
|
|
11490
|
+
const bytes = new TextEncoder().encode(text);
|
|
11491
|
+
let binary = "";
|
|
11492
|
+
for (const byte of bytes) binary += String.fromCharCode(byte);
|
|
11493
|
+
return btoa(binary);
|
|
11494
|
+
}
|
|
11495
|
+
function decodeCursorText(encoded) {
|
|
11496
|
+
if (typeof Buffer !== "undefined") return Buffer.from(encoded, "base64").toString("utf-8");
|
|
11497
|
+
const binary = atob(encoded);
|
|
11498
|
+
const bytes = new Uint8Array(binary.length);
|
|
11499
|
+
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
|
11500
|
+
return new TextDecoder().decode(bytes);
|
|
11501
|
+
}
|
|
11502
|
+
|
|
11503
|
+
// src/search/sorting.ts
|
|
11504
|
+
function isFieldList(sort) {
|
|
11505
|
+
return Array.isArray(sort);
|
|
11506
|
+
}
|
|
11507
|
+
function normalizeSort(sort) {
|
|
11508
|
+
if (sort === void 0) return [];
|
|
11509
|
+
if (isFieldList(sort)) {
|
|
11510
|
+
return sort.map(
|
|
11511
|
+
(entry) => entry.mode === void 0 ? { field: entry.field, direction: entry.direction } : { field: entry.field, direction: entry.direction, mode: entry.mode }
|
|
11512
|
+
);
|
|
11513
|
+
}
|
|
11514
|
+
return Object.entries(sort).map(([field, direction]) => ({ field, direction }));
|
|
11515
|
+
}
|
|
11516
|
+
function sortModesOf(fields) {
|
|
11517
|
+
return fields.map(sortModeOf);
|
|
11518
|
+
}
|
|
11519
|
+
function sortSignatureEntry(entry) {
|
|
11520
|
+
const mode = sortModeOf(entry);
|
|
11521
|
+
return mode === defaultSortMode(entry.direction) ? [entry.field, entry.direction] : [entry.field, entry.direction, mode];
|
|
11522
|
+
}
|
|
11523
|
+
function requireSortMode(entry, fieldType) {
|
|
11524
|
+
if (entry.mode === void 0) return;
|
|
11525
|
+
if (!isSortMode(entry.mode)) {
|
|
11526
|
+
throw new NarsilError(
|
|
11527
|
+
ErrorCodes.SEARCH_INVALID_MODE,
|
|
11528
|
+
`A sort mode is one of ${SORT_MODES.map((mode) => `"${mode}"`).join(", ")}, and the mode on "${entry.field}" is "${String(entry.mode)}"`,
|
|
11529
|
+
{ field: entry.field, mode: String(entry.mode) }
|
|
11530
|
+
);
|
|
11531
|
+
}
|
|
11532
|
+
const averages = entry.mode === "avg" || entry.mode === "median";
|
|
11533
|
+
if (averages && fieldType !== void 0 && fieldType !== "number" && fieldType !== "number[]") {
|
|
11534
|
+
throw new NarsilError(
|
|
11535
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
11536
|
+
`The engine applies the "${entry.mode}" mode to a number field alone, and "${entry.field}" is a ${fieldType} field`,
|
|
11537
|
+
{ field: entry.field, fieldType, mode: entry.mode }
|
|
11538
|
+
);
|
|
11539
|
+
}
|
|
11540
|
+
}
|
|
11541
|
+
function requireSortableFields(sort, schema, strict = false) {
|
|
11542
|
+
const fields = normalizeSort(sort);
|
|
11543
|
+
if (fields.length === 0) return;
|
|
11544
|
+
const flatSchema = flattenSchema(schema);
|
|
11545
|
+
for (const entry of fields) {
|
|
11546
|
+
const fieldType = flatSchema[entry.field];
|
|
11547
|
+
if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(entry.field, "sort");
|
|
11548
|
+
if (entry.direction !== "asc" && entry.direction !== "desc") {
|
|
11549
|
+
throw new NarsilError(
|
|
11550
|
+
ErrorCodes.SEARCH_INVALID_MODE,
|
|
11551
|
+
`A sort direction is "asc" or "desc", and the direction on "${entry.field}" is "${String(entry.direction)}"`,
|
|
11552
|
+
{ field: entry.field, direction: String(entry.direction) }
|
|
11553
|
+
);
|
|
11554
|
+
}
|
|
11555
|
+
requireSortMode(entry, fieldType);
|
|
11556
|
+
const parsed = parsedTypeOf(fieldType);
|
|
11557
|
+
if (parsed !== void 0 && isStringValuedBase(parsed.base) && !parsed.sortable) {
|
|
11558
|
+
throw new NarsilError(
|
|
11559
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
11560
|
+
`The engine sorts by text field "${entry.field}" only where its type includes "sortable", as in "${withSortableOption(parsed)}", because it keeps up to the first ${SORT_VALUE_MAX_CODE_POINTS} code points of every document's value in memory to sort by a text field`,
|
|
11561
|
+
{ field: entry.field, fieldType }
|
|
11562
|
+
);
|
|
11563
|
+
}
|
|
11564
|
+
if (isGeopointOrVectorType(fieldType)) {
|
|
11565
|
+
throw new NarsilError(
|
|
11566
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
11567
|
+
`The engine sorts by a number, boolean, enum, or sortable text field, and "${entry.field}" is a ${fieldType} field`,
|
|
11568
|
+
{ field: entry.field, fieldType }
|
|
11569
|
+
);
|
|
11570
|
+
}
|
|
11571
|
+
}
|
|
11572
|
+
}
|
|
11573
|
+
function applySorting(hits, sort, sortKeyOf) {
|
|
11574
|
+
const normalized = normalizeSort(sort);
|
|
11575
|
+
if (normalized.length === 0) return hits;
|
|
11576
|
+
const directions = normalized.map((entry) => entry.direction);
|
|
11577
|
+
const keyCache = /* @__PURE__ */ new Map();
|
|
11578
|
+
for (const hit of hits) {
|
|
11579
|
+
keyCache.set(hit.id, sortKeyOf(hit.id));
|
|
11580
|
+
}
|
|
11581
|
+
const sorted = hits.slice();
|
|
11582
|
+
sorted.sort(
|
|
11583
|
+
(a, b) => compareComparableKeys(keyCache.get(a.id) ?? [], keyCache.get(b.id) ?? [], directions) || compareCodePoints(a.id, b.id)
|
|
11584
|
+
);
|
|
11585
|
+
return sorted;
|
|
11586
|
+
}
|
|
11587
|
+
|
|
11588
|
+
// src/search/cursor.ts
|
|
11589
|
+
var CURSOR_VERSION = 4;
|
|
11590
|
+
function invalidCursor(cursor, reason) {
|
|
11591
|
+
const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
|
|
11592
|
+
return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
|
|
11593
|
+
}
|
|
11594
|
+
function sortSignatureOf(sort) {
|
|
11595
|
+
const fields = normalizeSort(sort);
|
|
11596
|
+
if (fields.length === 0) return null;
|
|
11597
|
+
if (fields.length > MAX_SORT_FIELDS) {
|
|
11598
|
+
throw new NarsilError(
|
|
11599
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
11600
|
+
`A sort names at most ${MAX_SORT_FIELDS} fields, because the paging cursor carries one value per field`,
|
|
11601
|
+
{ count: fields.length, limit: MAX_SORT_FIELDS }
|
|
11602
|
+
);
|
|
11603
|
+
}
|
|
11604
|
+
for (const entry of fields) {
|
|
11605
|
+
if (entry.field.length > MAX_SORT_FIELD_NAME_LENGTH) {
|
|
11606
|
+
throw new NarsilError(
|
|
11607
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
11608
|
+
`A sort field name holds at most ${MAX_SORT_FIELD_NAME_LENGTH} characters, because the paging cursor carries the sort with every page`,
|
|
11609
|
+
{ field: entry.field.slice(0, 64), length: entry.field.length, limit: MAX_SORT_FIELD_NAME_LENGTH }
|
|
11610
|
+
);
|
|
11611
|
+
}
|
|
11612
|
+
}
|
|
11613
|
+
return JSON.stringify(fields.map(sortSignatureEntry));
|
|
11614
|
+
}
|
|
11615
|
+
function encodePageCursor(cursor) {
|
|
11616
|
+
const payload = { v: CURSOR_VERSION, a: cursor.anchor };
|
|
11617
|
+
if (cursor.sortKey !== null && cursor.sortSignature !== null) {
|
|
11618
|
+
payload.k = cursor.sortKey.map(toComparableSortValue);
|
|
11619
|
+
payload.o = cursor.sortSignature;
|
|
11620
|
+
} else if (cursor.score !== null) {
|
|
11621
|
+
payload.s = cursor.score;
|
|
11622
|
+
}
|
|
11623
|
+
if (cursor.depth !== void 0 && cursor.depth > 0) payload.d = cursor.depth;
|
|
11624
|
+
payload.q = cursor.binding;
|
|
11625
|
+
return encodeCursorText(JSON.stringify(payload));
|
|
11626
|
+
}
|
|
11627
|
+
function decodeSortKey(value, cursor) {
|
|
11628
|
+
if (!Array.isArray(value)) throw invalidCursor(cursor, '"k" must be an array');
|
|
11629
|
+
if (value.length > MAX_SORT_FIELDS) throw invalidCursor(cursor, '"k" carries too many sort values');
|
|
11630
|
+
const key = [];
|
|
11631
|
+
for (const entry of value) {
|
|
11632
|
+
if (entry === null || typeof entry === "boolean") {
|
|
11633
|
+
key.push(entry);
|
|
11634
|
+
continue;
|
|
11635
|
+
}
|
|
11636
|
+
if (typeof entry === "number") {
|
|
11637
|
+
if (!Number.isFinite(entry)) throw invalidCursor(cursor, '"k" holds a number that is not finite');
|
|
11638
|
+
key.push(entry);
|
|
11639
|
+
continue;
|
|
11640
|
+
}
|
|
11641
|
+
if (typeof entry === "string") {
|
|
11642
|
+
if (exceedsCodePoints(entry, SORT_VALUE_MAX_CODE_POINTS)) {
|
|
11643
|
+
throw invalidCursor(cursor, '"k" holds an oversized sort value');
|
|
11886
11644
|
}
|
|
11887
|
-
|
|
11888
|
-
|
|
11889
|
-
|
|
11890
|
-
|
|
11891
|
-
|
|
11892
|
-
|
|
11893
|
-
|
|
11894
|
-
|
|
11895
|
-
|
|
11896
|
-
|
|
11897
|
-
|
|
11898
|
-
|
|
11899
|
-
|
|
11900
|
-
|
|
11901
|
-
|
|
11645
|
+
key.push(entry);
|
|
11646
|
+
continue;
|
|
11647
|
+
}
|
|
11648
|
+
throw invalidCursor(cursor, '"k" accepts a string, a finite number, a boolean, or null');
|
|
11649
|
+
}
|
|
11650
|
+
return key;
|
|
11651
|
+
}
|
|
11652
|
+
function decodePageCursor(cursor) {
|
|
11653
|
+
if (cursor.length > MAX_CURSOR_LENGTH) throw invalidCursor(cursor, "the cursor is too long");
|
|
11654
|
+
let json;
|
|
11655
|
+
try {
|
|
11656
|
+
json = decodeCursorText(cursor);
|
|
11657
|
+
} catch {
|
|
11658
|
+
throw invalidCursor(cursor, "invalid base64 encoding");
|
|
11659
|
+
}
|
|
11660
|
+
let parsed;
|
|
11661
|
+
try {
|
|
11662
|
+
parsed = JSON.parse(json);
|
|
11663
|
+
} catch {
|
|
11664
|
+
throw invalidCursor(cursor, "invalid JSON");
|
|
11665
|
+
}
|
|
11666
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
11667
|
+
throw invalidCursor(cursor, "expected an object");
|
|
11668
|
+
}
|
|
11669
|
+
const { v, a, s, k, o, d, q } = parsed;
|
|
11670
|
+
if (v !== CURSOR_VERSION) throw invalidCursor(cursor, `unsupported cursor version ${String(v)}`);
|
|
11671
|
+
if (typeof q !== "string" || !/^[0-9a-f]{8}$/.test(q)) {
|
|
11672
|
+
throw invalidCursor(cursor, '"q" must be 8 lowercase hex digits');
|
|
11673
|
+
}
|
|
11674
|
+
if (typeof a !== "string" || a.length === 0) throw invalidCursor(cursor, '"a" must be a non-empty string');
|
|
11675
|
+
if (exceedsCodePoints(a, MAX_CURSOR_ANCHOR_CODE_POINTS)) {
|
|
11676
|
+
throw invalidCursor(cursor, '"a" is too long to be a document id');
|
|
11677
|
+
}
|
|
11678
|
+
if (s !== void 0 && (typeof s !== "number" || !Number.isFinite(s))) {
|
|
11679
|
+
throw invalidCursor(cursor, '"s" must be a finite number');
|
|
11680
|
+
}
|
|
11681
|
+
if (s !== void 0 && k !== void 0) throw invalidCursor(cursor, 'a cursor carries "s" or "k", never both');
|
|
11682
|
+
if (d !== void 0 && (typeof d !== "number" || !Number.isSafeInteger(d) || d < 1)) {
|
|
11683
|
+
throw invalidCursor(cursor, '"d" must be a positive integer');
|
|
11684
|
+
}
|
|
11685
|
+
const sortKey = k === void 0 ? null : decodeSortKey(k, cursor);
|
|
11686
|
+
if (o !== void 0 && typeof o !== "string") throw invalidCursor(cursor, '"o" must be a string');
|
|
11687
|
+
if (sortKey === null !== (o === void 0)) {
|
|
11688
|
+
throw invalidCursor(cursor, '"k" and "o" have to arrive together');
|
|
11689
|
+
}
|
|
11690
|
+
return {
|
|
11691
|
+
anchor: a,
|
|
11692
|
+
score: typeof s === "number" ? s : null,
|
|
11693
|
+
sortKey,
|
|
11694
|
+
sortSignature: typeof o === "string" ? o : null,
|
|
11695
|
+
depth: typeof d === "number" ? d : 0,
|
|
11696
|
+
binding: q
|
|
11697
|
+
};
|
|
11698
|
+
}
|
|
11699
|
+
function requireMatchingCursor(cursor, encoded, signature, scoreAnchored, binding) {
|
|
11700
|
+
if (cursor.binding !== binding) {
|
|
11701
|
+
throw invalidCursor(encoded, "the cursor belongs to a different query");
|
|
11702
|
+
}
|
|
11703
|
+
if (signature !== null) {
|
|
11704
|
+
if (cursor.sortSignature !== signature) {
|
|
11705
|
+
throw invalidCursor(encoded, "the cursor belongs to a different sort order");
|
|
11706
|
+
}
|
|
11707
|
+
return;
|
|
11708
|
+
}
|
|
11709
|
+
if (cursor.sortSignature !== null) {
|
|
11710
|
+
throw invalidCursor(encoded, "the cursor belongs to a sorted request");
|
|
11711
|
+
}
|
|
11712
|
+
if (scoreAnchored && cursor.score === null) {
|
|
11713
|
+
throw invalidCursor(encoded, "the cursor carries no score to seek from");
|
|
11714
|
+
}
|
|
11715
|
+
if (!scoreAnchored && cursor.score !== null) {
|
|
11716
|
+
throw invalidCursor(encoded, "the cursor belongs to a search rather than a listing");
|
|
11717
|
+
}
|
|
11718
|
+
}
|
|
11719
|
+
|
|
11720
|
+
// src/search/pagination.ts
|
|
11721
|
+
function clampRowCount(value, fallback) {
|
|
11722
|
+
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
11723
|
+
return Math.max(0, Math.floor(value));
|
|
11724
|
+
}
|
|
11725
|
+
function requireWithinResultWindow(limit, offset) {
|
|
11726
|
+
const depth = offset + limit;
|
|
11727
|
+
if (depth <= RESULT_WINDOW) return;
|
|
11728
|
+
throw new NarsilError(
|
|
11729
|
+
ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
|
|
11730
|
+
`A request reaches the first ${RESULT_WINDOW} results, and offset + limit is ${depth}. Page past that with the cursor each result carries`,
|
|
11731
|
+
{ limit, offset, window: RESULT_WINDOW }
|
|
11732
|
+
);
|
|
11733
|
+
}
|
|
11734
|
+
function lastOrganicResult(sliced, pinnedIds) {
|
|
11735
|
+
for (let i = sliced.length - 1; i >= 0; i--) {
|
|
11736
|
+
if (pinnedIds === void 0 || !pinnedIds.has(sliced[i].id)) return sliced[i];
|
|
11737
|
+
}
|
|
11738
|
+
return void 0;
|
|
11739
|
+
}
|
|
11740
|
+
function ordersAfterAnchor(result, anchor, sort) {
|
|
11741
|
+
if (sort !== void 0 && anchor.sortKey !== null) {
|
|
11742
|
+
const comparison = compareSortValues(sort.sortKeyOf(result.id), anchor.sortKey, sort.directions) || compareCodePoints(result.id, anchor.anchor);
|
|
11743
|
+
return comparison > 0;
|
|
11744
|
+
}
|
|
11745
|
+
if (anchor.score === null) return true;
|
|
11746
|
+
if (result.score === void 0) return true;
|
|
11747
|
+
if (result.score < anchor.score) return true;
|
|
11748
|
+
return result.score === anchor.score && compareCodePoints(result.id, anchor.anchor) > 0;
|
|
11749
|
+
}
|
|
11750
|
+
function applyPagination(results, limit, offset, binding, cursor, sort, pinnedIds) {
|
|
11751
|
+
const decoded = cursor ? decodePageCursor(cursor) : null;
|
|
11752
|
+
if (decoded !== null && cursor !== void 0) {
|
|
11753
|
+
requireMatchingCursor(decoded, cursor, sort?.signature ?? null, true, binding);
|
|
11754
|
+
}
|
|
11755
|
+
if (limit <= 0) {
|
|
11756
|
+
return { paginated: [] };
|
|
11757
|
+
}
|
|
11758
|
+
let startIndex = 0;
|
|
11759
|
+
if (decoded !== null) {
|
|
11760
|
+
startIndex = results.length;
|
|
11761
|
+
for (let i = 0; i < results.length; i++) {
|
|
11762
|
+
if (ordersAfterAnchor(results[i], decoded, sort)) {
|
|
11763
|
+
startIndex = i;
|
|
11764
|
+
break;
|
|
11902
11765
|
}
|
|
11903
|
-
}
|
|
11904
|
-
|
|
11905
|
-
|
|
11906
|
-
|
|
11907
|
-
|
|
11908
|
-
|
|
11909
|
-
|
|
11910
|
-
|
|
11911
|
-
|
|
11912
|
-
|
|
11913
|
-
|
|
11914
|
-
|
|
11915
|
-
|
|
11916
|
-
|
|
11766
|
+
}
|
|
11767
|
+
}
|
|
11768
|
+
const afterOffset = startIndex + offset;
|
|
11769
|
+
const sliced = results.slice(afterOffset, afterOffset + limit);
|
|
11770
|
+
let nextCursor;
|
|
11771
|
+
const hasMore = afterOffset + limit < results.length;
|
|
11772
|
+
const lastResult = lastOrganicResult(sliced, pinnedIds);
|
|
11773
|
+
const depth = afterOffset + sliced.length;
|
|
11774
|
+
if (hasMore && lastResult !== void 0) {
|
|
11775
|
+
nextCursor = sort !== void 0 ? encodePageCursor({
|
|
11776
|
+
anchor: lastResult.id,
|
|
11777
|
+
score: null,
|
|
11778
|
+
sortKey: sort.sortKeyOf(lastResult.id).map(toComparableSortValue),
|
|
11779
|
+
sortSignature: sort.signature,
|
|
11780
|
+
depth,
|
|
11781
|
+
binding
|
|
11782
|
+
}) : encodePageCursor({
|
|
11783
|
+
anchor: lastResult.id,
|
|
11784
|
+
score: lastResult.score ?? null,
|
|
11785
|
+
sortKey: null,
|
|
11786
|
+
sortSignature: null,
|
|
11787
|
+
depth,
|
|
11788
|
+
binding
|
|
11789
|
+
});
|
|
11790
|
+
}
|
|
11791
|
+
return { paginated: sliced, nextCursor };
|
|
11792
|
+
}
|
|
11793
|
+
|
|
11794
|
+
// src/search/fulltext.ts
|
|
11795
|
+
function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
11796
|
+
if (!params.term || params.term.trim().length === 0) return null;
|
|
11797
|
+
if (params.fields && params.fields.length === 0) return null;
|
|
11798
|
+
const flatSchema = flattenSchema(schema);
|
|
11799
|
+
if (params.fields) {
|
|
11800
|
+
validateSearchFields(params.fields, flatSchema);
|
|
11801
|
+
}
|
|
11802
|
+
const queryTokenResult = tokenize(params.term, language, {
|
|
11803
|
+
stem: true,
|
|
11804
|
+
removeStopWords: true,
|
|
11805
|
+
customTokenizer: options?.customTokenizer,
|
|
11806
|
+
stopWordOverride: options?.stopWords
|
|
11807
|
+
});
|
|
11808
|
+
if (queryTokenResult.tokens.length === 0) return null;
|
|
11809
|
+
const queryTokens = deduplicateTokens(queryTokenResult.tokens);
|
|
11810
|
+
let filterBitset;
|
|
11811
|
+
if (params.filters) {
|
|
11812
|
+
filterBitset = partition.applyFiltersBitset(
|
|
11813
|
+
params.filters,
|
|
11814
|
+
schema,
|
|
11815
|
+
options?.patternWork ?? createPatternWorkMeter()
|
|
11816
|
+
);
|
|
11817
|
+
if (bitsetIsEmpty(filterBitset)) return null;
|
|
11818
|
+
}
|
|
11819
|
+
const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
|
|
11820
|
+
const synonyms = params.exact === true ? void 0 : options?.synonyms?.expand(queryTokens.map((qt) => qt.token));
|
|
11821
|
+
const groups = resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, options, synonyms);
|
|
11822
|
+
return { queryTokens, ...groups, filterBitset };
|
|
11823
|
+
}
|
|
11824
|
+
function fulltextMatches(partition, params, language, schema, options) {
|
|
11825
|
+
const prepared = prepareFulltextQuery(partition, params, language, schema, options);
|
|
11826
|
+
if (prepared === null) return null;
|
|
11827
|
+
return partition.searchFulltextMatches({
|
|
11828
|
+
queryTokens: prepared.queryTokens,
|
|
11829
|
+
termGroups: prepared.termGroups,
|
|
11830
|
+
partialFields: prepared.partialFields,
|
|
11831
|
+
fields: params.fields,
|
|
11832
|
+
tolerance: params.tolerance ?? 0,
|
|
11833
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
11834
|
+
exact: params.exact ?? false,
|
|
11835
|
+
filterBitset: prepared.filterBitset
|
|
11836
|
+
});
|
|
11837
|
+
}
|
|
11838
|
+
function fulltextSearch(partition, params, language, schema, options) {
|
|
11839
|
+
const prepared = prepareFulltextQuery(partition, params, language, schema, options);
|
|
11840
|
+
if (prepared === null) {
|
|
11841
|
+
return { scored: [], totalMatched: 0 };
|
|
11842
|
+
}
|
|
11843
|
+
const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
|
|
11844
|
+
const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
11845
|
+
const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
|
|
11846
|
+
const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
|
|
11847
|
+
const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
|
|
11848
|
+
const rawResult = partition.searchFulltext({
|
|
11849
|
+
queryTokens,
|
|
11850
|
+
termGroups,
|
|
11851
|
+
partialFields,
|
|
11852
|
+
fields: params.fields,
|
|
11853
|
+
boost: params.boost,
|
|
11854
|
+
tolerance: params.tolerance ?? 0,
|
|
11855
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
11856
|
+
exact: params.exact ?? false,
|
|
11857
|
+
bm25Params: options?.bm25Params,
|
|
11858
|
+
globalStats: options?.globalStats,
|
|
11859
|
+
maxResults,
|
|
11860
|
+
termMatch: params.termMatch,
|
|
11861
|
+
filterBitset,
|
|
11862
|
+
collectComponents,
|
|
11863
|
+
collectMatchedSet
|
|
11864
|
+
});
|
|
11865
|
+
let scored = rawResult.scored;
|
|
11866
|
+
const termMatch = params.termMatch ?? "any";
|
|
11867
|
+
if (termMatch !== "any") {
|
|
11868
|
+
scored = filterByTermCoverage(
|
|
11869
|
+
scored,
|
|
11870
|
+
queryTokens,
|
|
11871
|
+
termMatch,
|
|
11872
|
+
params.tolerance ?? 0,
|
|
11873
|
+
params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
11874
|
+
params.exact ?? false,
|
|
11875
|
+
termGroups,
|
|
11876
|
+
partialFields ?? []
|
|
11877
|
+
);
|
|
11878
|
+
}
|
|
11879
|
+
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
11880
|
+
const threshold = params.minScore;
|
|
11881
|
+
scored = scored.filter((doc) => doc.score >= threshold);
|
|
11882
|
+
}
|
|
11883
|
+
const totalMatched = needsAllResults ? scored.length : rawResult.totalMatched;
|
|
11884
|
+
if (params.facets === void 0) {
|
|
11885
|
+
return { scored, totalMatched };
|
|
11886
|
+
}
|
|
11887
|
+
if (needsAllResults) {
|
|
11888
|
+
return { scored, totalMatched, matchedIds: scored.map((doc) => doc.docId) };
|
|
11889
|
+
}
|
|
11890
|
+
return { scored, totalMatched, matchedOrdinalBitset: rawResult.matchedOrdinalBitset };
|
|
11891
|
+
}
|
|
11892
|
+
function validateSearchFields(fields, flatSchema) {
|
|
11893
|
+
for (const field of fields) {
|
|
11894
|
+
const fieldType = flatSchema[field];
|
|
11895
|
+
if (!fieldType) {
|
|
11896
|
+
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FIELD, `Field "${field}" does not exist in the schema`, { field });
|
|
11897
|
+
}
|
|
11898
|
+
if (!isWordIndexedFieldType(fieldType)) {
|
|
11899
|
+
throw new NarsilError(
|
|
11900
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
11901
|
+
`Field "${field}" has type "${fieldType}" which cannot be used for full-text search`,
|
|
11902
|
+
{ field, fieldType }
|
|
11917
11903
|
);
|
|
11918
|
-
|
|
11919
|
-
|
|
11920
|
-
|
|
11921
|
-
|
|
11922
|
-
|
|
11923
|
-
|
|
11924
|
-
|
|
11925
|
-
|
|
11926
|
-
|
|
11927
|
-
|
|
11928
|
-
|
|
11929
|
-
|
|
11930
|
-
|
|
11931
|
-
|
|
11932
|
-
|
|
11933
|
-
|
|
11934
|
-
|
|
11935
|
-
|
|
11936
|
-
|
|
11937
|
-
|
|
11938
|
-
|
|
11939
|
-
|
|
11940
|
-
|
|
11941
|
-
|
|
11942
|
-
|
|
11943
|
-
|
|
11944
|
-
|
|
11945
|
-
|
|
11946
|
-
|
|
11947
|
-
|
|
11948
|
-
|
|
11949
|
-
|
|
11950
|
-
|
|
11951
|
-
|
|
11952
|
-
|
|
11904
|
+
}
|
|
11905
|
+
}
|
|
11906
|
+
}
|
|
11907
|
+
function deduplicateTokens(tokens) {
|
|
11908
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11909
|
+
const result = [];
|
|
11910
|
+
for (const t of tokens) {
|
|
11911
|
+
if (!seen.has(t.token)) {
|
|
11912
|
+
seen.add(t.token);
|
|
11913
|
+
result.push(t);
|
|
11914
|
+
}
|
|
11915
|
+
}
|
|
11916
|
+
return result;
|
|
11917
|
+
}
|
|
11918
|
+
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
|
|
11919
|
+
const requiredCount = policy === "all" ? queryTokens.length : policy;
|
|
11920
|
+
if (requiredCount <= 0) return scored;
|
|
11921
|
+
if (requiredCount > queryTokens.length) return [];
|
|
11922
|
+
const groups = /* @__PURE__ */ new Map();
|
|
11923
|
+
for (const group of termGroups ?? []) {
|
|
11924
|
+
const partialOnly = /* @__PURE__ */ new Set();
|
|
11925
|
+
for (const member of group.members) {
|
|
11926
|
+
if (member.factor === 0) partialOnly.add(member.token);
|
|
11927
|
+
}
|
|
11928
|
+
groups.set(group.token, {
|
|
11929
|
+
members: /* @__PURE__ */ new Set([...group.members.map((member) => member.token), ...group.synonyms?.tokens ?? []]),
|
|
11930
|
+
partialOnly,
|
|
11931
|
+
typoMatches: group.typoMatches
|
|
11932
|
+
});
|
|
11933
|
+
}
|
|
11934
|
+
return scored.filter((doc) => {
|
|
11935
|
+
const matched = countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields);
|
|
11936
|
+
return matched >= requiredCount;
|
|
11937
|
+
});
|
|
11938
|
+
}
|
|
11939
|
+
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
|
|
11940
|
+
const indexTokens = Object.keys(doc.idf);
|
|
11941
|
+
if (indexTokens.length === 0) return 0;
|
|
11942
|
+
let count = 0;
|
|
11943
|
+
for (const qt of queryTokens) {
|
|
11944
|
+
const group = groups.get(qt.token);
|
|
11945
|
+
const satisfied = group === void 0 ? queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact) : groupSatisfied(group, doc, indexTokens, partialFields) || group.typoMatches && queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact);
|
|
11946
|
+
if (satisfied) count++;
|
|
11947
|
+
}
|
|
11948
|
+
return count;
|
|
11949
|
+
}
|
|
11950
|
+
function matchedInPartialField(doc, token, partialFields) {
|
|
11951
|
+
for (const field of partialFields) {
|
|
11952
|
+
if (doc.termFrequencies[`${field}:${token}`] !== void 0) return true;
|
|
11953
|
+
}
|
|
11954
|
+
return false;
|
|
11955
|
+
}
|
|
11956
|
+
function groupSatisfied(group, doc, indexTokens, partialFields) {
|
|
11957
|
+
for (const indexToken of indexTokens) {
|
|
11958
|
+
if (!group.members.has(indexToken)) continue;
|
|
11959
|
+
if (!group.partialOnly.has(indexToken) || matchedInPartialField(doc, indexToken, partialFields)) return true;
|
|
11960
|
+
}
|
|
11961
|
+
return false;
|
|
11962
|
+
}
|
|
11963
|
+
function queryTermSatisfied(queryToken, indexTokens, tolerance, prefixLength, exact) {
|
|
11964
|
+
if (exact || tolerance === 0) {
|
|
11965
|
+
return indexTokens.includes(queryToken);
|
|
11966
|
+
}
|
|
11967
|
+
for (const indexToken of indexTokens) {
|
|
11968
|
+
if (fuzzyTermMatches(queryToken, indexToken, tolerance, prefixLength)) {
|
|
11969
|
+
return true;
|
|
11970
|
+
}
|
|
11971
|
+
}
|
|
11972
|
+
return false;
|
|
11973
|
+
}
|
|
11974
|
+
|
|
11975
|
+
// src/partitioning/fan-out.ts
|
|
11976
|
+
async function fanOutQuery(manager, params, language, schema, config, searchOptions) {
|
|
11977
|
+
const partitions = partitionsIn(manager, config.partitionIds);
|
|
11978
|
+
if (partitions.length === 0) {
|
|
11979
|
+
return { scored: [], totalMatched: 0 };
|
|
11980
|
+
}
|
|
11981
|
+
let globalStats;
|
|
11982
|
+
const effectiveMode = resolveEffectiveMode(config);
|
|
11983
|
+
if (effectiveMode === "dfs") {
|
|
11984
|
+
if (params.term !== void 0 && params.term.trim().length > 0) {
|
|
11985
|
+
const analysis = {
|
|
11986
|
+
stopWords: searchOptions?.stopWords,
|
|
11987
|
+
customTokenizer: searchOptions?.customTokenizer,
|
|
11988
|
+
synonyms: searchOptions?.synonyms
|
|
11989
|
+
};
|
|
11990
|
+
globalStats = collectQueryTermStats(manager, params.term, language, analysis, config.partitionIds, {
|
|
11991
|
+
expansion: { options: params, schema }
|
|
11992
|
+
});
|
|
11993
|
+
}
|
|
11994
|
+
} else if (effectiveMode === "broadcast") {
|
|
11995
|
+
globalStats = config.globalStats;
|
|
11996
|
+
}
|
|
11997
|
+
const options = buildSearchOptions(searchOptions, globalStats);
|
|
11998
|
+
if (partitions.length === 1 && !config.dispatcher) {
|
|
11999
|
+
const result = dispatchSinglePartition(partitions[0], params, language, schema, options);
|
|
12000
|
+
let facets2;
|
|
12001
|
+
if (params.facets) {
|
|
12002
|
+
facets2 = partitions[0].computeFacets(facetMatchSetOf(result), params.facets, schema);
|
|
12003
|
+
}
|
|
12004
|
+
return { scored: result.scored, totalMatched: result.totalMatched, facets: facets2 };
|
|
12005
|
+
}
|
|
12006
|
+
let outcomes;
|
|
12007
|
+
let facets;
|
|
12008
|
+
if (config.dispatcher) {
|
|
12009
|
+
outcomes = await dispatchWithDispatcher(partitions, params, language, schema, options, config.dispatcher);
|
|
12010
|
+
if (params.facets) {
|
|
12011
|
+
facets = collectAndMergeFacets(outcomes, params.facets, schema);
|
|
12012
|
+
}
|
|
12013
|
+
} else {
|
|
12014
|
+
outcomes = [];
|
|
12015
|
+
const partitionFacets = [];
|
|
12016
|
+
const everyValue = params.facets ? everyValueFacetConfig(params.facets) : void 0;
|
|
12017
|
+
for (const partition of partitions) {
|
|
12018
|
+
const result = dispatchSinglePartition(partition, params, language, schema, options);
|
|
12019
|
+
if (everyValue) {
|
|
12020
|
+
partitionFacets.push(partition.computeFacets(facetMatchSetOf(result), everyValue, schema));
|
|
11953
12021
|
}
|
|
12022
|
+
outcomes.push({ result, partition });
|
|
11954
12023
|
}
|
|
12024
|
+
if (params.facets) {
|
|
12025
|
+
facets = mergeFacets(partitionFacets, params.facets);
|
|
12026
|
+
}
|
|
12027
|
+
}
|
|
12028
|
+
const allScoredArrays = outcomes.map((o) => o.result.scored);
|
|
12029
|
+
const merged = kWayMerge(allScoredArrays);
|
|
12030
|
+
let totalMatched = 0;
|
|
12031
|
+
for (const outcome of outcomes) {
|
|
12032
|
+
totalMatched += outcome.result.totalMatched;
|
|
12033
|
+
}
|
|
12034
|
+
return { scored: merged, totalMatched, facets };
|
|
12035
|
+
}
|
|
12036
|
+
function facetMatchSetOf(result) {
|
|
12037
|
+
if (result.matchedOrdinalBitset !== void 0) {
|
|
12038
|
+
return { ordinalBitset: result.matchedOrdinalBitset };
|
|
12039
|
+
}
|
|
12040
|
+
return new Set(result.matchedIds ?? []);
|
|
12041
|
+
}
|
|
12042
|
+
function resolveEffectiveMode(config) {
|
|
12043
|
+
if (config.scoringMode === "broadcast" && !config.globalStats) {
|
|
12044
|
+
return "dfs";
|
|
12045
|
+
}
|
|
12046
|
+
return config.scoringMode;
|
|
12047
|
+
}
|
|
12048
|
+
function buildSearchOptions(base, globalStats) {
|
|
12049
|
+
return {
|
|
12050
|
+
...base,
|
|
12051
|
+
globalStats: globalStats ?? base?.globalStats,
|
|
12052
|
+
patternWork: base?.patternWork ?? createPatternWorkMeter()
|
|
11955
12053
|
};
|
|
11956
|
-
|
|
12054
|
+
}
|
|
12055
|
+
function dispatchWithDispatcher(partitions, params, language, schema, options, dispatcher) {
|
|
12056
|
+
const promises = partitions.map(
|
|
12057
|
+
(partition) => dispatcher.dispatch(partition, params, language, schema, options).then((result) => ({ result, partition }))
|
|
12058
|
+
);
|
|
12059
|
+
return Promise.all(promises);
|
|
12060
|
+
}
|
|
12061
|
+
function dispatchSinglePartition(partition, params, language, schema, options) {
|
|
12062
|
+
return fulltextSearch(partition, params, language, schema, options);
|
|
12063
|
+
}
|
|
12064
|
+
function collectAndMergeFacets(outcomes, facetConfig, schema) {
|
|
12065
|
+
const everyValue = everyValueFacetConfig(facetConfig);
|
|
12066
|
+
const partitionFacets = [];
|
|
12067
|
+
for (const outcome of outcomes) {
|
|
12068
|
+
partitionFacets.push(outcome.partition.computeFacets(facetMatchSetOf(outcome.result), everyValue, schema));
|
|
12069
|
+
}
|
|
12070
|
+
return mergeFacets(partitionFacets, facetConfig);
|
|
11957
12071
|
}
|
|
11958
12072
|
|
|
11959
12073
|
// src/partitioning/match-count.ts
|
|
@@ -11973,6 +12087,43 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
11973
12087
|
return count;
|
|
11974
12088
|
}
|
|
11975
12089
|
|
|
12090
|
+
// src/workers/query-actions.ts
|
|
12091
|
+
async function runQueryAction(entry, action) {
|
|
12092
|
+
const patternWork = joinPatternWorkMeter(action.patternWork);
|
|
12093
|
+
const searchOptions = { ...entry.searchOptions, patternWork };
|
|
12094
|
+
if (action.type === "query") {
|
|
12095
|
+
const result2 = await fanOutQuery(
|
|
12096
|
+
entry.manager,
|
|
12097
|
+
action.params,
|
|
12098
|
+
entry.language,
|
|
12099
|
+
entry.config.schema,
|
|
12100
|
+
{
|
|
12101
|
+
scoringMode: action.params.scoring ?? entry.config.defaultScoring ?? "local",
|
|
12102
|
+
partitionIds: action.partitionIds,
|
|
12103
|
+
...action.globalStats !== void 0 ? { globalStats: sanitizeGlobalStats(action.globalStats) } : {}
|
|
12104
|
+
},
|
|
12105
|
+
searchOptions
|
|
12106
|
+
);
|
|
12107
|
+
result2.patternWorkSpent = patternWork.spent();
|
|
12108
|
+
return result2;
|
|
12109
|
+
}
|
|
12110
|
+
if (countsWithoutScores(action.params)) {
|
|
12111
|
+
const count = fanOutMatchCount(entry.manager, action.params, entry.language, entry.config.schema, {
|
|
12112
|
+
searchOptions
|
|
12113
|
+
});
|
|
12114
|
+
return { count };
|
|
12115
|
+
}
|
|
12116
|
+
const result = await fanOutQuery(
|
|
12117
|
+
entry.manager,
|
|
12118
|
+
action.params,
|
|
12119
|
+
entry.language,
|
|
12120
|
+
entry.config.schema,
|
|
12121
|
+
{ scoringMode: entry.config.defaultScoring ?? "local" },
|
|
12122
|
+
searchOptions
|
|
12123
|
+
);
|
|
12124
|
+
return { count: result.totalMatched };
|
|
12125
|
+
}
|
|
12126
|
+
|
|
11976
12127
|
// src/core/partition/frozen/doc-table.ts
|
|
11977
12128
|
function createFrozenDocTable(idTable, lengthColumns, documents, tombstones) {
|
|
11978
12129
|
const capacity = idTable.count;
|
|
@@ -14835,8 +14986,11 @@ function createDirectExecutor(options) {
|
|
|
14835
14986
|
}
|
|
14836
14987
|
return vectorIndexes;
|
|
14837
14988
|
}
|
|
14838
|
-
function createIndex(indexName, config, language, analysisStale = false) {
|
|
14839
|
-
addIndex(indexName, config, language, analysisStale, null);
|
|
14989
|
+
function createIndex(indexName, config, language, analysisStale = false, synonyms = null) {
|
|
14990
|
+
addIndex(indexName, config, language, analysisStale, null).synonyms = synonyms;
|
|
14991
|
+
}
|
|
14992
|
+
function lookupOf(config, language, encoded) {
|
|
14993
|
+
return encoded === void 0 ? null : synonymModule().storedLookup(encoded, config, language);
|
|
14840
14994
|
}
|
|
14841
14995
|
function addIndex(indexName, config, language, analysisStale, awaitedCopyLoad) {
|
|
14842
14996
|
if (indexes.has(indexName)) {
|
|
@@ -14855,13 +15009,17 @@ function createDirectExecutor(options) {
|
|
|
14855
15009
|
searchOptions: {
|
|
14856
15010
|
bm25Params: config.bm25,
|
|
14857
15011
|
stopWords: manager.analysis.stopWords,
|
|
14858
|
-
customTokenizer: manager.analysis.customTokenizer
|
|
15012
|
+
customTokenizer: manager.analysis.customTokenizer,
|
|
15013
|
+
get synonyms() {
|
|
15014
|
+
return manager.synonyms;
|
|
15015
|
+
}
|
|
14859
15016
|
},
|
|
14860
15017
|
vectorIndexes,
|
|
14861
15018
|
vectorCopies: createHeldVectorCopies(),
|
|
14862
15019
|
analysisStale,
|
|
14863
15020
|
awaitedCopyLoad
|
|
14864
15021
|
});
|
|
15022
|
+
return manager;
|
|
14865
15023
|
}
|
|
14866
15024
|
function dropIndex(indexName) {
|
|
14867
15025
|
const entry = requireIndex(indexName);
|
|
@@ -14914,7 +15072,19 @@ function createDirectExecutor(options) {
|
|
|
14914
15072
|
const loading2 = loadVectorCodeFor(action.config.schema);
|
|
14915
15073
|
if (loading2 !== null) await loading2;
|
|
14916
15074
|
const language = getLanguage(action.config.language ?? "english");
|
|
14917
|
-
|
|
15075
|
+
const manager = addIndex(
|
|
15076
|
+
action.indexName,
|
|
15077
|
+
action.config,
|
|
15078
|
+
language,
|
|
15079
|
+
action.analysisStale ?? false,
|
|
15080
|
+
action.copyLoad ?? null
|
|
15081
|
+
);
|
|
15082
|
+
try {
|
|
15083
|
+
manager.synonyms = lookupOf(action.config, language, action.synonyms);
|
|
15084
|
+
} catch (err) {
|
|
15085
|
+
dropIndex(action.indexName);
|
|
15086
|
+
throw err;
|
|
15087
|
+
}
|
|
14918
15088
|
return void 0;
|
|
14919
15089
|
}
|
|
14920
15090
|
case "copyLoaded": {
|
|
@@ -14963,42 +15133,13 @@ function createDirectExecutor(options) {
|
|
|
14963
15133
|
entry.manager.update(action.docId, action.document, void 0, action.removedSpellings);
|
|
14964
15134
|
return void 0;
|
|
14965
15135
|
}
|
|
14966
|
-
case "query":
|
|
14967
|
-
|
|
14968
|
-
|
|
14969
|
-
|
|
14970
|
-
|
|
14971
|
-
|
|
14972
|
-
|
|
14973
|
-
entry.config.schema,
|
|
14974
|
-
{
|
|
14975
|
-
scoringMode: action.params.scoring ?? entry.config.defaultScoring ?? "local",
|
|
14976
|
-
partitionIds: action.partitionIds,
|
|
14977
|
-
...action.globalStats !== void 0 ? { globalStats: sanitizeGlobalStats(action.globalStats) } : {}
|
|
14978
|
-
},
|
|
14979
|
-
{ ...entry.searchOptions, patternWork }
|
|
14980
|
-
);
|
|
14981
|
-
result.patternWorkSpent = patternWork.spent();
|
|
14982
|
-
return result;
|
|
14983
|
-
}
|
|
14984
|
-
case "preflight": {
|
|
14985
|
-
const entry = requireIndex(action.indexName);
|
|
14986
|
-
const searchOptions = { ...entry.searchOptions, patternWork: joinPatternWorkMeter(action.patternWork) };
|
|
14987
|
-
if (countsWithoutScores(action.params)) {
|
|
14988
|
-
const count = fanOutMatchCount(entry.manager, action.params, entry.language, entry.config.schema, {
|
|
14989
|
-
searchOptions
|
|
14990
|
-
});
|
|
14991
|
-
return { count };
|
|
14992
|
-
}
|
|
14993
|
-
const result = await fanOutQuery(
|
|
14994
|
-
entry.manager,
|
|
14995
|
-
action.params,
|
|
14996
|
-
entry.language,
|
|
14997
|
-
entry.config.schema,
|
|
14998
|
-
{ scoringMode: entry.config.defaultScoring ?? "local" },
|
|
14999
|
-
searchOptions
|
|
15000
|
-
);
|
|
15001
|
-
return { count: result.totalMatched };
|
|
15136
|
+
case "query":
|
|
15137
|
+
case "preflight":
|
|
15138
|
+
return await runQueryAction(requireIndex(action.indexName), action);
|
|
15139
|
+
case "replaceSynonyms": {
|
|
15140
|
+
const entry = indexes.get(action.indexName);
|
|
15141
|
+
if (entry !== void 0) entry.manager.synonyms = lookupOf(entry.config, entry.language, action.synonyms);
|
|
15142
|
+
return void 0;
|
|
15002
15143
|
}
|
|
15003
15144
|
case "get": {
|
|
15004
15145
|
const entry = requireIndex(action.indexName);
|
|
@@ -15061,7 +15202,7 @@ function createDirectExecutor(options) {
|
|
|
15061
15202
|
}
|
|
15062
15203
|
}
|
|
15063
15204
|
async function shutdown() {
|
|
15064
|
-
await execute({ type: "shutdown"});
|
|
15205
|
+
await execute({ type: "shutdown", requestId: "internal-shutdown" });
|
|
15065
15206
|
}
|
|
15066
15207
|
return {
|
|
15067
15208
|
execute,
|
|
@@ -15483,6 +15624,25 @@ function reconstructSchemaFromMetadata(metadata) {
|
|
|
15483
15624
|
return config;
|
|
15484
15625
|
}
|
|
15485
15626
|
|
|
15627
|
+
// src/engine/synonym-sets.ts
|
|
15628
|
+
function withoutSynonyms(config) {
|
|
15629
|
+
if (!Object.hasOwn(config, "synonyms")) return config;
|
|
15630
|
+
const { synonyms: _synonyms, ...rest } = config;
|
|
15631
|
+
return rest;
|
|
15632
|
+
}
|
|
15633
|
+
function prepareStoredSynonyms(value, config, language, raise, strictWords = false) {
|
|
15634
|
+
return isEmptySynonymSet(value) ? null : synonymModule().prepare(value, config, language, strictWords, raise);
|
|
15635
|
+
}
|
|
15636
|
+
function prepareConfiguredSynonyms(config, language, strictWords = true) {
|
|
15637
|
+
return prepareStoredSynonyms(config.synonyms, config, language, void 0, strictWords);
|
|
15638
|
+
}
|
|
15639
|
+
function lookupOfStoredSynonyms(encoded, config, language) {
|
|
15640
|
+
return encoded === null || encoded === void 0 ? null : synonymModule().storedLookup(encoded, config, language);
|
|
15641
|
+
}
|
|
15642
|
+
function rulesOfStoredSynonyms(encoded) {
|
|
15643
|
+
return encoded === null ? [] : synonymModule().decodeRules(encoded);
|
|
15644
|
+
}
|
|
15645
|
+
|
|
15486
15646
|
// src/engine/validation.ts
|
|
15487
15647
|
function now() {
|
|
15488
15648
|
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
@@ -15680,7 +15840,13 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
15680
15840
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
15681
15841
|
if (existing !== void 0) {
|
|
15682
15842
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
15683
|
-
deps.executor.createIndex(
|
|
15843
|
+
deps.executor.createIndex(
|
|
15844
|
+
metadata.indexName,
|
|
15845
|
+
existing.config,
|
|
15846
|
+
existing.language,
|
|
15847
|
+
false,
|
|
15848
|
+
lookupOfStoredSynonyms(existing.synonyms, existing.config, existing.language)
|
|
15849
|
+
);
|
|
15684
15850
|
}
|
|
15685
15851
|
return;
|
|
15686
15852
|
}
|
|
@@ -15694,9 +15860,18 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
15694
15860
|
throw recoveryFailure(metadata, err);
|
|
15695
15861
|
}
|
|
15696
15862
|
const embedding = resolveEmbeddingAdapter(deps, metadata, indexConfig.schema);
|
|
15863
|
+
const prepared = prepareStoredSynonyms(
|
|
15864
|
+
metadata.synonyms,
|
|
15865
|
+
indexConfig,
|
|
15866
|
+
language,
|
|
15867
|
+
"The synonym set in the index metadata is invalid: "
|
|
15868
|
+
);
|
|
15869
|
+
const synonyms = prepared?.encoded ?? null;
|
|
15697
15870
|
try {
|
|
15698
15871
|
validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
|
|
15699
|
-
if (loadData)
|
|
15872
|
+
if (loadData) {
|
|
15873
|
+
deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
|
|
15874
|
+
}
|
|
15700
15875
|
} catch (err) {
|
|
15701
15876
|
throw recoveryFailure(metadata, err);
|
|
15702
15877
|
}
|
|
@@ -15710,7 +15885,8 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
15710
15885
|
heldPartitions: metadata.heldPartitions ?? null,
|
|
15711
15886
|
documentCount: metadata.documentCount ?? 0,
|
|
15712
15887
|
partitionCount: metadata.partitionCount,
|
|
15713
|
-
analysisRevision: metadata.analysisRevision ?? null
|
|
15888
|
+
analysisRevision: metadata.analysisRevision ?? null,
|
|
15889
|
+
synonyms
|
|
15714
15890
|
});
|
|
15715
15891
|
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
15716
15892
|
else deps.indexState().registerClosed(metadata.indexName);
|
|
@@ -16536,17 +16712,17 @@ async function sendPartition(manager, executors, captured) {
|
|
|
16536
16712
|
)
|
|
16537
16713
|
);
|
|
16538
16714
|
}
|
|
16539
|
-
function transferIndexToPool(indexName, pool,
|
|
16715
|
+
function transferIndexToPool(indexName, pool, source, manager, analysisStale = false) {
|
|
16540
16716
|
return loadIndexOntoWorkers(
|
|
16541
16717
|
indexName,
|
|
16542
16718
|
pool.getAllExecutors(),
|
|
16543
|
-
|
|
16719
|
+
source,
|
|
16544
16720
|
manager,
|
|
16545
16721
|
() => pool.addIndexToAll(indexName),
|
|
16546
16722
|
analysisStale
|
|
16547
16723
|
);
|
|
16548
16724
|
}
|
|
16549
|
-
async function loadIndexOntoWorkers(indexName, executors,
|
|
16725
|
+
async function loadIndexOntoWorkers(indexName, executors, source, manager, onDropped, analysisStale = false) {
|
|
16550
16726
|
copyLoadSequence += 1;
|
|
16551
16727
|
const copyLoad = `${indexName}#${copyLoadSequence}`;
|
|
16552
16728
|
const captured = capturePartitions(manager);
|
|
@@ -16561,10 +16737,11 @@ async function loadIndexOntoWorkers(indexName, executors, config, manager, onDro
|
|
|
16561
16737
|
(workerExecutor) => workerExecutor.execute({
|
|
16562
16738
|
type: "createIndex",
|
|
16563
16739
|
indexName,
|
|
16564
|
-
config,
|
|
16740
|
+
config: source.config,
|
|
16565
16741
|
requestId: `resync-create-${indexName}`,
|
|
16566
16742
|
analysisStale,
|
|
16567
|
-
copyLoad
|
|
16743
|
+
copyLoad,
|
|
16744
|
+
...source.synonyms !== null ? { synonyms: source.synonyms } : {}
|
|
16568
16745
|
})
|
|
16569
16746
|
)
|
|
16570
16747
|
);
|
|
@@ -17951,7 +18128,7 @@ async function loadOntoReplacement(state, replacement, indexName) {
|
|
|
17951
18128
|
copyLoad = await loadIndexOntoWorkers(
|
|
17952
18129
|
indexName,
|
|
17953
18130
|
[replacement.executor],
|
|
17954
|
-
entry
|
|
18131
|
+
entry,
|
|
17955
18132
|
manager,
|
|
17956
18133
|
void 0,
|
|
17957
18134
|
state.callbacks?.isAnalysisStale?.(indexName)
|
|
@@ -18141,7 +18318,7 @@ async function loadCopies(state, indexName, reason) {
|
|
|
18141
18318
|
const copyLoad = await transferIndexToPool(
|
|
18142
18319
|
indexName,
|
|
18143
18320
|
pool,
|
|
18144
|
-
entry
|
|
18321
|
+
entry,
|
|
18145
18322
|
manager,
|
|
18146
18323
|
state.callbacks?.isAnalysisStale?.(indexName)
|
|
18147
18324
|
);
|
|
@@ -18564,7 +18741,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
18564
18741
|
const copyLoad = await transferIndexToPool(
|
|
18565
18742
|
indexName,
|
|
18566
18743
|
pool,
|
|
18567
|
-
entry
|
|
18744
|
+
entry,
|
|
18568
18745
|
manager,
|
|
18569
18746
|
callbacks?.isAnalysisStale?.(indexName)
|
|
18570
18747
|
);
|
|
@@ -18708,7 +18885,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
18708
18885
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
18709
18886
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
18710
18887
|
}
|
|
18711
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
18888
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-F4OGQUSU.mjs');
|
|
18712
18889
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
18713
18890
|
const executor = createDirectExecutor({
|
|
18714
18891
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -19315,7 +19492,14 @@ function refuseCode(entry) {
|
|
|
19315
19492
|
}
|
|
19316
19493
|
function indexConfigurationOf(entry) {
|
|
19317
19494
|
refuseCode(entry);
|
|
19318
|
-
|
|
19495
|
+
const synonyms = entry.synonyms === null ? void 0 : rulesOfStoredSynonyms(entry.synonyms);
|
|
19496
|
+
return configurationPayloadOf(
|
|
19497
|
+
entry.config,
|
|
19498
|
+
entry.language,
|
|
19499
|
+
entry.embeddingAdapterName,
|
|
19500
|
+
entry.analysisRevision,
|
|
19501
|
+
synonyms
|
|
19502
|
+
);
|
|
19319
19503
|
}
|
|
19320
19504
|
function requireConfigurationWithinSizeLimit(config, language) {
|
|
19321
19505
|
const adapter = config.embedding?.adapter;
|
|
@@ -19334,7 +19518,7 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
19334
19518
|
{ bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
|
|
19335
19519
|
);
|
|
19336
19520
|
}
|
|
19337
|
-
function configurationPayloadOf(config, language, adapterName, analysisRevision) {
|
|
19521
|
+
function configurationPayloadOf(config, language, adapterName, analysisRevision, synonyms) {
|
|
19338
19522
|
const bm25 = config.bm25;
|
|
19339
19523
|
return {
|
|
19340
19524
|
schema: config.schema,
|
|
@@ -19357,6 +19541,7 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision)
|
|
|
19357
19541
|
...config.required !== void 0 ? { required: config.required } : {},
|
|
19358
19542
|
...config.vectorPromotion !== void 0 ? { vectorPromotion: config.vectorPromotion } : {},
|
|
19359
19543
|
...config.patternValueLimit !== void 0 ? { patternValueLimit: config.patternValueLimit } : {},
|
|
19544
|
+
...synonyms !== void 0 ? { synonyms } : {},
|
|
19360
19545
|
...config.embedding !== void 0 ? {
|
|
19361
19546
|
embedding: {
|
|
19362
19547
|
fields: config.embedding.fields,
|
|
@@ -19429,7 +19614,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
19429
19614
|
}
|
|
19430
19615
|
const schema = validateSchema(requestedConfig.schema);
|
|
19431
19616
|
if (appliesCreationLimits) requireCreatableSchema(schema);
|
|
19432
|
-
const
|
|
19617
|
+
const configured = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
|
|
19618
|
+
const indexConfig = withoutSynonyms(configured);
|
|
19433
19619
|
requirePatternSearchFor(schema);
|
|
19434
19620
|
requirePartialWordSearchFor(schema);
|
|
19435
19621
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
@@ -19463,10 +19649,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
19463
19649
|
const fields = indexConfig.embedding.fields;
|
|
19464
19650
|
if (typeof configuredAdapter === "string") {
|
|
19465
19651
|
embeddingAdapterName = configuredAdapter;
|
|
19466
|
-
const
|
|
19467
|
-
if (
|
|
19652
|
+
const registered4 = core.embeddingAdapters.get(configuredAdapter);
|
|
19653
|
+
if (registered4 !== void 0) {
|
|
19468
19654
|
resolvedEmbeddingAdapter = validateEmbeddingConfig(
|
|
19469
|
-
{ fields, adapter:
|
|
19655
|
+
{ fields, adapter: registered4 },
|
|
19470
19656
|
indexConfig.schema,
|
|
19471
19657
|
config?.embedding
|
|
19472
19658
|
);
|
|
@@ -19490,7 +19676,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
19490
19676
|
}
|
|
19491
19677
|
const language = getLanguage(indexConfig.language ?? "english");
|
|
19492
19678
|
if (appliesCreationLimits) requireConfigurationWithinSizeLimit(indexConfig, language);
|
|
19493
|
-
|
|
19679
|
+
const synonyms = prepareConfiguredSynonyms(configured, language, appliesCreationLimits);
|
|
19680
|
+
core.executor.createIndex(name, indexConfig, language, false, synonyms?.lookup ?? null);
|
|
19494
19681
|
core.indexRegistry.set(name, {
|
|
19495
19682
|
config: indexConfig,
|
|
19496
19683
|
language,
|
|
@@ -19501,7 +19688,8 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
19501
19688
|
heldPartitions: null,
|
|
19502
19689
|
documentCount: 0,
|
|
19503
19690
|
partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
|
|
19504
|
-
analysisRevision: language.revision
|
|
19691
|
+
analysisRevision: language.revision,
|
|
19692
|
+
synonyms: synonyms?.encoded ?? null
|
|
19505
19693
|
});
|
|
19506
19694
|
await core.indexState.registerOpen(name);
|
|
19507
19695
|
if (core.durability) {
|
|
@@ -22103,7 +22291,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
22103
22291
|
};
|
|
22104
22292
|
}
|
|
22105
22293
|
async function restoreIndex(indexName, data) {
|
|
22106
|
-
const { restoreFromSnapshot } = await import('./snapshot-
|
|
22294
|
+
const { restoreFromSnapshot } = await import('./snapshot-5EJ7I7QY.mjs');
|
|
22107
22295
|
await restoreFromSnapshot(indexName, data, {
|
|
22108
22296
|
executor: core.executor,
|
|
22109
22297
|
indexRegistry: core.indexRegistry,
|
|
@@ -22123,7 +22311,7 @@ function portabilityOperations(core, config, engine) {
|
|
|
22123
22311
|
return {
|
|
22124
22312
|
snapshot(indexName) {
|
|
22125
22313
|
return withOpenIndex(indexName, async () => {
|
|
22126
|
-
const { createSnapshot } = await import('./snapshot-
|
|
22314
|
+
const { createSnapshot } = await import('./snapshot-5EJ7I7QY.mjs');
|
|
22127
22315
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
22128
22316
|
});
|
|
22129
22317
|
},
|
|
@@ -22135,8 +22323,8 @@ function portabilityOperations(core, config, engine) {
|
|
|
22135
22323
|
exportDocuments(indexName) {
|
|
22136
22324
|
return withOpenIndex(indexName, async () => {
|
|
22137
22325
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
22138
|
-
import('./snapshot-
|
|
22139
|
-
import('./write-
|
|
22326
|
+
import('./snapshot-5EJ7I7QY.mjs'),
|
|
22327
|
+
import('./write-K5V4XQ72.mjs')
|
|
22140
22328
|
]);
|
|
22141
22329
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
22142
22330
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -22443,13 +22631,19 @@ function broadcastStatsForWorker(params, context, scoring) {
|
|
|
22443
22631
|
if (term === void 0 || term.trim().length === 0) {
|
|
22444
22632
|
return void 0;
|
|
22445
22633
|
}
|
|
22446
|
-
|
|
22634
|
+
const { manager } = context;
|
|
22635
|
+
const analysis = { ...manager.analysis, synonyms: manager.synonyms };
|
|
22636
|
+
return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, analysis, {
|
|
22637
|
+
partitions: partitionsFor(manager, context.partitionIds),
|
|
22638
|
+
expansion: { options: params, schema: context.config.schema }
|
|
22639
|
+
});
|
|
22447
22640
|
}
|
|
22448
22641
|
function searchOptionsFor(manager, patternWork) {
|
|
22449
22642
|
return {
|
|
22450
22643
|
bm25Params: manager.config.bm25,
|
|
22451
22644
|
stopWords: manager.analysis.stopWords,
|
|
22452
22645
|
customTokenizer: manager.analysis.customTokenizer,
|
|
22646
|
+
synonyms: manager.synonyms,
|
|
22453
22647
|
patternWork
|
|
22454
22648
|
};
|
|
22455
22649
|
}
|
|
@@ -22864,12 +23058,13 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
22864
23058
|
const prefixLength = options?.prefixLength ?? 0;
|
|
22865
23059
|
const completedToken = options?.completedToken;
|
|
22866
23060
|
const queryTolerances = queryTokens.map((qt) => qt.token === completedToken ? 0 : tolerance);
|
|
23061
|
+
const synonyms = options?.synonyms;
|
|
22867
23062
|
const matchByFieldToken = /* @__PURE__ */ new Map();
|
|
22868
23063
|
const matchesQuery = (fieldToken) => {
|
|
22869
23064
|
const known = matchByFieldToken.get(fieldToken);
|
|
22870
23065
|
if (known !== void 0) return known;
|
|
22871
23066
|
const stemmedField = language.stemmer ? language.stemmer(fieldToken) : fieldToken;
|
|
22872
|
-
const matches = stemmedQueryTokens.some(
|
|
23067
|
+
const matches = synonyms?.includes(stemmedField) === true || stemmedQueryTokens.some(
|
|
22873
23068
|
(stemmedQuery, index) => fuzzyTermMatches(stemmedQuery, stemmedField, queryTolerances[index], prefixLength)
|
|
22874
23069
|
);
|
|
22875
23070
|
matchByFieldToken.set(fieldToken, matches);
|
|
@@ -22932,7 +23127,7 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
|
22932
23127
|
highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
|
|
22933
23128
|
};
|
|
22934
23129
|
}
|
|
22935
|
-
function applyHighlights(hits, params, language, analysis, schema, readStoredDocument) {
|
|
23130
|
+
function applyHighlights(hits, params, language, analysis, schema, readStoredDocument, synonymLookup) {
|
|
22936
23131
|
if (!params.highlight) return;
|
|
22937
23132
|
const queryTokenResult = tokenize(params.term ?? "", language, {
|
|
22938
23133
|
stem: true,
|
|
@@ -22957,6 +23152,7 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
22957
23152
|
}
|
|
22958
23153
|
}
|
|
22959
23154
|
const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
|
|
23155
|
+
const synonyms = params.exact === true ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
|
|
22960
23156
|
for (const hit of hits) {
|
|
22961
23157
|
const highlights = {};
|
|
22962
23158
|
const projected = hit.document;
|
|
@@ -22976,7 +23172,8 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
22976
23172
|
completedToken,
|
|
22977
23173
|
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
22978
23174
|
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
22979
|
-
partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0
|
|
23175
|
+
partialWords: partialWords?.fields.has(field) === true ? partialWords.highlight : void 0,
|
|
23176
|
+
synonyms
|
|
22980
23177
|
});
|
|
22981
23178
|
}
|
|
22982
23179
|
}
|
|
@@ -23489,7 +23686,7 @@ async function executeQuery(request, callerContext) {
|
|
|
23489
23686
|
if (params.highlight) {
|
|
23490
23687
|
const dropped = params.highlight.fields.some((field) => !projectionKeepsField(projection, field));
|
|
23491
23688
|
const readStoredDocument = dropped ? (docId) => manager.get(docId, { kind: "full" }) : void 0;
|
|
23492
|
-
applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument);
|
|
23689
|
+
applyHighlights(paginated, params, language, manager.analysis, manager.schema, readStoredDocument, manager.synonyms);
|
|
23493
23690
|
}
|
|
23494
23691
|
const elapsed = now() - startTime;
|
|
23495
23692
|
return {
|
|
@@ -23806,6 +24003,22 @@ async function runEngineListDocuments(core, indexName, params, partitionIds) {
|
|
|
23806
24003
|
}
|
|
23807
24004
|
}
|
|
23808
24005
|
|
|
24006
|
+
// src/narsil/synonyms.ts
|
|
24007
|
+
function synonymOperations(core) {
|
|
24008
|
+
return {
|
|
24009
|
+
async updateSynonyms(indexName, synonyms) {
|
|
24010
|
+
core.guardShutdown();
|
|
24011
|
+
const absent = core.requireIndex(indexName).synonyms === null && isEmptySynonymSet(synonyms);
|
|
24012
|
+
if (absent && availableSynonymModule() === null) return;
|
|
24013
|
+
await synonymModule().replace(core, indexName, synonyms, true);
|
|
24014
|
+
},
|
|
24015
|
+
async getSynonyms(indexName) {
|
|
24016
|
+
core.guardShutdown();
|
|
24017
|
+
return rulesOfStoredSynonyms(core.requireIndex(indexName).synonyms);
|
|
24018
|
+
}
|
|
24019
|
+
};
|
|
24020
|
+
}
|
|
24021
|
+
|
|
23809
24022
|
// src/narsil/operations.ts
|
|
23810
24023
|
function createNarsilFromCore(core, config) {
|
|
23811
24024
|
const {
|
|
@@ -23960,6 +24173,7 @@ function createNarsilFromCore(core, config) {
|
|
|
23960
24173
|
await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
|
|
23961
24174
|
},
|
|
23962
24175
|
...portabilityOperations(core, config, () => narsil),
|
|
24176
|
+
...synonymOperations(core),
|
|
23963
24177
|
async checkpoint(indexName) {
|
|
23964
24178
|
guardShutdown();
|
|
23965
24179
|
requireIndex(indexName);
|
|
@@ -24100,4 +24314,4 @@ async function createNarsil(config) {
|
|
|
24100
24314
|
// src/index.ts
|
|
24101
24315
|
var VERSION = "0.1.0";
|
|
24102
24316
|
|
|
24103
|
-
export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, raiseEarlierReleaseGraphSetting, readHeader, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerTokenizer, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, segmentArraysOf, selectionKeeps, setNestedValue, shareSegmentArrays, startsWithTagPrefix, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
|
|
24317
|
+
export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, setNestedValue, shareSegmentArrays, startsWithTagPrefix, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
|