@delali/narsil 0.3.1-next.127 → 0.3.1-next.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-LJS6PREA.mjs → chunk-3NVZD52E.mjs} +3 -1
- package/dist/browser/{chunk-4Y2EGA3U.mjs → chunk-5D6NJD6C.mjs} +554 -127
- package/dist/browser/{chunk-6POI2KNI.mjs → chunk-II5POT55.mjs} +3 -3
- package/dist/browser/{chunk-2CBQ2PCC.mjs → chunk-WLD6DDIX.mjs} +1 -1
- package/dist/browser/{chunk-V5DCQIOT.mjs → chunk-WZBTVTSU.mjs} +2 -2
- package/dist/browser/{chunk-B65QVTST.mjs → chunk-XTFJZHQP.mjs} +14 -9
- package/dist/browser/{chunk-P7C2S5JX.mjs → chunk-YAGXZU4W.mjs} +2 -2
- package/dist/browser/{durability-wiring-PUC4QSCR.mjs → durability-wiring-HCCC6BJB.mjs} +6 -6
- package/dist/browser/{module-ACWDC7YI.mjs → module-S2VDU2JE.mjs} +5 -5
- package/dist/browser/{restore-queue-MLUEUEAN.mjs → restore-queue-BZ7WWEC4.mjs} +1 -1
- package/dist/browser/{runtime-BX6XRFYR.mjs → runtime-PL4TTFJE.mjs} +1 -1
- package/dist/browser/{snapshot-U5Y346XG.mjs → snapshot-RTUCHI2L.mjs} +7 -7
- package/dist/browser/{write-GZHR6AVG.mjs → write-IMU7G5NU.mjs} +5 -5
- package/dist/{capabilities-9YwwCcqP.d.ts → capabilities-BTjv-tK8.d.ts} +10 -3
- package/dist/{chunk-N57XXGPG.mjs → chunk-2HQQ7Y5D.mjs} +23 -4
- package/dist/{chunk-ENBN5VQ6.mjs → chunk-2YOAGAGF.mjs} +115 -46
- package/dist/{chunk-QSCVJJW5.mjs → chunk-37RTD6KD.mjs} +8 -8
- package/dist/{chunk-N6J6EJW3.mjs → chunk-3LJVM6XM.mjs} +2 -2
- package/dist/{chunk-VK45J47R.mjs → chunk-3XHFVYIQ.mjs} +1 -1
- package/dist/{chunk-LOBYK2FG.mjs → chunk-43Q7HAI2.mjs} +20 -9
- package/dist/{chunk-6PN3U6XW.mjs → chunk-44TB6AIZ.mjs} +1 -1
- package/dist/{chunk-6ODIY6ZB.mjs → chunk-4XEVROSP.mjs} +1 -1
- package/dist/{chunk-4IHMMDRF.mjs → chunk-5LC5VMDQ.mjs} +4 -4
- package/dist/{chunk-MFPJVITN.mjs → chunk-5VSTAJET.mjs} +2 -2
- package/dist/{chunk-DG62H7OD.mjs → chunk-AFOZR4ZQ.mjs} +2 -2
- package/dist/{chunk-P34VZEXB.mjs → chunk-AN3CUMPZ.mjs} +2 -2
- package/dist/{chunk-4PQNDHSA.mjs → chunk-CSSCLRGY.mjs} +2 -2
- package/dist/{chunk-HWQDZB2U.mjs → chunk-CTF6WFTX.mjs} +3 -3
- package/dist/{chunk-E74YLAIG.mjs → chunk-DX2FI4LU.mjs} +2 -1
- package/dist/{chunk-22CK2LQO.mjs → chunk-EZIGAOUJ.mjs} +5 -5
- package/dist/{chunk-KCQG2PJT.mjs → chunk-F4UKMTKM.mjs} +4 -4
- package/dist/{chunk-542TD4BL.mjs → chunk-FRSWYMXH.mjs} +1 -1
- package/dist/{chunk-ANFNYCQG.mjs → chunk-J4LRE5Q7.mjs} +6 -6
- package/dist/{chunk-OGFSUXYY.mjs → chunk-K73PZTBR.mjs} +4 -4
- package/dist/{chunk-IBGMFR5Y.mjs → chunk-K76F5YYX.mjs} +274 -75
- package/dist/{chunk-FUUP45IS.mjs → chunk-M6MBSVIJ.mjs} +1 -1
- package/dist/{chunk-IR4KQ4WB.mjs → chunk-MGD7XETB.mjs} +32 -30
- package/dist/{chunk-7ZZQNOIY.mjs → chunk-OPO5SE4J.mjs} +1 -1
- package/dist/{chunk-UKXMSO5H.mjs → chunk-PCRWCOZC.mjs} +155 -31
- package/dist/{chunk-HFJXEFFV.mjs → chunk-PTIJAZWL.mjs} +3 -3
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/{chunk-SEG4CL6W.mjs → chunk-QQS4QZQA.mjs} +4 -4
- package/dist/{chunk-OFYKBKKV.mjs → chunk-RC27KF7F.mjs} +1 -1
- package/dist/{chunk-JEXJDCQH.mjs → chunk-RYH3PASD.mjs} +17 -12
- package/dist/{chunk-IE43SRL3.mjs → chunk-TB7YMDD5.mjs} +10 -6
- package/dist/{chunk-ZR63OGT2.mjs → chunk-TY64A3ZW.mjs} +2 -2
- package/dist/{chunk-Z4DZ4KK2.mjs → chunk-UJWGQIXU.mjs} +1 -0
- package/dist/{chunk-OBWUWZRA.mjs → chunk-WI5Q3L43.mjs} +1 -0
- package/dist/{chunk-SJEIIR56.mjs → chunk-WQYB4TNR.mjs} +2 -2
- package/dist/{chunk-JIXO442E.mjs → chunk-WUIDPBSY.mjs} +16 -10
- package/dist/{chunk-62IPM62F.mjs → chunk-XRXBCOA4.mjs} +2 -2
- package/dist/{chunk-U6NGTWI3.mjs → chunk-XTER67YM.mjs} +1 -1
- package/dist/{chunk-LXWDQLTW.mjs → chunk-Y4NOHICF.mjs} +1 -1
- package/dist/{chunk-OCB7TMUQ.mjs → chunk-Y4US5EGH.mjs} +1 -1
- package/dist/{client-DyiYkQ8o.d.ts → client-BOcveV7A.d.ts} +34 -11
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +42 -20
- package/dist/{config-DLVnlByZ.d.ts → config-C3YXk5Rn.d.ts} +2 -2
- package/dist/{config-CtTfxbpV.d.ts → config-CPmIY3aH.d.ts} +2 -2
- package/dist/{direct-executor-6KCVVHYT.mjs → direct-executor-ULA6V5HC.mjs} +12 -12
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +18 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +33 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +71 -43
- package/dist/distribution.mjs +359 -115
- package/dist/{durability-wiring-UGGUNATA.mjs → durability-wiring-6R26AJDI.mjs} +26 -26
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-Bq5wz6c1.d.ts → engine-JjRAXpT3.d.ts} +38 -8
- package/dist/{events-x2aVu0_o.d.ts → events-pM0lna7M.d.ts} +12 -1
- package/dist/{index-Cl5qQauj.d.ts → index-DkQ2DlgR.d.ts} +37 -3
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +19 -19
- package/dist/{long-query-thread-5BIPZVS6.mjs → long-query-thread-6VLA6YJI.mjs} +15 -15
- package/dist/{module-7OHZ2OPI.mjs → module-S4WGMFNW.mjs} +11 -11
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +22 -22
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +1 -0
- package/dist/{rerank-NYX5S7OG.mjs → rerank-V4J4MIWR.mjs} +6 -6
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-NVMWJMHI.mjs → restore-queue-PUGWCPCX.mjs} +1 -1
- package/dist/{runtime-XNYW6AWX.mjs → runtime-TBFBQQQO.mjs} +7 -7
- package/dist/{schema-4W_7tk1f.d.ts → schema-DGAP_8HJ.d.ts} +50 -24
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +27 -15
- package/dist/{snapshot-P7IQGNNQ.mjs → snapshot-PUVSMV7E.mjs} +27 -27
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-DLQIfXqy.d.ts → tasks-txGvNz6h.d.ts} +4 -6
- package/dist/{thread-QPETQEPK.mjs → thread-TA7QQBWP.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-WZVM7ZL6.mjs → write-TGKYT3C4.mjs} +21 -21
- package/package.json +7 -7
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-
|
|
1
|
+
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-WLD6DDIX.mjs';
|
|
2
2
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
|
|
3
|
-
import { ErrorCodes, NarsilError, createNarsilError, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, MIN_DOCUMENTS_FOR_SEGMENTS, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-
|
|
3
|
+
import { ErrorCodes, NarsilError, createNarsilError, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, MIN_DOCUMENTS_FOR_SEGMENTS, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, HEAP_PRESSURE_REARM_FRACTION, MAX_TOLERANCE, MAX_PREFIX_LENGTH, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-3NVZD52E.mjs';
|
|
4
4
|
import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
6
6
|
// src/analysis/registry.ts
|
|
@@ -3246,6 +3246,10 @@ function createDocumentStore() {
|
|
|
3246
3246
|
// src/core/constants.ts
|
|
3247
3247
|
var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
3248
3248
|
var MAX_TERM_FREQUENCY = 65535;
|
|
3249
|
+
var DEFAULT_BM25_K1 = 1.2;
|
|
3250
|
+
var DEFAULT_BM25_B = 0.75;
|
|
3251
|
+
var MAX_BM25_K1 = 1e3;
|
|
3252
|
+
var MAX_BM25_B = 1;
|
|
3249
3253
|
var PENDING_REMOVAL_MERGE_FLOOR = 64;
|
|
3250
3254
|
var PENDING_REMOVAL_MERGE_DIVISOR = 256;
|
|
3251
3255
|
|
|
@@ -7546,19 +7550,63 @@ function partitionFilterMatches(state, filters, schema, meter) {
|
|
|
7546
7550
|
}
|
|
7547
7551
|
|
|
7548
7552
|
// src/core/scorer.ts
|
|
7549
|
-
var DEFAULT_K1 = 1.2;
|
|
7550
|
-
var DEFAULT_B = 0.75;
|
|
7551
7553
|
function computeIDF(docFrequency, totalDocs) {
|
|
7552
7554
|
if (totalDocs === 0) return 0;
|
|
7553
7555
|
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
7554
7556
|
}
|
|
7557
|
+
function pairKeepsBoundSound(k1, b) {
|
|
7558
|
+
return k1 >= 0 && k1 <= MAX_BM25_K1 && b >= 0 && b <= MAX_BM25_B;
|
|
7559
|
+
}
|
|
7555
7560
|
function bm25PruningSound(params) {
|
|
7556
|
-
const k1 = params?.k1 ??
|
|
7557
|
-
const b = params?.b ??
|
|
7558
|
-
|
|
7561
|
+
const k1 = params?.k1 ?? DEFAULT_BM25_K1;
|
|
7562
|
+
const b = params?.b ?? DEFAULT_BM25_B;
|
|
7563
|
+
if (!pairKeepsBoundSound(k1, b)) return false;
|
|
7564
|
+
const fields = params?.fields;
|
|
7565
|
+
if (fields === void 0) return true;
|
|
7566
|
+
for (const name of Object.keys(fields)) {
|
|
7567
|
+
if (!pairKeepsBoundSound(fields[name].k1 ?? k1, fields[name].b ?? b)) return false;
|
|
7568
|
+
}
|
|
7569
|
+
return true;
|
|
7570
|
+
}
|
|
7571
|
+
function bm25SetsValue(params) {
|
|
7572
|
+
if (typeof params !== "object" || params === null) return params !== void 0;
|
|
7573
|
+
if (params.k1 !== void 0 || params.b !== void 0) return true;
|
|
7574
|
+
const fields = params.fields;
|
|
7575
|
+
if (fields === void 0) return false;
|
|
7576
|
+
if (typeof fields !== "object" || fields === null) return true;
|
|
7577
|
+
for (const name of Object.keys(fields)) {
|
|
7578
|
+
const pair = fields[name];
|
|
7579
|
+
if (typeof pair !== "object" || pair === null) return true;
|
|
7580
|
+
if (pair.k1 !== void 0 || pair.b !== void 0) return true;
|
|
7581
|
+
}
|
|
7582
|
+
return false;
|
|
7559
7583
|
}
|
|
7560
|
-
function
|
|
7561
|
-
return
|
|
7584
|
+
function ownField(fields, name) {
|
|
7585
|
+
return fields !== void 0 && Object.hasOwn(fields, name) ? fields[name] : void 0;
|
|
7586
|
+
}
|
|
7587
|
+
function mergeBM25Levels(search, index) {
|
|
7588
|
+
if (search === void 0 || !bm25SetsValue(search)) return index;
|
|
7589
|
+
const k1 = search.k1 ?? index?.k1;
|
|
7590
|
+
const b = search.b ?? index?.b;
|
|
7591
|
+
const merged = {
|
|
7592
|
+
...k1 !== void 0 ? { k1 } : {},
|
|
7593
|
+
...b !== void 0 ? { b } : {}
|
|
7594
|
+
};
|
|
7595
|
+
const names = [...Object.keys(search.fields ?? {}), ...Object.keys(index?.fields ?? {})];
|
|
7596
|
+
if (names.length === 0) return merged;
|
|
7597
|
+
const fields = /* @__PURE__ */ Object.create(null);
|
|
7598
|
+
for (const name of names) {
|
|
7599
|
+
const fromSearch = ownField(search.fields, name);
|
|
7600
|
+
const fromIndex = ownField(index?.fields, name);
|
|
7601
|
+
const fieldK1 = fromSearch?.k1 ?? search.k1 ?? fromIndex?.k1;
|
|
7602
|
+
const fieldB = fromSearch?.b ?? search.b ?? fromIndex?.b;
|
|
7603
|
+
fields[name] = {
|
|
7604
|
+
...fieldK1 !== void 0 ? { k1: fieldK1 } : {},
|
|
7605
|
+
...fieldB !== void 0 ? { b: fieldB } : {}
|
|
7606
|
+
};
|
|
7607
|
+
}
|
|
7608
|
+
merged.fields = fields;
|
|
7609
|
+
return merged;
|
|
7562
7610
|
}
|
|
7563
7611
|
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
7564
7612
|
if (avgFieldLength === 0) return 0;
|
|
@@ -7567,29 +7615,49 @@ function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1,
|
|
|
7567
7615
|
if (denominator === 0) return 0;
|
|
7568
7616
|
return idf * (numerator / denominator);
|
|
7569
7617
|
}
|
|
7570
|
-
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
7571
|
-
if (totalDocs === 0) return 0;
|
|
7572
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
7573
|
-
const b = params?.b ?? DEFAULT_B;
|
|
7574
|
-
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
7575
|
-
}
|
|
7576
7618
|
|
|
7577
7619
|
// src/core/partition/field-scoring.ts
|
|
7578
|
-
|
|
7620
|
+
var NO_FIELD_VALUES = new Float64Array(0);
|
|
7621
|
+
function differentPairsOf(searchable, k1s, bs) {
|
|
7622
|
+
const pairs = [];
|
|
7623
|
+
for (let fieldIndex = 0; fieldIndex < searchable.length; fieldIndex++) {
|
|
7624
|
+
if (searchable[fieldIndex] !== 1) continue;
|
|
7625
|
+
let seen = false;
|
|
7626
|
+
for (let at = 0; at < pairs.length && !seen; at += 2) {
|
|
7627
|
+
seen = pairs[at] === k1s[fieldIndex] && pairs[at + 1] === bs[fieldIndex];
|
|
7628
|
+
}
|
|
7629
|
+
if (!seen) pairs.push(k1s[fieldIndex], bs[fieldIndex]);
|
|
7630
|
+
}
|
|
7631
|
+
return pairs;
|
|
7632
|
+
}
|
|
7633
|
+
function loadFieldScoring(state, fields, boost, averageFieldLengths, bm25) {
|
|
7579
7634
|
const fieldNames = state.fieldNameTable.names;
|
|
7635
|
+
const fieldCount = fieldNames.length;
|
|
7636
|
+
const k1 = bm25?.k1 ?? DEFAULT_BM25_K1;
|
|
7637
|
+
const b = bm25?.b ?? DEFAULT_BM25_B;
|
|
7638
|
+
const perField = bm25?.fields;
|
|
7580
7639
|
const scoring = {
|
|
7581
|
-
searchable: new Uint8Array(
|
|
7582
|
-
boosts: new Float64Array(
|
|
7583
|
-
averageLengths: new Float64Array(
|
|
7584
|
-
lengthColumns: new Array(
|
|
7640
|
+
searchable: new Uint8Array(fieldCount),
|
|
7641
|
+
boosts: new Float64Array(fieldCount),
|
|
7642
|
+
averageLengths: new Float64Array(fieldCount),
|
|
7643
|
+
lengthColumns: new Array(fieldCount).fill(null),
|
|
7644
|
+
k1s: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
7645
|
+
bs: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
7646
|
+
boundPairs: [k1, b]
|
|
7585
7647
|
};
|
|
7586
|
-
for (let fieldIndex = 0; fieldIndex <
|
|
7648
|
+
for (let fieldIndex = 0; fieldIndex < fieldCount; fieldIndex++) {
|
|
7587
7649
|
const fieldName = fieldNames[fieldIndex];
|
|
7588
7650
|
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
7589
7651
|
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
7590
7652
|
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
7591
7653
|
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
7654
|
+
if (perField !== void 0) {
|
|
7655
|
+
const fieldPair = Object.hasOwn(perField, fieldName) ? perField[fieldName] : void 0;
|
|
7656
|
+
scoring.k1s[fieldIndex] = fieldPair?.k1 ?? k1;
|
|
7657
|
+
scoring.bs[fieldIndex] = fieldPair?.b ?? b;
|
|
7658
|
+
}
|
|
7592
7659
|
}
|
|
7660
|
+
if (perField !== void 0) scoring.boundPairs = differentPairsOf(scoring.searchable, scoring.k1s, scoring.bs);
|
|
7593
7661
|
return scoring;
|
|
7594
7662
|
}
|
|
7595
7663
|
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
@@ -7605,6 +7673,15 @@ function bestSearchable(searchable, values) {
|
|
|
7605
7673
|
}
|
|
7606
7674
|
return best;
|
|
7607
7675
|
}
|
|
7676
|
+
function blockEntryBound(fields, maxTermFrequency, idf, minFieldLength, maxAverageLength) {
|
|
7677
|
+
const pairs = fields.boundPairs;
|
|
7678
|
+
let best = 0;
|
|
7679
|
+
for (let at = 0; at < pairs.length; at += 2) {
|
|
7680
|
+
const bound = computeBM25WithIDF(maxTermFrequency, idf, minFieldLength, maxAverageLength, pairs[at], pairs[at + 1]);
|
|
7681
|
+
if (bound > best) best = bound;
|
|
7682
|
+
}
|
|
7683
|
+
return best;
|
|
7684
|
+
}
|
|
7608
7685
|
|
|
7609
7686
|
// src/core/partition/posting-columns.ts
|
|
7610
7687
|
function postingColumns(list) {
|
|
@@ -7698,8 +7775,9 @@ function groupPostingCount(matches) {
|
|
|
7698
7775
|
return total;
|
|
7699
7776
|
}
|
|
7700
7777
|
function computeGroupContributions(context, matches, collect) {
|
|
7701
|
-
const { fields, fieldNames, filterBitset, partialMarks
|
|
7778
|
+
const { fields, fieldNames, filterBitset, partialMarks } = context;
|
|
7702
7779
|
const scoresAreZero = context.totalDocs === 0;
|
|
7780
|
+
const onePair = fields.boundPairs.length === 2;
|
|
7703
7781
|
const best = /* @__PURE__ */ new Map();
|
|
7704
7782
|
for (const match of matches) {
|
|
7705
7783
|
const perTerm = /* @__PURE__ */ new Map();
|
|
@@ -7719,7 +7797,14 @@ function computeGroupContributions(context, matches, collect) {
|
|
|
7719
7797
|
const fieldBoost = fields.boosts[fieldIndex];
|
|
7720
7798
|
const avgLen = fields.averageLengths[fieldIndex];
|
|
7721
7799
|
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
7722
|
-
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
7800
|
+
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
7801
|
+
termFrequency,
|
|
7802
|
+
match.idf,
|
|
7803
|
+
actualFieldLength,
|
|
7804
|
+
avgLen,
|
|
7805
|
+
onePair ? fields.boundPairs[0] : fields.k1s[fieldIndex],
|
|
7806
|
+
onePair ? fields.boundPairs[1] : fields.bs[fieldIndex]
|
|
7807
|
+
)) * fieldBoost * factor;
|
|
7723
7808
|
const existing = perTerm.get(internalId);
|
|
7724
7809
|
if (existing) {
|
|
7725
7810
|
existing.score += termScore;
|
|
@@ -8283,7 +8368,7 @@ function boostsKeepScoresNonNegative(fields) {
|
|
|
8283
8368
|
}
|
|
8284
8369
|
return true;
|
|
8285
8370
|
}
|
|
8286
|
-
function
|
|
8371
|
+
function samePairUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
8287
8372
|
if (scan2.scoresAreZero) return 0;
|
|
8288
8373
|
const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
|
|
8289
8374
|
let best = 0;
|
|
@@ -8301,7 +8386,24 @@ function termUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
|
8301
8386
|
}
|
|
8302
8387
|
return best;
|
|
8303
8388
|
}
|
|
8304
|
-
function
|
|
8389
|
+
function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
8390
|
+
if (scan2.scoresAreZero) return 0;
|
|
8391
|
+
const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
|
|
8392
|
+
let best = 0;
|
|
8393
|
+
for (let block = 0; block < bounds.blockCount; block++) {
|
|
8394
|
+
const bestEntry = blockEntryBound(
|
|
8395
|
+
scan2.fields,
|
|
8396
|
+
bounds.maxTermFrequency[block],
|
|
8397
|
+
scan2.idf,
|
|
8398
|
+
bounds.minFieldLength[block],
|
|
8399
|
+
maxAverageLength
|
|
8400
|
+
);
|
|
8401
|
+
const blockBound = bestEntry * maxBoost * bounds.maxEntriesPerDocument[block];
|
|
8402
|
+
if (blockBound > best) best = blockBound;
|
|
8403
|
+
}
|
|
8404
|
+
return best;
|
|
8405
|
+
}
|
|
8406
|
+
function samePairEntryScore(scan2, entry, internalId) {
|
|
8305
8407
|
const { fields, columns } = scan2;
|
|
8306
8408
|
const fieldIndex = columns.fieldNameIndices[entry];
|
|
8307
8409
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
@@ -8309,17 +8411,55 @@ function entryScore(scan2, entry, internalId) {
|
|
|
8309
8411
|
const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan2.idf, fieldLength, averageLength, scan2.k1, scan2.b);
|
|
8310
8412
|
return bm25 * fields.boosts[fieldIndex];
|
|
8311
8413
|
}
|
|
8312
|
-
function
|
|
8414
|
+
function fieldPairEntryScore(scan2, entry, internalId) {
|
|
8415
|
+
const { fields, columns } = scan2;
|
|
8416
|
+
const fieldIndex = columns.fieldNameIndices[entry];
|
|
8417
|
+
const averageLength = fields.averageLengths[fieldIndex];
|
|
8418
|
+
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
8419
|
+
const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8420
|
+
columns.termFrequencies[entry],
|
|
8421
|
+
scan2.idf,
|
|
8422
|
+
fieldLength,
|
|
8423
|
+
averageLength,
|
|
8424
|
+
fields.k1s[fieldIndex],
|
|
8425
|
+
fields.bs[fieldIndex]
|
|
8426
|
+
);
|
|
8427
|
+
return bm25 * fields.boosts[fieldIndex];
|
|
8428
|
+
}
|
|
8429
|
+
function samePairScoreEveryDocument(scan2, buffer) {
|
|
8313
8430
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8314
8431
|
const searchable = scan2.fields.searchable;
|
|
8315
8432
|
for (let entry = 0; entry < count; entry++) {
|
|
8316
8433
|
const internalId = docIds[entry];
|
|
8317
8434
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8318
8435
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8319
|
-
addScore(buffer, internalId,
|
|
8436
|
+
addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
|
|
8320
8437
|
}
|
|
8321
8438
|
}
|
|
8322
|
-
function
|
|
8439
|
+
function fieldPairScoreEveryDocument(scan2, buffer) {
|
|
8440
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8441
|
+
const searchable = scan2.fields.searchable;
|
|
8442
|
+
for (let entry = 0; entry < count; entry++) {
|
|
8443
|
+
const internalId = docIds[entry];
|
|
8444
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8445
|
+
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8446
|
+
addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
|
|
8447
|
+
}
|
|
8448
|
+
}
|
|
8449
|
+
function samePairScoreKnownDocuments(scan2, buffer) {
|
|
8450
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8451
|
+
const searchable = scan2.fields.searchable;
|
|
8452
|
+
let counted = 0;
|
|
8453
|
+
for (let entry = 0; entry < count; entry++) {
|
|
8454
|
+
const internalId = docIds[entry];
|
|
8455
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8456
|
+
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8457
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
|
|
8458
|
+
else if (markCounted(buffer, internalId)) counted++;
|
|
8459
|
+
}
|
|
8460
|
+
return counted;
|
|
8461
|
+
}
|
|
8462
|
+
function fieldPairScoreKnownDocuments(scan2, buffer) {
|
|
8323
8463
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8324
8464
|
const searchable = scan2.fields.searchable;
|
|
8325
8465
|
let counted = 0;
|
|
@@ -8327,31 +8467,58 @@ function scoreKnownDocumentsAndCountTheRest(scan2, buffer) {
|
|
|
8327
8467
|
const internalId = docIds[entry];
|
|
8328
8468
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8329
8469
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8330
|
-
if (hasScore(buffer, internalId)) addScore(buffer, internalId,
|
|
8470
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
|
|
8331
8471
|
else if (markCounted(buffer, internalId)) counted++;
|
|
8332
8472
|
}
|
|
8333
8473
|
return counted;
|
|
8334
8474
|
}
|
|
8335
|
-
function
|
|
8475
|
+
function firstEntryOf(docIds, count, internalId) {
|
|
8476
|
+
let low = 0;
|
|
8477
|
+
let high = count;
|
|
8478
|
+
while (low < high) {
|
|
8479
|
+
const middle = low + high >>> 1;
|
|
8480
|
+
if (docIds[middle] < internalId) low = middle + 1;
|
|
8481
|
+
else high = middle;
|
|
8482
|
+
}
|
|
8483
|
+
return low;
|
|
8484
|
+
}
|
|
8485
|
+
function samePairQueryOrderScore(scans, internalId) {
|
|
8336
8486
|
let score = 0;
|
|
8337
8487
|
for (const scan2 of scans) {
|
|
8338
8488
|
const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8339
8489
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8340
|
-
let
|
|
8341
|
-
|
|
8342
|
-
|
|
8343
|
-
const middle = low + high >>> 1;
|
|
8344
|
-
if (docIds[middle] < internalId) low = middle + 1;
|
|
8345
|
-
else high = middle;
|
|
8490
|
+
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
8491
|
+
if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
8492
|
+
score += samePairEntryScore(scan2, entry, internalId);
|
|
8346
8493
|
}
|
|
8347
|
-
|
|
8494
|
+
}
|
|
8495
|
+
return score;
|
|
8496
|
+
}
|
|
8497
|
+
function fieldPairQueryOrderScore(scans, internalId) {
|
|
8498
|
+
let score = 0;
|
|
8499
|
+
for (const scan2 of scans) {
|
|
8500
|
+
const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8501
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8502
|
+
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
8348
8503
|
if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
8349
|
-
score +=
|
|
8504
|
+
score += fieldPairEntryScore(scan2, entry, internalId);
|
|
8350
8505
|
}
|
|
8351
8506
|
}
|
|
8352
8507
|
return score;
|
|
8353
8508
|
}
|
|
8354
|
-
|
|
8509
|
+
var SAME_PAIR_SCORER = {
|
|
8510
|
+
upperBound: samePairUpperBound,
|
|
8511
|
+
scoreEveryDocument: samePairScoreEveryDocument,
|
|
8512
|
+
scoreKnownDocumentsAndCountTheRest: samePairScoreKnownDocuments,
|
|
8513
|
+
queryOrderScore: samePairQueryOrderScore
|
|
8514
|
+
};
|
|
8515
|
+
var FIELD_PAIR_SCORER = {
|
|
8516
|
+
upperBound: fieldPairUpperBound,
|
|
8517
|
+
scoreEveryDocument: fieldPairScoreEveryDocument,
|
|
8518
|
+
scoreKnownDocumentsAndCountTheRest: fieldPairScoreKnownDocuments,
|
|
8519
|
+
queryOrderScore: fieldPairQueryOrderScore
|
|
8520
|
+
};
|
|
8521
|
+
function pageFromBuffer(scorer, scans, buffer, selection, resolver) {
|
|
8355
8522
|
const approximateKth = kthBestScore(buffer, selection, resolver);
|
|
8356
8523
|
const gatherFrom = approximateKth - Math.abs(approximateKth) * PRUNING_REJECTION_SLACK;
|
|
8357
8524
|
const heap = [];
|
|
@@ -8363,7 +8530,7 @@ function pageFromBuffer(scans, buffer, selection, resolver) {
|
|
|
8363
8530
|
if (scores[internalId] < gatherFrom) continue;
|
|
8364
8531
|
const externalId = resolver.toExternal(internalId);
|
|
8365
8532
|
if (externalId === void 0) continue;
|
|
8366
|
-
const score = queryOrderScore(scans, internalId);
|
|
8533
|
+
const score = scorer.queryOrderScore(scans, internalId);
|
|
8367
8534
|
if (full && score < threshold) continue;
|
|
8368
8535
|
const candidate = { internalId, externalId, score };
|
|
8369
8536
|
if (!full) {
|
|
@@ -8393,18 +8560,28 @@ function pageFromBuffer(scans, buffer, selection, resolver) {
|
|
|
8393
8560
|
return scored;
|
|
8394
8561
|
}
|
|
8395
8562
|
function multiTermTopK(request) {
|
|
8396
|
-
const { terms, docFrequencies, totalDocs,
|
|
8563
|
+
const { terms, docFrequencies, totalDocs, limit, fields, resolver, buffer } = request;
|
|
8397
8564
|
if (!boostsKeepScoresNonNegative(fields)) return null;
|
|
8398
|
-
const
|
|
8565
|
+
const onePair = fields.boundPairs.length === 2;
|
|
8566
|
+
const scorer = onePair ? SAME_PAIR_SCORER : FIELD_PAIR_SCORER;
|
|
8567
|
+
const k1 = onePair ? fields.boundPairs[0] : 0;
|
|
8568
|
+
const b = onePair ? fields.boundPairs[1] : 0;
|
|
8399
8569
|
const scans = new Array(terms.length);
|
|
8400
8570
|
for (let term = 0; term < terms.length; term++) {
|
|
8401
8571
|
const idf = computeIDF(docFrequencies[term], totalDocs);
|
|
8402
8572
|
if (!(idf >= 0)) return null;
|
|
8403
|
-
scans[term] = {
|
|
8573
|
+
scans[term] = {
|
|
8574
|
+
columns: postingColumns(terms[term].list),
|
|
8575
|
+
idf,
|
|
8576
|
+
scoresAreZero: totalDocs === 0,
|
|
8577
|
+
fields,
|
|
8578
|
+
k1,
|
|
8579
|
+
b
|
|
8580
|
+
};
|
|
8404
8581
|
}
|
|
8405
8582
|
const maxBoost = bestSearchable(fields.searchable, fields.boosts);
|
|
8406
8583
|
const maxAverageLength = bestSearchable(fields.searchable, fields.averageLengths);
|
|
8407
|
-
const upperBounds = scans.map((scan2, term) =>
|
|
8584
|
+
const upperBounds = scans.map((scan2, term) => scorer.upperBound(scan2, terms[term].list, maxBoost, maxAverageLength));
|
|
8408
8585
|
const order = scans.map((_, term) => term).sort((left, right) => upperBounds[right] - upperBounds[left]);
|
|
8409
8586
|
const remainingBound = new Float64Array(order.length + 1);
|
|
8410
8587
|
for (let position2 = order.length - 1; position2 >= 0; position2--) {
|
|
@@ -8419,15 +8596,15 @@ function multiTermTopK(request) {
|
|
|
8419
8596
|
const threshold = kthBestScore(buffer, selection, resolver);
|
|
8420
8597
|
if (remaining < threshold - threshold * PRUNING_REJECTION_SLACK) break;
|
|
8421
8598
|
}
|
|
8422
|
-
scoreEveryDocument(scans[order[position]], buffer);
|
|
8599
|
+
scorer.scoreEveryDocument(scans[order[position]], buffer);
|
|
8423
8600
|
}
|
|
8424
8601
|
let countedOnly = 0;
|
|
8425
8602
|
for (let rest = position; rest < order.length; rest++) {
|
|
8426
|
-
countedOnly += scoreKnownDocumentsAndCountTheRest(scans[order[rest]], buffer);
|
|
8603
|
+
countedOnly += scorer.scoreKnownDocumentsAndCountTheRest(scans[order[rest]], buffer);
|
|
8427
8604
|
}
|
|
8428
8605
|
const totalMatched = buffer.touchedCount + countedOnly;
|
|
8429
8606
|
if (wanted === 0) return { scored: [], totalMatched };
|
|
8430
|
-
return { scored: pageFromBuffer(scans, buffer, selection, resolver), totalMatched };
|
|
8607
|
+
return { scored: pageFromBuffer(scorer, scans, buffer, selection, resolver), totalMatched };
|
|
8431
8608
|
}
|
|
8432
8609
|
|
|
8433
8610
|
// src/core/partition/single-term-topk.ts
|
|
@@ -8455,20 +8632,25 @@ function prunableSingleTermList(params, index, fieldNames) {
|
|
|
8455
8632
|
return list;
|
|
8456
8633
|
}
|
|
8457
8634
|
function singleTermTopK(request) {
|
|
8458
|
-
const { list, docFrequency, totalDocs,
|
|
8635
|
+
const { list, docFrequency, totalDocs, limit, resolver } = request;
|
|
8459
8636
|
const {
|
|
8460
8637
|
searchable: fieldSearchable,
|
|
8461
8638
|
boosts: fieldBoosts,
|
|
8462
8639
|
averageLengths: fieldAvgLengths,
|
|
8463
|
-
lengthColumns: fieldLengthColumns
|
|
8640
|
+
lengthColumns: fieldLengthColumns,
|
|
8641
|
+
k1s,
|
|
8642
|
+
bs,
|
|
8643
|
+
boundPairs
|
|
8464
8644
|
} = request.fields;
|
|
8645
|
+
const onePair = boundPairs.length === 2;
|
|
8646
|
+
const onlyK1 = boundPairs[0];
|
|
8647
|
+
const onlyB = boundPairs[1];
|
|
8465
8648
|
const wanted = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 0;
|
|
8466
8649
|
const { docIds, termFrequencies, fieldNameIndices, deletedDocs: deleted, hasDeleted } = postingColumns(list);
|
|
8467
8650
|
const bounds = blockBoundsFor(list, fieldLengthColumns);
|
|
8468
8651
|
const maxBoost = bestSearchable(fieldSearchable, fieldBoosts);
|
|
8469
8652
|
const maxAverageLength = bestSearchable(fieldSearchable, fieldAvgLengths);
|
|
8470
8653
|
const idf = computeIDF(docFrequency, totalDocs);
|
|
8471
|
-
const { k1, b } = resolveBM25Params(bm25Params);
|
|
8472
8654
|
const scoresAreZero = totalDocs === 0;
|
|
8473
8655
|
const heap = [];
|
|
8474
8656
|
let full = false;
|
|
@@ -8478,14 +8660,12 @@ function singleTermTopK(request) {
|
|
|
8478
8660
|
const start = block === 0 ? 0 : bounds.entryEnd[block - 1];
|
|
8479
8661
|
const end = bounds.entryEnd[block];
|
|
8480
8662
|
if (full) {
|
|
8481
|
-
|
|
8482
|
-
|
|
8483
|
-
|
|
8484
|
-
|
|
8485
|
-
|
|
8486
|
-
|
|
8487
|
-
bm25Params
|
|
8488
|
-
);
|
|
8663
|
+
let bestEntry = 0;
|
|
8664
|
+
if (!scoresAreZero) {
|
|
8665
|
+
const blockMaxTf = bounds.maxTermFrequency[block];
|
|
8666
|
+
const blockMinLength = bounds.minFieldLength[block];
|
|
8667
|
+
bestEntry = onePair ? computeBM25WithIDF(blockMaxTf, idf, blockMinLength, maxAverageLength, onlyK1, onlyB) : blockEntryBound(request.fields, blockMaxTf, idf, blockMinLength, maxAverageLength);
|
|
8668
|
+
}
|
|
8489
8669
|
if (bestEntry * maxBoost * bounds.maxEntriesPerDocument[block] < threshold) {
|
|
8490
8670
|
totalMatched += bounds.documentCount[block];
|
|
8491
8671
|
continue;
|
|
@@ -8504,7 +8684,14 @@ function singleTermTopK(request) {
|
|
|
8504
8684
|
const fieldIndex = fieldNameIndices[entry];
|
|
8505
8685
|
if (fieldSearchable[fieldIndex] === 1) {
|
|
8506
8686
|
const fieldLength = fieldLengthOf(fieldLengthColumns, fieldIndex, internalId, fieldAvgLengths[fieldIndex]);
|
|
8507
|
-
score += (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8687
|
+
score += (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8688
|
+
termFrequencies[entry],
|
|
8689
|
+
idf,
|
|
8690
|
+
fieldLength,
|
|
8691
|
+
fieldAvgLengths[fieldIndex],
|
|
8692
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
8693
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
8694
|
+
)) * fieldBoosts[fieldIndex];
|
|
8508
8695
|
scored2 = true;
|
|
8509
8696
|
}
|
|
8510
8697
|
entry++;
|
|
@@ -8572,7 +8759,6 @@ function searchFulltext(state, params) {
|
|
|
8572
8759
|
}
|
|
8573
8760
|
const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
|
|
8574
8761
|
const avgFieldLengths = globalStats?.averageFieldLengths ?? state.stats.averageFieldLengths;
|
|
8575
|
-
const { k1, b } = resolveBM25Params(bm25Params);
|
|
8576
8762
|
const scoresAreZero = totalDocs === 0;
|
|
8577
8763
|
if (state.scoreBuffer === null) state.scoreBuffer = createScoreBuffer(state.docStore.internalIdCapacity());
|
|
8578
8764
|
const scoreBuffer = state.scoreBuffer;
|
|
@@ -8581,8 +8767,11 @@ function searchFulltext(state, params) {
|
|
|
8581
8767
|
const useIntersection = termMatch === "all" && queryTokens.length > 1;
|
|
8582
8768
|
const fieldNames = state.fieldNameTable.names;
|
|
8583
8769
|
const resolver = state.docStore.resolver();
|
|
8584
|
-
const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths);
|
|
8585
|
-
const { searchable, boosts, averageLengths, lengthColumns } = fieldScoring;
|
|
8770
|
+
const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths, bm25Params);
|
|
8771
|
+
const { searchable, boosts, averageLengths, lengthColumns, k1s, bs, boundPairs } = fieldScoring;
|
|
8772
|
+
const onePair = boundPairs.length === 2;
|
|
8773
|
+
const onlyK1 = boundPairs[0];
|
|
8774
|
+
const onlyB = boundPairs[1];
|
|
8586
8775
|
function documentFrequency(token, list) {
|
|
8587
8776
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
8588
8777
|
}
|
|
@@ -8592,8 +8781,6 @@ function searchFulltext(state, params) {
|
|
|
8592
8781
|
partialMarks: partialFieldMarks(fieldNames, partialFields),
|
|
8593
8782
|
fieldNames,
|
|
8594
8783
|
totalDocs,
|
|
8595
|
-
k1,
|
|
8596
|
-
b,
|
|
8597
8784
|
filterBitset,
|
|
8598
8785
|
globalFrequency(token) {
|
|
8599
8786
|
if (globalStats === void 0 || !Object.hasOwn(globalStats.docFrequencies, token)) return void 0;
|
|
@@ -8619,7 +8806,14 @@ function searchFulltext(state, params) {
|
|
|
8619
8806
|
if (searchable[fieldIndex] === 0) continue;
|
|
8620
8807
|
const avgLen = averageLengths[fieldIndex];
|
|
8621
8808
|
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8622
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8809
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8810
|
+
termFrequencies[pi],
|
|
8811
|
+
match.idf,
|
|
8812
|
+
actualFieldLength,
|
|
8813
|
+
avgLen,
|
|
8814
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
8815
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
8816
|
+
);
|
|
8623
8817
|
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
8624
8818
|
}
|
|
8625
8819
|
return;
|
|
@@ -8634,7 +8828,14 @@ function searchFulltext(state, params) {
|
|
|
8634
8828
|
const termFrequency = termFrequencies[pi];
|
|
8635
8829
|
const avgLen = averageLengths[fieldIndex];
|
|
8636
8830
|
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8637
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8831
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8832
|
+
termFrequency,
|
|
8833
|
+
match.idf,
|
|
8834
|
+
actualFieldLength,
|
|
8835
|
+
avgLen,
|
|
8836
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
8837
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
8838
|
+
);
|
|
8638
8839
|
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
8639
8840
|
if (components !== null) {
|
|
8640
8841
|
recordComponents(
|
|
@@ -8662,7 +8863,6 @@ function searchFulltext(state, params) {
|
|
|
8662
8863
|
list: prunableList,
|
|
8663
8864
|
docFrequency: documentFrequency(queryTokens[0].token, prunableList),
|
|
8664
8865
|
totalDocs,
|
|
8665
|
-
bm25Params,
|
|
8666
8866
|
limit: maxResults,
|
|
8667
8867
|
fields: fieldScoring,
|
|
8668
8868
|
resolver
|
|
@@ -8674,7 +8874,6 @@ function searchFulltext(state, params) {
|
|
|
8674
8874
|
terms: prunableLists,
|
|
8675
8875
|
docFrequencies: prunableLists.map((term) => documentFrequency(term.token, term.list)),
|
|
8676
8876
|
totalDocs,
|
|
8677
|
-
bm25Params,
|
|
8678
8877
|
limit: maxResults,
|
|
8679
8878
|
fields: fieldScoring,
|
|
8680
8879
|
resolver,
|
|
@@ -8695,7 +8894,7 @@ function searchFulltext(state, params) {
|
|
|
8695
8894
|
}
|
|
8696
8895
|
resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
|
|
8697
8896
|
}
|
|
8698
|
-
resolved.sort((a,
|
|
8897
|
+
resolved.sort((a, b) => a.totalPostings - b.totalPostings);
|
|
8699
8898
|
for (let tokenIndex = 0; tokenIndex < resolved.length; tokenIndex++) {
|
|
8700
8899
|
const onlyScored = tokenIndex > 0;
|
|
8701
8900
|
for (const match of resolved[tokenIndex].matches) scorePostings(match, onlyScored);
|
|
@@ -12329,7 +12528,8 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
12329
12528
|
// src/workers/query-actions.ts
|
|
12330
12529
|
async function runQueryAction(entry, action) {
|
|
12331
12530
|
const patternWork = joinPatternWorkMeter(action.patternWork, action.params.filters);
|
|
12332
|
-
const
|
|
12531
|
+
const bm25Params = action.type === "query" ? action.bm25 ?? void 0 : entry.config.bm25;
|
|
12532
|
+
const searchOptions = { ...entry.searchOptions, patternWork, bm25Params };
|
|
12333
12533
|
if (action.type === "query") {
|
|
12334
12534
|
const result2 = await fanOutQuery(
|
|
12335
12535
|
entry.manager,
|
|
@@ -15103,7 +15303,7 @@ function requireLoadedVectorCode() {
|
|
|
15103
15303
|
function loadVectorCode() {
|
|
15104
15304
|
if (loaded !== null) return Promise.resolve(loaded);
|
|
15105
15305
|
if (loading === null) {
|
|
15106
|
-
loading = import('./module-
|
|
15306
|
+
loading = import('./module-S2VDU2JE.mjs').then(
|
|
15107
15307
|
(code) => {
|
|
15108
15308
|
loaded = code;
|
|
15109
15309
|
return code;
|
|
@@ -15254,7 +15454,6 @@ function createDirectExecutor(options) {
|
|
|
15254
15454
|
config,
|
|
15255
15455
|
language,
|
|
15256
15456
|
searchOptions: {
|
|
15257
|
-
bm25Params: config.bm25,
|
|
15258
15457
|
stopWords: manager.analysis.stopWords,
|
|
15259
15458
|
customTokenizer: manager.analysis.customTokenizer,
|
|
15260
15459
|
get synonyms() {
|
|
@@ -15395,6 +15594,11 @@ function createDirectExecutor(options) {
|
|
|
15395
15594
|
if (entry !== void 0) entry.manager.synonyms = lookupOf(entry.config, entry.language, action.synonyms);
|
|
15396
15595
|
return void 0;
|
|
15397
15596
|
}
|
|
15597
|
+
case "replaceBm25": {
|
|
15598
|
+
const entry = indexes.get(action.indexName);
|
|
15599
|
+
if (entry !== void 0) entry.config.bm25 = action.bm25;
|
|
15600
|
+
return void 0;
|
|
15601
|
+
}
|
|
15398
15602
|
case "get": {
|
|
15399
15603
|
const entry = requireIndex(action.indexName);
|
|
15400
15604
|
return entry.manager.get(action.docId);
|
|
@@ -15842,7 +16046,11 @@ function reconstructSchemaFromMetadata(metadata) {
|
|
|
15842
16046
|
const config = {
|
|
15843
16047
|
schema: unflattenSchema(metadata.schema),
|
|
15844
16048
|
language: metadata.language,
|
|
15845
|
-
bm25: {
|
|
16049
|
+
bm25: {
|
|
16050
|
+
k1: metadata.bm25Params.k1,
|
|
16051
|
+
b: metadata.bm25Params.b,
|
|
16052
|
+
...metadata.bm25Params.fields !== void 0 ? { fields: metadata.bm25Params.fields } : {}
|
|
16053
|
+
}
|
|
15846
16054
|
};
|
|
15847
16055
|
if (metadata.embedding) {
|
|
15848
16056
|
config.embedding = metadata.embedding.adapter !== void 0 ? { adapter: metadata.embedding.adapter, fields: metadata.embedding.fields } : { fields: metadata.embedding.fields };
|
|
@@ -16037,16 +16245,6 @@ function validateWorkerConfig(workers, lifecycle) {
|
|
|
16037
16245
|
);
|
|
16038
16246
|
}
|
|
16039
16247
|
}
|
|
16040
|
-
function validateBM25Params(bm25) {
|
|
16041
|
-
if (bm25 === void 0) return;
|
|
16042
|
-
const { k1, b } = bm25;
|
|
16043
|
-
if (k1 !== void 0 && (!Number.isFinite(k1) || k1 < 0)) {
|
|
16044
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.k1 must be a finite number at or above 0", { k1 });
|
|
16045
|
-
}
|
|
16046
|
-
if (b !== void 0 && (!Number.isFinite(b) || b < 0 || b > 1)) {
|
|
16047
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.b must be between 0 and 1", { b });
|
|
16048
|
-
}
|
|
16049
|
-
}
|
|
16050
16248
|
function clampLimit(limit) {
|
|
16051
16249
|
return clampRowCount(limit, DEFAULT_LIMIT);
|
|
16052
16250
|
}
|
|
@@ -18796,23 +18994,24 @@ function scoresPerPartition(state, indexName, params, globalStats) {
|
|
|
18796
18994
|
if (mode === "local") return true;
|
|
18797
18995
|
return mode === "broadcast" && globalStats !== void 0;
|
|
18798
18996
|
}
|
|
18799
|
-
function queryAction(indexName, params, globalStats, partitionIds, patternWork) {
|
|
18997
|
+
function queryAction(indexName, params, globalStats, partitionIds, patternWork, bm25) {
|
|
18800
18998
|
return {
|
|
18801
18999
|
type: "query",
|
|
18802
19000
|
indexName,
|
|
18803
19001
|
params,
|
|
18804
19002
|
requestId: createRequestId(),
|
|
18805
19003
|
patternWork,
|
|
19004
|
+
bm25: bm25 ?? null,
|
|
18806
19005
|
...partitionIds !== void 0 ? { partitionIds } : {},
|
|
18807
19006
|
...globalStats !== void 0 ? { globalStats } : {}
|
|
18808
19007
|
};
|
|
18809
19008
|
}
|
|
18810
|
-
async function runSplit(leases, scope, indexName, params, globalStats, patternWork) {
|
|
19009
|
+
async function runSplit(leases, scope, indexName, params, globalStats, patternWork, bm25) {
|
|
18811
19010
|
const assignments = partitionsPerLease(scope, leases.length);
|
|
18812
19011
|
const workerParams = params.facets !== void 0 ? { ...params, facets: oversampledFacetConfig(params.facets) } : params;
|
|
18813
19012
|
const outcomes = await Promise.allSettled(
|
|
18814
19013
|
assignments.map(
|
|
18815
|
-
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds, patternWork)).finally(() => leases[at].release())
|
|
19014
|
+
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds, patternWork, bm25)).finally(() => leases[at].release())
|
|
18816
19015
|
)
|
|
18817
19016
|
);
|
|
18818
19017
|
const results = [];
|
|
@@ -18848,7 +19047,7 @@ function answersFasterOnMainCopy(state, pool, manager) {
|
|
|
18848
19047
|
if (manager.countDocuments() > MAIN_COPY_LONE_QUERY_DOCUMENTS) return false;
|
|
18849
19048
|
return takeMainCopyTurn(state);
|
|
18850
19049
|
}
|
|
18851
|
-
async function searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork) {
|
|
19050
|
+
async function searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork, bm25) {
|
|
18852
19051
|
const pool = state.workerPool;
|
|
18853
19052
|
if (!pool) return null;
|
|
18854
19053
|
if (!state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName)) return null;
|
|
@@ -18863,7 +19062,7 @@ async function searchViaWorker(state, indexName, params, globalStats, partitionI
|
|
|
18863
19062
|
if (idle.length >= 2 && scoresPerPartition(state, indexName, params, globalStats)) {
|
|
18864
19063
|
leases.push(...idle);
|
|
18865
19064
|
fork = patternWork.fork(leases.length);
|
|
18866
|
-
const merged = await runSplit(leases, scope, indexName, params, globalStats, fork.shared);
|
|
19065
|
+
const merged = await runSplit(leases, scope, indexName, params, globalStats, fork.shared, bm25);
|
|
18867
19066
|
fork.settle(merged.patternWorkSpent ?? 0);
|
|
18868
19067
|
return merged;
|
|
18869
19068
|
}
|
|
@@ -18873,7 +19072,7 @@ async function searchViaWorker(state, indexName, params, globalStats, partitionI
|
|
|
18873
19072
|
leases.push(lease);
|
|
18874
19073
|
fork = patternWork.fork(1);
|
|
18875
19074
|
const result = await lease.executor.execute(
|
|
18876
|
-
queryAction(indexName, params, globalStats, partitionIds, fork.shared)
|
|
19075
|
+
queryAction(indexName, params, globalStats, partitionIds, fork.shared, bm25)
|
|
18877
19076
|
);
|
|
18878
19077
|
fork.settle(result.patternWorkSpent ?? 0);
|
|
18879
19078
|
return result;
|
|
@@ -19134,7 +19333,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
19134
19333
|
segmentBuildConcurrency: (indexName) => segmentBuildConcurrency(state, indexName),
|
|
19135
19334
|
holdUnbroadcastSegments: (indexName, segmentIds) => holdUnbroadcastSegments(state, indexName, segmentIds),
|
|
19136
19335
|
releaseUnbroadcastSegments: (indexName, segmentIds) => releaseUnbroadcastSegments(state, indexName, segmentIds),
|
|
19137
|
-
searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork),
|
|
19336
|
+
searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork, bm25) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork, bm25),
|
|
19138
19337
|
hasWorkerPool: () => state.workerPool !== null,
|
|
19139
19338
|
mainCopyQueries: () => state.mainCopyQueries ?? "lone",
|
|
19140
19339
|
shareMainThread: () => {
|
|
@@ -19198,7 +19397,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
19198
19397
|
// src/engine/core.ts
|
|
19199
19398
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
19200
19399
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
19201
|
-
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-
|
|
19400
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-PL4TTFJE.mjs')).resolveRerankSettings(config);
|
|
19202
19401
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
19203
19402
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
19204
19403
|
const vectorCopyPolicy = {
|
|
@@ -19209,7 +19408,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19209
19408
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
19210
19409
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
19211
19410
|
}
|
|
19212
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
19411
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-HCCC6BJB.mjs');
|
|
19213
19412
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
19214
19413
|
const executor = createDirectExecutor({
|
|
19215
19414
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -19506,7 +19705,7 @@ async function shutdownEngine(core) {
|
|
|
19506
19705
|
}
|
|
19507
19706
|
}
|
|
19508
19707
|
if (core.rerankAdapters.size > 0) {
|
|
19509
|
-
await (await import('./runtime-
|
|
19708
|
+
await (await import('./runtime-PL4TTFJE.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
19510
19709
|
}
|
|
19511
19710
|
await orchestrator.shutdown();
|
|
19512
19711
|
await executor.shutdown();
|
|
@@ -19539,6 +19738,105 @@ async function resolveRunnableConfig(config) {
|
|
|
19539
19738
|
}
|
|
19540
19739
|
}
|
|
19541
19740
|
|
|
19741
|
+
// src/search/bm25-params.ts
|
|
19742
|
+
var BM25_KEYS = /* @__PURE__ */ new Set(["k1", "b", "fields"]);
|
|
19743
|
+
var BM25_FIELD_KEYS = /* @__PURE__ */ new Set(["k1", "b"]);
|
|
19744
|
+
function displayed(value) {
|
|
19745
|
+
if (typeof value === "string") return `"${value.slice(0, BM25_DISPLAYED_NAME_LENGTH)}"`;
|
|
19746
|
+
if (typeof value === "number" || typeof value === "boolean" || value === null) return String(value);
|
|
19747
|
+
return `a value of type ${typeof value}`;
|
|
19748
|
+
}
|
|
19749
|
+
function requireValue(label, value, maximum, ranges) {
|
|
19750
|
+
if (value === void 0) return;
|
|
19751
|
+
const usable = typeof value === "number" && Number.isFinite(value) && (ranges === "stored" || value >= 0 && value <= maximum);
|
|
19752
|
+
if (usable) return;
|
|
19753
|
+
throw new NarsilError(
|
|
19754
|
+
ErrorCodes.CONFIG_INVALID,
|
|
19755
|
+
`${label} must be a finite number from 0 to ${maximum}, and this request sets ${displayed(value)}`,
|
|
19756
|
+
{ parameter: label }
|
|
19757
|
+
);
|
|
19758
|
+
}
|
|
19759
|
+
function requireKnownKeys2(label, value, known) {
|
|
19760
|
+
for (const key of Object.keys(value)) {
|
|
19761
|
+
if (known.has(key)) continue;
|
|
19762
|
+
throw new NarsilError(
|
|
19763
|
+
ErrorCodes.CONFIG_INVALID,
|
|
19764
|
+
`${label} takes ${[...known].join(", ")}, and this request sets ${displayed(key)}`,
|
|
19765
|
+
{ parameter: label }
|
|
19766
|
+
);
|
|
19767
|
+
}
|
|
19768
|
+
}
|
|
19769
|
+
function requireFieldPair(name, pair, ranges) {
|
|
19770
|
+
const label = `bm25.fields.${name.slice(0, BM25_DISPLAYED_NAME_LENGTH)}`;
|
|
19771
|
+
if (!isPlainObject(pair)) {
|
|
19772
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${label} must be an object that sets k1 or b`, {
|
|
19773
|
+
parameter: label
|
|
19774
|
+
});
|
|
19775
|
+
}
|
|
19776
|
+
requireKnownKeys2(label, pair, BM25_FIELD_KEYS);
|
|
19777
|
+
requireValue(`${label}.k1`, pair.k1, MAX_BM25_K1, ranges);
|
|
19778
|
+
requireValue(`${label}.b`, pair.b, MAX_BM25_B, ranges);
|
|
19779
|
+
}
|
|
19780
|
+
function requireScoredField(name, flatSchema, unknownFieldCode) {
|
|
19781
|
+
const fieldType = Object.hasOwn(flatSchema, name) ? flatSchema[name] : void 0;
|
|
19782
|
+
if (typeof fieldType === "string" && isWordIndexedFieldType(fieldType)) return;
|
|
19783
|
+
throw new NarsilError(
|
|
19784
|
+
unknownFieldCode,
|
|
19785
|
+
fieldType === void 0 ? `BM25 parameters apply to a text field that the schema declares, and the schema declares no field ${displayed(name)}` : `BM25 parameters apply to a text field alone, and the schema declares field ${displayed(name)} as ${displayed(fieldType)}`,
|
|
19786
|
+
{ field: name.slice(0, BM25_DISPLAYED_NAME_LENGTH) }
|
|
19787
|
+
);
|
|
19788
|
+
}
|
|
19789
|
+
function fieldsOfCheckedSetting(raw, ranges) {
|
|
19790
|
+
if (!isPlainObject(raw)) {
|
|
19791
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25 must be an object that sets k1, b, or fields");
|
|
19792
|
+
}
|
|
19793
|
+
requireKnownKeys2("bm25", raw, BM25_KEYS);
|
|
19794
|
+
requireValue("bm25.k1", raw.k1, MAX_BM25_K1, ranges);
|
|
19795
|
+
requireValue("bm25.b", raw.b, MAX_BM25_B, ranges);
|
|
19796
|
+
const fields = raw.fields;
|
|
19797
|
+
if (fields === void 0) return void 0;
|
|
19798
|
+
if (!isPlainObject(fields)) {
|
|
19799
|
+
throw new NarsilError(
|
|
19800
|
+
ErrorCodes.CONFIG_INVALID,
|
|
19801
|
+
"bm25.fields must be an object that maps a field to its parameters"
|
|
19802
|
+
);
|
|
19803
|
+
}
|
|
19804
|
+
return fields;
|
|
19805
|
+
}
|
|
19806
|
+
function validateBM25Params(raw, schema, unknownFieldCode, ranges = "settable") {
|
|
19807
|
+
const fields = fieldsOfCheckedSetting(raw, ranges);
|
|
19808
|
+
if (fields === void 0) return;
|
|
19809
|
+
const names = Object.keys(fields);
|
|
19810
|
+
if (names.length === 0) return;
|
|
19811
|
+
const flatSchema = flattenSchema(schema);
|
|
19812
|
+
for (const name of names) {
|
|
19813
|
+
requireScoredField(name, flatSchema, unknownFieldCode);
|
|
19814
|
+
requireFieldPair(name, fields[name], ranges);
|
|
19815
|
+
}
|
|
19816
|
+
}
|
|
19817
|
+
function presentPair(pair) {
|
|
19818
|
+
return {
|
|
19819
|
+
...pair.k1 !== void 0 ? { k1: pair.k1 } : {},
|
|
19820
|
+
...pair.b !== void 0 ? { b: pair.b } : {}
|
|
19821
|
+
};
|
|
19822
|
+
}
|
|
19823
|
+
function storedBM25Params(params) {
|
|
19824
|
+
const stored = presentPair(params);
|
|
19825
|
+
const fields = params.fields;
|
|
19826
|
+
if (fields !== void 0) {
|
|
19827
|
+
const kept = /* @__PURE__ */ Object.create(null);
|
|
19828
|
+
let holdsField = false;
|
|
19829
|
+
for (const name of Object.keys(fields)) {
|
|
19830
|
+
const pair = presentPair(fields[name]);
|
|
19831
|
+
if (pair.k1 === void 0 && pair.b === void 0) continue;
|
|
19832
|
+
kept[name] = pair;
|
|
19833
|
+
holdsField = true;
|
|
19834
|
+
}
|
|
19835
|
+
if (holdsField) stored.fields = kept;
|
|
19836
|
+
}
|
|
19837
|
+
return Object.keys(stored).length === 0 ? void 0 : stored;
|
|
19838
|
+
}
|
|
19839
|
+
|
|
19542
19840
|
// src/serialization/crc32.ts
|
|
19543
19841
|
var IEEE_POLYNOMIAL = 3988292384;
|
|
19544
19842
|
var ALL_BITS = 4294967295;
|
|
@@ -19868,7 +20166,8 @@ function configurationPayloadOf(config, language, adapterName, analysisRevision,
|
|
|
19868
20166
|
...bm25 !== void 0 ? {
|
|
19869
20167
|
bm25: {
|
|
19870
20168
|
...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
|
|
19871
|
-
...bm25.b !== void 0 ? { b: bm25.b } : {}
|
|
20169
|
+
...bm25.b !== void 0 ? { b: bm25.b } : {},
|
|
20170
|
+
...bm25.fields !== void 0 ? { fields: bm25.fields } : {}
|
|
19872
20171
|
}
|
|
19873
20172
|
} : {},
|
|
19874
20173
|
surfaceForms: config.surfaceForms !== false,
|
|
@@ -19956,14 +20255,21 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
19956
20255
|
const schema = validateSchema(requestedConfig.schema);
|
|
19957
20256
|
if (appliesCreationLimits) requireCreatableSchema(schema);
|
|
19958
20257
|
const configured = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
|
|
19959
|
-
const indexConfig = withoutSynonyms(configured);
|
|
20258
|
+
const indexConfig = { ...withoutSynonyms(configured) };
|
|
19960
20259
|
requirePatternSearchFor(schema);
|
|
19961
20260
|
requirePartialWordSearchFor(schema);
|
|
19962
20261
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
19963
20262
|
requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
|
|
19964
20263
|
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
19965
20264
|
validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
|
|
19966
|
-
|
|
20265
|
+
if (indexConfig.bm25 !== void 0) {
|
|
20266
|
+
validateBM25Params(
|
|
20267
|
+
indexConfig.bm25,
|
|
20268
|
+
schema,
|
|
20269
|
+
ErrorCodes.CONFIG_INVALID,
|
|
20270
|
+
appliesCreationLimits ? "settable" : "stored"
|
|
20271
|
+
);
|
|
20272
|
+
}
|
|
19967
20273
|
if (indexConfig.partitions) {
|
|
19968
20274
|
validatePartitionConfig(indexConfig.partitions);
|
|
19969
20275
|
}
|
|
@@ -21985,6 +22291,75 @@ function readProcessMemory() {
|
|
|
21985
22291
|
}
|
|
21986
22292
|
}
|
|
21987
22293
|
|
|
22294
|
+
// src/engine/serial-index-update.ts
|
|
22295
|
+
var updatingByCore = /* @__PURE__ */ new WeakMap();
|
|
22296
|
+
async function withOpenIndex(core, indexName, action) {
|
|
22297
|
+
core.guardShutdown();
|
|
22298
|
+
core.requireIndex(indexName);
|
|
22299
|
+
const release = await core.indexState.acquire(indexName);
|
|
22300
|
+
try {
|
|
22301
|
+
return await action();
|
|
22302
|
+
} finally {
|
|
22303
|
+
release();
|
|
22304
|
+
}
|
|
22305
|
+
}
|
|
22306
|
+
async function oneAtATime(core, indexName, action) {
|
|
22307
|
+
let chains = updatingByCore.get(core);
|
|
22308
|
+
if (chains === void 0) {
|
|
22309
|
+
chains = /* @__PURE__ */ new Map();
|
|
22310
|
+
updatingByCore.set(core, chains);
|
|
22311
|
+
}
|
|
22312
|
+
const updating = chains;
|
|
22313
|
+
const earlier = updating.get(indexName) ?? Promise.resolve();
|
|
22314
|
+
const current = earlier.catch(() => void 0).then(action);
|
|
22315
|
+
updating.set(indexName, current);
|
|
22316
|
+
try {
|
|
22317
|
+
return await current;
|
|
22318
|
+
} finally {
|
|
22319
|
+
if (updating.get(indexName) === current) updating.delete(indexName);
|
|
22320
|
+
}
|
|
22321
|
+
}
|
|
22322
|
+
|
|
22323
|
+
// src/engine/bm25-update.ts
|
|
22324
|
+
async function replaceNow(core, indexName, bm25) {
|
|
22325
|
+
const { config } = core.requireIndex(indexName);
|
|
22326
|
+
const previous = config.bm25;
|
|
22327
|
+
config.bm25 = bm25;
|
|
22328
|
+
try {
|
|
22329
|
+
if (core.durability) await core.durability.manager.persistMetadata(indexName);
|
|
22330
|
+
} catch (err) {
|
|
22331
|
+
config.bm25 = previous;
|
|
22332
|
+
throw err;
|
|
22333
|
+
}
|
|
22334
|
+
await core.orchestrator.replicateToWorkers({
|
|
22335
|
+
type: "replaceBm25",
|
|
22336
|
+
indexName,
|
|
22337
|
+
requestId: `replace-bm25-${indexName}`,
|
|
22338
|
+
...bm25 !== void 0 ? { bm25 } : {}
|
|
22339
|
+
});
|
|
22340
|
+
await core.orchestrator.awaitWrites(indexName);
|
|
22341
|
+
}
|
|
22342
|
+
function updateIndexBm25(core, indexName, value) {
|
|
22343
|
+
return withOpenIndex(
|
|
22344
|
+
core,
|
|
22345
|
+
indexName,
|
|
22346
|
+
() => oneAtATime(core, indexName, () => {
|
|
22347
|
+
const entry = core.requireIndex(indexName);
|
|
22348
|
+
validateBM25Params(value, entry.config.schema, ErrorCodes.CONFIG_INVALID);
|
|
22349
|
+
return replaceNow(core, indexName, storedBM25Params(value));
|
|
22350
|
+
})
|
|
22351
|
+
);
|
|
22352
|
+
}
|
|
22353
|
+
|
|
22354
|
+
// src/narsil/bm25.ts
|
|
22355
|
+
function bm25Operations(core) {
|
|
22356
|
+
return {
|
|
22357
|
+
async updateBM25(indexName, bm25) {
|
|
22358
|
+
await updateIndexBm25(core, indexName, bm25);
|
|
22359
|
+
}
|
|
22360
|
+
};
|
|
22361
|
+
}
|
|
22362
|
+
|
|
21988
22363
|
// src/narsil/internals.ts
|
|
21989
22364
|
var engineCores = /* @__PURE__ */ new WeakMap();
|
|
21990
22365
|
function bindEngineCore(engine, core) {
|
|
@@ -22005,18 +22380,37 @@ function restoredBm25(raw, reject2) {
|
|
|
22005
22380
|
if (!isPlainObject3(raw)) {
|
|
22006
22381
|
reject2("bm25 must be an object");
|
|
22007
22382
|
}
|
|
22383
|
+
const fields = restoredBm25Fields(raw.fields, reject2);
|
|
22384
|
+
return {
|
|
22385
|
+
...restoredBm25Pair(raw, "bm25", reject2),
|
|
22386
|
+
...fields !== void 0 ? { fields } : {}
|
|
22387
|
+
};
|
|
22388
|
+
}
|
|
22389
|
+
function restoredBm25Pair(raw, label, reject2) {
|
|
22008
22390
|
const { k1, b } = raw;
|
|
22009
22391
|
if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
|
|
22010
|
-
reject2(
|
|
22392
|
+
reject2(`${label} k1 must be a finite number`);
|
|
22011
22393
|
}
|
|
22012
22394
|
if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
|
|
22013
|
-
reject2(
|
|
22395
|
+
reject2(`${label} b must be a finite number`);
|
|
22014
22396
|
}
|
|
22015
22397
|
return {
|
|
22016
22398
|
...k1 !== void 0 ? { k1 } : {},
|
|
22017
22399
|
...b !== void 0 ? { b } : {}
|
|
22018
22400
|
};
|
|
22019
22401
|
}
|
|
22402
|
+
function restoredBm25Fields(raw, reject2) {
|
|
22403
|
+
if (raw === void 0) return void 0;
|
|
22404
|
+
if (!isPlainObject3(raw)) reject2("bm25 fields must be an object");
|
|
22405
|
+
const source = raw;
|
|
22406
|
+
const fields = /* @__PURE__ */ Object.create(null);
|
|
22407
|
+
for (const name of Object.keys(source)) {
|
|
22408
|
+
const pair = source[name];
|
|
22409
|
+
if (!isPlainObject3(pair)) reject2("bm25 fields must map each field to an object");
|
|
22410
|
+
fields[name] = restoredBm25Pair(pair, "bm25 field", reject2);
|
|
22411
|
+
}
|
|
22412
|
+
return fields;
|
|
22413
|
+
}
|
|
22020
22414
|
function restoredStopWords(name, list, reject2) {
|
|
22021
22415
|
if (name !== void 0 && typeof name !== "string") {
|
|
22022
22416
|
reject2("stopWords must be a name");
|
|
@@ -22614,7 +23008,7 @@ async function readDocumentExport(engine, indexName, source) {
|
|
|
22614
23008
|
|
|
22615
23009
|
// src/narsil/portability.ts
|
|
22616
23010
|
function portabilityOperations(core, config, engine) {
|
|
22617
|
-
async function
|
|
23011
|
+
async function withOpenIndex2(indexName, action) {
|
|
22618
23012
|
core.guardShutdown();
|
|
22619
23013
|
core.requireIndex(indexName);
|
|
22620
23014
|
const release = await core.indexState.acquire(indexName);
|
|
@@ -22642,25 +23036,25 @@ function portabilityOperations(core, config, engine) {
|
|
|
22642
23036
|
}
|
|
22643
23037
|
return {
|
|
22644
23038
|
snapshot(indexName) {
|
|
22645
|
-
return
|
|
22646
|
-
const { createSnapshot } = await import('./snapshot-
|
|
23039
|
+
return withOpenIndex2(indexName, async () => {
|
|
23040
|
+
const { createSnapshot } = await import('./snapshot-RTUCHI2L.mjs');
|
|
22647
23041
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
22648
23042
|
});
|
|
22649
23043
|
},
|
|
22650
23044
|
async restore(indexName, data) {
|
|
22651
23045
|
core.guardShutdown();
|
|
22652
23046
|
validateIndexName(indexName);
|
|
22653
|
-
const restored = import('./restore-queue-
|
|
23047
|
+
const restored = import('./restore-queue-BZ7WWEC4.mjs').then(
|
|
22654
23048
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
22655
23049
|
);
|
|
22656
23050
|
await emitWriteEventAfter(core, indexName, restored);
|
|
22657
23051
|
core.heapPressureNotifier.check(indexName);
|
|
22658
23052
|
},
|
|
22659
23053
|
exportDocuments(indexName) {
|
|
22660
|
-
return
|
|
23054
|
+
return withOpenIndex2(indexName, async () => {
|
|
22661
23055
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
22662
|
-
import('./snapshot-
|
|
22663
|
-
import('./write-
|
|
23056
|
+
import('./snapshot-RTUCHI2L.mjs'),
|
|
23057
|
+
import('./write-IMU7G5NU.mjs')
|
|
22664
23058
|
]);
|
|
22665
23059
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
22666
23060
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -22816,6 +23210,27 @@ function boundRerankOf(rerank) {
|
|
|
22816
23210
|
}
|
|
22817
23211
|
return present;
|
|
22818
23212
|
}
|
|
23213
|
+
function presentMembersOf(record) {
|
|
23214
|
+
const present = {};
|
|
23215
|
+
for (const [key, value] of Object.entries(record)) {
|
|
23216
|
+
if (value !== void 0) present[key] = value;
|
|
23217
|
+
}
|
|
23218
|
+
return present;
|
|
23219
|
+
}
|
|
23220
|
+
function boundBm25Of(bm25) {
|
|
23221
|
+
const bound = presentMembersOf(bm25);
|
|
23222
|
+
const fields = bm25.fields;
|
|
23223
|
+
if (typeof fields !== "object" || fields === null) return bound;
|
|
23224
|
+
const boundFields = {};
|
|
23225
|
+
for (const [name, pair] of Object.entries(fields)) {
|
|
23226
|
+
const members = typeof pair === "object" && pair !== null ? presentMembersOf(pair) : pair;
|
|
23227
|
+
if (typeof members === "object" && members !== null && Object.keys(members).length === 0) continue;
|
|
23228
|
+
boundFields[name] = members;
|
|
23229
|
+
}
|
|
23230
|
+
if (Object.keys(boundFields).length === 0) delete bound.fields;
|
|
23231
|
+
else bound.fields = boundFields;
|
|
23232
|
+
return bound;
|
|
23233
|
+
}
|
|
22819
23234
|
var FILTERS_SLOT = 2;
|
|
22820
23235
|
var BOUND_SLOT_COUNT = 16;
|
|
22821
23236
|
function bindingOf(values, clock) {
|
|
@@ -22846,6 +23261,7 @@ function queryBindingOf(params, clock) {
|
|
|
22846
23261
|
params.scoring ?? "local"
|
|
22847
23262
|
];
|
|
22848
23263
|
if (params.rerank !== void 0) values.push(boundRerankOf(params.rerank));
|
|
23264
|
+
if (params.bm25 !== void 0 && bm25SetsValue(params.bm25)) values.push(boundBm25Of(params.bm25));
|
|
22849
23265
|
return bindingOf(values, clock);
|
|
22850
23266
|
}
|
|
22851
23267
|
function lazyQueryBinding(params, clock) {
|
|
@@ -22908,6 +23324,7 @@ function requireGroupableFields(fields, flatSchema, strict) {
|
|
|
22908
23324
|
}
|
|
22909
23325
|
}
|
|
22910
23326
|
function requireUsableQueryFields(params, schema, strict = false) {
|
|
23327
|
+
if (params.bm25 !== void 0) validateBM25Params(params.bm25, schema, ErrorCodes.SEARCH_INVALID_FIELD);
|
|
22911
23328
|
const groupFields = params.group?.fields;
|
|
22912
23329
|
const checksGroup = Array.isArray(groupFields) && groupFields.length > 0;
|
|
22913
23330
|
if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
|
|
@@ -22986,9 +23403,13 @@ function broadcastStatsForWorker(params, context, scoring) {
|
|
|
22986
23403
|
expansion: { options: params, schema: context.config.schema }
|
|
22987
23404
|
});
|
|
22988
23405
|
}
|
|
22989
|
-
function
|
|
23406
|
+
function bm25SettingFor(params, context) {
|
|
23407
|
+
return mergeBM25Levels(params.bm25, context.indexBm25 ?? context.config.bm25);
|
|
23408
|
+
}
|
|
23409
|
+
function searchOptionsFor(params, context) {
|
|
23410
|
+
const { manager, patternWork } = context;
|
|
22990
23411
|
return {
|
|
22991
|
-
bm25Params:
|
|
23412
|
+
bm25Params: bm25SettingFor(params, context),
|
|
22992
23413
|
stopWords: manager.analysis.stopWords,
|
|
22993
23414
|
customTokenizer: manager.analysis.customTokenizer,
|
|
22994
23415
|
synonyms: manager.synonyms,
|
|
@@ -23749,7 +24170,8 @@ async function textLeg(textOnlyParams, context) {
|
|
|
23749
24170
|
textOnlyParams,
|
|
23750
24171
|
broadcastStatsForWorker(textOnlyParams, context, scoring),
|
|
23751
24172
|
context.partitionIds,
|
|
23752
|
-
context.patternWork
|
|
24173
|
+
context.patternWork,
|
|
24174
|
+
bm25SettingFor(textOnlyParams, context)
|
|
23753
24175
|
);
|
|
23754
24176
|
if (viaWorker) return viaWorker;
|
|
23755
24177
|
}
|
|
@@ -23759,7 +24181,7 @@ async function textLeg(textOnlyParams, context) {
|
|
|
23759
24181
|
language,
|
|
23760
24182
|
config.schema,
|
|
23761
24183
|
scoring,
|
|
23762
|
-
searchOptionsFor(
|
|
24184
|
+
searchOptionsFor(textOnlyParams, context)
|
|
23763
24185
|
);
|
|
23764
24186
|
}
|
|
23765
24187
|
async function vectorLeg(vectorConfig, queryVector, context, filtered, k, rescoreWindow) {
|
|
@@ -23857,7 +24279,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
23857
24279
|
const flatSchema = flattenSchema(schema);
|
|
23858
24280
|
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
23859
24281
|
const modes = sortModesOf(normalized);
|
|
23860
|
-
const options = searchOptionsFor(
|
|
24282
|
+
const options = searchOptionsFor(params, context);
|
|
23861
24283
|
const partitions = partitionsFor(manager, context.partitionIds);
|
|
23862
24284
|
const partitionLimit = offset + limit + 1;
|
|
23863
24285
|
const merged = [];
|
|
@@ -23925,13 +24347,13 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
23925
24347
|
}
|
|
23926
24348
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
23927
24349
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
23928
|
-
return import('./runtime-
|
|
24350
|
+
return import('./runtime-PL4TTFJE.mjs').then(
|
|
23929
24351
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
23930
24352
|
);
|
|
23931
24353
|
}
|
|
23932
24354
|
function executeQuery(request, callerContext) {
|
|
23933
24355
|
if (request.rerank === void 0) return runQuery(request, callerContext);
|
|
23934
|
-
return import('./runtime-
|
|
24356
|
+
return import('./runtime-PL4TTFJE.mjs').then(
|
|
23935
24357
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
23936
24358
|
);
|
|
23937
24359
|
}
|
|
@@ -23988,7 +24410,8 @@ async function runQuery(request, callerContext) {
|
|
|
23988
24410
|
params,
|
|
23989
24411
|
broadcastStatsForWorker(params, context, scoring),
|
|
23990
24412
|
context.partitionIds,
|
|
23991
|
-
context.patternWork
|
|
24413
|
+
context.patternWork,
|
|
24414
|
+
bm25SettingFor(params, context)
|
|
23992
24415
|
) : null;
|
|
23993
24416
|
if (workerResult) {
|
|
23994
24417
|
fanOutResult = workerResult;
|
|
@@ -23999,7 +24422,7 @@ async function runQuery(request, callerContext) {
|
|
|
23999
24422
|
language,
|
|
24000
24423
|
config.schema,
|
|
24001
24424
|
scoring,
|
|
24002
|
-
searchOptionsFor(
|
|
24425
|
+
searchOptionsFor(params, context)
|
|
24003
24426
|
);
|
|
24004
24427
|
}
|
|
24005
24428
|
}
|
|
@@ -24149,7 +24572,7 @@ async function executePreflight(request, callerContext) {
|
|
|
24149
24572
|
countExact = result.matchedExact !== false;
|
|
24150
24573
|
} else if (countsWithoutScores(params)) {
|
|
24151
24574
|
totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
|
|
24152
|
-
searchOptions: searchOptionsFor(
|
|
24575
|
+
searchOptions: searchOptionsFor(params, context),
|
|
24153
24576
|
partitionIds: context.partitionIds
|
|
24154
24577
|
});
|
|
24155
24578
|
} else {
|
|
@@ -24159,7 +24582,8 @@ async function executePreflight(request, callerContext) {
|
|
|
24159
24582
|
params,
|
|
24160
24583
|
broadcastStatsForWorker(params, context, scoring),
|
|
24161
24584
|
context.partitionIds,
|
|
24162
|
-
context.patternWork
|
|
24585
|
+
context.patternWork,
|
|
24586
|
+
bm25SettingFor(params, context)
|
|
24163
24587
|
) : null;
|
|
24164
24588
|
if (workerResult) {
|
|
24165
24589
|
totalMatched = workerResult.totalMatched;
|
|
@@ -24170,7 +24594,7 @@ async function executePreflight(request, callerContext) {
|
|
|
24170
24594
|
language,
|
|
24171
24595
|
config.schema,
|
|
24172
24596
|
scoring,
|
|
24173
|
-
searchOptionsFor(
|
|
24597
|
+
searchOptionsFor(params, context)
|
|
24174
24598
|
);
|
|
24175
24599
|
totalMatched = fanOutResult.totalMatched;
|
|
24176
24600
|
}
|
|
@@ -24312,7 +24736,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24312
24736
|
const manager = core.requireManager(indexName);
|
|
24313
24737
|
const scoped = scopedParams(params, options);
|
|
24314
24738
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
24315
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-
|
|
24739
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-PL4TTFJE.mjs');
|
|
24316
24740
|
const resolvedParams = await resolveVectorText(
|
|
24317
24741
|
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
24318
24742
|
entry.embeddingAdapter,
|
|
@@ -24329,6 +24753,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24329
24753
|
broadcastStats: broadcastStatsFor(core, options),
|
|
24330
24754
|
partitionIds: options?.partitionIds,
|
|
24331
24755
|
rescoreWindow: options?.rescoreWindow,
|
|
24756
|
+
indexBm25: options?.indexBm25,
|
|
24332
24757
|
cursorBinding: lazyQueryBinding(params),
|
|
24333
24758
|
patternWorkCap: core.patternWorkCap,
|
|
24334
24759
|
rerank: rerank?.rerankRunnerFor(core)
|
|
@@ -24378,6 +24803,7 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
24378
24803
|
indexName,
|
|
24379
24804
|
broadcastStats: broadcastStatsFor(core, options),
|
|
24380
24805
|
partitionIds: options?.partitionIds,
|
|
24806
|
+
indexBm25: options?.indexBm25,
|
|
24381
24807
|
cursorBinding: lazyQueryBinding(params),
|
|
24382
24808
|
patternWorkCap: core.patternWorkCap
|
|
24383
24809
|
};
|
|
@@ -24462,7 +24888,7 @@ function createNarsilFromCore(core, config) {
|
|
|
24462
24888
|
mutationCtx,
|
|
24463
24889
|
rebalanceCtx
|
|
24464
24890
|
} = core;
|
|
24465
|
-
async function
|
|
24891
|
+
async function withOpenIndex2(indexName, action) {
|
|
24466
24892
|
guardShutdown();
|
|
24467
24893
|
requireIndex(indexName);
|
|
24468
24894
|
const release = await core.indexState.acquire(indexName);
|
|
@@ -24473,7 +24899,7 @@ function createNarsilFromCore(core, config) {
|
|
|
24473
24899
|
}
|
|
24474
24900
|
}
|
|
24475
24901
|
function withOpenIndexWrite(indexName, action) {
|
|
24476
|
-
return emitWriteEventAfter(core, indexName,
|
|
24902
|
+
return emitWriteEventAfter(core, indexName, withOpenIndex2(indexName, action));
|
|
24477
24903
|
}
|
|
24478
24904
|
const narsil = {
|
|
24479
24905
|
createIndex(name, indexConfig) {
|
|
@@ -24559,12 +24985,12 @@ function createNarsilFromCore(core, config) {
|
|
|
24559
24985
|
return orchestrator.awaitWrites(indexName);
|
|
24560
24986
|
},
|
|
24561
24987
|
async get(indexName, docId) {
|
|
24562
|
-
return
|
|
24988
|
+
return withOpenIndex2(indexName, () => executor.execute({ type: "get", indexName, docId, requestId: docId }));
|
|
24563
24989
|
},
|
|
24564
24990
|
async getMultiple(indexName, docIds) {
|
|
24565
24991
|
guardShutdown();
|
|
24566
24992
|
requireIndex(indexName);
|
|
24567
|
-
return
|
|
24993
|
+
return withOpenIndex2(indexName, async () => {
|
|
24568
24994
|
const result = /* @__PURE__ */ new Map();
|
|
24569
24995
|
for (const docId of docIds) {
|
|
24570
24996
|
const doc = await executor.execute({
|
|
@@ -24579,10 +25005,10 @@ function createNarsilFromCore(core, config) {
|
|
|
24579
25005
|
});
|
|
24580
25006
|
},
|
|
24581
25007
|
async has(indexName, docId) {
|
|
24582
|
-
return
|
|
25008
|
+
return withOpenIndex2(indexName, () => executor.execute({ type: "has", indexName, docId, requestId: docId }));
|
|
24583
25009
|
},
|
|
24584
25010
|
async countDocuments(indexName) {
|
|
24585
|
-
return
|
|
25011
|
+
return withOpenIndex2(indexName, () => executor.execute({ type: "count", indexName, requestId: indexName }));
|
|
24586
25012
|
},
|
|
24587
25013
|
async listDocuments(indexName, params) {
|
|
24588
25014
|
return runEngineListDocuments(core, indexName, params ?? {});
|
|
@@ -24603,10 +25029,11 @@ function createNarsilFromCore(core, config) {
|
|
|
24603
25029
|
},
|
|
24604
25030
|
...portabilityOperations(core, config, () => narsil),
|
|
24605
25031
|
...synonymOperations(core),
|
|
25032
|
+
...bm25Operations(core),
|
|
24606
25033
|
async checkpoint(indexName) {
|
|
24607
25034
|
guardShutdown();
|
|
24608
25035
|
requireIndex(indexName);
|
|
24609
|
-
await
|
|
25036
|
+
await withOpenIndex2(indexName, async () => {
|
|
24610
25037
|
if (durability) await durability.manager.checkpoint(indexName);
|
|
24611
25038
|
});
|
|
24612
25039
|
},
|
|
@@ -24629,7 +25056,7 @@ function createNarsilFromCore(core, config) {
|
|
|
24629
25056
|
},
|
|
24630
25057
|
async updatePartitionConfig(indexName, partitionConfig) {
|
|
24631
25058
|
guardShutdown();
|
|
24632
|
-
return
|
|
25059
|
+
return withOpenIndex2(indexName, async () => {
|
|
24633
25060
|
const entry = requireIndex(indexName);
|
|
24634
25061
|
const manager = requireManager(indexName);
|
|
24635
25062
|
if (rebalancingIndexes.has(indexName)) {
|
|
@@ -24744,4 +25171,4 @@ async function createNarsil(config) {
|
|
|
24744
25171
|
// src/index.ts
|
|
24745
25172
|
var VERSION = "0.1.0";
|
|
24746
25173
|
|
|
24747
|
-
export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
|
|
25174
|
+
export { BYTES_TAG, DATE_TAG, DEFAULT_BM25_B, DEFAULT_BM25_K1, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
|