@delali/narsil 0.3.1-next.128 → 0.3.1-next.130
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/chunk-26A5JFT2.mjs +89 -0
- package/dist/browser/{chunk-4Y2EGA3U.mjs → chunk-2Y3CO4EQ.mjs} +878 -310
- package/dist/browser/{chunk-2CBQ2PCC.mjs → chunk-AIIJDWMG.mjs} +1 -1
- package/dist/browser/{chunk-P7C2S5JX.mjs → chunk-D7WQF6DV.mjs} +2 -2
- package/dist/browser/{chunk-V5DCQIOT.mjs → chunk-M5UAAXGU.mjs} +2 -2
- package/dist/browser/{chunk-B65QVTST.mjs → chunk-NSCSR6MA.mjs} +24 -9
- package/dist/browser/{chunk-LJS6PREA.mjs → chunk-UDGB2MWT.mjs} +7 -28
- package/dist/browser/{chunk-6POI2KNI.mjs → chunk-VBBLTD7O.mjs} +4 -3
- package/dist/browser/chunk-VEGHFP2P.mjs +30 -0
- package/dist/browser/{durability-wiring-PUC4QSCR.mjs → durability-wiring-FNZSRH4N.mjs} +8 -6
- package/dist/browser/{module-ACWDC7YI.mjs → module-4HIMOQM5.mjs} +6 -5
- package/dist/browser/phrase-module-WESYVU7K.mjs +327 -0
- package/dist/browser/{restore-queue-MLUEUEAN.mjs → restore-queue-GEZ54CA5.mjs} +1 -1
- package/dist/browser/{runtime-BX6XRFYR.mjs → runtime-3X7ZJ5T3.mjs} +2 -1
- package/dist/browser/{snapshot-U5Y346XG.mjs → snapshot-V7VF6BN6.mjs} +25 -10
- package/dist/browser/{write-GZHR6AVG.mjs → write-YIC2MKRY.mjs} +7 -5
- package/dist/{capabilities-9YwwCcqP.d.ts → capabilities-Bl8G9taj.d.ts} +10 -3
- package/dist/{chunk-VK45J47R.mjs → chunk-2C3WXBQS.mjs} +1 -1
- package/dist/{chunk-QSCVJJW5.mjs → chunk-2JWNJSNE.mjs} +8 -8
- package/dist/{chunk-IBGMFR5Y.mjs → chunk-2MKX2SCO.mjs} +481 -121
- package/dist/{chunk-P34VZEXB.mjs → chunk-2SEL6SBS.mjs} +4 -95
- package/dist/{chunk-7ZZQNOIY.mjs → chunk-2WBY4Y4M.mjs} +1 -1
- package/dist/{chunk-HFJXEFFV.mjs → chunk-4GDLFDPZ.mjs} +3 -3
- package/dist/chunk-4UMZ7KQL.mjs +17 -0
- package/dist/{chunk-ZR63OGT2.mjs → chunk-5LX6MOIB.mjs} +2 -2
- package/dist/{chunk-UKXMSO5H.mjs → chunk-6M6SDH3S.mjs} +189 -45
- package/dist/{chunk-IR4KQ4WB.mjs → chunk-6SSVIW7Y.mjs} +49 -31
- package/dist/{chunk-4IHMMDRF.mjs → chunk-7OIQ2SIO.mjs} +4 -4
- package/dist/{chunk-6ODIY6ZB.mjs → chunk-7XY6VD7B.mjs} +1 -1
- package/dist/{chunk-OBWUWZRA.mjs → chunk-CZ7ISLXA.mjs} +3 -0
- package/dist/{chunk-OFYKBKKV.mjs → chunk-D5TFRB5G.mjs} +3 -15
- package/dist/{chunk-KCQG2PJT.mjs → chunk-EUCX5K74.mjs} +4 -4
- package/dist/{chunk-HWQDZB2U.mjs → chunk-F6AAXI7T.mjs} +3 -3
- package/dist/{chunk-E74YLAIG.mjs → chunk-FQBFZAWG.mjs} +4 -1
- package/dist/{chunk-U6NGTWI3.mjs → chunk-HCGJKDZV.mjs} +1 -1
- package/dist/{chunk-SJEIIR56.mjs → chunk-JJ6EUMXB.mjs} +2 -2
- package/dist/{chunk-N57XXGPG.mjs → chunk-JT3YZGCE.mjs} +23 -4
- package/dist/chunk-JUKYYXSP.mjs +93 -0
- package/dist/{chunk-JIXO442E.mjs → chunk-K77FJXKS.mjs} +26 -10
- package/dist/{chunk-4PQNDHSA.mjs → chunk-NMXRXFGZ.mjs} +2 -2
- package/dist/{chunk-6PN3U6XW.mjs → chunk-NNO7CO3A.mjs} +1 -1
- package/dist/{chunk-542TD4BL.mjs → chunk-PATMDIMN.mjs} +1 -1
- package/dist/{chunk-DG62H7OD.mjs → chunk-PU6SKH7O.mjs} +2 -2
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/{chunk-22CK2LQO.mjs → chunk-QQTXT65X.mjs} +6 -5
- package/dist/{chunk-LXWDQLTW.mjs → chunk-RNSC4NFI.mjs} +1 -1
- package/dist/{chunk-JEXJDCQH.mjs → chunk-RWESFSJI.mjs} +28 -13
- package/dist/{chunk-SEG4CL6W.mjs → chunk-SOHMMNCH.mjs} +4 -4
- package/dist/{chunk-OGFSUXYY.mjs → chunk-SZGRYEIC.mjs} +4 -4
- package/dist/{chunk-LOBYK2FG.mjs → chunk-T4OFG4IQ.mjs} +20 -9
- package/dist/{chunk-ENBN5VQ6.mjs → chunk-TGHNXN7W.mjs} +150 -52
- package/dist/{chunk-Z4DZ4KK2.mjs → chunk-UJWGQIXU.mjs} +1 -0
- package/dist/{chunk-ANFNYCQG.mjs → chunk-WVRDRLIC.mjs} +6 -6
- package/dist/{chunk-62IPM62F.mjs → chunk-X5CNUKD5.mjs} +2 -2
- package/dist/{chunk-IE43SRL3.mjs → chunk-XABMCBHA.mjs} +12 -6
- package/dist/{chunk-OCB7TMUQ.mjs → chunk-Y2U5VVWX.mjs} +1 -1
- package/dist/{chunk-MFPJVITN.mjs → chunk-Y4BVTQ53.mjs} +2 -2
- package/dist/{chunk-N6J6EJW3.mjs → chunk-YBWSV5DV.mjs} +2 -2
- package/dist/{chunk-FUUP45IS.mjs → chunk-ZR4VM4A6.mjs} +1 -1
- package/dist/{client-CsuhZ13c.d.ts → client-Cx-TiqPE.d.ts} +19 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +10 -2
- package/dist/{config-CtTfxbpV.d.ts → config-CzB9YvF_.d.ts} +2 -2
- package/dist/{config-DLVnlByZ.d.ts → config-hOWGBeC6.d.ts} +7 -5
- package/dist/direct-executor-HQ4QVDX7.mjs +28 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +18 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +33 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +71 -43
- package/dist/distribution.mjs +411 -127
- package/dist/{durability-wiring-UGGUNATA.mjs → durability-wiring-S7OX2MAI.mjs} +28 -26
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-Bq5wz6c1.d.ts → engine-B_W9sdZK.d.ts} +38 -8
- package/dist/{events-x2aVu0_o.d.ts → events-DErop1LM.d.ts} +23 -2
- package/dist/{index-Cl5qQauj.d.ts → index-CKI1fDEl.d.ts} +37 -3
- package/dist/index.browser.mjs +5 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +21 -19
- package/dist/{long-query-thread-5BIPZVS6.mjs → long-query-thread-WVSMIBYP.mjs} +17 -15
- package/dist/{module-7OHZ2OPI.mjs → module-2E4XYSGT.mjs} +11 -11
- package/dist/partial-words.mjs +7 -6
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +3 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +24 -22
- package/dist/phrase-module-JERF2IDG.mjs +329 -0
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +3 -0
- package/dist/{rerank-NYX5S7OG.mjs → rerank-DMMT4FZA.mjs} +6 -6
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-NVMWJMHI.mjs → restore-queue-JAMNRC6T.mjs} +1 -1
- package/dist/{runtime-XNYW6AWX.mjs → runtime-7MLQ7J64.mjs} +7 -7
- package/dist/{schema-4W_7tk1f.d.ts → schema-CS-clLun.d.ts} +54 -28
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +27 -15
- package/dist/{snapshot-P7IQGNNQ.mjs → snapshot-DOQLGGEX.mjs} +45 -30
- package/dist/synonyms.mjs +4 -3
- package/dist/{tasks-DLQIfXqy.d.ts → tasks-BfF-VRhm.d.ts} +4 -6
- package/dist/{thread-QPETQEPK.mjs → thread-DNK5XWXG.mjs} +16 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-WZVM7ZL6.mjs → write-ZKR7SNRT.mjs} +23 -21
- package/package.json +7 -7
- package/dist/direct-executor-6KCVVHYT.mjs +0 -26
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { GrowableUint32, SEGMENT_TOKEN_TABLE_START, SEGMENT_POSTING_LOG_START, SEGMENT_POSITION_POOL_START, SEGMENT_FIELD_SCRATCH_START, LOOKUP_KEY_SCRATCH_BYTES, bitsetHas, bitsetSet, createBitSet, bitsetIsEmpty, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT, bitsetFromSet } from './chunk-26A5JFT2.mjs';
|
|
2
|
+
import { MAX_DOCUMENT_ENCODING_DEPTH, singleFoldTable, appendFoldedCodePoint, compareCodePoints, EARLIER_RELEASE_MIN_M, MIN_M, MAX_M, DOCUMENT_ENCODER_RETAINED_BUFFER_BYTES, MAX_FILE_BYTES, detectRuntime, splitWorkerBudget, resolveWorkerCount, CHECKSUM_YIELD_CHUNK_BYTES, resolveWorkerEntry, missingWorkerEntry, DEFAULT_REQUEST_TIMEOUT_MS, toReducedSortValue, resolveRequestThreadCount, defaultSortMode, holdsUnpairedSurrogate, compareComparableValues, readSortField, exceedsCodePoints, toComparableSortValue, SORT_VALUE_MAX_CODE_POINTS, compareComparableKeys, truncateSortString, sortModeOf, isSortMode, SORT_MODES, SHUTDOWN_TIMEOUT_MS, MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS, compareSortValues, DEFAULT_BACKPRESSURE_LIMIT, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, codePointCount, noteThreadSlotGone, threadSlotOfWorker } from './chunk-AIIJDWMG.mjs';
|
|
2
3
|
import { DEFAULT_MAX_RERANK_WINDOW, DEFAULT_RERANK_TIMEOUT_MS } from './chunk-JPTQZWJQ.mjs';
|
|
3
|
-
import {
|
|
4
|
+
import { ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, MAX_SUGGEST_LIMIT, MAX_INDEX_CONFIGURATION_BYTES, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_IMPORT_REFUSALS_REPORTED, DEFAULT_SUGGEST_LIMIT, DEFAULT_LIMIT, DEFAULT_OFFSET, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, BATCH_CHUNK_SIZE, MAX_DOC_ID_LENGTH } from './chunk-VEGHFP2P.mjs';
|
|
5
|
+
import { ErrorCodes, NarsilError, createNarsilError, DEFAULT_FUZZY_PREFIX_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, BM25_DISPLAYED_NAME_LENGTH, RESULT_WINDOW, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_PAGE_SIZE, DEFAULT_MAX_PER_GROUP, PREFIX_MAX_EXPANSIONS, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-UDGB2MWT.mjs';
|
|
4
6
|
import { Encoder, Decoder, decode, encode } from '@msgpack/msgpack';
|
|
5
7
|
|
|
6
8
|
// src/analysis/registry.ts
|
|
@@ -55,10 +57,26 @@ function getStopWords(name) {
|
|
|
55
57
|
}
|
|
56
58
|
return stopWords2;
|
|
57
59
|
}
|
|
60
|
+
function holdsPositionsOutOfOrder(entries) {
|
|
61
|
+
for (let index = 1; index < entries.length; index++) {
|
|
62
|
+
if (entries[index].position < entries[index - 1].position) return true;
|
|
63
|
+
}
|
|
64
|
+
return false;
|
|
65
|
+
}
|
|
66
|
+
function inPositionOrder(tokenizer) {
|
|
67
|
+
return {
|
|
68
|
+
tokenize(text) {
|
|
69
|
+
const entries = tokenizer.tokenize(text);
|
|
70
|
+
if (!holdsPositionsOutOfOrder(entries)) return entries;
|
|
71
|
+
return [...entries].sort((first, second) => first.position - second.position);
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
}
|
|
58
75
|
function resolveIndexAnalysis(config) {
|
|
76
|
+
const tokenizer = typeof config.tokenizer === "string" ? getTokenizer(config.tokenizer) : config.tokenizer;
|
|
59
77
|
return {
|
|
60
78
|
stopWords: typeof config.stopWords === "string" ? getStopWords(config.stopWords) : config.stopWords,
|
|
61
|
-
customTokenizer:
|
|
79
|
+
customTokenizer: tokenizer === void 0 ? void 0 : inPositionOrder(tokenizer)
|
|
62
80
|
};
|
|
63
81
|
}
|
|
64
82
|
|
|
@@ -2232,6 +2250,17 @@ function requireValidPatternWorkCap(cap) {
|
|
|
2232
2250
|
);
|
|
2233
2251
|
}
|
|
2234
2252
|
|
|
2253
|
+
// src/core/constants.ts
|
|
2254
|
+
var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
2255
|
+
var MAX_TERM_FREQUENCY = 65535;
|
|
2256
|
+
var ARRAY_POSITION_GAP = 100;
|
|
2257
|
+
var DEFAULT_BM25_K1 = 1.2;
|
|
2258
|
+
var DEFAULT_BM25_B = 0.75;
|
|
2259
|
+
var MAX_BM25_K1 = 1e3;
|
|
2260
|
+
var MAX_BM25_B = 1;
|
|
2261
|
+
var PENDING_REMOVAL_MERGE_FLOOR = 64;
|
|
2262
|
+
var PENDING_REMOVAL_MERGE_DIVISOR = 256;
|
|
2263
|
+
|
|
2235
2264
|
// src/core/id-generator.ts
|
|
2236
2265
|
function generateId() {
|
|
2237
2266
|
const bytes = new Uint8Array(16);
|
|
@@ -3243,12 +3272,6 @@ function createDocumentStore() {
|
|
|
3243
3272
|
};
|
|
3244
3273
|
}
|
|
3245
3274
|
|
|
3246
|
-
// src/core/constants.ts
|
|
3247
|
-
var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
3248
|
-
var MAX_TERM_FREQUENCY = 65535;
|
|
3249
|
-
var PENDING_REMOVAL_MERGE_FLOOR = 64;
|
|
3250
|
-
var PENDING_REMOVAL_MERGE_DIVISOR = 256;
|
|
3251
|
-
|
|
3252
3275
|
// src/core/fuzzy.ts
|
|
3253
3276
|
var FIRST_SURROGATE = 55296;
|
|
3254
3277
|
var LAST_SURROGATE = 57343;
|
|
@@ -3746,6 +3769,13 @@ var LivePostingView = class {
|
|
|
3746
3769
|
const starts = this.store.pool.positionStarts;
|
|
3747
3770
|
return starts === null ? 0 : this.store.positions.countAt(starts[this.syncedStart + row]);
|
|
3748
3771
|
}
|
|
3772
|
+
get positionStore() {
|
|
3773
|
+
return this.holdsPositions ? this.store.positions.values : null;
|
|
3774
|
+
}
|
|
3775
|
+
positionStartAt(row) {
|
|
3776
|
+
const starts = this.store.pool.positionStarts;
|
|
3777
|
+
return starts === null ? 0 : starts[this.syncedStart + row] + 1;
|
|
3778
|
+
}
|
|
3749
3779
|
copyPositionsAt(row, target, offset) {
|
|
3750
3780
|
if (!this.holdsPositions) return 0;
|
|
3751
3781
|
const starts = this.store.pool.positionStarts;
|
|
@@ -4468,48 +4498,6 @@ function createSurfaceRegistry() {
|
|
|
4468
4498
|
};
|
|
4469
4499
|
}
|
|
4470
4500
|
|
|
4471
|
-
// src/core/bitset.ts
|
|
4472
|
-
function createBitSet(capacity) {
|
|
4473
|
-
return new Uint32Array(capacity + 31 >>> 5 || 1);
|
|
4474
|
-
}
|
|
4475
|
-
function bitsetSet(bs, index) {
|
|
4476
|
-
const word = index >>> 5;
|
|
4477
|
-
if (word < bs.length) {
|
|
4478
|
-
bs[word] |= 1 << (index & 31);
|
|
4479
|
-
}
|
|
4480
|
-
}
|
|
4481
|
-
function bitsetHas(bs, index) {
|
|
4482
|
-
const word = index >>> 5;
|
|
4483
|
-
if (word >= bs.length) return false;
|
|
4484
|
-
return (bs[word] & 1 << (index & 31)) !== 0;
|
|
4485
|
-
}
|
|
4486
|
-
function bitsetAndInto(target, other) {
|
|
4487
|
-
const shared2 = Math.min(target.length, other.length);
|
|
4488
|
-
for (let i = 0; i < shared2; i++) target[i] &= other[i];
|
|
4489
|
-
target.fill(0, shared2);
|
|
4490
|
-
}
|
|
4491
|
-
function bitsetOrInto(target, other) {
|
|
4492
|
-
const shared2 = Math.min(target.length, other.length);
|
|
4493
|
-
for (let i = 0; i < shared2; i++) target[i] |= other[i];
|
|
4494
|
-
}
|
|
4495
|
-
function bitsetAndNotInto(target, removed) {
|
|
4496
|
-
const shared2 = Math.min(target.length, removed.length);
|
|
4497
|
-
for (let i = 0; i < shared2; i++) target[i] &= ~removed[i];
|
|
4498
|
-
}
|
|
4499
|
-
function bitsetIsEmpty(bs) {
|
|
4500
|
-
for (let i = 0; i < bs.length; i++) {
|
|
4501
|
-
if (bs[i] !== 0) return false;
|
|
4502
|
-
}
|
|
4503
|
-
return true;
|
|
4504
|
-
}
|
|
4505
|
-
function bitsetFromSet(set, capacity) {
|
|
4506
|
-
const bs = createBitSet(capacity);
|
|
4507
|
-
for (const id of set) {
|
|
4508
|
-
bs[id >>> 5] |= 1 << (id & 31);
|
|
4509
|
-
}
|
|
4510
|
-
return bs;
|
|
4511
|
-
}
|
|
4512
|
-
|
|
4513
4501
|
// src/core/partition/facets.ts
|
|
4514
4502
|
var UNDECLARED_FIELD_TYPE = "";
|
|
4515
4503
|
function keptFacetValues(limit) {
|
|
@@ -5780,7 +5768,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5780
5768
|
if (state.trackPositions) {
|
|
5781
5769
|
const tokenFreqs = /* @__PURE__ */ new Map();
|
|
5782
5770
|
let positionOffset = 0;
|
|
5783
|
-
let
|
|
5771
|
+
let lastItemPosition = -1;
|
|
5784
5772
|
const addToken = (token, position, surface) => {
|
|
5785
5773
|
const existing = tokenFreqs.get(token);
|
|
5786
5774
|
if (existing) {
|
|
@@ -5791,12 +5779,12 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
5791
5779
|
}
|
|
5792
5780
|
fieldTokenList.push(token);
|
|
5793
5781
|
countOccurrence(surfaceCounts, spellings, surface, token);
|
|
5794
|
-
|
|
5782
|
+
lastItemPosition = position;
|
|
5795
5783
|
};
|
|
5796
5784
|
for (const item of arr) {
|
|
5797
|
-
|
|
5785
|
+
lastItemPosition = -1;
|
|
5798
5786
|
visitTokens(item, language, opts, addToken);
|
|
5799
|
-
positionOffset +=
|
|
5787
|
+
if (lastItemPosition >= 0) positionOffset += lastItemPosition + 1 + ARRAY_POSITION_GAP;
|
|
5800
5788
|
}
|
|
5801
5789
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
5802
5790
|
for (const [token, freq] of tokenFreqs) {
|
|
@@ -6793,18 +6781,6 @@ function evaluateFilters(expression, context) {
|
|
|
6793
6781
|
return evaluateExpression(validatedFilterOf(expression, context.fieldTypes), context);
|
|
6794
6782
|
}
|
|
6795
6783
|
|
|
6796
|
-
// src/core/partition/constants.ts
|
|
6797
|
-
var TARGET_BLOCK_ENTRIES = 128;
|
|
6798
|
-
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
6799
|
-
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
6800
|
-
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
6801
|
-
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
6802
|
-
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
6803
|
-
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
6804
|
-
var LOOKUP_KEY_SCRATCH_BYTES = 4096;
|
|
6805
|
-
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
6806
|
-
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
6807
|
-
|
|
6808
6784
|
// src/core/partition/sort-columns/order.ts
|
|
6809
6785
|
var MISSING_RANK = -1;
|
|
6810
6786
|
var BYTES_PER_RANK = 4;
|
|
@@ -7545,20 +7521,115 @@ function partitionFilterMatches(state, filters, schema, meter) {
|
|
|
7545
7521
|
};
|
|
7546
7522
|
}
|
|
7547
7523
|
|
|
7524
|
+
// src/search/quotes.ts
|
|
7525
|
+
var QUOTE_CHARACTER = /["“”„]/;
|
|
7526
|
+
function holdsQuoteCharacter(term) {
|
|
7527
|
+
return QUOTE_CHARACTER.test(term);
|
|
7528
|
+
}
|
|
7529
|
+
|
|
7530
|
+
// src/search/phrase-code.ts
|
|
7531
|
+
var loaded = null;
|
|
7532
|
+
var loading = null;
|
|
7533
|
+
function requireLoadedPhraseCode() {
|
|
7534
|
+
if (loaded === null) {
|
|
7535
|
+
throw new NarsilError(ErrorCodes.SEARCH_PHRASE_NOT_READY, "The phrase code has not loaded. Try the query again");
|
|
7536
|
+
}
|
|
7537
|
+
return loaded;
|
|
7538
|
+
}
|
|
7539
|
+
function loadPhraseCode() {
|
|
7540
|
+
if (loading === null) {
|
|
7541
|
+
loading = import('./phrase-module-WESYVU7K.mjs').then(
|
|
7542
|
+
(code) => {
|
|
7543
|
+
loaded = code;
|
|
7544
|
+
return code;
|
|
7545
|
+
},
|
|
7546
|
+
(err) => {
|
|
7547
|
+
loading = null;
|
|
7548
|
+
throw new NarsilError(
|
|
7549
|
+
ErrorCodes.SEARCH_PHRASE_NOT_READY,
|
|
7550
|
+
"The phrase code failed to load. Remove the quotes, or try again once it can load",
|
|
7551
|
+
{ cause: err instanceof Error ? err.message : String(err) }
|
|
7552
|
+
);
|
|
7553
|
+
}
|
|
7554
|
+
);
|
|
7555
|
+
}
|
|
7556
|
+
return loading;
|
|
7557
|
+
}
|
|
7558
|
+
function loadPhraseCodeFor(term) {
|
|
7559
|
+
if (loaded !== null || term === void 0 || !holdsQuoteCharacter(term)) return null;
|
|
7560
|
+
return loadPhraseCode();
|
|
7561
|
+
}
|
|
7562
|
+
function analyseQueryPhrases(term, language, analysis) {
|
|
7563
|
+
if (!holdsQuoteCharacter(term)) return null;
|
|
7564
|
+
return requireLoadedPhraseCode().analyseQueryPhrases(
|
|
7565
|
+
term.split(QUOTE_CHARACTER),
|
|
7566
|
+
(text) => tokenize(text, language, {
|
|
7567
|
+
stem: true,
|
|
7568
|
+
removeStopWords: true,
|
|
7569
|
+
customTokenizer: analysis.customTokenizer,
|
|
7570
|
+
stopWordOverride: analysis.stopWords
|
|
7571
|
+
}).tokens
|
|
7572
|
+
);
|
|
7573
|
+
}
|
|
7574
|
+
|
|
7548
7575
|
// src/core/scorer.ts
|
|
7549
|
-
var DEFAULT_K1 = 1.2;
|
|
7550
|
-
var DEFAULT_B = 0.75;
|
|
7551
7576
|
function computeIDF(docFrequency, totalDocs) {
|
|
7552
7577
|
if (totalDocs === 0) return 0;
|
|
7553
7578
|
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
7554
7579
|
}
|
|
7580
|
+
function pairKeepsBoundSound(k1, b) {
|
|
7581
|
+
return k1 >= 0 && k1 <= MAX_BM25_K1 && b >= 0 && b <= MAX_BM25_B;
|
|
7582
|
+
}
|
|
7555
7583
|
function bm25PruningSound(params) {
|
|
7556
|
-
const k1 = params?.k1 ??
|
|
7557
|
-
const b = params?.b ??
|
|
7558
|
-
|
|
7584
|
+
const k1 = params?.k1 ?? DEFAULT_BM25_K1;
|
|
7585
|
+
const b = params?.b ?? DEFAULT_BM25_B;
|
|
7586
|
+
if (!pairKeepsBoundSound(k1, b)) return false;
|
|
7587
|
+
const fields = params?.fields;
|
|
7588
|
+
if (fields === void 0) return true;
|
|
7589
|
+
for (const name of Object.keys(fields)) {
|
|
7590
|
+
if (!pairKeepsBoundSound(fields[name].k1 ?? k1, fields[name].b ?? b)) return false;
|
|
7591
|
+
}
|
|
7592
|
+
return true;
|
|
7593
|
+
}
|
|
7594
|
+
function bm25SetsValue(params) {
|
|
7595
|
+
if (typeof params !== "object" || params === null) return params !== void 0;
|
|
7596
|
+
if (params.k1 !== void 0 || params.b !== void 0) return true;
|
|
7597
|
+
const fields = params.fields;
|
|
7598
|
+
if (fields === void 0) return false;
|
|
7599
|
+
if (typeof fields !== "object" || fields === null) return true;
|
|
7600
|
+
for (const name of Object.keys(fields)) {
|
|
7601
|
+
const pair = fields[name];
|
|
7602
|
+
if (typeof pair !== "object" || pair === null) return true;
|
|
7603
|
+
if (pair.k1 !== void 0 || pair.b !== void 0) return true;
|
|
7604
|
+
}
|
|
7605
|
+
return false;
|
|
7559
7606
|
}
|
|
7560
|
-
function
|
|
7561
|
-
return
|
|
7607
|
+
function ownField(fields, name) {
|
|
7608
|
+
return fields !== void 0 && Object.hasOwn(fields, name) ? fields[name] : void 0;
|
|
7609
|
+
}
|
|
7610
|
+
function mergeBM25Levels(search, index) {
|
|
7611
|
+
if (search === void 0 || !bm25SetsValue(search)) return index;
|
|
7612
|
+
const k1 = search.k1 ?? index?.k1;
|
|
7613
|
+
const b = search.b ?? index?.b;
|
|
7614
|
+
const merged = {
|
|
7615
|
+
...k1 !== void 0 ? { k1 } : {},
|
|
7616
|
+
...b !== void 0 ? { b } : {}
|
|
7617
|
+
};
|
|
7618
|
+
const names = [...Object.keys(search.fields ?? {}), ...Object.keys(index?.fields ?? {})];
|
|
7619
|
+
if (names.length === 0) return merged;
|
|
7620
|
+
const fields = /* @__PURE__ */ Object.create(null);
|
|
7621
|
+
for (const name of names) {
|
|
7622
|
+
const fromSearch = ownField(search.fields, name);
|
|
7623
|
+
const fromIndex = ownField(index?.fields, name);
|
|
7624
|
+
const fieldK1 = fromSearch?.k1 ?? search.k1 ?? fromIndex?.k1;
|
|
7625
|
+
const fieldB = fromSearch?.b ?? search.b ?? fromIndex?.b;
|
|
7626
|
+
fields[name] = {
|
|
7627
|
+
...fieldK1 !== void 0 ? { k1: fieldK1 } : {},
|
|
7628
|
+
...fieldB !== void 0 ? { b: fieldB } : {}
|
|
7629
|
+
};
|
|
7630
|
+
}
|
|
7631
|
+
merged.fields = fields;
|
|
7632
|
+
return merged;
|
|
7562
7633
|
}
|
|
7563
7634
|
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
7564
7635
|
if (avgFieldLength === 0) return 0;
|
|
@@ -7567,29 +7638,49 @@ function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1,
|
|
|
7567
7638
|
if (denominator === 0) return 0;
|
|
7568
7639
|
return idf * (numerator / denominator);
|
|
7569
7640
|
}
|
|
7570
|
-
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
7571
|
-
if (totalDocs === 0) return 0;
|
|
7572
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
7573
|
-
const b = params?.b ?? DEFAULT_B;
|
|
7574
|
-
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
7575
|
-
}
|
|
7576
7641
|
|
|
7577
7642
|
// src/core/partition/field-scoring.ts
|
|
7578
|
-
|
|
7643
|
+
var NO_FIELD_VALUES = new Float64Array(0);
|
|
7644
|
+
function differentPairsOf(searchable, k1s, bs) {
|
|
7645
|
+
const pairs = [];
|
|
7646
|
+
for (let fieldIndex = 0; fieldIndex < searchable.length; fieldIndex++) {
|
|
7647
|
+
if (searchable[fieldIndex] !== 1) continue;
|
|
7648
|
+
let seen = false;
|
|
7649
|
+
for (let at = 0; at < pairs.length && !seen; at += 2) {
|
|
7650
|
+
seen = pairs[at] === k1s[fieldIndex] && pairs[at + 1] === bs[fieldIndex];
|
|
7651
|
+
}
|
|
7652
|
+
if (!seen) pairs.push(k1s[fieldIndex], bs[fieldIndex]);
|
|
7653
|
+
}
|
|
7654
|
+
return pairs;
|
|
7655
|
+
}
|
|
7656
|
+
function loadFieldScoring(state, fields, boost, averageFieldLengths, bm25) {
|
|
7579
7657
|
const fieldNames = state.fieldNameTable.names;
|
|
7658
|
+
const fieldCount = fieldNames.length;
|
|
7659
|
+
const k1 = bm25?.k1 ?? DEFAULT_BM25_K1;
|
|
7660
|
+
const b = bm25?.b ?? DEFAULT_BM25_B;
|
|
7661
|
+
const perField = bm25?.fields;
|
|
7580
7662
|
const scoring = {
|
|
7581
|
-
searchable: new Uint8Array(
|
|
7582
|
-
boosts: new Float64Array(
|
|
7583
|
-
averageLengths: new Float64Array(
|
|
7584
|
-
lengthColumns: new Array(
|
|
7663
|
+
searchable: new Uint8Array(fieldCount),
|
|
7664
|
+
boosts: new Float64Array(fieldCount),
|
|
7665
|
+
averageLengths: new Float64Array(fieldCount),
|
|
7666
|
+
lengthColumns: new Array(fieldCount).fill(null),
|
|
7667
|
+
k1s: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
7668
|
+
bs: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
7669
|
+
boundPairs: [k1, b]
|
|
7585
7670
|
};
|
|
7586
|
-
for (let fieldIndex = 0; fieldIndex <
|
|
7671
|
+
for (let fieldIndex = 0; fieldIndex < fieldCount; fieldIndex++) {
|
|
7587
7672
|
const fieldName = fieldNames[fieldIndex];
|
|
7588
7673
|
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
7589
7674
|
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
7590
7675
|
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
7591
7676
|
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
7677
|
+
if (perField !== void 0) {
|
|
7678
|
+
const fieldPair = Object.hasOwn(perField, fieldName) ? perField[fieldName] : void 0;
|
|
7679
|
+
scoring.k1s[fieldIndex] = fieldPair?.k1 ?? k1;
|
|
7680
|
+
scoring.bs[fieldIndex] = fieldPair?.b ?? b;
|
|
7681
|
+
}
|
|
7592
7682
|
}
|
|
7683
|
+
if (perField !== void 0) scoring.boundPairs = differentPairsOf(scoring.searchable, scoring.k1s, scoring.bs);
|
|
7593
7684
|
return scoring;
|
|
7594
7685
|
}
|
|
7595
7686
|
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
@@ -7605,6 +7696,15 @@ function bestSearchable(searchable, values) {
|
|
|
7605
7696
|
}
|
|
7606
7697
|
return best;
|
|
7607
7698
|
}
|
|
7699
|
+
function blockEntryBound(fields, maxTermFrequency, idf, minFieldLength, maxAverageLength) {
|
|
7700
|
+
const pairs = fields.boundPairs;
|
|
7701
|
+
let best = 0;
|
|
7702
|
+
for (let at = 0; at < pairs.length; at += 2) {
|
|
7703
|
+
const bound = computeBM25WithIDF(maxTermFrequency, idf, minFieldLength, maxAverageLength, pairs[at], pairs[at + 1]);
|
|
7704
|
+
if (bound > best) best = bound;
|
|
7705
|
+
}
|
|
7706
|
+
return best;
|
|
7707
|
+
}
|
|
7608
7708
|
|
|
7609
7709
|
// src/core/partition/posting-columns.ts
|
|
7610
7710
|
function postingColumns(list) {
|
|
@@ -7698,8 +7798,9 @@ function groupPostingCount(matches) {
|
|
|
7698
7798
|
return total;
|
|
7699
7799
|
}
|
|
7700
7800
|
function computeGroupContributions(context, matches, collect) {
|
|
7701
|
-
const { fields, fieldNames, filterBitset, partialMarks
|
|
7801
|
+
const { fields, fieldNames, filterBitset, partialMarks } = context;
|
|
7702
7802
|
const scoresAreZero = context.totalDocs === 0;
|
|
7803
|
+
const onePair = fields.boundPairs.length === 2;
|
|
7703
7804
|
const best = /* @__PURE__ */ new Map();
|
|
7704
7805
|
for (const match of matches) {
|
|
7705
7806
|
const perTerm = /* @__PURE__ */ new Map();
|
|
@@ -7719,7 +7820,14 @@ function computeGroupContributions(context, matches, collect) {
|
|
|
7719
7820
|
const fieldBoost = fields.boosts[fieldIndex];
|
|
7720
7821
|
const avgLen = fields.averageLengths[fieldIndex];
|
|
7721
7822
|
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
7722
|
-
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
7823
|
+
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
7824
|
+
termFrequency,
|
|
7825
|
+
match.idf,
|
|
7826
|
+
actualFieldLength,
|
|
7827
|
+
avgLen,
|
|
7828
|
+
onePair ? fields.boundPairs[0] : fields.k1s[fieldIndex],
|
|
7829
|
+
onePair ? fields.boundPairs[1] : fields.bs[fieldIndex]
|
|
7830
|
+
)) * fieldBoost * factor;
|
|
7723
7831
|
const existing = perTerm.get(internalId);
|
|
7724
7832
|
if (existing) {
|
|
7725
7833
|
existing.score += termScore;
|
|
@@ -7776,11 +7884,20 @@ function searchedFieldMarks(fieldNames, fields) {
|
|
|
7776
7884
|
function searchFulltextMatches(state, params) {
|
|
7777
7885
|
const { queryTokens, termGroups, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
|
|
7778
7886
|
const fieldNames = state.fieldNameTable.names;
|
|
7779
|
-
const matched = createBitSet(state.docStore.internalIdCapacity());
|
|
7780
7887
|
const searched = searchedFieldMarks(fieldNames, fields);
|
|
7781
7888
|
const partial = partialFieldMarks(fieldNames, params.partialFields);
|
|
7782
7889
|
const searchedPartial = searched.map((mark, fieldIndex) => mark & partial[fieldIndex]);
|
|
7783
|
-
|
|
7890
|
+
const capacity = state.docStore.internalIdCapacity();
|
|
7891
|
+
const matched = params.phrases === void 0 ? createBitSet(capacity) : requireLoadedPhraseCode().matchPhrases(
|
|
7892
|
+
state.invertedIdx,
|
|
7893
|
+
params.phrases,
|
|
7894
|
+
searched,
|
|
7895
|
+
filterBitset,
|
|
7896
|
+
capacity,
|
|
7897
|
+
params.phraseWork,
|
|
7898
|
+
false
|
|
7899
|
+
).matched;
|
|
7900
|
+
for (const qt of params.phrases === void 0 ? queryTokens : []) {
|
|
7784
7901
|
const group = termGroupOf(termGroups, qt.token);
|
|
7785
7902
|
for (const match of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
7786
7903
|
markPostingList(matched, match.postingList, searched, filterBitset);
|
|
@@ -8283,7 +8400,7 @@ function boostsKeepScoresNonNegative(fields) {
|
|
|
8283
8400
|
}
|
|
8284
8401
|
return true;
|
|
8285
8402
|
}
|
|
8286
|
-
function
|
|
8403
|
+
function samePairUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
8287
8404
|
if (scan2.scoresAreZero) return 0;
|
|
8288
8405
|
const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
|
|
8289
8406
|
let best = 0;
|
|
@@ -8301,7 +8418,24 @@ function termUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
|
8301
8418
|
}
|
|
8302
8419
|
return best;
|
|
8303
8420
|
}
|
|
8304
|
-
function
|
|
8421
|
+
function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
8422
|
+
if (scan2.scoresAreZero) return 0;
|
|
8423
|
+
const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
|
|
8424
|
+
let best = 0;
|
|
8425
|
+
for (let block = 0; block < bounds.blockCount; block++) {
|
|
8426
|
+
const bestEntry = blockEntryBound(
|
|
8427
|
+
scan2.fields,
|
|
8428
|
+
bounds.maxTermFrequency[block],
|
|
8429
|
+
scan2.idf,
|
|
8430
|
+
bounds.minFieldLength[block],
|
|
8431
|
+
maxAverageLength
|
|
8432
|
+
);
|
|
8433
|
+
const blockBound = bestEntry * maxBoost * bounds.maxEntriesPerDocument[block];
|
|
8434
|
+
if (blockBound > best) best = blockBound;
|
|
8435
|
+
}
|
|
8436
|
+
return best;
|
|
8437
|
+
}
|
|
8438
|
+
function samePairEntryScore(scan2, entry, internalId) {
|
|
8305
8439
|
const { fields, columns } = scan2;
|
|
8306
8440
|
const fieldIndex = columns.fieldNameIndices[entry];
|
|
8307
8441
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
@@ -8309,17 +8443,55 @@ function entryScore(scan2, entry, internalId) {
|
|
|
8309
8443
|
const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan2.idf, fieldLength, averageLength, scan2.k1, scan2.b);
|
|
8310
8444
|
return bm25 * fields.boosts[fieldIndex];
|
|
8311
8445
|
}
|
|
8312
|
-
function
|
|
8446
|
+
function fieldPairEntryScore(scan2, entry, internalId) {
|
|
8447
|
+
const { fields, columns } = scan2;
|
|
8448
|
+
const fieldIndex = columns.fieldNameIndices[entry];
|
|
8449
|
+
const averageLength = fields.averageLengths[fieldIndex];
|
|
8450
|
+
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
8451
|
+
const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8452
|
+
columns.termFrequencies[entry],
|
|
8453
|
+
scan2.idf,
|
|
8454
|
+
fieldLength,
|
|
8455
|
+
averageLength,
|
|
8456
|
+
fields.k1s[fieldIndex],
|
|
8457
|
+
fields.bs[fieldIndex]
|
|
8458
|
+
);
|
|
8459
|
+
return bm25 * fields.boosts[fieldIndex];
|
|
8460
|
+
}
|
|
8461
|
+
function samePairScoreEveryDocument(scan2, buffer) {
|
|
8462
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8463
|
+
const searchable = scan2.fields.searchable;
|
|
8464
|
+
for (let entry = 0; entry < count; entry++) {
|
|
8465
|
+
const internalId = docIds[entry];
|
|
8466
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8467
|
+
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8468
|
+
addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
|
|
8469
|
+
}
|
|
8470
|
+
}
|
|
8471
|
+
function fieldPairScoreEveryDocument(scan2, buffer) {
|
|
8472
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8473
|
+
const searchable = scan2.fields.searchable;
|
|
8474
|
+
for (let entry = 0; entry < count; entry++) {
|
|
8475
|
+
const internalId = docIds[entry];
|
|
8476
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8477
|
+
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8478
|
+
addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
|
|
8479
|
+
}
|
|
8480
|
+
}
|
|
8481
|
+
function samePairScoreKnownDocuments(scan2, buffer) {
|
|
8313
8482
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8314
8483
|
const searchable = scan2.fields.searchable;
|
|
8484
|
+
let counted = 0;
|
|
8315
8485
|
for (let entry = 0; entry < count; entry++) {
|
|
8316
8486
|
const internalId = docIds[entry];
|
|
8317
8487
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8318
8488
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8319
|
-
addScore(buffer, internalId,
|
|
8489
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
|
|
8490
|
+
else if (markCounted(buffer, internalId)) counted++;
|
|
8320
8491
|
}
|
|
8492
|
+
return counted;
|
|
8321
8493
|
}
|
|
8322
|
-
function
|
|
8494
|
+
function fieldPairScoreKnownDocuments(scan2, buffer) {
|
|
8323
8495
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8324
8496
|
const searchable = scan2.fields.searchable;
|
|
8325
8497
|
let counted = 0;
|
|
@@ -8327,31 +8499,58 @@ function scoreKnownDocumentsAndCountTheRest(scan2, buffer) {
|
|
|
8327
8499
|
const internalId = docIds[entry];
|
|
8328
8500
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8329
8501
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
8330
|
-
if (hasScore(buffer, internalId)) addScore(buffer, internalId,
|
|
8502
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
|
|
8331
8503
|
else if (markCounted(buffer, internalId)) counted++;
|
|
8332
8504
|
}
|
|
8333
8505
|
return counted;
|
|
8334
8506
|
}
|
|
8335
|
-
function
|
|
8507
|
+
function firstEntryOf(docIds, count, internalId) {
|
|
8508
|
+
let low = 0;
|
|
8509
|
+
let high = count;
|
|
8510
|
+
while (low < high) {
|
|
8511
|
+
const middle = low + high >>> 1;
|
|
8512
|
+
if (docIds[middle] < internalId) low = middle + 1;
|
|
8513
|
+
else high = middle;
|
|
8514
|
+
}
|
|
8515
|
+
return low;
|
|
8516
|
+
}
|
|
8517
|
+
function samePairQueryOrderScore(scans, internalId) {
|
|
8336
8518
|
let score = 0;
|
|
8337
8519
|
for (const scan2 of scans) {
|
|
8338
8520
|
const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8339
8521
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8340
|
-
let
|
|
8341
|
-
|
|
8342
|
-
|
|
8343
|
-
const middle = low + high >>> 1;
|
|
8344
|
-
if (docIds[middle] < internalId) low = middle + 1;
|
|
8345
|
-
else high = middle;
|
|
8522
|
+
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
8523
|
+
if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
8524
|
+
score += samePairEntryScore(scan2, entry, internalId);
|
|
8346
8525
|
}
|
|
8347
|
-
|
|
8526
|
+
}
|
|
8527
|
+
return score;
|
|
8528
|
+
}
|
|
8529
|
+
function fieldPairQueryOrderScore(scans, internalId) {
|
|
8530
|
+
let score = 0;
|
|
8531
|
+
for (const scan2 of scans) {
|
|
8532
|
+
const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
8533
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
8534
|
+
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
8348
8535
|
if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
8349
|
-
score +=
|
|
8536
|
+
score += fieldPairEntryScore(scan2, entry, internalId);
|
|
8350
8537
|
}
|
|
8351
8538
|
}
|
|
8352
8539
|
return score;
|
|
8353
8540
|
}
|
|
8354
|
-
|
|
8541
|
+
var SAME_PAIR_SCORER = {
|
|
8542
|
+
upperBound: samePairUpperBound,
|
|
8543
|
+
scoreEveryDocument: samePairScoreEveryDocument,
|
|
8544
|
+
scoreKnownDocumentsAndCountTheRest: samePairScoreKnownDocuments,
|
|
8545
|
+
queryOrderScore: samePairQueryOrderScore
|
|
8546
|
+
};
|
|
8547
|
+
var FIELD_PAIR_SCORER = {
|
|
8548
|
+
upperBound: fieldPairUpperBound,
|
|
8549
|
+
scoreEveryDocument: fieldPairScoreEveryDocument,
|
|
8550
|
+
scoreKnownDocumentsAndCountTheRest: fieldPairScoreKnownDocuments,
|
|
8551
|
+
queryOrderScore: fieldPairQueryOrderScore
|
|
8552
|
+
};
|
|
8553
|
+
function pageFromBuffer(scorer, scans, buffer, selection, resolver) {
|
|
8355
8554
|
const approximateKth = kthBestScore(buffer, selection, resolver);
|
|
8356
8555
|
const gatherFrom = approximateKth - Math.abs(approximateKth) * PRUNING_REJECTION_SLACK;
|
|
8357
8556
|
const heap = [];
|
|
@@ -8363,7 +8562,7 @@ function pageFromBuffer(scans, buffer, selection, resolver) {
|
|
|
8363
8562
|
if (scores[internalId] < gatherFrom) continue;
|
|
8364
8563
|
const externalId = resolver.toExternal(internalId);
|
|
8365
8564
|
if (externalId === void 0) continue;
|
|
8366
|
-
const score = queryOrderScore(scans, internalId);
|
|
8565
|
+
const score = scorer.queryOrderScore(scans, internalId);
|
|
8367
8566
|
if (full && score < threshold) continue;
|
|
8368
8567
|
const candidate = { internalId, externalId, score };
|
|
8369
8568
|
if (!full) {
|
|
@@ -8393,18 +8592,28 @@ function pageFromBuffer(scans, buffer, selection, resolver) {
|
|
|
8393
8592
|
return scored;
|
|
8394
8593
|
}
|
|
8395
8594
|
function multiTermTopK(request) {
|
|
8396
|
-
const { terms, docFrequencies, totalDocs,
|
|
8595
|
+
const { terms, docFrequencies, totalDocs, limit, fields, resolver, buffer } = request;
|
|
8397
8596
|
if (!boostsKeepScoresNonNegative(fields)) return null;
|
|
8398
|
-
const
|
|
8597
|
+
const onePair = fields.boundPairs.length === 2;
|
|
8598
|
+
const scorer = onePair ? SAME_PAIR_SCORER : FIELD_PAIR_SCORER;
|
|
8599
|
+
const k1 = onePair ? fields.boundPairs[0] : 0;
|
|
8600
|
+
const b = onePair ? fields.boundPairs[1] : 0;
|
|
8399
8601
|
const scans = new Array(terms.length);
|
|
8400
8602
|
for (let term = 0; term < terms.length; term++) {
|
|
8401
8603
|
const idf = computeIDF(docFrequencies[term], totalDocs);
|
|
8402
8604
|
if (!(idf >= 0)) return null;
|
|
8403
|
-
scans[term] = {
|
|
8605
|
+
scans[term] = {
|
|
8606
|
+
columns: postingColumns(terms[term].list),
|
|
8607
|
+
idf,
|
|
8608
|
+
scoresAreZero: totalDocs === 0,
|
|
8609
|
+
fields,
|
|
8610
|
+
k1,
|
|
8611
|
+
b
|
|
8612
|
+
};
|
|
8404
8613
|
}
|
|
8405
8614
|
const maxBoost = bestSearchable(fields.searchable, fields.boosts);
|
|
8406
8615
|
const maxAverageLength = bestSearchable(fields.searchable, fields.averageLengths);
|
|
8407
|
-
const upperBounds = scans.map((scan2, term) =>
|
|
8616
|
+
const upperBounds = scans.map((scan2, term) => scorer.upperBound(scan2, terms[term].list, maxBoost, maxAverageLength));
|
|
8408
8617
|
const order = scans.map((_, term) => term).sort((left, right) => upperBounds[right] - upperBounds[left]);
|
|
8409
8618
|
const remainingBound = new Float64Array(order.length + 1);
|
|
8410
8619
|
for (let position2 = order.length - 1; position2 >= 0; position2--) {
|
|
@@ -8419,15 +8628,15 @@ function multiTermTopK(request) {
|
|
|
8419
8628
|
const threshold = kthBestScore(buffer, selection, resolver);
|
|
8420
8629
|
if (remaining < threshold - threshold * PRUNING_REJECTION_SLACK) break;
|
|
8421
8630
|
}
|
|
8422
|
-
scoreEveryDocument(scans[order[position]], buffer);
|
|
8631
|
+
scorer.scoreEveryDocument(scans[order[position]], buffer);
|
|
8423
8632
|
}
|
|
8424
8633
|
let countedOnly = 0;
|
|
8425
8634
|
for (let rest = position; rest < order.length; rest++) {
|
|
8426
|
-
countedOnly += scoreKnownDocumentsAndCountTheRest(scans[order[rest]], buffer);
|
|
8635
|
+
countedOnly += scorer.scoreKnownDocumentsAndCountTheRest(scans[order[rest]], buffer);
|
|
8427
8636
|
}
|
|
8428
8637
|
const totalMatched = buffer.touchedCount + countedOnly;
|
|
8429
8638
|
if (wanted === 0) return { scored: [], totalMatched };
|
|
8430
|
-
return { scored: pageFromBuffer(scans, buffer, selection, resolver), totalMatched };
|
|
8639
|
+
return { scored: pageFromBuffer(scorer, scans, buffer, selection, resolver), totalMatched };
|
|
8431
8640
|
}
|
|
8432
8641
|
|
|
8433
8642
|
// src/core/partition/single-term-topk.ts
|
|
@@ -8455,20 +8664,25 @@ function prunableSingleTermList(params, index, fieldNames) {
|
|
|
8455
8664
|
return list;
|
|
8456
8665
|
}
|
|
8457
8666
|
function singleTermTopK(request) {
|
|
8458
|
-
const { list, docFrequency, totalDocs,
|
|
8667
|
+
const { list, docFrequency, totalDocs, limit, resolver } = request;
|
|
8459
8668
|
const {
|
|
8460
8669
|
searchable: fieldSearchable,
|
|
8461
8670
|
boosts: fieldBoosts,
|
|
8462
8671
|
averageLengths: fieldAvgLengths,
|
|
8463
|
-
lengthColumns: fieldLengthColumns
|
|
8672
|
+
lengthColumns: fieldLengthColumns,
|
|
8673
|
+
k1s,
|
|
8674
|
+
bs,
|
|
8675
|
+
boundPairs
|
|
8464
8676
|
} = request.fields;
|
|
8677
|
+
const onePair = boundPairs.length === 2;
|
|
8678
|
+
const onlyK1 = boundPairs[0];
|
|
8679
|
+
const onlyB = boundPairs[1];
|
|
8465
8680
|
const wanted = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 0;
|
|
8466
8681
|
const { docIds, termFrequencies, fieldNameIndices, deletedDocs: deleted, hasDeleted } = postingColumns(list);
|
|
8467
8682
|
const bounds = blockBoundsFor(list, fieldLengthColumns);
|
|
8468
8683
|
const maxBoost = bestSearchable(fieldSearchable, fieldBoosts);
|
|
8469
8684
|
const maxAverageLength = bestSearchable(fieldSearchable, fieldAvgLengths);
|
|
8470
8685
|
const idf = computeIDF(docFrequency, totalDocs);
|
|
8471
|
-
const { k1, b } = resolveBM25Params(bm25Params);
|
|
8472
8686
|
const scoresAreZero = totalDocs === 0;
|
|
8473
8687
|
const heap = [];
|
|
8474
8688
|
let full = false;
|
|
@@ -8478,14 +8692,12 @@ function singleTermTopK(request) {
|
|
|
8478
8692
|
const start = block === 0 ? 0 : bounds.entryEnd[block - 1];
|
|
8479
8693
|
const end = bounds.entryEnd[block];
|
|
8480
8694
|
if (full) {
|
|
8481
|
-
|
|
8482
|
-
|
|
8483
|
-
|
|
8484
|
-
|
|
8485
|
-
|
|
8486
|
-
|
|
8487
|
-
bm25Params
|
|
8488
|
-
);
|
|
8695
|
+
let bestEntry = 0;
|
|
8696
|
+
if (!scoresAreZero) {
|
|
8697
|
+
const blockMaxTf = bounds.maxTermFrequency[block];
|
|
8698
|
+
const blockMinLength = bounds.minFieldLength[block];
|
|
8699
|
+
bestEntry = onePair ? computeBM25WithIDF(blockMaxTf, idf, blockMinLength, maxAverageLength, onlyK1, onlyB) : blockEntryBound(request.fields, blockMaxTf, idf, blockMinLength, maxAverageLength);
|
|
8700
|
+
}
|
|
8489
8701
|
if (bestEntry * maxBoost * bounds.maxEntriesPerDocument[block] < threshold) {
|
|
8490
8702
|
totalMatched += bounds.documentCount[block];
|
|
8491
8703
|
continue;
|
|
@@ -8504,7 +8716,14 @@ function singleTermTopK(request) {
|
|
|
8504
8716
|
const fieldIndex = fieldNameIndices[entry];
|
|
8505
8717
|
if (fieldSearchable[fieldIndex] === 1) {
|
|
8506
8718
|
const fieldLength = fieldLengthOf(fieldLengthColumns, fieldIndex, internalId, fieldAvgLengths[fieldIndex]);
|
|
8507
|
-
score += (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8719
|
+
score += (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8720
|
+
termFrequencies[entry],
|
|
8721
|
+
idf,
|
|
8722
|
+
fieldLength,
|
|
8723
|
+
fieldAvgLengths[fieldIndex],
|
|
8724
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
8725
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
8726
|
+
)) * fieldBoosts[fieldIndex];
|
|
8508
8727
|
scored2 = true;
|
|
8509
8728
|
}
|
|
8510
8729
|
entry++;
|
|
@@ -8564,15 +8783,14 @@ function searchFulltext(state, params) {
|
|
|
8564
8783
|
globalStats,
|
|
8565
8784
|
maxResults,
|
|
8566
8785
|
termMatch,
|
|
8567
|
-
|
|
8786
|
+
phrases
|
|
8568
8787
|
} = params;
|
|
8569
8788
|
const collectComponents = params.collectComponents !== false;
|
|
8570
|
-
if (queryTokens.length === 0) {
|
|
8789
|
+
if (queryTokens.length === 0 && phrases === void 0) {
|
|
8571
8790
|
return { scored: [], totalMatched: 0 };
|
|
8572
8791
|
}
|
|
8573
8792
|
const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
|
|
8574
8793
|
const avgFieldLengths = globalStats?.averageFieldLengths ?? state.stats.averageFieldLengths;
|
|
8575
|
-
const { k1, b } = resolveBM25Params(bm25Params);
|
|
8576
8794
|
const scoresAreZero = totalDocs === 0;
|
|
8577
8795
|
if (state.scoreBuffer === null) state.scoreBuffer = createScoreBuffer(state.docStore.internalIdCapacity());
|
|
8578
8796
|
const scoreBuffer = state.scoreBuffer;
|
|
@@ -8581,19 +8799,59 @@ function searchFulltext(state, params) {
|
|
|
8581
8799
|
const useIntersection = termMatch === "all" && queryTokens.length > 1;
|
|
8582
8800
|
const fieldNames = state.fieldNameTable.names;
|
|
8583
8801
|
const resolver = state.docStore.resolver();
|
|
8584
|
-
const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths);
|
|
8585
|
-
const { searchable, boosts, averageLengths, lengthColumns } = fieldScoring;
|
|
8802
|
+
const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths, bm25Params);
|
|
8803
|
+
const { searchable, boosts, averageLengths, lengthColumns, k1s, bs, boundPairs } = fieldScoring;
|
|
8804
|
+
const onePair = boundPairs.length === 2;
|
|
8805
|
+
const onlyK1 = boundPairs[0];
|
|
8806
|
+
const onlyB = boundPairs[1];
|
|
8807
|
+
const phraseMatches = phrases === void 0 ? null : requireLoadedPhraseCode().matchPhrases(
|
|
8808
|
+
state.invertedIdx,
|
|
8809
|
+
phrases,
|
|
8810
|
+
searchable,
|
|
8811
|
+
params.filterBitset,
|
|
8812
|
+
state.docStore.internalIdCapacity(),
|
|
8813
|
+
params.phraseWork,
|
|
8814
|
+
true
|
|
8815
|
+
);
|
|
8816
|
+
if (phraseMatches !== null && bitsetIsEmpty(phraseMatches.matched)) {
|
|
8817
|
+
return { scored: [], totalMatched: 0 };
|
|
8818
|
+
}
|
|
8819
|
+
const filterBitset = phraseMatches?.matched ?? params.filterBitset;
|
|
8586
8820
|
function documentFrequency(token, list) {
|
|
8587
8821
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
8588
8822
|
}
|
|
8823
|
+
function scorePhraseRows(rows, matched) {
|
|
8824
|
+
let idf = 0;
|
|
8825
|
+
for (let entry = 0; entry < rows.phrase.tokens.length; entry++) {
|
|
8826
|
+
idf += computeIDF(documentFrequency(rows.phrase.tokens[entry], rows.lists[entry]), totalDocs);
|
|
8827
|
+
}
|
|
8828
|
+
for (let row = 0; row < rows.docIds.length; row++) {
|
|
8829
|
+
const internalId = rows.docIds.values[row];
|
|
8830
|
+
if (!bitsetHas(matched, internalId)) continue;
|
|
8831
|
+
const fieldIndex = rows.fieldIndices.values[row];
|
|
8832
|
+
const frequency = rows.frequencies.values[row];
|
|
8833
|
+
const avgLen = averageLengths[fieldIndex];
|
|
8834
|
+
const fieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8835
|
+
const score = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8836
|
+
frequency,
|
|
8837
|
+
idf,
|
|
8838
|
+
fieldLength,
|
|
8839
|
+
avgLen,
|
|
8840
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
8841
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
8842
|
+
);
|
|
8843
|
+
addScore(scoreBuffer, internalId, score * boosts[fieldIndex]);
|
|
8844
|
+
if (components !== null) {
|
|
8845
|
+
recordComponents(components, internalId, fieldNames[fieldIndex], rows.phrase.key, frequency, fieldLength, idf);
|
|
8846
|
+
}
|
|
8847
|
+
}
|
|
8848
|
+
}
|
|
8589
8849
|
const groupContext = {
|
|
8590
8850
|
index: state.invertedIdx,
|
|
8591
8851
|
fields: fieldScoring,
|
|
8592
8852
|
partialMarks: partialFieldMarks(fieldNames, partialFields),
|
|
8593
8853
|
fieldNames,
|
|
8594
8854
|
totalDocs,
|
|
8595
|
-
k1,
|
|
8596
|
-
b,
|
|
8597
8855
|
filterBitset,
|
|
8598
8856
|
globalFrequency(token) {
|
|
8599
8857
|
if (globalStats === void 0 || !Object.hasOwn(globalStats.docFrequencies, token)) return void 0;
|
|
@@ -8619,7 +8877,14 @@ function searchFulltext(state, params) {
|
|
|
8619
8877
|
if (searchable[fieldIndex] === 0) continue;
|
|
8620
8878
|
const avgLen = averageLengths[fieldIndex];
|
|
8621
8879
|
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8622
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8880
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8881
|
+
termFrequencies[pi],
|
|
8882
|
+
match.idf,
|
|
8883
|
+
actualFieldLength,
|
|
8884
|
+
avgLen,
|
|
8885
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
8886
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
8887
|
+
);
|
|
8623
8888
|
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
8624
8889
|
}
|
|
8625
8890
|
return;
|
|
@@ -8634,7 +8899,14 @@ function searchFulltext(state, params) {
|
|
|
8634
8899
|
const termFrequency = termFrequencies[pi];
|
|
8635
8900
|
const avgLen = averageLengths[fieldIndex];
|
|
8636
8901
|
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
8637
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8902
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
8903
|
+
termFrequency,
|
|
8904
|
+
match.idf,
|
|
8905
|
+
actualFieldLength,
|
|
8906
|
+
avgLen,
|
|
8907
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
8908
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
8909
|
+
);
|
|
8638
8910
|
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
8639
8911
|
if (components !== null) {
|
|
8640
8912
|
recordComponents(
|
|
@@ -8656,25 +8928,23 @@ function searchFulltext(state, params) {
|
|
|
8656
8928
|
mergeGroupContribution(internalId, contribution);
|
|
8657
8929
|
}
|
|
8658
8930
|
}
|
|
8659
|
-
const prunableList = prunableSingleTermList(params, state.invertedIdx, fieldNames);
|
|
8931
|
+
const prunableList = phrases === void 0 ? prunableSingleTermList(params, state.invertedIdx, fieldNames) : null;
|
|
8660
8932
|
if (prunableList !== null && maxResults !== void 0) {
|
|
8661
8933
|
return singleTermTopK({
|
|
8662
8934
|
list: prunableList,
|
|
8663
8935
|
docFrequency: documentFrequency(queryTokens[0].token, prunableList),
|
|
8664
8936
|
totalDocs,
|
|
8665
|
-
bm25Params,
|
|
8666
8937
|
limit: maxResults,
|
|
8667
8938
|
fields: fieldScoring,
|
|
8668
8939
|
resolver
|
|
8669
8940
|
});
|
|
8670
8941
|
}
|
|
8671
|
-
const prunableLists = prunableMultiTermLists(params, state.invertedIdx);
|
|
8942
|
+
const prunableLists = phrases === void 0 ? prunableMultiTermLists(params, state.invertedIdx) : null;
|
|
8672
8943
|
if (prunableLists !== null && maxResults !== void 0) {
|
|
8673
8944
|
const pruned = multiTermTopK({
|
|
8674
8945
|
terms: prunableLists,
|
|
8675
8946
|
docFrequencies: prunableLists.map((term) => documentFrequency(term.token, term.list)),
|
|
8676
8947
|
totalDocs,
|
|
8677
|
-
bm25Params,
|
|
8678
8948
|
limit: maxResults,
|
|
8679
8949
|
fields: fieldScoring,
|
|
8680
8950
|
resolver,
|
|
@@ -8695,7 +8965,7 @@ function searchFulltext(state, params) {
|
|
|
8695
8965
|
}
|
|
8696
8966
|
resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
|
|
8697
8967
|
}
|
|
8698
|
-
resolved.sort((a,
|
|
8968
|
+
resolved.sort((a, b) => a.totalPostings - b.totalPostings);
|
|
8699
8969
|
for (let tokenIndex = 0; tokenIndex < resolved.length; tokenIndex++) {
|
|
8700
8970
|
const onlyScored = tokenIndex > 0;
|
|
8701
8971
|
for (const match of resolved[tokenIndex].matches) scorePostings(match, onlyScored);
|
|
@@ -8710,6 +8980,9 @@ function searchFulltext(state, params) {
|
|
|
8710
8980
|
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), false);
|
|
8711
8981
|
}
|
|
8712
8982
|
}
|
|
8983
|
+
if (phraseMatches !== null) {
|
|
8984
|
+
for (const rows of phraseMatches.rows) scorePhraseRows(rows, phraseMatches.matched);
|
|
8985
|
+
}
|
|
8713
8986
|
const totalMatched = scoreBuffer.touchedCount;
|
|
8714
8987
|
const k = maxResults === void 0 ? totalMatched : Math.max(0, Math.min(maxResults, totalMatched));
|
|
8715
8988
|
const scored = topKFromBuffer(scoreBuffer, k, resolver, components);
|
|
@@ -10367,6 +10640,9 @@ function heapPop(heap) {
|
|
|
10367
10640
|
function collectCandidateTerms(subs, params) {
|
|
10368
10641
|
const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
|
|
10369
10642
|
const terms = /* @__PURE__ */ new Set();
|
|
10643
|
+
for (const phrase of params.phrases ?? []) {
|
|
10644
|
+
for (const token of phrase.tokens) terms.add(token);
|
|
10645
|
+
}
|
|
10370
10646
|
for (const qt of queryTokens) {
|
|
10371
10647
|
const group = termGroupOf(termGroups, qt.token);
|
|
10372
10648
|
if (group !== void 0) {
|
|
@@ -10468,7 +10744,7 @@ function searchSingleSub(subs, layout, params, index) {
|
|
|
10468
10744
|
return { ...result, matchedOrdinalBitset };
|
|
10469
10745
|
}
|
|
10470
10746
|
function compositeSearchFulltext(subs, layout, params) {
|
|
10471
|
-
if (params.queryTokens.length === 0) {
|
|
10747
|
+
if (params.queryTokens.length === 0 && params.phrases === void 0) {
|
|
10472
10748
|
return { scored: [], totalMatched: 0 };
|
|
10473
10749
|
}
|
|
10474
10750
|
const single = onlyPopulatedSub(subs);
|
|
@@ -11357,7 +11633,7 @@ var PREPARED_TYPO_TEST_BYTES = Math.ceil(MAX_FILTER_TESTS / TYPO_CLAIM_SLOTS_PER
|
|
|
11357
11633
|
function capExceeded(cap) {
|
|
11358
11634
|
return new NarsilError(
|
|
11359
11635
|
ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
|
|
11360
|
-
`The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
|
|
11636
|
+
`The engine stopped this search before it checked every candidate, because the work on its pattern tests and phrases passed the cap of ${cap} units. Narrow the tests or the phrases, or raise the patternWorkCap engine setting`,
|
|
11361
11637
|
{ cap }
|
|
11362
11638
|
);
|
|
11363
11639
|
}
|
|
@@ -11588,15 +11864,19 @@ function addPartitionTokens(tokens, addedMembers, partition, term, analysed, exp
|
|
|
11588
11864
|
}
|
|
11589
11865
|
}
|
|
11590
11866
|
}
|
|
11591
|
-
function addReachedTokens(tokens, partitions, term,
|
|
11867
|
+
function addReachedTokens(tokens, partitions, term, requested, language, analysis) {
|
|
11868
|
+
if (requested.options.exact === true) return [];
|
|
11869
|
+
const quoted = analyseQueryPhrases(term, language, analysis);
|
|
11870
|
+
const words = quoted?.words ?? term;
|
|
11871
|
+
if (quoted !== null && words.trim().length === 0) return [];
|
|
11872
|
+
const expansion = quoted?.endsWithPhrase === true ? { schema: requested.schema, options: { ...requested.options, prefix: false } } : requested;
|
|
11592
11873
|
const { options } = expansion;
|
|
11593
|
-
if (options.exact === true) return [];
|
|
11594
11874
|
if (!((options.tolerance ?? 0) > 0 || options.prefix === true || options.partialWords === true)) return [];
|
|
11595
|
-
const analysed = queryTokensOf(
|
|
11875
|
+
const analysed = queryTokensOf(words, language, analysis);
|
|
11596
11876
|
if (analysed === null) return [];
|
|
11597
11877
|
const addedMembers = /* @__PURE__ */ new Set();
|
|
11598
11878
|
for (const partition of partitions) {
|
|
11599
|
-
addPartitionTokens(tokens, addedMembers, partition,
|
|
11879
|
+
addPartitionTokens(tokens, addedMembers, partition, words, analysed, expansion, language, analysis);
|
|
11600
11880
|
}
|
|
11601
11881
|
for (const token of addedMembers) tokens.add(token);
|
|
11602
11882
|
return [...addedMembers];
|
|
@@ -11619,17 +11899,24 @@ function partitionsIn(manager, partitionIds) {
|
|
|
11619
11899
|
|
|
11620
11900
|
// src/partitioning/distributed-scoring.ts
|
|
11621
11901
|
function analysedQueryTokens(term, language, analysis) {
|
|
11622
|
-
const
|
|
11623
|
-
|
|
11624
|
-
removeStopWords: true,
|
|
11625
|
-
customTokenizer: analysis.customTokenizer,
|
|
11626
|
-
stopWordOverride: analysis.stopWords
|
|
11627
|
-
});
|
|
11902
|
+
const quoted = analyseQueryPhrases(term, language, analysis);
|
|
11903
|
+
const words = quoted?.words ?? term;
|
|
11628
11904
|
const unique = /* @__PURE__ */ new Set();
|
|
11629
|
-
|
|
11630
|
-
|
|
11905
|
+
if (quoted === null || words.trim().length > 0) {
|
|
11906
|
+
const { tokens } = tokenize(words, language, {
|
|
11907
|
+
stem: true,
|
|
11908
|
+
removeStopWords: true,
|
|
11909
|
+
customTokenizer: analysis.customTokenizer,
|
|
11910
|
+
stopWordOverride: analysis.stopWords
|
|
11911
|
+
});
|
|
11912
|
+
for (const { token } of tokens) {
|
|
11913
|
+
unique.add(token);
|
|
11914
|
+
}
|
|
11631
11915
|
}
|
|
11632
11916
|
for (const synonym of analysis.synonyms?.expand(unique)?.tokens ?? []) unique.add(synonym);
|
|
11917
|
+
for (const phrase of quoted?.phrases ?? []) {
|
|
11918
|
+
for (const token of phrase.tokens) unique.add(token);
|
|
11919
|
+
}
|
|
11633
11920
|
return unique;
|
|
11634
11921
|
}
|
|
11635
11922
|
function copyNumericRecord(source) {
|
|
@@ -12028,6 +12315,14 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
|
|
|
12028
12315
|
}
|
|
12029
12316
|
|
|
12030
12317
|
// src/search/fulltext.ts
|
|
12318
|
+
function analysedWords(term, language, options) {
|
|
12319
|
+
return tokenize(term, language, {
|
|
12320
|
+
stem: true,
|
|
12321
|
+
removeStopWords: true,
|
|
12322
|
+
customTokenizer: options?.customTokenizer,
|
|
12323
|
+
stopWordOverride: options?.stopWords
|
|
12324
|
+
}).tokens;
|
|
12325
|
+
}
|
|
12031
12326
|
function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
12032
12327
|
if (!params.term || params.term.trim().length === 0) return null;
|
|
12033
12328
|
if (params.fields && params.fields.length === 0) return null;
|
|
@@ -12035,14 +12330,16 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
12035
12330
|
if (params.fields) {
|
|
12036
12331
|
validateSearchFields(params.fields, flatSchema);
|
|
12037
12332
|
}
|
|
12038
|
-
const
|
|
12039
|
-
|
|
12040
|
-
|
|
12041
|
-
customTokenizer: options?.customTokenizer,
|
|
12042
|
-
stopWordOverride: options?.stopWords
|
|
12333
|
+
const quoted = analyseQueryPhrases(params.term, language, {
|
|
12334
|
+
stopWords: options?.stopWords,
|
|
12335
|
+
customTokenizer: options?.customTokenizer
|
|
12043
12336
|
});
|
|
12044
|
-
|
|
12045
|
-
const
|
|
12337
|
+
const wordParams = quoted === null ? params : requireLoadedPhraseCode().withWordsOnly(params, quoted);
|
|
12338
|
+
const phrases = quoted !== null && quoted.phrases.length > 0 ? quoted.phrases : void 0;
|
|
12339
|
+
const words = wordParams.term ?? "";
|
|
12340
|
+
const analysedTokens = quoted !== null && words.trim().length === 0 ? [] : analysedWords(words, language, options);
|
|
12341
|
+
if (analysedTokens.length === 0 && phrases === void 0) return null;
|
|
12342
|
+
const queryTokens = deduplicateTokens(analysedTokens);
|
|
12046
12343
|
let filterBitset;
|
|
12047
12344
|
if (params.filters) {
|
|
12048
12345
|
filterBitset = partition.applyFiltersBitset(
|
|
@@ -12052,16 +12349,19 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
12052
12349
|
);
|
|
12053
12350
|
if (bitsetIsEmpty(filterBitset)) return null;
|
|
12054
12351
|
}
|
|
12055
|
-
|
|
12352
|
+
if (analysedTokens.length === 0) return { queryTokens, phrases, filterBitset };
|
|
12353
|
+
const lastToken = analysedTokens[analysedTokens.length - 1];
|
|
12056
12354
|
const synonyms = params.exact === true ? void 0 : options?.synonyms?.expand(queryTokens.map((qt) => qt.token));
|
|
12057
|
-
const groups = resolveTermGroups(partition,
|
|
12058
|
-
return { queryTokens, ...groups, filterBitset };
|
|
12355
|
+
const groups = resolveTermGroups(partition, wordParams, language, schema, queryTokens, lastToken, options, synonyms);
|
|
12356
|
+
return { queryTokens, phrases, ...groups, filterBitset };
|
|
12059
12357
|
}
|
|
12060
12358
|
function fulltextMatches(partition, params, language, schema, options) {
|
|
12061
12359
|
const prepared = prepareFulltextQuery(partition, params, language, schema, options);
|
|
12062
12360
|
if (prepared === null) return null;
|
|
12063
12361
|
return partition.searchFulltextMatches({
|
|
12064
12362
|
queryTokens: prepared.queryTokens,
|
|
12363
|
+
phrases: prepared.phrases,
|
|
12364
|
+
phraseWork: options?.patternWork,
|
|
12065
12365
|
termGroups: prepared.termGroups,
|
|
12066
12366
|
partialFields: prepared.partialFields,
|
|
12067
12367
|
fields: params.fields,
|
|
@@ -12076,13 +12376,15 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
12076
12376
|
if (prepared === null) {
|
|
12077
12377
|
return { scored: [], totalMatched: 0 };
|
|
12078
12378
|
}
|
|
12079
|
-
const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
|
|
12379
|
+
const { queryTokens, phrases, termGroups, partialFields, filterBitset } = prepared;
|
|
12080
12380
|
const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
12081
12381
|
const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
|
|
12082
12382
|
const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
|
|
12083
12383
|
const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
|
|
12084
12384
|
const rawResult = partition.searchFulltext({
|
|
12085
12385
|
queryTokens,
|
|
12386
|
+
phrases,
|
|
12387
|
+
phraseWork: options?.patternWork,
|
|
12086
12388
|
termGroups,
|
|
12087
12389
|
partialFields,
|
|
12088
12390
|
fields: params.fields,
|
|
@@ -12100,7 +12402,7 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
12100
12402
|
});
|
|
12101
12403
|
let scored = rawResult.scored;
|
|
12102
12404
|
const termMatch = params.termMatch ?? "any";
|
|
12103
|
-
if (termMatch !== "any") {
|
|
12405
|
+
if (termMatch !== "any" && queryTokens.length > 0) {
|
|
12104
12406
|
scored = filterByTermCoverage(
|
|
12105
12407
|
scored,
|
|
12106
12408
|
queryTokens,
|
|
@@ -12109,7 +12411,8 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
12109
12411
|
params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
12110
12412
|
params.exact ?? false,
|
|
12111
12413
|
termGroups,
|
|
12112
|
-
partialFields ?? []
|
|
12414
|
+
partialFields ?? [],
|
|
12415
|
+
phrases
|
|
12113
12416
|
);
|
|
12114
12417
|
}
|
|
12115
12418
|
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
@@ -12151,8 +12454,9 @@ function deduplicateTokens(tokens) {
|
|
|
12151
12454
|
}
|
|
12152
12455
|
return result;
|
|
12153
12456
|
}
|
|
12154
|
-
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
|
|
12457
|
+
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields, phrases) {
|
|
12155
12458
|
const requiredCount = policy === "all" ? queryTokens.length : policy;
|
|
12459
|
+
const phraseKeys = phrases === void 0 ? null : new Set(phrases.map((phrase) => phrase.key));
|
|
12156
12460
|
if (requiredCount <= 0) return scored;
|
|
12157
12461
|
if (requiredCount > queryTokens.length) return [];
|
|
12158
12462
|
const groups = /* @__PURE__ */ new Map();
|
|
@@ -12168,12 +12472,22 @@ function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLeng
|
|
|
12168
12472
|
});
|
|
12169
12473
|
}
|
|
12170
12474
|
return scored.filter((doc) => {
|
|
12171
|
-
const matched = countDocTermMatches(
|
|
12475
|
+
const matched = countDocTermMatches(
|
|
12476
|
+
doc,
|
|
12477
|
+
queryTokens,
|
|
12478
|
+
tolerance,
|
|
12479
|
+
prefixLength,
|
|
12480
|
+
exact,
|
|
12481
|
+
groups,
|
|
12482
|
+
partialFields,
|
|
12483
|
+
phraseKeys
|
|
12484
|
+
);
|
|
12172
12485
|
return matched >= requiredCount;
|
|
12173
12486
|
});
|
|
12174
12487
|
}
|
|
12175
|
-
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
|
|
12176
|
-
const
|
|
12488
|
+
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields, phraseKeys) {
|
|
12489
|
+
const componentKeys = Object.keys(doc.idf);
|
|
12490
|
+
const indexTokens = phraseKeys === null ? componentKeys : componentKeys.filter((key) => !phraseKeys.has(key));
|
|
12177
12491
|
if (indexTokens.length === 0) return 0;
|
|
12178
12492
|
let count = 0;
|
|
12179
12493
|
for (const qt of queryTokens) {
|
|
@@ -12328,8 +12642,11 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
12328
12642
|
|
|
12329
12643
|
// src/workers/query-actions.ts
|
|
12330
12644
|
async function runQueryAction(entry, action) {
|
|
12645
|
+
const loadingPhraseCode = loadPhraseCodeFor(action.params.term);
|
|
12646
|
+
if (loadingPhraseCode !== null) await loadingPhraseCode;
|
|
12331
12647
|
const patternWork = joinPatternWorkMeter(action.patternWork, action.params.filters);
|
|
12332
|
-
const
|
|
12648
|
+
const bm25Params = action.type === "query" ? action.bm25 ?? void 0 : entry.config.bm25;
|
|
12649
|
+
const searchOptions = { ...entry.searchOptions, patternWork, bm25Params };
|
|
12333
12650
|
if (action.type === "query") {
|
|
12334
12651
|
const result2 = await fanOutQuery(
|
|
12335
12652
|
entry.manager,
|
|
@@ -13447,6 +13764,10 @@ function createFrozenPostingViews(payload, tombstones, counts) {
|
|
|
13447
13764
|
if (positionOffsets === null) return 0;
|
|
13448
13765
|
return positionOffsets[start + row + 1] - positionOffsets[start + row];
|
|
13449
13766
|
},
|
|
13767
|
+
positionStore: positionValues,
|
|
13768
|
+
positionStartAt(row) {
|
|
13769
|
+
return positionOffsets === null ? 0 : positionOffsets[start + row];
|
|
13770
|
+
},
|
|
13450
13771
|
docIdSet: {
|
|
13451
13772
|
get size() {
|
|
13452
13773
|
return counts.documentFrequency(sortedIndex);
|
|
@@ -13565,7 +13886,7 @@ function addForms(target, forms) {
|
|
|
13565
13886
|
function createFrozenSurfaceSlot(load, documents) {
|
|
13566
13887
|
let registry = null;
|
|
13567
13888
|
let uncounted = null;
|
|
13568
|
-
function
|
|
13889
|
+
function loaded3() {
|
|
13569
13890
|
if (registry === null) {
|
|
13570
13891
|
const built = createSurfaceRegistry();
|
|
13571
13892
|
const forms = load();
|
|
@@ -13579,26 +13900,26 @@ function createFrozenSurfaceSlot(load, documents) {
|
|
|
13579
13900
|
}
|
|
13580
13901
|
return {
|
|
13581
13902
|
reader: {
|
|
13582
|
-
candidatesForPrefix: (prefix) =>
|
|
13583
|
-
stemChangedTotalFor: (token) =>
|
|
13584
|
-
size: () =>
|
|
13585
|
-
serialize: () =>
|
|
13903
|
+
candidatesForPrefix: (prefix) => loaded3().candidatesForPrefix(prefix),
|
|
13904
|
+
stemChangedTotalFor: (token) => loaded3().stemChangedTotalFor(token),
|
|
13905
|
+
size: () => loaded3().size(),
|
|
13906
|
+
serialize: () => loaded3().serialize()
|
|
13586
13907
|
},
|
|
13587
13908
|
holdsAny() {
|
|
13588
|
-
return
|
|
13909
|
+
return loaded3().size() > 0;
|
|
13589
13910
|
},
|
|
13590
13911
|
holds(spelling) {
|
|
13591
|
-
return
|
|
13912
|
+
return loaded3().holds(spelling);
|
|
13592
13913
|
},
|
|
13593
13914
|
subtractSpellings(ordinal, spellings) {
|
|
13594
13915
|
uncounted ??= new Uint8Array(documents.count);
|
|
13595
13916
|
uncounted[ordinal] = 1;
|
|
13596
13917
|
if (spellings.length === 0) return;
|
|
13597
|
-
const target =
|
|
13918
|
+
const target = loaded3();
|
|
13598
13919
|
for (const spelling of spellings) target.subtract(spelling, 1);
|
|
13599
13920
|
},
|
|
13600
13921
|
addKeptForms(target, mergedOrdinals, analysis) {
|
|
13601
|
-
const source =
|
|
13922
|
+
const source = loaded3();
|
|
13602
13923
|
if (analysis === null) {
|
|
13603
13924
|
addForms(target, source.serialize());
|
|
13604
13925
|
return;
|
|
@@ -14653,36 +14974,6 @@ function mergeSegmentContents(selections, analysis = null, textAnalysis = null)
|
|
|
14653
14974
|
return { payload, documents: mergeDocuments2(inputs, documentCount), reusedPartialWords: partialWords.reused };
|
|
14654
14975
|
}
|
|
14655
14976
|
|
|
14656
|
-
// src/core/partition/growable-uint32.ts
|
|
14657
|
-
var GrowableUint32 = class {
|
|
14658
|
-
values;
|
|
14659
|
-
length = 0;
|
|
14660
|
-
constructor(capacity) {
|
|
14661
|
-
this.values = new Uint32Array(capacity);
|
|
14662
|
-
}
|
|
14663
|
-
growTo(needed, kept) {
|
|
14664
|
-
let capacity = this.values.length * 2;
|
|
14665
|
-
while (capacity < needed) capacity *= 2;
|
|
14666
|
-
const next = new Uint32Array(capacity);
|
|
14667
|
-
next.set(this.values.subarray(0, kept));
|
|
14668
|
-
this.values = next;
|
|
14669
|
-
}
|
|
14670
|
-
reserve(extra) {
|
|
14671
|
-
const needed = this.length + extra;
|
|
14672
|
-
if (needed > this.values.length) this.growTo(needed, this.length);
|
|
14673
|
-
}
|
|
14674
|
-
push(value) {
|
|
14675
|
-
if (this.length === this.values.length) this.growTo(this.length + 1, this.length);
|
|
14676
|
-
this.values[this.length++] = value;
|
|
14677
|
-
}
|
|
14678
|
-
ensureIndex(index) {
|
|
14679
|
-
if (index >= this.values.length) this.growTo(index + 1, this.values.length);
|
|
14680
|
-
}
|
|
14681
|
-
clear() {
|
|
14682
|
-
this.length = 0;
|
|
14683
|
-
}
|
|
14684
|
-
};
|
|
14685
|
-
|
|
14686
14977
|
// src/core/partition/segment-postings.ts
|
|
14687
14978
|
var TextPostings = class {
|
|
14688
14979
|
constructor(trackPositions) {
|
|
@@ -14952,8 +15243,10 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
14952
15243
|
const spellings = /* @__PURE__ */ new Map();
|
|
14953
15244
|
let fieldSpellings = null;
|
|
14954
15245
|
let positionOffset = 0;
|
|
15246
|
+
let lastItemPosition = -1;
|
|
14955
15247
|
const addToken = (token, position, surface) => {
|
|
14956
15248
|
postings.addToken(token, positionOffset + position);
|
|
15249
|
+
lastItemPosition = position;
|
|
14957
15250
|
surfaces.count(surface, token);
|
|
14958
15251
|
if (fieldSpellings !== null) countPartialSpelling(fieldSpellings, token, surface ?? token, 1);
|
|
14959
15252
|
};
|
|
@@ -14972,8 +15265,9 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
14972
15265
|
postings.beginField();
|
|
14973
15266
|
positionOffset = 0;
|
|
14974
15267
|
for (const text of fieldType === "string" ? [value] : value) {
|
|
15268
|
+
lastItemPosition = -1;
|
|
14975
15269
|
visitTokens(text, language, tokenizing, addToken);
|
|
14976
|
-
positionOffset
|
|
15270
|
+
if (lastItemPosition >= 0) positionOffset += lastItemPosition + 1 + ARRAY_POSITION_GAP;
|
|
14977
15271
|
}
|
|
14978
15272
|
postings.endField(ordinal, fieldPath);
|
|
14979
15273
|
lengths.record(ordinal, fieldPath, postings.fieldLength);
|
|
@@ -15091,30 +15385,30 @@ function runSegmentAction(entry, action) {
|
|
|
15091
15385
|
|
|
15092
15386
|
// src/workers/vector-code.ts
|
|
15093
15387
|
var VECTOR_TYPE_NAME = "vector";
|
|
15094
|
-
var
|
|
15095
|
-
var
|
|
15388
|
+
var loaded2 = null;
|
|
15389
|
+
var loading2 = null;
|
|
15096
15390
|
function loadedVectorCode() {
|
|
15097
|
-
return
|
|
15391
|
+
return loaded2;
|
|
15098
15392
|
}
|
|
15099
15393
|
function requireLoadedVectorCode() {
|
|
15100
|
-
if (
|
|
15101
|
-
return
|
|
15394
|
+
if (loaded2 === null) throw new Error("The engine reached vector code before it loaded that code");
|
|
15395
|
+
return loaded2;
|
|
15102
15396
|
}
|
|
15103
15397
|
function loadVectorCode() {
|
|
15104
|
-
if (
|
|
15105
|
-
if (
|
|
15106
|
-
|
|
15398
|
+
if (loaded2 !== null) return Promise.resolve(loaded2);
|
|
15399
|
+
if (loading2 === null) {
|
|
15400
|
+
loading2 = import('./module-4HIMOQM5.mjs').then(
|
|
15107
15401
|
(code) => {
|
|
15108
|
-
|
|
15402
|
+
loaded2 = code;
|
|
15109
15403
|
return code;
|
|
15110
15404
|
},
|
|
15111
15405
|
(err) => {
|
|
15112
|
-
|
|
15406
|
+
loading2 = null;
|
|
15113
15407
|
throw err;
|
|
15114
15408
|
}
|
|
15115
15409
|
);
|
|
15116
15410
|
}
|
|
15117
|
-
return
|
|
15411
|
+
return loading2;
|
|
15118
15412
|
}
|
|
15119
15413
|
function mayDeclareVectorField(schema, depth) {
|
|
15120
15414
|
if (!isPlainObject(schema) || depth > MAX_NESTING_DEPTH + 1) return false;
|
|
@@ -15126,7 +15420,7 @@ function mayDeclareVectorField(schema, depth) {
|
|
|
15126
15420
|
return false;
|
|
15127
15421
|
}
|
|
15128
15422
|
function loadVectorCodeFor(schema) {
|
|
15129
|
-
if (
|
|
15423
|
+
if (loaded2 !== null || !mayDeclareVectorField(schema, 1)) return null;
|
|
15130
15424
|
return loadVectorCode();
|
|
15131
15425
|
}
|
|
15132
15426
|
|
|
@@ -15254,7 +15548,6 @@ function createDirectExecutor(options) {
|
|
|
15254
15548
|
config,
|
|
15255
15549
|
language,
|
|
15256
15550
|
searchOptions: {
|
|
15257
|
-
bm25Params: config.bm25,
|
|
15258
15551
|
stopWords: manager.analysis.stopWords,
|
|
15259
15552
|
customTokenizer: manager.analysis.customTokenizer,
|
|
15260
15553
|
get synonyms() {
|
|
@@ -15323,8 +15616,8 @@ function createDirectExecutor(options) {
|
|
|
15323
15616
|
);
|
|
15324
15617
|
}
|
|
15325
15618
|
case "createIndex": {
|
|
15326
|
-
const
|
|
15327
|
-
if (
|
|
15619
|
+
const loading3 = loadVectorCodeFor(action.config.schema);
|
|
15620
|
+
if (loading3 !== null) await loading3;
|
|
15328
15621
|
const language = getLanguage(action.config.language ?? "english");
|
|
15329
15622
|
const manager = addIndex(
|
|
15330
15623
|
action.indexName,
|
|
@@ -15395,6 +15688,11 @@ function createDirectExecutor(options) {
|
|
|
15395
15688
|
if (entry !== void 0) entry.manager.synonyms = lookupOf(entry.config, entry.language, action.synonyms);
|
|
15396
15689
|
return void 0;
|
|
15397
15690
|
}
|
|
15691
|
+
case "replaceBm25": {
|
|
15692
|
+
const entry = indexes.get(action.indexName);
|
|
15693
|
+
if (entry !== void 0) entry.config.bm25 = action.bm25;
|
|
15694
|
+
return void 0;
|
|
15695
|
+
}
|
|
15398
15696
|
case "get": {
|
|
15399
15697
|
const entry = requireIndex(action.indexName);
|
|
15400
15698
|
return entry.manager.get(action.docId);
|
|
@@ -15688,6 +15986,7 @@ function createAnalysisRebuildCoordinator(config, deps) {
|
|
|
15688
15986
|
}
|
|
15689
15987
|
}
|
|
15690
15988
|
return {
|
|
15989
|
+
rebuildMode,
|
|
15691
15990
|
markStale(index) {
|
|
15692
15991
|
const existing = stale.get(index.indexName);
|
|
15693
15992
|
if (existing !== void 0 && existing.run !== null) {
|
|
@@ -15816,6 +16115,17 @@ function resolveDurabilityTier(config) {
|
|
|
15816
16115
|
return { kind: "snapshot", config: {}, adapter: config.persistence };
|
|
15817
16116
|
}
|
|
15818
16117
|
|
|
16118
|
+
// src/engine/array-positions.ts
|
|
16119
|
+
function holdsStringArrayField(schema) {
|
|
16120
|
+
return schemaFieldsOf(schema).some((field) => field.base === "string[]");
|
|
16121
|
+
}
|
|
16122
|
+
function arrayPositionsAreStale(config, storedGap) {
|
|
16123
|
+
return storedGap !== ARRAY_POSITION_GAP && config.trackPositions !== false && holdsStringArrayField(config.schema);
|
|
16124
|
+
}
|
|
16125
|
+
function storedArrayPositionGap(value) {
|
|
16126
|
+
return typeof value === "number" ? value : null;
|
|
16127
|
+
}
|
|
16128
|
+
|
|
15819
16129
|
// src/engine/recovery-schema.ts
|
|
15820
16130
|
function unflattenSchema(flat) {
|
|
15821
16131
|
const root = {};
|
|
@@ -15842,7 +16152,11 @@ function reconstructSchemaFromMetadata(metadata) {
|
|
|
15842
16152
|
const config = {
|
|
15843
16153
|
schema: unflattenSchema(metadata.schema),
|
|
15844
16154
|
language: metadata.language,
|
|
15845
|
-
bm25: {
|
|
16155
|
+
bm25: {
|
|
16156
|
+
k1: metadata.bm25Params.k1,
|
|
16157
|
+
b: metadata.bm25Params.b,
|
|
16158
|
+
...metadata.bm25Params.fields !== void 0 ? { fields: metadata.bm25Params.fields } : {}
|
|
16159
|
+
}
|
|
15846
16160
|
};
|
|
15847
16161
|
if (metadata.embedding) {
|
|
15848
16162
|
config.embedding = metadata.embedding.adapter !== void 0 ? { adapter: metadata.embedding.adapter, fields: metadata.embedding.fields } : { fields: metadata.embedding.fields };
|
|
@@ -16037,16 +16351,6 @@ function validateWorkerConfig(workers, lifecycle) {
|
|
|
16037
16351
|
);
|
|
16038
16352
|
}
|
|
16039
16353
|
}
|
|
16040
|
-
function validateBM25Params(bm25) {
|
|
16041
|
-
if (bm25 === void 0) return;
|
|
16042
|
-
const { k1, b } = bm25;
|
|
16043
|
-
if (k1 !== void 0 && (!Number.isFinite(k1) || k1 < 0)) {
|
|
16044
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.k1 must be a finite number at or above 0", { k1 });
|
|
16045
|
-
}
|
|
16046
|
-
if (b !== void 0 && (!Number.isFinite(b) || b < 0 || b > 1)) {
|
|
16047
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25.b must be between 0 and 1", { b });
|
|
16048
|
-
}
|
|
16049
|
-
}
|
|
16050
16354
|
function clampLimit(limit) {
|
|
16051
16355
|
return clampRowCount(limit, DEFAULT_LIMIT);
|
|
16052
16356
|
}
|
|
@@ -16144,11 +16448,12 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
16144
16448
|
documentCount: metadata.documentCount ?? 0,
|
|
16145
16449
|
partitionCount: metadata.partitionCount,
|
|
16146
16450
|
analysisRevision: metadata.analysisRevision ?? null,
|
|
16451
|
+
arrayPositionGap: metadata.arrayPositionGap ?? null,
|
|
16147
16452
|
synonyms
|
|
16148
16453
|
});
|
|
16149
16454
|
if (loadData) await deps.indexState().registerOpen(metadata.indexName);
|
|
16150
16455
|
else deps.indexState().registerClosed(metadata.indexName);
|
|
16151
|
-
if (metadata.analysisRevision !== language.revision) {
|
|
16456
|
+
if (metadata.analysisRevision !== language.revision || arrayPositionsAreStale(indexConfig, metadata.arrayPositionGap ?? null)) {
|
|
16152
16457
|
deps.analysisRebuild().markStale({
|
|
16153
16458
|
indexName: metadata.indexName,
|
|
16154
16459
|
language: language.name,
|
|
@@ -17464,10 +17769,10 @@ async function replicateToWorkers(state, action) {
|
|
|
17464
17769
|
return;
|
|
17465
17770
|
}
|
|
17466
17771
|
const detached = detachCallerDocuments(state, action);
|
|
17467
|
-
const
|
|
17468
|
-
if (
|
|
17772
|
+
const loading3 = state.copyLoadBuffers.get(action.indexName);
|
|
17773
|
+
if (loading3 !== void 0) {
|
|
17469
17774
|
queueForCopies(state, detached);
|
|
17470
|
-
if (
|
|
17775
|
+
if (loading3.length % LOAD_BUFFER_YIELD_INTERVAL === 0) await yieldToEventLoop();
|
|
17471
17776
|
return;
|
|
17472
17777
|
}
|
|
17473
17778
|
const queue = state.replicationQueues.get(action.indexName);
|
|
@@ -17478,9 +17783,9 @@ async function replicateToWorkers(state, action) {
|
|
|
17478
17783
|
}
|
|
17479
17784
|
function queueForCopies(state, action) {
|
|
17480
17785
|
if (!("indexName" in action)) return;
|
|
17481
|
-
const
|
|
17482
|
-
if (
|
|
17483
|
-
|
|
17786
|
+
const loading3 = state.copyLoadBuffers.get(action.indexName);
|
|
17787
|
+
if (loading3 !== void 0) {
|
|
17788
|
+
loading3.push(action);
|
|
17484
17789
|
return;
|
|
17485
17790
|
}
|
|
17486
17791
|
if (state.workerPool === null || !state.scaledOutIndexes.has(action.indexName)) return;
|
|
@@ -18796,23 +19101,24 @@ function scoresPerPartition(state, indexName, params, globalStats) {
|
|
|
18796
19101
|
if (mode === "local") return true;
|
|
18797
19102
|
return mode === "broadcast" && globalStats !== void 0;
|
|
18798
19103
|
}
|
|
18799
|
-
function queryAction(indexName, params, globalStats, partitionIds, patternWork) {
|
|
19104
|
+
function queryAction(indexName, params, globalStats, partitionIds, patternWork, bm25) {
|
|
18800
19105
|
return {
|
|
18801
19106
|
type: "query",
|
|
18802
19107
|
indexName,
|
|
18803
19108
|
params,
|
|
18804
19109
|
requestId: createRequestId(),
|
|
18805
19110
|
patternWork,
|
|
19111
|
+
bm25: bm25 ?? null,
|
|
18806
19112
|
...partitionIds !== void 0 ? { partitionIds } : {},
|
|
18807
19113
|
...globalStats !== void 0 ? { globalStats } : {}
|
|
18808
19114
|
};
|
|
18809
19115
|
}
|
|
18810
|
-
async function runSplit(leases, scope, indexName, params, globalStats, patternWork) {
|
|
19116
|
+
async function runSplit(leases, scope, indexName, params, globalStats, patternWork, bm25) {
|
|
18811
19117
|
const assignments = partitionsPerLease(scope, leases.length);
|
|
18812
19118
|
const workerParams = params.facets !== void 0 ? { ...params, facets: oversampledFacetConfig(params.facets) } : params;
|
|
18813
19119
|
const outcomes = await Promise.allSettled(
|
|
18814
19120
|
assignments.map(
|
|
18815
|
-
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds, patternWork)).finally(() => leases[at].release())
|
|
19121
|
+
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds, patternWork, bm25)).finally(() => leases[at].release())
|
|
18816
19122
|
)
|
|
18817
19123
|
);
|
|
18818
19124
|
const results = [];
|
|
@@ -18848,7 +19154,7 @@ function answersFasterOnMainCopy(state, pool, manager) {
|
|
|
18848
19154
|
if (manager.countDocuments() > MAIN_COPY_LONE_QUERY_DOCUMENTS) return false;
|
|
18849
19155
|
return takeMainCopyTurn(state);
|
|
18850
19156
|
}
|
|
18851
|
-
async function searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork) {
|
|
19157
|
+
async function searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork, bm25) {
|
|
18852
19158
|
const pool = state.workerPool;
|
|
18853
19159
|
if (!pool) return null;
|
|
18854
19160
|
if (!state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName)) return null;
|
|
@@ -18863,7 +19169,7 @@ async function searchViaWorker(state, indexName, params, globalStats, partitionI
|
|
|
18863
19169
|
if (idle.length >= 2 && scoresPerPartition(state, indexName, params, globalStats)) {
|
|
18864
19170
|
leases.push(...idle);
|
|
18865
19171
|
fork = patternWork.fork(leases.length);
|
|
18866
|
-
const merged = await runSplit(leases, scope, indexName, params, globalStats, fork.shared);
|
|
19172
|
+
const merged = await runSplit(leases, scope, indexName, params, globalStats, fork.shared, bm25);
|
|
18867
19173
|
fork.settle(merged.patternWorkSpent ?? 0);
|
|
18868
19174
|
return merged;
|
|
18869
19175
|
}
|
|
@@ -18873,7 +19179,7 @@ async function searchViaWorker(state, indexName, params, globalStats, partitionI
|
|
|
18873
19179
|
leases.push(lease);
|
|
18874
19180
|
fork = patternWork.fork(1);
|
|
18875
19181
|
const result = await lease.executor.execute(
|
|
18876
|
-
queryAction(indexName, params, globalStats, partitionIds, fork.shared)
|
|
19182
|
+
queryAction(indexName, params, globalStats, partitionIds, fork.shared, bm25)
|
|
18877
19183
|
);
|
|
18878
19184
|
fork.settle(result.patternWorkSpent ?? 0);
|
|
18879
19185
|
return result;
|
|
@@ -19134,7 +19440,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
19134
19440
|
segmentBuildConcurrency: (indexName) => segmentBuildConcurrency(state, indexName),
|
|
19135
19441
|
holdUnbroadcastSegments: (indexName, segmentIds) => holdUnbroadcastSegments(state, indexName, segmentIds),
|
|
19136
19442
|
releaseUnbroadcastSegments: (indexName, segmentIds) => releaseUnbroadcastSegments(state, indexName, segmentIds),
|
|
19137
|
-
searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork),
|
|
19443
|
+
searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork, bm25) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork, bm25),
|
|
19138
19444
|
hasWorkerPool: () => state.workerPool !== null,
|
|
19139
19445
|
mainCopyQueries: () => state.mainCopyQueries ?? "lone",
|
|
19140
19446
|
shareMainThread: () => {
|
|
@@ -19146,10 +19452,10 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
19146
19452
|
stopRequestThreads: () => stopRequestThreads(state),
|
|
19147
19453
|
desyncIndex: (indexName) => {
|
|
19148
19454
|
state.desyncedIndexes.add(indexName);
|
|
19149
|
-
const
|
|
19455
|
+
const loading3 = state.copyTransitions.get(indexName);
|
|
19150
19456
|
const wasScaledOut = state.scaledOutIndexes.delete(indexName);
|
|
19151
19457
|
if (wasScaledOut) withdrawCopies(indexName);
|
|
19152
|
-
return wasScaledOut ||
|
|
19458
|
+
return wasScaledOut || loading3 !== void 0 && loading3.kind !== "drop";
|
|
19153
19459
|
},
|
|
19154
19460
|
async releaseIndexCopies(indexName) {
|
|
19155
19461
|
const hold = state.copyReleaseHolds.get(indexName);
|
|
@@ -19198,7 +19504,7 @@ async function emitWriteEventAfter(source, indexName, write) {
|
|
|
19198
19504
|
// src/engine/core.ts
|
|
19199
19505
|
async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
19200
19506
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
19201
|
-
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-
|
|
19507
|
+
const rerankSettings = config?.rerankTimeout === void 0 && config?.maxRerankWindow === void 0 ? { timeoutMs: DEFAULT_RERANK_TIMEOUT_MS, maxWindow: DEFAULT_MAX_RERANK_WINDOW } : (await import('./runtime-3X7ZJ5T3.mjs')).resolveRerankSettings(config);
|
|
19202
19508
|
const eventHandlers = eventHandlersOf(config?.events);
|
|
19203
19509
|
const vectorWorkerCount = splitWorkerBudget(resolveWorkerCount(config?.workers?.count)).vector;
|
|
19204
19510
|
const vectorCopyPolicy = {
|
|
@@ -19209,7 +19515,7 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19209
19515
|
if (config?.lifecycle !== void 0 && durabilityTier === null) {
|
|
19210
19516
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Index lifecycle settings require durability");
|
|
19211
19517
|
}
|
|
19212
|
-
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-
|
|
19518
|
+
const durabilityWiring = durabilityTier === null ? null : await import('./durability-wiring-FNZSRH4N.mjs');
|
|
19213
19519
|
const filesystemDurability = durabilityTier?.kind === "wal";
|
|
19214
19520
|
const executor = createDirectExecutor({
|
|
19215
19521
|
vectorWorkerCopies: vectorCopyPolicy,
|
|
@@ -19375,7 +19681,9 @@ async function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
19375
19681
|
resyncIndex: (indexName, wasPromoted) => orchestrator.resyncIndex(indexName, wasPromoted),
|
|
19376
19682
|
recordCurrentAnalysis: (indexName) => {
|
|
19377
19683
|
const entry = indexRegistry.get(indexName);
|
|
19378
|
-
if (entry
|
|
19684
|
+
if (entry === void 0 || analysisRebuild.isStale(indexName)) return;
|
|
19685
|
+
entry.analysisRevision = entry.language.revision;
|
|
19686
|
+
entry.arrayPositionGap = ARRAY_POSITION_GAP;
|
|
19379
19687
|
},
|
|
19380
19688
|
durabilityManager: durability?.manager ?? null
|
|
19381
19689
|
});
|
|
@@ -19506,7 +19814,7 @@ async function shutdownEngine(core) {
|
|
|
19506
19814
|
}
|
|
19507
19815
|
}
|
|
19508
19816
|
if (core.rerankAdapters.size > 0) {
|
|
19509
|
-
await (await import('./runtime-
|
|
19817
|
+
await (await import('./runtime-3X7ZJ5T3.mjs')).shutdownRerankAdapters(core.rerankAdapters);
|
|
19510
19818
|
}
|
|
19511
19819
|
await orchestrator.shutdown();
|
|
19512
19820
|
await executor.shutdown();
|
|
@@ -19539,6 +19847,105 @@ async function resolveRunnableConfig(config) {
|
|
|
19539
19847
|
}
|
|
19540
19848
|
}
|
|
19541
19849
|
|
|
19850
|
+
// src/search/bm25-params.ts
|
|
19851
|
+
var BM25_KEYS = /* @__PURE__ */ new Set(["k1", "b", "fields"]);
|
|
19852
|
+
var BM25_FIELD_KEYS = /* @__PURE__ */ new Set(["k1", "b"]);
|
|
19853
|
+
function displayed(value) {
|
|
19854
|
+
if (typeof value === "string") return `"${value.slice(0, BM25_DISPLAYED_NAME_LENGTH)}"`;
|
|
19855
|
+
if (typeof value === "number" || typeof value === "boolean" || value === null) return String(value);
|
|
19856
|
+
return `a value of type ${typeof value}`;
|
|
19857
|
+
}
|
|
19858
|
+
function requireValue(label, value, maximum, ranges) {
|
|
19859
|
+
if (value === void 0) return;
|
|
19860
|
+
const usable = typeof value === "number" && Number.isFinite(value) && (ranges === "stored" || value >= 0 && value <= maximum);
|
|
19861
|
+
if (usable) return;
|
|
19862
|
+
throw new NarsilError(
|
|
19863
|
+
ErrorCodes.CONFIG_INVALID,
|
|
19864
|
+
`${label} must be a finite number from 0 to ${maximum}, and this request sets ${displayed(value)}`,
|
|
19865
|
+
{ parameter: label }
|
|
19866
|
+
);
|
|
19867
|
+
}
|
|
19868
|
+
function requireKnownKeys2(label, value, known) {
|
|
19869
|
+
for (const key of Object.keys(value)) {
|
|
19870
|
+
if (known.has(key)) continue;
|
|
19871
|
+
throw new NarsilError(
|
|
19872
|
+
ErrorCodes.CONFIG_INVALID,
|
|
19873
|
+
`${label} takes ${[...known].join(", ")}, and this request sets ${displayed(key)}`,
|
|
19874
|
+
{ parameter: label }
|
|
19875
|
+
);
|
|
19876
|
+
}
|
|
19877
|
+
}
|
|
19878
|
+
function requireFieldPair(name, pair, ranges) {
|
|
19879
|
+
const label = `bm25.fields.${name.slice(0, BM25_DISPLAYED_NAME_LENGTH)}`;
|
|
19880
|
+
if (!isPlainObject(pair)) {
|
|
19881
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${label} must be an object that sets k1 or b`, {
|
|
19882
|
+
parameter: label
|
|
19883
|
+
});
|
|
19884
|
+
}
|
|
19885
|
+
requireKnownKeys2(label, pair, BM25_FIELD_KEYS);
|
|
19886
|
+
requireValue(`${label}.k1`, pair.k1, MAX_BM25_K1, ranges);
|
|
19887
|
+
requireValue(`${label}.b`, pair.b, MAX_BM25_B, ranges);
|
|
19888
|
+
}
|
|
19889
|
+
function requireScoredField(name, flatSchema, unknownFieldCode) {
|
|
19890
|
+
const fieldType = Object.hasOwn(flatSchema, name) ? flatSchema[name] : void 0;
|
|
19891
|
+
if (typeof fieldType === "string" && isWordIndexedFieldType(fieldType)) return;
|
|
19892
|
+
throw new NarsilError(
|
|
19893
|
+
unknownFieldCode,
|
|
19894
|
+
fieldType === void 0 ? `BM25 parameters apply to a text field that the schema declares, and the schema declares no field ${displayed(name)}` : `BM25 parameters apply to a text field alone, and the schema declares field ${displayed(name)} as ${displayed(fieldType)}`,
|
|
19895
|
+
{ field: name.slice(0, BM25_DISPLAYED_NAME_LENGTH) }
|
|
19896
|
+
);
|
|
19897
|
+
}
|
|
19898
|
+
function fieldsOfCheckedSetting(raw, ranges) {
|
|
19899
|
+
if (!isPlainObject(raw)) {
|
|
19900
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "bm25 must be an object that sets k1, b, or fields");
|
|
19901
|
+
}
|
|
19902
|
+
requireKnownKeys2("bm25", raw, BM25_KEYS);
|
|
19903
|
+
requireValue("bm25.k1", raw.k1, MAX_BM25_K1, ranges);
|
|
19904
|
+
requireValue("bm25.b", raw.b, MAX_BM25_B, ranges);
|
|
19905
|
+
const fields = raw.fields;
|
|
19906
|
+
if (fields === void 0) return void 0;
|
|
19907
|
+
if (!isPlainObject(fields)) {
|
|
19908
|
+
throw new NarsilError(
|
|
19909
|
+
ErrorCodes.CONFIG_INVALID,
|
|
19910
|
+
"bm25.fields must be an object that maps a field to its parameters"
|
|
19911
|
+
);
|
|
19912
|
+
}
|
|
19913
|
+
return fields;
|
|
19914
|
+
}
|
|
19915
|
+
function validateBM25Params(raw, schema, unknownFieldCode, ranges = "settable") {
|
|
19916
|
+
const fields = fieldsOfCheckedSetting(raw, ranges);
|
|
19917
|
+
if (fields === void 0) return;
|
|
19918
|
+
const names = Object.keys(fields);
|
|
19919
|
+
if (names.length === 0) return;
|
|
19920
|
+
const flatSchema = flattenSchema(schema);
|
|
19921
|
+
for (const name of names) {
|
|
19922
|
+
requireScoredField(name, flatSchema, unknownFieldCode);
|
|
19923
|
+
requireFieldPair(name, fields[name], ranges);
|
|
19924
|
+
}
|
|
19925
|
+
}
|
|
19926
|
+
function presentPair(pair) {
|
|
19927
|
+
return {
|
|
19928
|
+
...pair.k1 !== void 0 ? { k1: pair.k1 } : {},
|
|
19929
|
+
...pair.b !== void 0 ? { b: pair.b } : {}
|
|
19930
|
+
};
|
|
19931
|
+
}
|
|
19932
|
+
function storedBM25Params(params) {
|
|
19933
|
+
const stored = presentPair(params);
|
|
19934
|
+
const fields = params.fields;
|
|
19935
|
+
if (fields !== void 0) {
|
|
19936
|
+
const kept = /* @__PURE__ */ Object.create(null);
|
|
19937
|
+
let holdsField = false;
|
|
19938
|
+
for (const name of Object.keys(fields)) {
|
|
19939
|
+
const pair = presentPair(fields[name]);
|
|
19940
|
+
if (pair.k1 === void 0 && pair.b === void 0) continue;
|
|
19941
|
+
kept[name] = pair;
|
|
19942
|
+
holdsField = true;
|
|
19943
|
+
}
|
|
19944
|
+
if (holdsField) stored.fields = kept;
|
|
19945
|
+
}
|
|
19946
|
+
return Object.keys(stored).length === 0 ? void 0 : stored;
|
|
19947
|
+
}
|
|
19948
|
+
|
|
19542
19949
|
// src/serialization/crc32.ts
|
|
19543
19950
|
var IEEE_POLYNOMIAL = 3988292384;
|
|
19544
19951
|
var ALL_BITS = 4294967295;
|
|
@@ -19836,6 +20243,7 @@ function indexConfigurationOf(entry) {
|
|
|
19836
20243
|
entry.language,
|
|
19837
20244
|
entry.embeddingAdapterName,
|
|
19838
20245
|
entry.analysisRevision,
|
|
20246
|
+
entry.arrayPositionGap,
|
|
19839
20247
|
synonyms
|
|
19840
20248
|
);
|
|
19841
20249
|
}
|
|
@@ -19845,7 +20253,8 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
19845
20253
|
config,
|
|
19846
20254
|
language,
|
|
19847
20255
|
typeof adapter === "string" ? adapter : null,
|
|
19848
|
-
language.revision
|
|
20256
|
+
language.revision,
|
|
20257
|
+
ARRAY_POSITION_GAP
|
|
19849
20258
|
);
|
|
19850
20259
|
if (messagePackUpperBoundBytes(payload) <= MAX_INDEX_CONFIGURATION_BYTES) return;
|
|
19851
20260
|
const bytes = encode(payload).byteLength;
|
|
@@ -19856,19 +20265,21 @@ function requireConfigurationWithinSizeLimit(config, language) {
|
|
|
19856
20265
|
{ bytes, maximum: MAX_INDEX_CONFIGURATION_BYTES }
|
|
19857
20266
|
);
|
|
19858
20267
|
}
|
|
19859
|
-
function configurationPayloadOf(config, language, adapterName, analysisRevision, synonyms) {
|
|
20268
|
+
function configurationPayloadOf(config, language, adapterName, analysisRevision, arrayPositionGap, synonyms) {
|
|
19860
20269
|
const bm25 = config.bm25;
|
|
19861
20270
|
return {
|
|
19862
20271
|
schema: config.schema,
|
|
19863
20272
|
language: language.name,
|
|
19864
20273
|
...analysisRevision !== null ? { analysisRevision } : {},
|
|
20274
|
+
...arrayPositionGap !== null ? { arrayPositionGap } : {},
|
|
19865
20275
|
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
19866
20276
|
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
19867
20277
|
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {},
|
|
19868
20278
|
...bm25 !== void 0 ? {
|
|
19869
20279
|
bm25: {
|
|
19870
20280
|
...bm25.k1 !== void 0 ? { k1: bm25.k1 } : {},
|
|
19871
|
-
...bm25.b !== void 0 ? { b: bm25.b } : {}
|
|
20281
|
+
...bm25.b !== void 0 ? { b: bm25.b } : {},
|
|
20282
|
+
...bm25.fields !== void 0 ? { fields: bm25.fields } : {}
|
|
19872
20283
|
}
|
|
19873
20284
|
} : {},
|
|
19874
20285
|
surfaceForms: config.surfaceForms !== false,
|
|
@@ -19956,14 +20367,21 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
19956
20367
|
const schema = validateSchema(requestedConfig.schema);
|
|
19957
20368
|
if (appliesCreationLimits) requireCreatableSchema(schema);
|
|
19958
20369
|
const configured = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
|
|
19959
|
-
const indexConfig = withoutSynonyms(configured);
|
|
20370
|
+
const indexConfig = { ...withoutSynonyms(configured) };
|
|
19960
20371
|
requirePatternSearchFor(schema);
|
|
19961
20372
|
requirePartialWordSearchFor(schema);
|
|
19962
20373
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
19963
20374
|
requireSurfaceFormsForPartialFields(schema, indexConfig.surfaceForms);
|
|
19964
20375
|
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
19965
20376
|
validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
|
|
19966
|
-
|
|
20377
|
+
if (indexConfig.bm25 !== void 0) {
|
|
20378
|
+
validateBM25Params(
|
|
20379
|
+
indexConfig.bm25,
|
|
20380
|
+
schema,
|
|
20381
|
+
ErrorCodes.CONFIG_INVALID,
|
|
20382
|
+
appliesCreationLimits ? "settable" : "stored"
|
|
20383
|
+
);
|
|
20384
|
+
}
|
|
19967
20385
|
if (indexConfig.partitions) {
|
|
19968
20386
|
validatePartitionConfig(indexConfig.partitions);
|
|
19969
20387
|
}
|
|
@@ -20030,6 +20448,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
20030
20448
|
documentCount: 0,
|
|
20031
20449
|
partitionCount: indexConfig.partitions?.maxPartitions ?? 1,
|
|
20032
20450
|
analysisRevision: language.revision,
|
|
20451
|
+
arrayPositionGap: ARRAY_POSITION_GAP,
|
|
20033
20452
|
synonyms: synonyms?.encoded ?? null
|
|
20034
20453
|
});
|
|
20035
20454
|
await core.indexState.registerOpen(name);
|
|
@@ -21985,6 +22404,75 @@ function readProcessMemory() {
|
|
|
21985
22404
|
}
|
|
21986
22405
|
}
|
|
21987
22406
|
|
|
22407
|
+
// src/engine/serial-index-update.ts
|
|
22408
|
+
var updatingByCore = /* @__PURE__ */ new WeakMap();
|
|
22409
|
+
async function withOpenIndex(core, indexName, action) {
|
|
22410
|
+
core.guardShutdown();
|
|
22411
|
+
core.requireIndex(indexName);
|
|
22412
|
+
const release = await core.indexState.acquire(indexName);
|
|
22413
|
+
try {
|
|
22414
|
+
return await action();
|
|
22415
|
+
} finally {
|
|
22416
|
+
release();
|
|
22417
|
+
}
|
|
22418
|
+
}
|
|
22419
|
+
async function oneAtATime(core, indexName, action) {
|
|
22420
|
+
let chains = updatingByCore.get(core);
|
|
22421
|
+
if (chains === void 0) {
|
|
22422
|
+
chains = /* @__PURE__ */ new Map();
|
|
22423
|
+
updatingByCore.set(core, chains);
|
|
22424
|
+
}
|
|
22425
|
+
const updating = chains;
|
|
22426
|
+
const earlier = updating.get(indexName) ?? Promise.resolve();
|
|
22427
|
+
const current = earlier.catch(() => void 0).then(action);
|
|
22428
|
+
updating.set(indexName, current);
|
|
22429
|
+
try {
|
|
22430
|
+
return await current;
|
|
22431
|
+
} finally {
|
|
22432
|
+
if (updating.get(indexName) === current) updating.delete(indexName);
|
|
22433
|
+
}
|
|
22434
|
+
}
|
|
22435
|
+
|
|
22436
|
+
// src/engine/bm25-update.ts
|
|
22437
|
+
async function replaceNow(core, indexName, bm25) {
|
|
22438
|
+
const { config } = core.requireIndex(indexName);
|
|
22439
|
+
const previous = config.bm25;
|
|
22440
|
+
config.bm25 = bm25;
|
|
22441
|
+
try {
|
|
22442
|
+
if (core.durability) await core.durability.manager.persistMetadata(indexName);
|
|
22443
|
+
} catch (err) {
|
|
22444
|
+
config.bm25 = previous;
|
|
22445
|
+
throw err;
|
|
22446
|
+
}
|
|
22447
|
+
await core.orchestrator.replicateToWorkers({
|
|
22448
|
+
type: "replaceBm25",
|
|
22449
|
+
indexName,
|
|
22450
|
+
requestId: `replace-bm25-${indexName}`,
|
|
22451
|
+
...bm25 !== void 0 ? { bm25 } : {}
|
|
22452
|
+
});
|
|
22453
|
+
await core.orchestrator.awaitWrites(indexName);
|
|
22454
|
+
}
|
|
22455
|
+
function updateIndexBm25(core, indexName, value) {
|
|
22456
|
+
return withOpenIndex(
|
|
22457
|
+
core,
|
|
22458
|
+
indexName,
|
|
22459
|
+
() => oneAtATime(core, indexName, () => {
|
|
22460
|
+
const entry = core.requireIndex(indexName);
|
|
22461
|
+
validateBM25Params(value, entry.config.schema, ErrorCodes.CONFIG_INVALID);
|
|
22462
|
+
return replaceNow(core, indexName, storedBM25Params(value));
|
|
22463
|
+
})
|
|
22464
|
+
);
|
|
22465
|
+
}
|
|
22466
|
+
|
|
22467
|
+
// src/narsil/bm25.ts
|
|
22468
|
+
function bm25Operations(core) {
|
|
22469
|
+
return {
|
|
22470
|
+
async updateBM25(indexName, bm25) {
|
|
22471
|
+
await updateIndexBm25(core, indexName, bm25);
|
|
22472
|
+
}
|
|
22473
|
+
};
|
|
22474
|
+
}
|
|
22475
|
+
|
|
21988
22476
|
// src/narsil/internals.ts
|
|
21989
22477
|
var engineCores = /* @__PURE__ */ new WeakMap();
|
|
21990
22478
|
function bindEngineCore(engine, core) {
|
|
@@ -22005,18 +22493,37 @@ function restoredBm25(raw, reject2) {
|
|
|
22005
22493
|
if (!isPlainObject3(raw)) {
|
|
22006
22494
|
reject2("bm25 must be an object");
|
|
22007
22495
|
}
|
|
22496
|
+
const fields = restoredBm25Fields(raw.fields, reject2);
|
|
22497
|
+
return {
|
|
22498
|
+
...restoredBm25Pair(raw, "bm25", reject2),
|
|
22499
|
+
...fields !== void 0 ? { fields } : {}
|
|
22500
|
+
};
|
|
22501
|
+
}
|
|
22502
|
+
function restoredBm25Pair(raw, label, reject2) {
|
|
22008
22503
|
const { k1, b } = raw;
|
|
22009
22504
|
if (k1 !== void 0 && (typeof k1 !== "number" || !Number.isFinite(k1))) {
|
|
22010
|
-
reject2(
|
|
22505
|
+
reject2(`${label} k1 must be a finite number`);
|
|
22011
22506
|
}
|
|
22012
22507
|
if (b !== void 0 && (typeof b !== "number" || !Number.isFinite(b))) {
|
|
22013
|
-
reject2(
|
|
22508
|
+
reject2(`${label} b must be a finite number`);
|
|
22014
22509
|
}
|
|
22015
22510
|
return {
|
|
22016
22511
|
...k1 !== void 0 ? { k1 } : {},
|
|
22017
22512
|
...b !== void 0 ? { b } : {}
|
|
22018
22513
|
};
|
|
22019
22514
|
}
|
|
22515
|
+
function restoredBm25Fields(raw, reject2) {
|
|
22516
|
+
if (raw === void 0) return void 0;
|
|
22517
|
+
if (!isPlainObject3(raw)) reject2("bm25 fields must be an object");
|
|
22518
|
+
const source = raw;
|
|
22519
|
+
const fields = /* @__PURE__ */ Object.create(null);
|
|
22520
|
+
for (const name of Object.keys(source)) {
|
|
22521
|
+
const pair = source[name];
|
|
22522
|
+
if (!isPlainObject3(pair)) reject2("bm25 fields must map each field to an object");
|
|
22523
|
+
fields[name] = restoredBm25Pair(pair, "bm25 field", reject2);
|
|
22524
|
+
}
|
|
22525
|
+
return fields;
|
|
22526
|
+
}
|
|
22020
22527
|
function restoredStopWords(name, list, reject2) {
|
|
22021
22528
|
if (name !== void 0 && typeof name !== "string") {
|
|
22022
22529
|
reject2("stopWords must be a name");
|
|
@@ -22614,7 +23121,7 @@ async function readDocumentExport(engine, indexName, source) {
|
|
|
22614
23121
|
|
|
22615
23122
|
// src/narsil/portability.ts
|
|
22616
23123
|
function portabilityOperations(core, config, engine) {
|
|
22617
|
-
async function
|
|
23124
|
+
async function withOpenIndex2(indexName, action) {
|
|
22618
23125
|
core.guardShutdown();
|
|
22619
23126
|
core.requireIndex(indexName);
|
|
22620
23127
|
const release = await core.indexState.acquire(indexName);
|
|
@@ -22642,25 +23149,25 @@ function portabilityOperations(core, config, engine) {
|
|
|
22642
23149
|
}
|
|
22643
23150
|
return {
|
|
22644
23151
|
snapshot(indexName) {
|
|
22645
|
-
return
|
|
22646
|
-
const { createSnapshot } = await import('./snapshot-
|
|
23152
|
+
return withOpenIndex2(indexName, async () => {
|
|
23153
|
+
const { createSnapshot } = await import('./snapshot-V7VF6BN6.mjs');
|
|
22647
23154
|
return createSnapshot(snapshotDepsOf(indexName));
|
|
22648
23155
|
});
|
|
22649
23156
|
},
|
|
22650
23157
|
async restore(indexName, data) {
|
|
22651
23158
|
core.guardShutdown();
|
|
22652
23159
|
validateIndexName(indexName);
|
|
22653
|
-
const restored = import('./restore-queue-
|
|
23160
|
+
const restored = import('./restore-queue-GEZ54CA5.mjs').then(
|
|
22654
23161
|
({ queueRestore }) => queueRestore(core, restoreHost, restoresInFlight, indexName, data)
|
|
22655
23162
|
);
|
|
22656
23163
|
await emitWriteEventAfter(core, indexName, restored);
|
|
22657
23164
|
core.heapPressureNotifier.check(indexName);
|
|
22658
23165
|
},
|
|
22659
23166
|
exportDocuments(indexName) {
|
|
22660
|
-
return
|
|
23167
|
+
return withOpenIndex2(indexName, async () => {
|
|
22661
23168
|
const [{ checkpointForArchive }, { documentExportStream }] = await Promise.all([
|
|
22662
|
-
import('./snapshot-
|
|
22663
|
-
import('./write-
|
|
23169
|
+
import('./snapshot-V7VF6BN6.mjs'),
|
|
23170
|
+
import('./write-YIC2MKRY.mjs')
|
|
22664
23171
|
]);
|
|
22665
23172
|
const header = indexConfigurationOf(core.requireIndex(indexName));
|
|
22666
23173
|
return documentExportStream(header, await checkpointForArchive(snapshotDepsOf(indexName)));
|
|
@@ -22816,6 +23323,27 @@ function boundRerankOf(rerank) {
|
|
|
22816
23323
|
}
|
|
22817
23324
|
return present;
|
|
22818
23325
|
}
|
|
23326
|
+
function presentMembersOf(record) {
|
|
23327
|
+
const present = {};
|
|
23328
|
+
for (const [key, value] of Object.entries(record)) {
|
|
23329
|
+
if (value !== void 0) present[key] = value;
|
|
23330
|
+
}
|
|
23331
|
+
return present;
|
|
23332
|
+
}
|
|
23333
|
+
function boundBm25Of(bm25) {
|
|
23334
|
+
const bound = presentMembersOf(bm25);
|
|
23335
|
+
const fields = bm25.fields;
|
|
23336
|
+
if (typeof fields !== "object" || fields === null) return bound;
|
|
23337
|
+
const boundFields = {};
|
|
23338
|
+
for (const [name, pair] of Object.entries(fields)) {
|
|
23339
|
+
const members = typeof pair === "object" && pair !== null ? presentMembersOf(pair) : pair;
|
|
23340
|
+
if (typeof members === "object" && members !== null && Object.keys(members).length === 0) continue;
|
|
23341
|
+
boundFields[name] = members;
|
|
23342
|
+
}
|
|
23343
|
+
if (Object.keys(boundFields).length === 0) delete bound.fields;
|
|
23344
|
+
else bound.fields = boundFields;
|
|
23345
|
+
return bound;
|
|
23346
|
+
}
|
|
22819
23347
|
var FILTERS_SLOT = 2;
|
|
22820
23348
|
var BOUND_SLOT_COUNT = 16;
|
|
22821
23349
|
function bindingOf(values, clock) {
|
|
@@ -22846,6 +23374,7 @@ function queryBindingOf(params, clock) {
|
|
|
22846
23374
|
params.scoring ?? "local"
|
|
22847
23375
|
];
|
|
22848
23376
|
if (params.rerank !== void 0) values.push(boundRerankOf(params.rerank));
|
|
23377
|
+
if (params.bm25 !== void 0 && bm25SetsValue(params.bm25)) values.push(boundBm25Of(params.bm25));
|
|
22849
23378
|
return bindingOf(values, clock);
|
|
22850
23379
|
}
|
|
22851
23380
|
function lazyQueryBinding(params, clock) {
|
|
@@ -22908,6 +23437,7 @@ function requireGroupableFields(fields, flatSchema, strict) {
|
|
|
22908
23437
|
}
|
|
22909
23438
|
}
|
|
22910
23439
|
function requireUsableQueryFields(params, schema, strict = false) {
|
|
23440
|
+
if (params.bm25 !== void 0) validateBM25Params(params.bm25, schema, ErrorCodes.SEARCH_INVALID_FIELD);
|
|
22911
23441
|
const groupFields = params.group?.fields;
|
|
22912
23442
|
const checksGroup = Array.isArray(groupFields) && groupFields.length > 0;
|
|
22913
23443
|
if (params.boost === void 0 && params.facets === void 0 && !checksGroup) return;
|
|
@@ -22986,9 +23516,13 @@ function broadcastStatsForWorker(params, context, scoring) {
|
|
|
22986
23516
|
expansion: { options: params, schema: context.config.schema }
|
|
22987
23517
|
});
|
|
22988
23518
|
}
|
|
22989
|
-
function
|
|
23519
|
+
function bm25SettingFor(params, context) {
|
|
23520
|
+
return mergeBM25Levels(params.bm25, context.indexBm25 ?? context.config.bm25);
|
|
23521
|
+
}
|
|
23522
|
+
function searchOptionsFor(params, context) {
|
|
23523
|
+
const { manager, patternWork } = context;
|
|
22990
23524
|
return {
|
|
22991
|
-
bm25Params:
|
|
23525
|
+
bm25Params: bm25SettingFor(params, context),
|
|
22992
23526
|
stopWords: manager.analysis.stopWords,
|
|
22993
23527
|
customTokenizer: manager.analysis.customTokenizer,
|
|
22994
23528
|
synonyms: manager.synonyms,
|
|
@@ -23040,6 +23574,13 @@ function validQueryFilter(filters, schema) {
|
|
|
23040
23574
|
if (filters === void 0 || filters === null) return filters;
|
|
23041
23575
|
return validatedFilterOf(filters, flattenSchema(schema));
|
|
23042
23576
|
}
|
|
23577
|
+
function requireUsablePhrases(params, context) {
|
|
23578
|
+
if (params.term === void 0) return;
|
|
23579
|
+
const quoted = analyseQueryPhrases(params.term, context.language, context.manager.analysis);
|
|
23580
|
+
if (quoted === null || quoted.keptPhraseCount === 0) return;
|
|
23581
|
+
requireLoadedPhraseCode().requirePhrasesWithinLimits(quoted, context.config.trackPositions !== false);
|
|
23582
|
+
context.requirePhraseReady?.();
|
|
23583
|
+
}
|
|
23043
23584
|
function requireVectorSearchable(params, context, needsVector) {
|
|
23044
23585
|
if (!needsVector) return;
|
|
23045
23586
|
const field = params.vector?.field;
|
|
@@ -23445,8 +23986,8 @@ function highlightField(text, queryTokens, language, options) {
|
|
|
23445
23986
|
}
|
|
23446
23987
|
|
|
23447
23988
|
// src/engine/query/highlight.ts
|
|
23448
|
-
function writtenWordsOf(
|
|
23449
|
-
return tokenize(
|
|
23989
|
+
function writtenWordsOf(term, language, analysis) {
|
|
23990
|
+
return tokenize(term, language, {
|
|
23450
23991
|
stem: false,
|
|
23451
23992
|
removeStopWords: true,
|
|
23452
23993
|
customTokenizer: analysis.customTokenizer,
|
|
@@ -23464,18 +24005,21 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
|
|
|
23464
24005
|
highlight: { words, customTokenizer: analysis.customTokenizer, stopWordOverride: analysis.stopWords }
|
|
23465
24006
|
};
|
|
23466
24007
|
}
|
|
23467
|
-
function applyHighlights(hits,
|
|
23468
|
-
if (!
|
|
23469
|
-
const
|
|
24008
|
+
function applyHighlights(hits, request, language, analysis, schema, readStoredDocument, synonymLookup) {
|
|
24009
|
+
if (!request.highlight) return;
|
|
24010
|
+
const quoted = analyseQueryPhrases(request.term ?? "", language, analysis);
|
|
24011
|
+
const params = quoted === null ? request : requireLoadedPhraseCode().withWordsOnly(request, quoted);
|
|
24012
|
+
const words = params.term ?? "";
|
|
24013
|
+
const highlightConfig = request.highlight;
|
|
24014
|
+
const queryTokens = quoted !== null && words.trim().length === 0 ? [] : tokenize(words, language, {
|
|
23470
24015
|
stem: true,
|
|
23471
24016
|
removeStopWords: true,
|
|
23472
24017
|
customTokenizer: analysis.customTokenizer,
|
|
23473
24018
|
stopWordOverride: analysis.stopWords
|
|
23474
|
-
});
|
|
23475
|
-
const queryTokens = queryTokenResult.tokens;
|
|
24019
|
+
}).tokens;
|
|
23476
24020
|
const usesPrefix = params.prefix === true && params.exact !== true && queryTokens.length > 0;
|
|
23477
24021
|
const usesPartialWords = params.partialWords === true && params.exact !== true && queryTokens.length > 0;
|
|
23478
|
-
const written = usesPrefix || usesPartialWords ? writtenWordsOf(
|
|
24022
|
+
const written = usesPrefix || usesPartialWords ? writtenWordsOf(words, language, analysis) : [];
|
|
23479
24023
|
let prefixToken;
|
|
23480
24024
|
let completedToken;
|
|
23481
24025
|
if (usesPrefix) {
|
|
@@ -23489,12 +24033,14 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
23489
24033
|
}
|
|
23490
24034
|
}
|
|
23491
24035
|
const partialWords = usesPartialWords ? partialWordTagging(params, schema, queryTokens, written, analysis) : null;
|
|
23492
|
-
const
|
|
24036
|
+
const wordSynonyms = params.exact === true || queryTokens.length === 0 ? void 0 : synonymLookup?.expand(queryTokens.map((qt) => qt.token))?.tokens;
|
|
24037
|
+
const phraseTokens = quoted?.phrases.flatMap((phrase) => phrase.tokens) ?? [];
|
|
24038
|
+
const synonyms = phraseTokens.length === 0 ? wordSynonyms : [...wordSynonyms ?? [], ...phraseTokens];
|
|
23493
24039
|
for (const hit of hits) {
|
|
23494
24040
|
const highlights = {};
|
|
23495
24041
|
const projected = hit.document;
|
|
23496
24042
|
let stored;
|
|
23497
|
-
for (const field of
|
|
24043
|
+
for (const field of highlightConfig.fields) {
|
|
23498
24044
|
let fieldValue = projected[field];
|
|
23499
24045
|
if (typeof fieldValue !== "string" && readStoredDocument !== void 0) {
|
|
23500
24046
|
stored ??= readStoredDocument(hit.id);
|
|
@@ -23502,9 +24048,9 @@ function applyHighlights(hits, params, language, analysis, schema, readStoredDoc
|
|
|
23502
24048
|
}
|
|
23503
24049
|
if (typeof fieldValue === "string") {
|
|
23504
24050
|
highlights[field] = highlightField(fieldValue, queryTokens, language, {
|
|
23505
|
-
preTag:
|
|
23506
|
-
postTag:
|
|
23507
|
-
maxSnippetLength:
|
|
24051
|
+
preTag: highlightConfig.preTag,
|
|
24052
|
+
postTag: highlightConfig.postTag,
|
|
24053
|
+
maxSnippetLength: highlightConfig.maxSnippetLength,
|
|
23508
24054
|
prefixToken,
|
|
23509
24055
|
completedToken,
|
|
23510
24056
|
tolerance: params.exact === true ? 0 : params.tolerance ?? 0,
|
|
@@ -23749,7 +24295,8 @@ async function textLeg(textOnlyParams, context) {
|
|
|
23749
24295
|
textOnlyParams,
|
|
23750
24296
|
broadcastStatsForWorker(textOnlyParams, context, scoring),
|
|
23751
24297
|
context.partitionIds,
|
|
23752
|
-
context.patternWork
|
|
24298
|
+
context.patternWork,
|
|
24299
|
+
bm25SettingFor(textOnlyParams, context)
|
|
23753
24300
|
);
|
|
23754
24301
|
if (viaWorker) return viaWorker;
|
|
23755
24302
|
}
|
|
@@ -23759,7 +24306,7 @@ async function textLeg(textOnlyParams, context) {
|
|
|
23759
24306
|
language,
|
|
23760
24307
|
config.schema,
|
|
23761
24308
|
scoring,
|
|
23762
|
-
searchOptionsFor(
|
|
24309
|
+
searchOptionsFor(textOnlyParams, context)
|
|
23763
24310
|
);
|
|
23764
24311
|
}
|
|
23765
24312
|
async function vectorLeg(vectorConfig, queryVector, context, filtered, k, rescoreWindow) {
|
|
@@ -23857,7 +24404,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
23857
24404
|
const flatSchema = flattenSchema(schema);
|
|
23858
24405
|
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
23859
24406
|
const modes = sortModesOf(normalized);
|
|
23860
|
-
const options = searchOptionsFor(
|
|
24407
|
+
const options = searchOptionsFor(params, context);
|
|
23861
24408
|
const partitions = partitionsFor(manager, context.partitionIds);
|
|
23862
24409
|
const partitionLimit = offset + limit + 1;
|
|
23863
24410
|
const merged = [];
|
|
@@ -23925,13 +24472,17 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
23925
24472
|
}
|
|
23926
24473
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
23927
24474
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
23928
|
-
return import('./runtime-
|
|
24475
|
+
return import('./runtime-3X7ZJ5T3.mjs').then(
|
|
23929
24476
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
23930
24477
|
);
|
|
23931
24478
|
}
|
|
23932
24479
|
function executeQuery(request, callerContext) {
|
|
23933
|
-
if (request.rerank === void 0)
|
|
23934
|
-
|
|
24480
|
+
if (request.rerank === void 0) {
|
|
24481
|
+
const loadingPhraseCode = loadPhraseCodeFor(request.term);
|
|
24482
|
+
if (loadingPhraseCode === null) return runQuery(request, callerContext);
|
|
24483
|
+
return loadingPhraseCode.then(() => runQuery(request, callerContext));
|
|
24484
|
+
}
|
|
24485
|
+
return import('./runtime-3X7ZJ5T3.mjs').then(
|
|
23935
24486
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
23936
24487
|
);
|
|
23937
24488
|
}
|
|
@@ -23958,6 +24509,7 @@ async function runQuery(request, callerContext) {
|
|
|
23958
24509
|
}
|
|
23959
24510
|
const params = validQueryParams(request, config);
|
|
23960
24511
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
24512
|
+
if (!isVectorOnly) requireUsablePhrases(params, context);
|
|
23961
24513
|
const projection = resolveProjection(params.document);
|
|
23962
24514
|
let paginated;
|
|
23963
24515
|
let nextCursor;
|
|
@@ -23988,7 +24540,8 @@ async function runQuery(request, callerContext) {
|
|
|
23988
24540
|
params,
|
|
23989
24541
|
broadcastStatsForWorker(params, context, scoring),
|
|
23990
24542
|
context.partitionIds,
|
|
23991
|
-
context.patternWork
|
|
24543
|
+
context.patternWork,
|
|
24544
|
+
bm25SettingFor(params, context)
|
|
23992
24545
|
) : null;
|
|
23993
24546
|
if (workerResult) {
|
|
23994
24547
|
fanOutResult = workerResult;
|
|
@@ -23999,7 +24552,7 @@ async function runQuery(request, callerContext) {
|
|
|
23999
24552
|
language,
|
|
24000
24553
|
config.schema,
|
|
24001
24554
|
scoring,
|
|
24002
|
-
searchOptionsFor(
|
|
24555
|
+
searchOptionsFor(params, context)
|
|
24003
24556
|
);
|
|
24004
24557
|
}
|
|
24005
24558
|
}
|
|
@@ -24133,6 +24686,9 @@ async function executePreflight(request, callerContext) {
|
|
|
24133
24686
|
const isHybridMode = params.mode === "hybrid" || hasTerm && hasVector;
|
|
24134
24687
|
const isVectorOnly = (params.mode === "vector" || hasVector && !hasTerm) && !isHybridMode;
|
|
24135
24688
|
requireVectorSearchable(params, context, isVectorOnly || isHybridMode);
|
|
24689
|
+
const loadingPhraseCode = loadPhraseCodeFor(params.term);
|
|
24690
|
+
if (loadingPhraseCode !== null) await loadingPhraseCode;
|
|
24691
|
+
if (!isVectorOnly) requireUsablePhrases(params, context);
|
|
24136
24692
|
const requestedVectorField = params.vector?.field;
|
|
24137
24693
|
const hasGlobalVectorIndex = requestedVectorField !== void 0 && vectorSearchersOf(context).has(requestedVectorField);
|
|
24138
24694
|
let totalMatched;
|
|
@@ -24149,7 +24705,7 @@ async function executePreflight(request, callerContext) {
|
|
|
24149
24705
|
countExact = result.matchedExact !== false;
|
|
24150
24706
|
} else if (countsWithoutScores(params)) {
|
|
24151
24707
|
totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
|
|
24152
|
-
searchOptions: searchOptionsFor(
|
|
24708
|
+
searchOptions: searchOptionsFor(params, context),
|
|
24153
24709
|
partitionIds: context.partitionIds
|
|
24154
24710
|
});
|
|
24155
24711
|
} else {
|
|
@@ -24159,7 +24715,8 @@ async function executePreflight(request, callerContext) {
|
|
|
24159
24715
|
params,
|
|
24160
24716
|
broadcastStatsForWorker(params, context, scoring),
|
|
24161
24717
|
context.partitionIds,
|
|
24162
|
-
context.patternWork
|
|
24718
|
+
context.patternWork,
|
|
24719
|
+
bm25SettingFor(params, context)
|
|
24163
24720
|
) : null;
|
|
24164
24721
|
if (workerResult) {
|
|
24165
24722
|
totalMatched = workerResult.totalMatched;
|
|
@@ -24170,7 +24727,7 @@ async function executePreflight(request, callerContext) {
|
|
|
24170
24727
|
language,
|
|
24171
24728
|
config.schema,
|
|
24172
24729
|
scoring,
|
|
24173
|
-
searchOptionsFor(
|
|
24730
|
+
searchOptionsFor(params, context)
|
|
24174
24731
|
);
|
|
24175
24732
|
totalMatched = fanOutResult.totalMatched;
|
|
24176
24733
|
}
|
|
@@ -24292,6 +24849,12 @@ async function readUnderOneAnalysis(core, indexName, read) {
|
|
|
24292
24849
|
}
|
|
24293
24850
|
return { result, analysisStale };
|
|
24294
24851
|
}
|
|
24852
|
+
function phraseReadinessOf(core, indexName, config) {
|
|
24853
|
+
return () => {
|
|
24854
|
+
if (!core.analysisRebuild.isStale(indexName) || !holdsStringArrayField(config.schema)) return;
|
|
24855
|
+
requireLoadedPhraseCode().refusePhraseUntilRebuilt(indexName, core.analysisRebuild.rebuildMode);
|
|
24856
|
+
};
|
|
24857
|
+
}
|
|
24295
24858
|
function scopedParams(params, options) {
|
|
24296
24859
|
{
|
|
24297
24860
|
return params;
|
|
@@ -24312,7 +24875,7 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24312
24875
|
const manager = core.requireManager(indexName);
|
|
24313
24876
|
const scoped = scopedParams(params, options);
|
|
24314
24877
|
await core.pluginRegistry.runHook("beforeSearch", { indexName, params: scoped });
|
|
24315
|
-
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-
|
|
24878
|
+
const rerank = scoped.rerank === void 0 ? void 0 : await import('./runtime-3X7ZJ5T3.mjs');
|
|
24316
24879
|
const resolvedParams = await resolveVectorText(
|
|
24317
24880
|
rerank === void 0 ? scoped : rerank.withRerankQueryText(scoped),
|
|
24318
24881
|
entry.embeddingAdapter,
|
|
@@ -24329,9 +24892,11 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
24329
24892
|
broadcastStats: broadcastStatsFor(core, options),
|
|
24330
24893
|
partitionIds: options?.partitionIds,
|
|
24331
24894
|
rescoreWindow: options?.rescoreWindow,
|
|
24895
|
+
indexBm25: options?.indexBm25,
|
|
24332
24896
|
cursorBinding: lazyQueryBinding(params),
|
|
24333
24897
|
patternWorkCap: core.patternWorkCap,
|
|
24334
|
-
rerank: rerank?.rerankRunnerFor(core)
|
|
24898
|
+
rerank: rerank?.rerankRunnerFor(core),
|
|
24899
|
+
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24335
24900
|
};
|
|
24336
24901
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
24337
24902
|
core,
|
|
@@ -24378,8 +24943,10 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
24378
24943
|
indexName,
|
|
24379
24944
|
broadcastStats: broadcastStatsFor(core, options),
|
|
24380
24945
|
partitionIds: options?.partitionIds,
|
|
24946
|
+
indexBm25: options?.indexBm25,
|
|
24381
24947
|
cursorBinding: lazyQueryBinding(params),
|
|
24382
|
-
patternWorkCap: core.patternWorkCap
|
|
24948
|
+
patternWorkCap: core.patternWorkCap,
|
|
24949
|
+
requirePhraseReady: phraseReadinessOf(core, indexName, entry.config)
|
|
24383
24950
|
};
|
|
24384
24951
|
const { result, analysisStale } = await readUnderOneAnalysis(
|
|
24385
24952
|
core,
|
|
@@ -24462,7 +25029,7 @@ function createNarsilFromCore(core, config) {
|
|
|
24462
25029
|
mutationCtx,
|
|
24463
25030
|
rebalanceCtx
|
|
24464
25031
|
} = core;
|
|
24465
|
-
async function
|
|
25032
|
+
async function withOpenIndex2(indexName, action) {
|
|
24466
25033
|
guardShutdown();
|
|
24467
25034
|
requireIndex(indexName);
|
|
24468
25035
|
const release = await core.indexState.acquire(indexName);
|
|
@@ -24473,7 +25040,7 @@ function createNarsilFromCore(core, config) {
|
|
|
24473
25040
|
}
|
|
24474
25041
|
}
|
|
24475
25042
|
function withOpenIndexWrite(indexName, action) {
|
|
24476
|
-
return emitWriteEventAfter(core, indexName,
|
|
25043
|
+
return emitWriteEventAfter(core, indexName, withOpenIndex2(indexName, action));
|
|
24477
25044
|
}
|
|
24478
25045
|
const narsil = {
|
|
24479
25046
|
createIndex(name, indexConfig) {
|
|
@@ -24559,12 +25126,12 @@ function createNarsilFromCore(core, config) {
|
|
|
24559
25126
|
return orchestrator.awaitWrites(indexName);
|
|
24560
25127
|
},
|
|
24561
25128
|
async get(indexName, docId) {
|
|
24562
|
-
return
|
|
25129
|
+
return withOpenIndex2(indexName, () => executor.execute({ type: "get", indexName, docId, requestId: docId }));
|
|
24563
25130
|
},
|
|
24564
25131
|
async getMultiple(indexName, docIds) {
|
|
24565
25132
|
guardShutdown();
|
|
24566
25133
|
requireIndex(indexName);
|
|
24567
|
-
return
|
|
25134
|
+
return withOpenIndex2(indexName, async () => {
|
|
24568
25135
|
const result = /* @__PURE__ */ new Map();
|
|
24569
25136
|
for (const docId of docIds) {
|
|
24570
25137
|
const doc = await executor.execute({
|
|
@@ -24579,10 +25146,10 @@ function createNarsilFromCore(core, config) {
|
|
|
24579
25146
|
});
|
|
24580
25147
|
},
|
|
24581
25148
|
async has(indexName, docId) {
|
|
24582
|
-
return
|
|
25149
|
+
return withOpenIndex2(indexName, () => executor.execute({ type: "has", indexName, docId, requestId: docId }));
|
|
24583
25150
|
},
|
|
24584
25151
|
async countDocuments(indexName) {
|
|
24585
|
-
return
|
|
25152
|
+
return withOpenIndex2(indexName, () => executor.execute({ type: "count", indexName, requestId: indexName }));
|
|
24586
25153
|
},
|
|
24587
25154
|
async listDocuments(indexName, params) {
|
|
24588
25155
|
return runEngineListDocuments(core, indexName, params ?? {});
|
|
@@ -24603,10 +25170,11 @@ function createNarsilFromCore(core, config) {
|
|
|
24603
25170
|
},
|
|
24604
25171
|
...portabilityOperations(core, config, () => narsil),
|
|
24605
25172
|
...synonymOperations(core),
|
|
25173
|
+
...bm25Operations(core),
|
|
24606
25174
|
async checkpoint(indexName) {
|
|
24607
25175
|
guardShutdown();
|
|
24608
25176
|
requireIndex(indexName);
|
|
24609
|
-
await
|
|
25177
|
+
await withOpenIndex2(indexName, async () => {
|
|
24610
25178
|
if (durability) await durability.manager.checkpoint(indexName);
|
|
24611
25179
|
});
|
|
24612
25180
|
},
|
|
@@ -24629,7 +25197,7 @@ function createNarsilFromCore(core, config) {
|
|
|
24629
25197
|
},
|
|
24630
25198
|
async updatePartitionConfig(indexName, partitionConfig) {
|
|
24631
25199
|
guardShutdown();
|
|
24632
|
-
return
|
|
25200
|
+
return withOpenIndex2(indexName, async () => {
|
|
24633
25201
|
const entry = requireIndex(indexName);
|
|
24634
25202
|
const manager = requireManager(indexName);
|
|
24635
25203
|
if (rebalancingIndexes.has(indexName)) {
|
|
@@ -24744,4 +25312,4 @@ async function createNarsil(config) {
|
|
|
24744
25312
|
// src/index.ts
|
|
24745
25313
|
var VERSION = "0.1.0";
|
|
24746
25314
|
|
|
24747
|
-
export { BYTES_TAG, DATE_TAG, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
|
|
25315
|
+
export { ARRAY_POSITION_GAP, BYTES_TAG, DATE_TAG, DEFAULT_BM25_B, DEFAULT_BM25_K1, DOCUMENT_EXPORT_FORMAT, DOCUMENT_EXPORT_VERSION, HEADER_SIZE, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, NUMBER_TAG, OBJECT_TAG, VECTOR_PATTERN, VERSION, arrayPositionsAreStale, buildSegmentPayload, clearNormalizationCache, configureNormalizationCache, crc32, createByteReader, createFrozenSegment, createNarsil, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, decodeEveryStoredDocument, decodeIndexConfiguration, decodeMapWithoutFields, decodeStoredDocument, encodeBase64, encodeDocument, encodeIndexConfiguration, engineVersionParts, extractVectorFieldsFromSchema, flattenSchema, fnv1a, freezeSegmentShared, getLanguage, getNestedValue, getNormalizationCacheSize, getStopWords, getTokenizer, hasNrslMagic, hasStopWords, hasTokenizer, insertDocumentVectors, isCompositePartition, isQuantizationMode, isStorageMode, loadVectorCodeFor, mergeSegmentContents, messagePackUpperBoundBytes, openChecksummedEnvelope, packChecksummedEnvelope, parseVectorDimension, partialFieldsOf, partitionStateOf, prepareDocumentVectors, prepareStoredSynonyms, raiseEarlierReleaseGraphSetting, readHeader, readHeapStatistics, readResidentMemory, reconstructSchemaFromMetadata, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, rejectSnapshot, removeDocumentVectors, requireSurfaceFormsForPartialFields, requireValidFieldTypes, resetNormalizationCache, resolveIndexAnalysis, resolvePartitionInsertOptions, restoredConfigFields, restoredEmbedding, rulesOfStoredSynonyms, segmentArraysOf, selectionKeeps, shareSegmentArrays, startsWithTagPrefix, storedArrayPositionGap, survivorDocumentsOf, validateEmbeddingConfig, validatePatternValueLimit, validateRequiredFieldsInSchema, validateSchema, validateVectorPromotion, validateVectorStorage, writeHeader };
|