@delali/narsil 0.3.1-next.128 → 0.3.1-next.130
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/chunk-26A5JFT2.mjs +89 -0
- package/dist/browser/{chunk-4Y2EGA3U.mjs → chunk-2Y3CO4EQ.mjs} +878 -310
- package/dist/browser/{chunk-2CBQ2PCC.mjs → chunk-AIIJDWMG.mjs} +1 -1
- package/dist/browser/{chunk-P7C2S5JX.mjs → chunk-D7WQF6DV.mjs} +2 -2
- package/dist/browser/{chunk-V5DCQIOT.mjs → chunk-M5UAAXGU.mjs} +2 -2
- package/dist/browser/{chunk-B65QVTST.mjs → chunk-NSCSR6MA.mjs} +24 -9
- package/dist/browser/{chunk-LJS6PREA.mjs → chunk-UDGB2MWT.mjs} +7 -28
- package/dist/browser/{chunk-6POI2KNI.mjs → chunk-VBBLTD7O.mjs} +4 -3
- package/dist/browser/chunk-VEGHFP2P.mjs +30 -0
- package/dist/browser/{durability-wiring-PUC4QSCR.mjs → durability-wiring-FNZSRH4N.mjs} +8 -6
- package/dist/browser/{module-ACWDC7YI.mjs → module-4HIMOQM5.mjs} +6 -5
- package/dist/browser/phrase-module-WESYVU7K.mjs +327 -0
- package/dist/browser/{restore-queue-MLUEUEAN.mjs → restore-queue-GEZ54CA5.mjs} +1 -1
- package/dist/browser/{runtime-BX6XRFYR.mjs → runtime-3X7ZJ5T3.mjs} +2 -1
- package/dist/browser/{snapshot-U5Y346XG.mjs → snapshot-V7VF6BN6.mjs} +25 -10
- package/dist/browser/{write-GZHR6AVG.mjs → write-YIC2MKRY.mjs} +7 -5
- package/dist/{capabilities-9YwwCcqP.d.ts → capabilities-Bl8G9taj.d.ts} +10 -3
- package/dist/{chunk-VK45J47R.mjs → chunk-2C3WXBQS.mjs} +1 -1
- package/dist/{chunk-QSCVJJW5.mjs → chunk-2JWNJSNE.mjs} +8 -8
- package/dist/{chunk-IBGMFR5Y.mjs → chunk-2MKX2SCO.mjs} +481 -121
- package/dist/{chunk-P34VZEXB.mjs → chunk-2SEL6SBS.mjs} +4 -95
- package/dist/{chunk-7ZZQNOIY.mjs → chunk-2WBY4Y4M.mjs} +1 -1
- package/dist/{chunk-HFJXEFFV.mjs → chunk-4GDLFDPZ.mjs} +3 -3
- package/dist/chunk-4UMZ7KQL.mjs +17 -0
- package/dist/{chunk-ZR63OGT2.mjs → chunk-5LX6MOIB.mjs} +2 -2
- package/dist/{chunk-UKXMSO5H.mjs → chunk-6M6SDH3S.mjs} +189 -45
- package/dist/{chunk-IR4KQ4WB.mjs → chunk-6SSVIW7Y.mjs} +49 -31
- package/dist/{chunk-4IHMMDRF.mjs → chunk-7OIQ2SIO.mjs} +4 -4
- package/dist/{chunk-6ODIY6ZB.mjs → chunk-7XY6VD7B.mjs} +1 -1
- package/dist/{chunk-OBWUWZRA.mjs → chunk-CZ7ISLXA.mjs} +3 -0
- package/dist/{chunk-OFYKBKKV.mjs → chunk-D5TFRB5G.mjs} +3 -15
- package/dist/{chunk-KCQG2PJT.mjs → chunk-EUCX5K74.mjs} +4 -4
- package/dist/{chunk-HWQDZB2U.mjs → chunk-F6AAXI7T.mjs} +3 -3
- package/dist/{chunk-E74YLAIG.mjs → chunk-FQBFZAWG.mjs} +4 -1
- package/dist/{chunk-U6NGTWI3.mjs → chunk-HCGJKDZV.mjs} +1 -1
- package/dist/{chunk-SJEIIR56.mjs → chunk-JJ6EUMXB.mjs} +2 -2
- package/dist/{chunk-N57XXGPG.mjs → chunk-JT3YZGCE.mjs} +23 -4
- package/dist/chunk-JUKYYXSP.mjs +93 -0
- package/dist/{chunk-JIXO442E.mjs → chunk-K77FJXKS.mjs} +26 -10
- package/dist/{chunk-4PQNDHSA.mjs → chunk-NMXRXFGZ.mjs} +2 -2
- package/dist/{chunk-6PN3U6XW.mjs → chunk-NNO7CO3A.mjs} +1 -1
- package/dist/{chunk-542TD4BL.mjs → chunk-PATMDIMN.mjs} +1 -1
- package/dist/{chunk-DG62H7OD.mjs → chunk-PU6SKH7O.mjs} +2 -2
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/{chunk-22CK2LQO.mjs → chunk-QQTXT65X.mjs} +6 -5
- package/dist/{chunk-LXWDQLTW.mjs → chunk-RNSC4NFI.mjs} +1 -1
- package/dist/{chunk-JEXJDCQH.mjs → chunk-RWESFSJI.mjs} +28 -13
- package/dist/{chunk-SEG4CL6W.mjs → chunk-SOHMMNCH.mjs} +4 -4
- package/dist/{chunk-OGFSUXYY.mjs → chunk-SZGRYEIC.mjs} +4 -4
- package/dist/{chunk-LOBYK2FG.mjs → chunk-T4OFG4IQ.mjs} +20 -9
- package/dist/{chunk-ENBN5VQ6.mjs → chunk-TGHNXN7W.mjs} +150 -52
- package/dist/{chunk-Z4DZ4KK2.mjs → chunk-UJWGQIXU.mjs} +1 -0
- package/dist/{chunk-ANFNYCQG.mjs → chunk-WVRDRLIC.mjs} +6 -6
- package/dist/{chunk-62IPM62F.mjs → chunk-X5CNUKD5.mjs} +2 -2
- package/dist/{chunk-IE43SRL3.mjs → chunk-XABMCBHA.mjs} +12 -6
- package/dist/{chunk-OCB7TMUQ.mjs → chunk-Y2U5VVWX.mjs} +1 -1
- package/dist/{chunk-MFPJVITN.mjs → chunk-Y4BVTQ53.mjs} +2 -2
- package/dist/{chunk-N6J6EJW3.mjs → chunk-YBWSV5DV.mjs} +2 -2
- package/dist/{chunk-FUUP45IS.mjs → chunk-ZR4VM4A6.mjs} +1 -1
- package/dist/{client-CsuhZ13c.d.ts → client-Cx-TiqPE.d.ts} +19 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +10 -2
- package/dist/{config-CtTfxbpV.d.ts → config-CzB9YvF_.d.ts} +2 -2
- package/dist/{config-DLVnlByZ.d.ts → config-hOWGBeC6.d.ts} +7 -5
- package/dist/direct-executor-HQ4QVDX7.mjs +28 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +18 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +33 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +71 -43
- package/dist/distribution.mjs +411 -127
- package/dist/{durability-wiring-UGGUNATA.mjs → durability-wiring-S7OX2MAI.mjs} +28 -26
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-Bq5wz6c1.d.ts → engine-B_W9sdZK.d.ts} +38 -8
- package/dist/{events-x2aVu0_o.d.ts → events-DErop1LM.d.ts} +23 -2
- package/dist/{index-Cl5qQauj.d.ts → index-CKI1fDEl.d.ts} +37 -3
- package/dist/index.browser.mjs +5 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +21 -19
- package/dist/{long-query-thread-5BIPZVS6.mjs → long-query-thread-WVSMIBYP.mjs} +17 -15
- package/dist/{module-7OHZ2OPI.mjs → module-2E4XYSGT.mjs} +11 -11
- package/dist/partial-words.mjs +7 -6
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +3 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +24 -22
- package/dist/phrase-module-JERF2IDG.mjs +329 -0
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +3 -0
- package/dist/{rerank-NYX5S7OG.mjs → rerank-DMMT4FZA.mjs} +6 -6
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-NVMWJMHI.mjs → restore-queue-JAMNRC6T.mjs} +1 -1
- package/dist/{runtime-XNYW6AWX.mjs → runtime-7MLQ7J64.mjs} +7 -7
- package/dist/{schema-4W_7tk1f.d.ts → schema-CS-clLun.d.ts} +54 -28
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +27 -15
- package/dist/{snapshot-P7IQGNNQ.mjs → snapshot-DOQLGGEX.mjs} +45 -30
- package/dist/synonyms.mjs +4 -3
- package/dist/{tasks-DLQIfXqy.d.ts → tasks-BfF-VRhm.d.ts} +4 -6
- package/dist/{thread-QPETQEPK.mjs → thread-DNK5XWXG.mjs} +16 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-WZVM7ZL6.mjs → write-ZKR7SNRT.mjs} +23 -21
- package/package.json +7 -7
- package/dist/direct-executor-6KCVVHYT.mjs +0 -26
|
@@ -1,15 +1,17 @@
|
|
|
1
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP,
|
|
1
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, TYPO_CLAIM_SLOTS_PER_WORD, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-2SEL6SBS.mjs';
|
|
2
2
|
import { GEO_UNIT_METRES, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { bitsetHas, createBitSet, bitsetSet, bitsetIsEmpty, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet } from './chunk-JUKYYXSP.mjs';
|
|
4
|
+
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-EUCX5K74.mjs';
|
|
5
|
+
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-PU6SKH7O.mjs';
|
|
6
|
+
import { MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PREFIX_MAX_EXPANSIONS } from './chunk-FQBFZAWG.mjs';
|
|
6
7
|
import { fuzzyPrefixOf, boundedLevenshtein, fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-
|
|
8
|
-
import { tokenize,
|
|
8
|
+
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-7XY6VD7B.mjs';
|
|
9
|
+
import { tokenize, visitTokens } from './chunk-D5TFRB5G.mjs';
|
|
10
|
+
import { SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-4UMZ7KQL.mjs';
|
|
9
11
|
import { singleFoldTable, appendFoldedCodePoint, sortModeOf, defaultSortMode, SORT_VALUE_MAX_CODE_POINTS, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, isSortMode, SORT_MODES, compareSortValues, compareComparableValues, truncateSortString } from './chunk-R6A773WK.mjs';
|
|
10
12
|
import { compareCodePoints, exceedsCodePoints, codePointCount, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
13
|
import { isGeopointOrVectorType, PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
14
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
13
15
|
|
|
14
16
|
// src/core/partition/partial-words/platform.ts
|
|
15
17
|
function platformPartialWordSearch() {
|
|
@@ -165,6 +167,17 @@ function requirePatternSearch(fieldPath) {
|
|
|
165
167
|
);
|
|
166
168
|
}
|
|
167
169
|
|
|
170
|
+
// src/core/constants.ts
|
|
171
|
+
var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
172
|
+
var MAX_TERM_FREQUENCY = 65535;
|
|
173
|
+
var ARRAY_POSITION_GAP = 100;
|
|
174
|
+
var DEFAULT_BM25_K1 = 1.2;
|
|
175
|
+
var DEFAULT_BM25_B = 0.75;
|
|
176
|
+
var MAX_BM25_K1 = 1e3;
|
|
177
|
+
var MAX_BM25_B = 1;
|
|
178
|
+
var PENDING_REMOVAL_MERGE_FLOOR = 64;
|
|
179
|
+
var PENDING_REMOVAL_MERGE_DIVISOR = 256;
|
|
180
|
+
|
|
168
181
|
// src/core/projection.ts
|
|
169
182
|
function requireFieldPathList(paths, option) {
|
|
170
183
|
if (paths === void 0) return;
|
|
@@ -594,11 +607,64 @@ function createDocumentStore() {
|
|
|
594
607
|
};
|
|
595
608
|
}
|
|
596
609
|
|
|
597
|
-
// src/
|
|
598
|
-
var
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
610
|
+
// src/search/quotes.ts
|
|
611
|
+
var QUOTE_CHARACTER = /["“”„]/;
|
|
612
|
+
function holdsQuoteCharacter(term) {
|
|
613
|
+
return QUOTE_CHARACTER.test(term);
|
|
614
|
+
}
|
|
615
|
+
function holdsQuotedText(term) {
|
|
616
|
+
if (!QUOTE_CHARACTER.test(term)) return false;
|
|
617
|
+
const parts = term.split(QUOTE_CHARACTER);
|
|
618
|
+
for (let index = 1; index < parts.length; index += 2) {
|
|
619
|
+
if (parts[index].trim().length > 0) return true;
|
|
620
|
+
}
|
|
621
|
+
return false;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
// src/search/phrase-code.ts
|
|
625
|
+
var loaded = null;
|
|
626
|
+
var loading = null;
|
|
627
|
+
function requireLoadedPhraseCode() {
|
|
628
|
+
if (loaded === null) {
|
|
629
|
+
throw new NarsilError(ErrorCodes.SEARCH_PHRASE_NOT_READY, "The phrase code has not loaded. Try the query again");
|
|
630
|
+
}
|
|
631
|
+
return loaded;
|
|
632
|
+
}
|
|
633
|
+
function loadPhraseCode() {
|
|
634
|
+
if (loading === null) {
|
|
635
|
+
loading = import('./phrase-module-JERF2IDG.mjs').then(
|
|
636
|
+
(code) => {
|
|
637
|
+
loaded = code;
|
|
638
|
+
return code;
|
|
639
|
+
},
|
|
640
|
+
(err) => {
|
|
641
|
+
loading = null;
|
|
642
|
+
throw new NarsilError(
|
|
643
|
+
ErrorCodes.SEARCH_PHRASE_NOT_READY,
|
|
644
|
+
"The phrase code failed to load. Remove the quotes, or try again once it can load",
|
|
645
|
+
{ cause: err instanceof Error ? err.message : String(err) }
|
|
646
|
+
);
|
|
647
|
+
}
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
return loading;
|
|
651
|
+
}
|
|
652
|
+
function loadPhraseCodeFor(term) {
|
|
653
|
+
if (loaded !== null || term === void 0 || !holdsQuoteCharacter(term)) return null;
|
|
654
|
+
return loadPhraseCode();
|
|
655
|
+
}
|
|
656
|
+
function analyseQueryPhrases(term, language, analysis) {
|
|
657
|
+
if (!holdsQuoteCharacter(term)) return null;
|
|
658
|
+
return requireLoadedPhraseCode().analyseQueryPhrases(
|
|
659
|
+
term.split(QUOTE_CHARACTER),
|
|
660
|
+
(text) => tokenize(text, language, {
|
|
661
|
+
stem: true,
|
|
662
|
+
removeStopWords: true,
|
|
663
|
+
customTokenizer: analysis.customTokenizer,
|
|
664
|
+
stopWordOverride: analysis.stopWords
|
|
665
|
+
}).tokens
|
|
666
|
+
);
|
|
667
|
+
}
|
|
602
668
|
|
|
603
669
|
// src/core/inverted-index/constants.ts
|
|
604
670
|
var TERM_TABLE_START_CAPACITY = 1 << 10;
|
|
@@ -1030,6 +1096,13 @@ var LivePostingView = class {
|
|
|
1030
1096
|
const starts = this.store.pool.positionStarts;
|
|
1031
1097
|
return starts === null ? 0 : this.store.positions.countAt(starts[this.syncedStart + row]);
|
|
1032
1098
|
}
|
|
1099
|
+
get positionStore() {
|
|
1100
|
+
return this.holdsPositions ? this.store.positions.values : null;
|
|
1101
|
+
}
|
|
1102
|
+
positionStartAt(row) {
|
|
1103
|
+
const starts = this.store.pool.positionStarts;
|
|
1104
|
+
return starts === null ? 0 : starts[this.syncedStart + row] + 1;
|
|
1105
|
+
}
|
|
1033
1106
|
copyPositionsAt(row, target, offset) {
|
|
1034
1107
|
if (!this.holdsPositions) return 0;
|
|
1035
1108
|
const starts = this.store.pool.positionStarts;
|
|
@@ -3022,7 +3095,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
3022
3095
|
if (state.trackPositions) {
|
|
3023
3096
|
const tokenFreqs = /* @__PURE__ */ new Map();
|
|
3024
3097
|
let positionOffset = 0;
|
|
3025
|
-
let
|
|
3098
|
+
let lastItemPosition = -1;
|
|
3026
3099
|
const addToken = (token, position, surface) => {
|
|
3027
3100
|
const existing = tokenFreqs.get(token);
|
|
3028
3101
|
if (existing) {
|
|
@@ -3033,12 +3106,12 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
3033
3106
|
}
|
|
3034
3107
|
fieldTokenList.push(token);
|
|
3035
3108
|
countOccurrence(surfaceCounts, spellings, surface, token);
|
|
3036
|
-
|
|
3109
|
+
lastItemPosition = position;
|
|
3037
3110
|
};
|
|
3038
3111
|
for (const item of arr) {
|
|
3039
|
-
|
|
3112
|
+
lastItemPosition = -1;
|
|
3040
3113
|
visitTokens(item, language, opts, addToken);
|
|
3041
|
-
positionOffset +=
|
|
3114
|
+
if (lastItemPosition >= 0) positionOffset += lastItemPosition + 1 + ARRAY_POSITION_GAP;
|
|
3042
3115
|
}
|
|
3043
3116
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
3044
3117
|
for (const [token, freq] of tokenFreqs) {
|
|
@@ -4732,19 +4805,63 @@ function partitionFilterMatches(state, filters, schema, meter) {
|
|
|
4732
4805
|
}
|
|
4733
4806
|
|
|
4734
4807
|
// src/core/scorer.ts
|
|
4735
|
-
var DEFAULT_K1 = 1.2;
|
|
4736
|
-
var DEFAULT_B = 0.75;
|
|
4737
4808
|
function computeIDF(docFrequency, totalDocs) {
|
|
4738
4809
|
if (totalDocs === 0) return 0;
|
|
4739
4810
|
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
4740
4811
|
}
|
|
4812
|
+
function pairKeepsBoundSound(k1, b) {
|
|
4813
|
+
return k1 >= 0 && k1 <= MAX_BM25_K1 && b >= 0 && b <= MAX_BM25_B;
|
|
4814
|
+
}
|
|
4741
4815
|
function bm25PruningSound(params) {
|
|
4742
|
-
const k1 = params?.k1 ??
|
|
4743
|
-
const b = params?.b ??
|
|
4744
|
-
|
|
4816
|
+
const k1 = params?.k1 ?? DEFAULT_BM25_K1;
|
|
4817
|
+
const b = params?.b ?? DEFAULT_BM25_B;
|
|
4818
|
+
if (!pairKeepsBoundSound(k1, b)) return false;
|
|
4819
|
+
const fields = params?.fields;
|
|
4820
|
+
if (fields === void 0) return true;
|
|
4821
|
+
for (const name of Object.keys(fields)) {
|
|
4822
|
+
if (!pairKeepsBoundSound(fields[name].k1 ?? k1, fields[name].b ?? b)) return false;
|
|
4823
|
+
}
|
|
4824
|
+
return true;
|
|
4825
|
+
}
|
|
4826
|
+
function bm25SetsValue(params) {
|
|
4827
|
+
if (typeof params !== "object" || params === null) return params !== void 0;
|
|
4828
|
+
if (params.k1 !== void 0 || params.b !== void 0) return true;
|
|
4829
|
+
const fields = params.fields;
|
|
4830
|
+
if (fields === void 0) return false;
|
|
4831
|
+
if (typeof fields !== "object" || fields === null) return true;
|
|
4832
|
+
for (const name of Object.keys(fields)) {
|
|
4833
|
+
const pair = fields[name];
|
|
4834
|
+
if (typeof pair !== "object" || pair === null) return true;
|
|
4835
|
+
if (pair.k1 !== void 0 || pair.b !== void 0) return true;
|
|
4836
|
+
}
|
|
4837
|
+
return false;
|
|
4745
4838
|
}
|
|
4746
|
-
function
|
|
4747
|
-
return
|
|
4839
|
+
function ownField(fields, name) {
|
|
4840
|
+
return fields !== void 0 && Object.hasOwn(fields, name) ? fields[name] : void 0;
|
|
4841
|
+
}
|
|
4842
|
+
function mergeBM25Levels(search, index) {
|
|
4843
|
+
if (search === void 0 || !bm25SetsValue(search)) return index;
|
|
4844
|
+
const k1 = search.k1 ?? index?.k1;
|
|
4845
|
+
const b = search.b ?? index?.b;
|
|
4846
|
+
const merged = {
|
|
4847
|
+
...k1 !== void 0 ? { k1 } : {},
|
|
4848
|
+
...b !== void 0 ? { b } : {}
|
|
4849
|
+
};
|
|
4850
|
+
const names = [...Object.keys(search.fields ?? {}), ...Object.keys(index?.fields ?? {})];
|
|
4851
|
+
if (names.length === 0) return merged;
|
|
4852
|
+
const fields = /* @__PURE__ */ Object.create(null);
|
|
4853
|
+
for (const name of names) {
|
|
4854
|
+
const fromSearch = ownField(search.fields, name);
|
|
4855
|
+
const fromIndex = ownField(index?.fields, name);
|
|
4856
|
+
const fieldK1 = fromSearch?.k1 ?? search.k1 ?? fromIndex?.k1;
|
|
4857
|
+
const fieldB = fromSearch?.b ?? search.b ?? fromIndex?.b;
|
|
4858
|
+
fields[name] = {
|
|
4859
|
+
...fieldK1 !== void 0 ? { k1: fieldK1 } : {},
|
|
4860
|
+
...fieldB !== void 0 ? { b: fieldB } : {}
|
|
4861
|
+
};
|
|
4862
|
+
}
|
|
4863
|
+
merged.fields = fields;
|
|
4864
|
+
return merged;
|
|
4748
4865
|
}
|
|
4749
4866
|
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
4750
4867
|
if (avgFieldLength === 0) return 0;
|
|
@@ -4753,29 +4870,49 @@ function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1,
|
|
|
4753
4870
|
if (denominator === 0) return 0;
|
|
4754
4871
|
return idf * (numerator / denominator);
|
|
4755
4872
|
}
|
|
4756
|
-
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
4757
|
-
if (totalDocs === 0) return 0;
|
|
4758
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
4759
|
-
const b = params?.b ?? DEFAULT_B;
|
|
4760
|
-
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
4761
|
-
}
|
|
4762
4873
|
|
|
4763
4874
|
// src/core/partition/field-scoring.ts
|
|
4764
|
-
|
|
4875
|
+
var NO_FIELD_VALUES = new Float64Array(0);
|
|
4876
|
+
function differentPairsOf(searchable, k1s, bs) {
|
|
4877
|
+
const pairs = [];
|
|
4878
|
+
for (let fieldIndex = 0; fieldIndex < searchable.length; fieldIndex++) {
|
|
4879
|
+
if (searchable[fieldIndex] !== 1) continue;
|
|
4880
|
+
let seen = false;
|
|
4881
|
+
for (let at = 0; at < pairs.length && !seen; at += 2) {
|
|
4882
|
+
seen = pairs[at] === k1s[fieldIndex] && pairs[at + 1] === bs[fieldIndex];
|
|
4883
|
+
}
|
|
4884
|
+
if (!seen) pairs.push(k1s[fieldIndex], bs[fieldIndex]);
|
|
4885
|
+
}
|
|
4886
|
+
return pairs;
|
|
4887
|
+
}
|
|
4888
|
+
function loadFieldScoring(state, fields, boost, averageFieldLengths, bm25) {
|
|
4765
4889
|
const fieldNames = state.fieldNameTable.names;
|
|
4890
|
+
const fieldCount = fieldNames.length;
|
|
4891
|
+
const k1 = bm25?.k1 ?? DEFAULT_BM25_K1;
|
|
4892
|
+
const b = bm25?.b ?? DEFAULT_BM25_B;
|
|
4893
|
+
const perField = bm25?.fields;
|
|
4766
4894
|
const scoring = {
|
|
4767
|
-
searchable: new Uint8Array(
|
|
4768
|
-
boosts: new Float64Array(
|
|
4769
|
-
averageLengths: new Float64Array(
|
|
4770
|
-
lengthColumns: new Array(
|
|
4895
|
+
searchable: new Uint8Array(fieldCount),
|
|
4896
|
+
boosts: new Float64Array(fieldCount),
|
|
4897
|
+
averageLengths: new Float64Array(fieldCount),
|
|
4898
|
+
lengthColumns: new Array(fieldCount).fill(null),
|
|
4899
|
+
k1s: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
4900
|
+
bs: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
4901
|
+
boundPairs: [k1, b]
|
|
4771
4902
|
};
|
|
4772
|
-
for (let fieldIndex = 0; fieldIndex <
|
|
4903
|
+
for (let fieldIndex = 0; fieldIndex < fieldCount; fieldIndex++) {
|
|
4773
4904
|
const fieldName = fieldNames[fieldIndex];
|
|
4774
4905
|
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
4775
4906
|
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
4776
4907
|
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
4777
4908
|
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
4909
|
+
if (perField !== void 0) {
|
|
4910
|
+
const fieldPair = Object.hasOwn(perField, fieldName) ? perField[fieldName] : void 0;
|
|
4911
|
+
scoring.k1s[fieldIndex] = fieldPair?.k1 ?? k1;
|
|
4912
|
+
scoring.bs[fieldIndex] = fieldPair?.b ?? b;
|
|
4913
|
+
}
|
|
4778
4914
|
}
|
|
4915
|
+
if (perField !== void 0) scoring.boundPairs = differentPairsOf(scoring.searchable, scoring.k1s, scoring.bs);
|
|
4779
4916
|
return scoring;
|
|
4780
4917
|
}
|
|
4781
4918
|
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
@@ -4791,6 +4928,15 @@ function bestSearchable(searchable, values) {
|
|
|
4791
4928
|
}
|
|
4792
4929
|
return best;
|
|
4793
4930
|
}
|
|
4931
|
+
function blockEntryBound(fields, maxTermFrequency, idf, minFieldLength, maxAverageLength) {
|
|
4932
|
+
const pairs = fields.boundPairs;
|
|
4933
|
+
let best = 0;
|
|
4934
|
+
for (let at = 0; at < pairs.length; at += 2) {
|
|
4935
|
+
const bound = computeBM25WithIDF(maxTermFrequency, idf, minFieldLength, maxAverageLength, pairs[at], pairs[at + 1]);
|
|
4936
|
+
if (bound > best) best = bound;
|
|
4937
|
+
}
|
|
4938
|
+
return best;
|
|
4939
|
+
}
|
|
4794
4940
|
|
|
4795
4941
|
// src/core/partition/posting-columns.ts
|
|
4796
4942
|
function postingColumns(list) {
|
|
@@ -4884,8 +5030,9 @@ function groupPostingCount(matches) {
|
|
|
4884
5030
|
return total;
|
|
4885
5031
|
}
|
|
4886
5032
|
function computeGroupContributions(context, matches, collect) {
|
|
4887
|
-
const { fields, fieldNames, filterBitset, partialMarks
|
|
5033
|
+
const { fields, fieldNames, filterBitset, partialMarks } = context;
|
|
4888
5034
|
const scoresAreZero = context.totalDocs === 0;
|
|
5035
|
+
const onePair = fields.boundPairs.length === 2;
|
|
4889
5036
|
const best = /* @__PURE__ */ new Map();
|
|
4890
5037
|
for (const match of matches) {
|
|
4891
5038
|
const perTerm = /* @__PURE__ */ new Map();
|
|
@@ -4905,7 +5052,14 @@ function computeGroupContributions(context, matches, collect) {
|
|
|
4905
5052
|
const fieldBoost = fields.boosts[fieldIndex];
|
|
4906
5053
|
const avgLen = fields.averageLengths[fieldIndex];
|
|
4907
5054
|
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
4908
|
-
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
5055
|
+
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
5056
|
+
termFrequency,
|
|
5057
|
+
match.idf,
|
|
5058
|
+
actualFieldLength,
|
|
5059
|
+
avgLen,
|
|
5060
|
+
onePair ? fields.boundPairs[0] : fields.k1s[fieldIndex],
|
|
5061
|
+
onePair ? fields.boundPairs[1] : fields.bs[fieldIndex]
|
|
5062
|
+
)) * fieldBoost * factor;
|
|
4909
5063
|
const existing = perTerm.get(internalId);
|
|
4910
5064
|
if (existing) {
|
|
4911
5065
|
existing.score += termScore;
|
|
@@ -4962,11 +5116,20 @@ function searchedFieldMarks(fieldNames, fields) {
|
|
|
4962
5116
|
function searchFulltextMatches(state, params) {
|
|
4963
5117
|
const { queryTokens, termGroups, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
|
|
4964
5118
|
const fieldNames = state.fieldNameTable.names;
|
|
4965
|
-
const matched = createBitSet(state.docStore.internalIdCapacity());
|
|
4966
5119
|
const searched = searchedFieldMarks(fieldNames, fields);
|
|
4967
5120
|
const partial = partialFieldMarks(fieldNames, params.partialFields);
|
|
4968
5121
|
const searchedPartial = searched.map((mark, fieldIndex) => mark & partial[fieldIndex]);
|
|
4969
|
-
|
|
5122
|
+
const capacity = state.docStore.internalIdCapacity();
|
|
5123
|
+
const matched = params.phrases === void 0 ? createBitSet(capacity) : requireLoadedPhraseCode().matchPhrases(
|
|
5124
|
+
state.invertedIdx,
|
|
5125
|
+
params.phrases,
|
|
5126
|
+
searched,
|
|
5127
|
+
filterBitset,
|
|
5128
|
+
capacity,
|
|
5129
|
+
params.phraseWork,
|
|
5130
|
+
false
|
|
5131
|
+
).matched;
|
|
5132
|
+
for (const qt of params.phrases === void 0 ? queryTokens : []) {
|
|
4970
5133
|
const group = termGroupOf(termGroups, qt.token);
|
|
4971
5134
|
for (const match of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
4972
5135
|
markPostingList(matched, match.postingList, searched, filterBitset);
|
|
@@ -5469,7 +5632,7 @@ function boostsKeepScoresNonNegative(fields) {
|
|
|
5469
5632
|
}
|
|
5470
5633
|
return true;
|
|
5471
5634
|
}
|
|
5472
|
-
function
|
|
5635
|
+
function samePairUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
5473
5636
|
if (scan2.scoresAreZero) return 0;
|
|
5474
5637
|
const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
|
|
5475
5638
|
let best = 0;
|
|
@@ -5487,7 +5650,24 @@ function termUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
|
5487
5650
|
}
|
|
5488
5651
|
return best;
|
|
5489
5652
|
}
|
|
5490
|
-
function
|
|
5653
|
+
function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
5654
|
+
if (scan2.scoresAreZero) return 0;
|
|
5655
|
+
const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
|
|
5656
|
+
let best = 0;
|
|
5657
|
+
for (let block = 0; block < bounds.blockCount; block++) {
|
|
5658
|
+
const bestEntry = blockEntryBound(
|
|
5659
|
+
scan2.fields,
|
|
5660
|
+
bounds.maxTermFrequency[block],
|
|
5661
|
+
scan2.idf,
|
|
5662
|
+
bounds.minFieldLength[block],
|
|
5663
|
+
maxAverageLength
|
|
5664
|
+
);
|
|
5665
|
+
const blockBound = bestEntry * maxBoost * bounds.maxEntriesPerDocument[block];
|
|
5666
|
+
if (blockBound > best) best = blockBound;
|
|
5667
|
+
}
|
|
5668
|
+
return best;
|
|
5669
|
+
}
|
|
5670
|
+
function samePairEntryScore(scan2, entry, internalId) {
|
|
5491
5671
|
const { fields, columns } = scan2;
|
|
5492
5672
|
const fieldIndex = columns.fieldNameIndices[entry];
|
|
5493
5673
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
@@ -5495,17 +5675,42 @@ function entryScore(scan2, entry, internalId) {
|
|
|
5495
5675
|
const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan2.idf, fieldLength, averageLength, scan2.k1, scan2.b);
|
|
5496
5676
|
return bm25 * fields.boosts[fieldIndex];
|
|
5497
5677
|
}
|
|
5498
|
-
function
|
|
5678
|
+
function fieldPairEntryScore(scan2, entry, internalId) {
|
|
5679
|
+
const { fields, columns } = scan2;
|
|
5680
|
+
const fieldIndex = columns.fieldNameIndices[entry];
|
|
5681
|
+
const averageLength = fields.averageLengths[fieldIndex];
|
|
5682
|
+
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
5683
|
+
const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(
|
|
5684
|
+
columns.termFrequencies[entry],
|
|
5685
|
+
scan2.idf,
|
|
5686
|
+
fieldLength,
|
|
5687
|
+
averageLength,
|
|
5688
|
+
fields.k1s[fieldIndex],
|
|
5689
|
+
fields.bs[fieldIndex]
|
|
5690
|
+
);
|
|
5691
|
+
return bm25 * fields.boosts[fieldIndex];
|
|
5692
|
+
}
|
|
5693
|
+
function samePairScoreEveryDocument(scan2, buffer) {
|
|
5694
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5695
|
+
const searchable = scan2.fields.searchable;
|
|
5696
|
+
for (let entry = 0; entry < count; entry++) {
|
|
5697
|
+
const internalId = docIds[entry];
|
|
5698
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5699
|
+
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5700
|
+
addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
|
|
5701
|
+
}
|
|
5702
|
+
}
|
|
5703
|
+
function fieldPairScoreEveryDocument(scan2, buffer) {
|
|
5499
5704
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5500
5705
|
const searchable = scan2.fields.searchable;
|
|
5501
5706
|
for (let entry = 0; entry < count; entry++) {
|
|
5502
5707
|
const internalId = docIds[entry];
|
|
5503
5708
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5504
5709
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5505
|
-
addScore(buffer, internalId,
|
|
5710
|
+
addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
|
|
5506
5711
|
}
|
|
5507
5712
|
}
|
|
5508
|
-
function
|
|
5713
|
+
function samePairScoreKnownDocuments(scan2, buffer) {
|
|
5509
5714
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5510
5715
|
const searchable = scan2.fields.searchable;
|
|
5511
5716
|
let counted = 0;
|
|
@@ -5513,31 +5718,71 @@ function scoreKnownDocumentsAndCountTheRest(scan2, buffer) {
|
|
|
5513
5718
|
const internalId = docIds[entry];
|
|
5514
5719
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5515
5720
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5516
|
-
if (hasScore(buffer, internalId)) addScore(buffer, internalId,
|
|
5721
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
|
|
5517
5722
|
else if (markCounted(buffer, internalId)) counted++;
|
|
5518
5723
|
}
|
|
5519
5724
|
return counted;
|
|
5520
5725
|
}
|
|
5521
|
-
function
|
|
5726
|
+
function fieldPairScoreKnownDocuments(scan2, buffer) {
|
|
5727
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5728
|
+
const searchable = scan2.fields.searchable;
|
|
5729
|
+
let counted = 0;
|
|
5730
|
+
for (let entry = 0; entry < count; entry++) {
|
|
5731
|
+
const internalId = docIds[entry];
|
|
5732
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5733
|
+
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5734
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
|
|
5735
|
+
else if (markCounted(buffer, internalId)) counted++;
|
|
5736
|
+
}
|
|
5737
|
+
return counted;
|
|
5738
|
+
}
|
|
5739
|
+
function firstEntryOf(docIds, count, internalId) {
|
|
5740
|
+
let low = 0;
|
|
5741
|
+
let high = count;
|
|
5742
|
+
while (low < high) {
|
|
5743
|
+
const middle = low + high >>> 1;
|
|
5744
|
+
if (docIds[middle] < internalId) low = middle + 1;
|
|
5745
|
+
else high = middle;
|
|
5746
|
+
}
|
|
5747
|
+
return low;
|
|
5748
|
+
}
|
|
5749
|
+
function samePairQueryOrderScore(scans, internalId) {
|
|
5522
5750
|
let score = 0;
|
|
5523
5751
|
for (const scan2 of scans) {
|
|
5524
5752
|
const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5525
5753
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5526
|
-
let
|
|
5527
|
-
|
|
5528
|
-
|
|
5529
|
-
const middle = low + high >>> 1;
|
|
5530
|
-
if (docIds[middle] < internalId) low = middle + 1;
|
|
5531
|
-
else high = middle;
|
|
5754
|
+
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
5755
|
+
if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
5756
|
+
score += samePairEntryScore(scan2, entry, internalId);
|
|
5532
5757
|
}
|
|
5533
|
-
|
|
5758
|
+
}
|
|
5759
|
+
return score;
|
|
5760
|
+
}
|
|
5761
|
+
function fieldPairQueryOrderScore(scans, internalId) {
|
|
5762
|
+
let score = 0;
|
|
5763
|
+
for (const scan2 of scans) {
|
|
5764
|
+
const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5765
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5766
|
+
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
5534
5767
|
if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
5535
|
-
score +=
|
|
5768
|
+
score += fieldPairEntryScore(scan2, entry, internalId);
|
|
5536
5769
|
}
|
|
5537
5770
|
}
|
|
5538
5771
|
return score;
|
|
5539
5772
|
}
|
|
5540
|
-
|
|
5773
|
+
var SAME_PAIR_SCORER = {
|
|
5774
|
+
upperBound: samePairUpperBound,
|
|
5775
|
+
scoreEveryDocument: samePairScoreEveryDocument,
|
|
5776
|
+
scoreKnownDocumentsAndCountTheRest: samePairScoreKnownDocuments,
|
|
5777
|
+
queryOrderScore: samePairQueryOrderScore
|
|
5778
|
+
};
|
|
5779
|
+
var FIELD_PAIR_SCORER = {
|
|
5780
|
+
upperBound: fieldPairUpperBound,
|
|
5781
|
+
scoreEveryDocument: fieldPairScoreEveryDocument,
|
|
5782
|
+
scoreKnownDocumentsAndCountTheRest: fieldPairScoreKnownDocuments,
|
|
5783
|
+
queryOrderScore: fieldPairQueryOrderScore
|
|
5784
|
+
};
|
|
5785
|
+
function pageFromBuffer(scorer, scans, buffer, selection, resolver) {
|
|
5541
5786
|
const approximateKth = kthBestScore(buffer, selection, resolver);
|
|
5542
5787
|
const gatherFrom = approximateKth - Math.abs(approximateKth) * PRUNING_REJECTION_SLACK;
|
|
5543
5788
|
const heap = [];
|
|
@@ -5549,7 +5794,7 @@ function pageFromBuffer(scans, buffer, selection, resolver) {
|
|
|
5549
5794
|
if (scores[internalId] < gatherFrom) continue;
|
|
5550
5795
|
const externalId = resolver.toExternal(internalId);
|
|
5551
5796
|
if (externalId === void 0) continue;
|
|
5552
|
-
const score = queryOrderScore(scans, internalId);
|
|
5797
|
+
const score = scorer.queryOrderScore(scans, internalId);
|
|
5553
5798
|
if (full && score < threshold) continue;
|
|
5554
5799
|
const candidate = { internalId, externalId, score };
|
|
5555
5800
|
if (!full) {
|
|
@@ -5579,18 +5824,28 @@ function pageFromBuffer(scans, buffer, selection, resolver) {
|
|
|
5579
5824
|
return scored;
|
|
5580
5825
|
}
|
|
5581
5826
|
function multiTermTopK(request) {
|
|
5582
|
-
const { terms, docFrequencies, totalDocs,
|
|
5827
|
+
const { terms, docFrequencies, totalDocs, limit, fields, resolver, buffer } = request;
|
|
5583
5828
|
if (!boostsKeepScoresNonNegative(fields)) return null;
|
|
5584
|
-
const
|
|
5829
|
+
const onePair = fields.boundPairs.length === 2;
|
|
5830
|
+
const scorer = onePair ? SAME_PAIR_SCORER : FIELD_PAIR_SCORER;
|
|
5831
|
+
const k1 = onePair ? fields.boundPairs[0] : 0;
|
|
5832
|
+
const b = onePair ? fields.boundPairs[1] : 0;
|
|
5585
5833
|
const scans = new Array(terms.length);
|
|
5586
5834
|
for (let term = 0; term < terms.length; term++) {
|
|
5587
5835
|
const idf = computeIDF(docFrequencies[term], totalDocs);
|
|
5588
5836
|
if (!(idf >= 0)) return null;
|
|
5589
|
-
scans[term] = {
|
|
5837
|
+
scans[term] = {
|
|
5838
|
+
columns: postingColumns(terms[term].list),
|
|
5839
|
+
idf,
|
|
5840
|
+
scoresAreZero: totalDocs === 0,
|
|
5841
|
+
fields,
|
|
5842
|
+
k1,
|
|
5843
|
+
b
|
|
5844
|
+
};
|
|
5590
5845
|
}
|
|
5591
5846
|
const maxBoost = bestSearchable(fields.searchable, fields.boosts);
|
|
5592
5847
|
const maxAverageLength = bestSearchable(fields.searchable, fields.averageLengths);
|
|
5593
|
-
const upperBounds = scans.map((scan2, term) =>
|
|
5848
|
+
const upperBounds = scans.map((scan2, term) => scorer.upperBound(scan2, terms[term].list, maxBoost, maxAverageLength));
|
|
5594
5849
|
const order = scans.map((_, term) => term).sort((left, right) => upperBounds[right] - upperBounds[left]);
|
|
5595
5850
|
const remainingBound = new Float64Array(order.length + 1);
|
|
5596
5851
|
for (let position2 = order.length - 1; position2 >= 0; position2--) {
|
|
@@ -5605,15 +5860,15 @@ function multiTermTopK(request) {
|
|
|
5605
5860
|
const threshold = kthBestScore(buffer, selection, resolver);
|
|
5606
5861
|
if (remaining < threshold - threshold * PRUNING_REJECTION_SLACK) break;
|
|
5607
5862
|
}
|
|
5608
|
-
scoreEveryDocument(scans[order[position]], buffer);
|
|
5863
|
+
scorer.scoreEveryDocument(scans[order[position]], buffer);
|
|
5609
5864
|
}
|
|
5610
5865
|
let countedOnly = 0;
|
|
5611
5866
|
for (let rest = position; rest < order.length; rest++) {
|
|
5612
|
-
countedOnly += scoreKnownDocumentsAndCountTheRest(scans[order[rest]], buffer);
|
|
5867
|
+
countedOnly += scorer.scoreKnownDocumentsAndCountTheRest(scans[order[rest]], buffer);
|
|
5613
5868
|
}
|
|
5614
5869
|
const totalMatched = buffer.touchedCount + countedOnly;
|
|
5615
5870
|
if (wanted === 0) return { scored: [], totalMatched };
|
|
5616
|
-
return { scored: pageFromBuffer(scans, buffer, selection, resolver), totalMatched };
|
|
5871
|
+
return { scored: pageFromBuffer(scorer, scans, buffer, selection, resolver), totalMatched };
|
|
5617
5872
|
}
|
|
5618
5873
|
|
|
5619
5874
|
// src/core/partition/single-term-topk.ts
|
|
@@ -5641,20 +5896,25 @@ function prunableSingleTermList(params, index, fieldNames) {
|
|
|
5641
5896
|
return list;
|
|
5642
5897
|
}
|
|
5643
5898
|
function singleTermTopK(request) {
|
|
5644
|
-
const { list, docFrequency, totalDocs,
|
|
5899
|
+
const { list, docFrequency, totalDocs, limit, resolver } = request;
|
|
5645
5900
|
const {
|
|
5646
5901
|
searchable: fieldSearchable,
|
|
5647
5902
|
boosts: fieldBoosts,
|
|
5648
5903
|
averageLengths: fieldAvgLengths,
|
|
5649
|
-
lengthColumns: fieldLengthColumns
|
|
5904
|
+
lengthColumns: fieldLengthColumns,
|
|
5905
|
+
k1s,
|
|
5906
|
+
bs,
|
|
5907
|
+
boundPairs
|
|
5650
5908
|
} = request.fields;
|
|
5909
|
+
const onePair = boundPairs.length === 2;
|
|
5910
|
+
const onlyK1 = boundPairs[0];
|
|
5911
|
+
const onlyB = boundPairs[1];
|
|
5651
5912
|
const wanted = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 0;
|
|
5652
5913
|
const { docIds, termFrequencies, fieldNameIndices, deletedDocs: deleted, hasDeleted } = postingColumns(list);
|
|
5653
5914
|
const bounds = blockBoundsFor(list, fieldLengthColumns);
|
|
5654
5915
|
const maxBoost = bestSearchable(fieldSearchable, fieldBoosts);
|
|
5655
5916
|
const maxAverageLength = bestSearchable(fieldSearchable, fieldAvgLengths);
|
|
5656
5917
|
const idf = computeIDF(docFrequency, totalDocs);
|
|
5657
|
-
const { k1, b } = resolveBM25Params(bm25Params);
|
|
5658
5918
|
const scoresAreZero = totalDocs === 0;
|
|
5659
5919
|
const heap = [];
|
|
5660
5920
|
let full = false;
|
|
@@ -5664,14 +5924,12 @@ function singleTermTopK(request) {
|
|
|
5664
5924
|
const start = block === 0 ? 0 : bounds.entryEnd[block - 1];
|
|
5665
5925
|
const end = bounds.entryEnd[block];
|
|
5666
5926
|
if (full) {
|
|
5667
|
-
|
|
5668
|
-
|
|
5669
|
-
|
|
5670
|
-
|
|
5671
|
-
|
|
5672
|
-
|
|
5673
|
-
bm25Params
|
|
5674
|
-
);
|
|
5927
|
+
let bestEntry = 0;
|
|
5928
|
+
if (!scoresAreZero) {
|
|
5929
|
+
const blockMaxTf = bounds.maxTermFrequency[block];
|
|
5930
|
+
const blockMinLength = bounds.minFieldLength[block];
|
|
5931
|
+
bestEntry = onePair ? computeBM25WithIDF(blockMaxTf, idf, blockMinLength, maxAverageLength, onlyK1, onlyB) : blockEntryBound(request.fields, blockMaxTf, idf, blockMinLength, maxAverageLength);
|
|
5932
|
+
}
|
|
5675
5933
|
if (bestEntry * maxBoost * bounds.maxEntriesPerDocument[block] < threshold) {
|
|
5676
5934
|
totalMatched += bounds.documentCount[block];
|
|
5677
5935
|
continue;
|
|
@@ -5690,7 +5948,14 @@ function singleTermTopK(request) {
|
|
|
5690
5948
|
const fieldIndex = fieldNameIndices[entry];
|
|
5691
5949
|
if (fieldSearchable[fieldIndex] === 1) {
|
|
5692
5950
|
const fieldLength = fieldLengthOf(fieldLengthColumns, fieldIndex, internalId, fieldAvgLengths[fieldIndex]);
|
|
5693
|
-
score += (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
5951
|
+
score += (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
5952
|
+
termFrequencies[entry],
|
|
5953
|
+
idf,
|
|
5954
|
+
fieldLength,
|
|
5955
|
+
fieldAvgLengths[fieldIndex],
|
|
5956
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
5957
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
5958
|
+
)) * fieldBoosts[fieldIndex];
|
|
5694
5959
|
scored2 = true;
|
|
5695
5960
|
}
|
|
5696
5961
|
entry++;
|
|
@@ -5750,15 +6015,14 @@ function searchFulltext(state, params) {
|
|
|
5750
6015
|
globalStats,
|
|
5751
6016
|
maxResults,
|
|
5752
6017
|
termMatch,
|
|
5753
|
-
|
|
6018
|
+
phrases
|
|
5754
6019
|
} = params;
|
|
5755
6020
|
const collectComponents = params.collectComponents !== false;
|
|
5756
|
-
if (queryTokens.length === 0) {
|
|
6021
|
+
if (queryTokens.length === 0 && phrases === void 0) {
|
|
5757
6022
|
return { scored: [], totalMatched: 0 };
|
|
5758
6023
|
}
|
|
5759
6024
|
const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
|
|
5760
6025
|
const avgFieldLengths = globalStats?.averageFieldLengths ?? state.stats.averageFieldLengths;
|
|
5761
|
-
const { k1, b } = resolveBM25Params(bm25Params);
|
|
5762
6026
|
const scoresAreZero = totalDocs === 0;
|
|
5763
6027
|
if (state.scoreBuffer === null) state.scoreBuffer = createScoreBuffer(state.docStore.internalIdCapacity());
|
|
5764
6028
|
const scoreBuffer = state.scoreBuffer;
|
|
@@ -5767,19 +6031,59 @@ function searchFulltext(state, params) {
|
|
|
5767
6031
|
const useIntersection = termMatch === "all" && queryTokens.length > 1;
|
|
5768
6032
|
const fieldNames = state.fieldNameTable.names;
|
|
5769
6033
|
const resolver = state.docStore.resolver();
|
|
5770
|
-
const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths);
|
|
5771
|
-
const { searchable, boosts, averageLengths, lengthColumns } = fieldScoring;
|
|
6034
|
+
const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths, bm25Params);
|
|
6035
|
+
const { searchable, boosts, averageLengths, lengthColumns, k1s, bs, boundPairs } = fieldScoring;
|
|
6036
|
+
const onePair = boundPairs.length === 2;
|
|
6037
|
+
const onlyK1 = boundPairs[0];
|
|
6038
|
+
const onlyB = boundPairs[1];
|
|
6039
|
+
const phraseMatches = phrases === void 0 ? null : requireLoadedPhraseCode().matchPhrases(
|
|
6040
|
+
state.invertedIdx,
|
|
6041
|
+
phrases,
|
|
6042
|
+
searchable,
|
|
6043
|
+
params.filterBitset,
|
|
6044
|
+
state.docStore.internalIdCapacity(),
|
|
6045
|
+
params.phraseWork,
|
|
6046
|
+
true
|
|
6047
|
+
);
|
|
6048
|
+
if (phraseMatches !== null && bitsetIsEmpty(phraseMatches.matched)) {
|
|
6049
|
+
return { scored: [], totalMatched: 0 };
|
|
6050
|
+
}
|
|
6051
|
+
const filterBitset = phraseMatches?.matched ?? params.filterBitset;
|
|
5772
6052
|
function documentFrequency(token, list) {
|
|
5773
6053
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
5774
6054
|
}
|
|
6055
|
+
function scorePhraseRows(rows, matched) {
|
|
6056
|
+
let idf = 0;
|
|
6057
|
+
for (let entry = 0; entry < rows.phrase.tokens.length; entry++) {
|
|
6058
|
+
idf += computeIDF(documentFrequency(rows.phrase.tokens[entry], rows.lists[entry]), totalDocs);
|
|
6059
|
+
}
|
|
6060
|
+
for (let row = 0; row < rows.docIds.length; row++) {
|
|
6061
|
+
const internalId = rows.docIds.values[row];
|
|
6062
|
+
if (!bitsetHas(matched, internalId)) continue;
|
|
6063
|
+
const fieldIndex = rows.fieldIndices.values[row];
|
|
6064
|
+
const frequency = rows.frequencies.values[row];
|
|
6065
|
+
const avgLen = averageLengths[fieldIndex];
|
|
6066
|
+
const fieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
6067
|
+
const score = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
6068
|
+
frequency,
|
|
6069
|
+
idf,
|
|
6070
|
+
fieldLength,
|
|
6071
|
+
avgLen,
|
|
6072
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
6073
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
6074
|
+
);
|
|
6075
|
+
addScore(scoreBuffer, internalId, score * boosts[fieldIndex]);
|
|
6076
|
+
if (components !== null) {
|
|
6077
|
+
recordComponents(components, internalId, fieldNames[fieldIndex], rows.phrase.key, frequency, fieldLength, idf);
|
|
6078
|
+
}
|
|
6079
|
+
}
|
|
6080
|
+
}
|
|
5775
6081
|
const groupContext = {
|
|
5776
6082
|
index: state.invertedIdx,
|
|
5777
6083
|
fields: fieldScoring,
|
|
5778
6084
|
partialMarks: partialFieldMarks(fieldNames, partialFields),
|
|
5779
6085
|
fieldNames,
|
|
5780
6086
|
totalDocs,
|
|
5781
|
-
k1,
|
|
5782
|
-
b,
|
|
5783
6087
|
filterBitset,
|
|
5784
6088
|
globalFrequency(token) {
|
|
5785
6089
|
if (globalStats === void 0 || !Object.hasOwn(globalStats.docFrequencies, token)) return void 0;
|
|
@@ -5805,7 +6109,14 @@ function searchFulltext(state, params) {
|
|
|
5805
6109
|
if (searchable[fieldIndex] === 0) continue;
|
|
5806
6110
|
const avgLen = averageLengths[fieldIndex];
|
|
5807
6111
|
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
5808
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
6112
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
6113
|
+
termFrequencies[pi],
|
|
6114
|
+
match.idf,
|
|
6115
|
+
actualFieldLength,
|
|
6116
|
+
avgLen,
|
|
6117
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
6118
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
6119
|
+
);
|
|
5809
6120
|
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
5810
6121
|
}
|
|
5811
6122
|
return;
|
|
@@ -5820,7 +6131,14 @@ function searchFulltext(state, params) {
|
|
|
5820
6131
|
const termFrequency = termFrequencies[pi];
|
|
5821
6132
|
const avgLen = averageLengths[fieldIndex];
|
|
5822
6133
|
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
5823
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
6134
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
6135
|
+
termFrequency,
|
|
6136
|
+
match.idf,
|
|
6137
|
+
actualFieldLength,
|
|
6138
|
+
avgLen,
|
|
6139
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
6140
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
6141
|
+
);
|
|
5824
6142
|
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
5825
6143
|
if (components !== null) {
|
|
5826
6144
|
recordComponents(
|
|
@@ -5842,25 +6160,23 @@ function searchFulltext(state, params) {
|
|
|
5842
6160
|
mergeGroupContribution(internalId, contribution);
|
|
5843
6161
|
}
|
|
5844
6162
|
}
|
|
5845
|
-
const prunableList = prunableSingleTermList(params, state.invertedIdx, fieldNames);
|
|
6163
|
+
const prunableList = phrases === void 0 ? prunableSingleTermList(params, state.invertedIdx, fieldNames) : null;
|
|
5846
6164
|
if (prunableList !== null && maxResults !== void 0) {
|
|
5847
6165
|
return singleTermTopK({
|
|
5848
6166
|
list: prunableList,
|
|
5849
6167
|
docFrequency: documentFrequency(queryTokens[0].token, prunableList),
|
|
5850
6168
|
totalDocs,
|
|
5851
|
-
bm25Params,
|
|
5852
6169
|
limit: maxResults,
|
|
5853
6170
|
fields: fieldScoring,
|
|
5854
6171
|
resolver
|
|
5855
6172
|
});
|
|
5856
6173
|
}
|
|
5857
|
-
const prunableLists = prunableMultiTermLists(params, state.invertedIdx);
|
|
6174
|
+
const prunableLists = phrases === void 0 ? prunableMultiTermLists(params, state.invertedIdx) : null;
|
|
5858
6175
|
if (prunableLists !== null && maxResults !== void 0) {
|
|
5859
6176
|
const pruned = multiTermTopK({
|
|
5860
6177
|
terms: prunableLists,
|
|
5861
6178
|
docFrequencies: prunableLists.map((term) => documentFrequency(term.token, term.list)),
|
|
5862
6179
|
totalDocs,
|
|
5863
|
-
bm25Params,
|
|
5864
6180
|
limit: maxResults,
|
|
5865
6181
|
fields: fieldScoring,
|
|
5866
6182
|
resolver,
|
|
@@ -5881,7 +6197,7 @@ function searchFulltext(state, params) {
|
|
|
5881
6197
|
}
|
|
5882
6198
|
resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
|
|
5883
6199
|
}
|
|
5884
|
-
resolved.sort((a,
|
|
6200
|
+
resolved.sort((a, b) => a.totalPostings - b.totalPostings);
|
|
5885
6201
|
for (let tokenIndex = 0; tokenIndex < resolved.length; tokenIndex++) {
|
|
5886
6202
|
const onlyScored = tokenIndex > 0;
|
|
5887
6203
|
for (const match of resolved[tokenIndex].matches) scorePostings(match, onlyScored);
|
|
@@ -5896,6 +6212,9 @@ function searchFulltext(state, params) {
|
|
|
5896
6212
|
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), false);
|
|
5897
6213
|
}
|
|
5898
6214
|
}
|
|
6215
|
+
if (phraseMatches !== null) {
|
|
6216
|
+
for (const rows of phraseMatches.rows) scorePhraseRows(rows, phraseMatches.matched);
|
|
6217
|
+
}
|
|
5899
6218
|
const totalMatched = scoreBuffer.touchedCount;
|
|
5900
6219
|
const k = maxResults === void 0 ? totalMatched : Math.max(0, Math.min(maxResults, totalMatched));
|
|
5901
6220
|
const scored = topKFromBuffer(scoreBuffer, k, resolver, components);
|
|
@@ -7131,6 +7450,9 @@ function heapPop(heap) {
|
|
|
7131
7450
|
function collectCandidateTerms(subs, params) {
|
|
7132
7451
|
const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
|
|
7133
7452
|
const terms = /* @__PURE__ */ new Set();
|
|
7453
|
+
for (const phrase of params.phrases ?? []) {
|
|
7454
|
+
for (const token of phrase.tokens) terms.add(token);
|
|
7455
|
+
}
|
|
7134
7456
|
for (const qt of queryTokens) {
|
|
7135
7457
|
const group = termGroupOf(termGroups, qt.token);
|
|
7136
7458
|
if (group !== void 0) {
|
|
@@ -7232,7 +7554,7 @@ function searchSingleSub(subs, layout, params, index) {
|
|
|
7232
7554
|
return { ...result, matchedOrdinalBitset };
|
|
7233
7555
|
}
|
|
7234
7556
|
function compositeSearchFulltext(subs, layout, params) {
|
|
7235
|
-
if (params.queryTokens.length === 0) {
|
|
7557
|
+
if (params.queryTokens.length === 0 && params.phrases === void 0) {
|
|
7236
7558
|
return { scored: [], totalMatched: 0 };
|
|
7237
7559
|
}
|
|
7238
7560
|
const single = onlyPopulatedSub(subs);
|
|
@@ -7661,7 +7983,7 @@ var PREPARED_TYPO_TEST_BYTES = Math.ceil(MAX_FILTER_TESTS / TYPO_CLAIM_SLOTS_PER
|
|
|
7661
7983
|
function capExceeded(cap) {
|
|
7662
7984
|
return new NarsilError(
|
|
7663
7985
|
ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
|
|
7664
|
-
`The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
|
|
7986
|
+
`The engine stopped this search before it checked every candidate, because the work on its pattern tests and phrases passed the cap of ${cap} units. Narrow the tests or the phrases, or raise the patternWorkCap engine setting`,
|
|
7665
7987
|
{ cap }
|
|
7666
7988
|
);
|
|
7667
7989
|
}
|
|
@@ -7892,15 +8214,19 @@ function addPartitionTokens(tokens, addedMembers, partition, term, analysed, exp
|
|
|
7892
8214
|
}
|
|
7893
8215
|
}
|
|
7894
8216
|
}
|
|
7895
|
-
function addReachedTokens(tokens, partitions, term,
|
|
8217
|
+
function addReachedTokens(tokens, partitions, term, requested, language, analysis) {
|
|
8218
|
+
if (requested.options.exact === true) return [];
|
|
8219
|
+
const quoted = analyseQueryPhrases(term, language, analysis);
|
|
8220
|
+
const words = quoted?.words ?? term;
|
|
8221
|
+
if (quoted !== null && words.trim().length === 0) return [];
|
|
8222
|
+
const expansion = quoted?.endsWithPhrase === true ? { schema: requested.schema, options: { ...requested.options, prefix: false } } : requested;
|
|
7896
8223
|
const { options } = expansion;
|
|
7897
|
-
if (options.exact === true) return [];
|
|
7898
8224
|
if (!((options.tolerance ?? 0) > 0 || options.prefix === true || options.partialWords === true)) return [];
|
|
7899
|
-
const analysed = queryTokensOf(
|
|
8225
|
+
const analysed = queryTokensOf(words, language, analysis);
|
|
7900
8226
|
if (analysed === null) return [];
|
|
7901
8227
|
const addedMembers = /* @__PURE__ */ new Set();
|
|
7902
8228
|
for (const partition of partitions) {
|
|
7903
|
-
addPartitionTokens(tokens, addedMembers, partition,
|
|
8229
|
+
addPartitionTokens(tokens, addedMembers, partition, words, analysed, expansion, language, analysis);
|
|
7904
8230
|
}
|
|
7905
8231
|
for (const token of addedMembers) tokens.add(token);
|
|
7906
8232
|
return [...addedMembers];
|
|
@@ -7923,17 +8249,24 @@ function partitionsIn(manager, partitionIds) {
|
|
|
7923
8249
|
|
|
7924
8250
|
// src/partitioning/distributed-scoring.ts
|
|
7925
8251
|
function analysedQueryTokens(term, language, analysis) {
|
|
7926
|
-
const
|
|
7927
|
-
|
|
7928
|
-
removeStopWords: true,
|
|
7929
|
-
customTokenizer: analysis.customTokenizer,
|
|
7930
|
-
stopWordOverride: analysis.stopWords
|
|
7931
|
-
});
|
|
8252
|
+
const quoted = analyseQueryPhrases(term, language, analysis);
|
|
8253
|
+
const words = quoted?.words ?? term;
|
|
7932
8254
|
const unique = /* @__PURE__ */ new Set();
|
|
7933
|
-
|
|
7934
|
-
|
|
8255
|
+
if (quoted === null || words.trim().length > 0) {
|
|
8256
|
+
const { tokens } = tokenize(words, language, {
|
|
8257
|
+
stem: true,
|
|
8258
|
+
removeStopWords: true,
|
|
8259
|
+
customTokenizer: analysis.customTokenizer,
|
|
8260
|
+
stopWordOverride: analysis.stopWords
|
|
8261
|
+
});
|
|
8262
|
+
for (const { token } of tokens) {
|
|
8263
|
+
unique.add(token);
|
|
8264
|
+
}
|
|
7935
8265
|
}
|
|
7936
8266
|
for (const synonym of analysis.synonyms?.expand(unique)?.tokens ?? []) unique.add(synonym);
|
|
8267
|
+
for (const phrase of quoted?.phrases ?? []) {
|
|
8268
|
+
for (const token of phrase.tokens) unique.add(token);
|
|
8269
|
+
}
|
|
7937
8270
|
return unique;
|
|
7938
8271
|
}
|
|
7939
8272
|
function copyNumericRecord(source) {
|
|
@@ -8336,6 +8669,14 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
|
|
|
8336
8669
|
}
|
|
8337
8670
|
|
|
8338
8671
|
// src/search/fulltext.ts
|
|
8672
|
+
function analysedWords(term, language, options) {
|
|
8673
|
+
return tokenize(term, language, {
|
|
8674
|
+
stem: true,
|
|
8675
|
+
removeStopWords: true,
|
|
8676
|
+
customTokenizer: options?.customTokenizer,
|
|
8677
|
+
stopWordOverride: options?.stopWords
|
|
8678
|
+
}).tokens;
|
|
8679
|
+
}
|
|
8339
8680
|
function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
8340
8681
|
if (!params.term || params.term.trim().length === 0) return null;
|
|
8341
8682
|
if (params.fields && params.fields.length === 0) return null;
|
|
@@ -8343,14 +8684,16 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
8343
8684
|
if (params.fields) {
|
|
8344
8685
|
validateSearchFields(params.fields, flatSchema);
|
|
8345
8686
|
}
|
|
8346
|
-
const
|
|
8347
|
-
|
|
8348
|
-
|
|
8349
|
-
customTokenizer: options?.customTokenizer,
|
|
8350
|
-
stopWordOverride: options?.stopWords
|
|
8687
|
+
const quoted = analyseQueryPhrases(params.term, language, {
|
|
8688
|
+
stopWords: options?.stopWords,
|
|
8689
|
+
customTokenizer: options?.customTokenizer
|
|
8351
8690
|
});
|
|
8352
|
-
|
|
8353
|
-
const
|
|
8691
|
+
const wordParams = quoted === null ? params : requireLoadedPhraseCode().withWordsOnly(params, quoted);
|
|
8692
|
+
const phrases = quoted !== null && quoted.phrases.length > 0 ? quoted.phrases : void 0;
|
|
8693
|
+
const words = wordParams.term ?? "";
|
|
8694
|
+
const analysedTokens = quoted !== null && words.trim().length === 0 ? [] : analysedWords(words, language, options);
|
|
8695
|
+
if (analysedTokens.length === 0 && phrases === void 0) return null;
|
|
8696
|
+
const queryTokens = deduplicateTokens(analysedTokens);
|
|
8354
8697
|
let filterBitset;
|
|
8355
8698
|
if (params.filters) {
|
|
8356
8699
|
filterBitset = partition.applyFiltersBitset(
|
|
@@ -8360,16 +8703,19 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
8360
8703
|
);
|
|
8361
8704
|
if (bitsetIsEmpty(filterBitset)) return null;
|
|
8362
8705
|
}
|
|
8363
|
-
|
|
8706
|
+
if (analysedTokens.length === 0) return { queryTokens, phrases, filterBitset };
|
|
8707
|
+
const lastToken = analysedTokens[analysedTokens.length - 1];
|
|
8364
8708
|
const synonyms = params.exact === true ? void 0 : options?.synonyms?.expand(queryTokens.map((qt) => qt.token));
|
|
8365
|
-
const groups = resolveTermGroups(partition,
|
|
8366
|
-
return { queryTokens, ...groups, filterBitset };
|
|
8709
|
+
const groups = resolveTermGroups(partition, wordParams, language, schema, queryTokens, lastToken, options, synonyms);
|
|
8710
|
+
return { queryTokens, phrases, ...groups, filterBitset };
|
|
8367
8711
|
}
|
|
8368
8712
|
function fulltextMatches(partition, params, language, schema, options) {
|
|
8369
8713
|
const prepared = prepareFulltextQuery(partition, params, language, schema, options);
|
|
8370
8714
|
if (prepared === null) return null;
|
|
8371
8715
|
return partition.searchFulltextMatches({
|
|
8372
8716
|
queryTokens: prepared.queryTokens,
|
|
8717
|
+
phrases: prepared.phrases,
|
|
8718
|
+
phraseWork: options?.patternWork,
|
|
8373
8719
|
termGroups: prepared.termGroups,
|
|
8374
8720
|
partialFields: prepared.partialFields,
|
|
8375
8721
|
fields: params.fields,
|
|
@@ -8384,13 +8730,15 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
8384
8730
|
if (prepared === null) {
|
|
8385
8731
|
return { scored: [], totalMatched: 0 };
|
|
8386
8732
|
}
|
|
8387
|
-
const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
|
|
8733
|
+
const { queryTokens, phrases, termGroups, partialFields, filterBitset } = prepared;
|
|
8388
8734
|
const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
8389
8735
|
const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
|
|
8390
8736
|
const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
|
|
8391
8737
|
const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
|
|
8392
8738
|
const rawResult = partition.searchFulltext({
|
|
8393
8739
|
queryTokens,
|
|
8740
|
+
phrases,
|
|
8741
|
+
phraseWork: options?.patternWork,
|
|
8394
8742
|
termGroups,
|
|
8395
8743
|
partialFields,
|
|
8396
8744
|
fields: params.fields,
|
|
@@ -8408,7 +8756,7 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
8408
8756
|
});
|
|
8409
8757
|
let scored = rawResult.scored;
|
|
8410
8758
|
const termMatch = params.termMatch ?? "any";
|
|
8411
|
-
if (termMatch !== "any") {
|
|
8759
|
+
if (termMatch !== "any" && queryTokens.length > 0) {
|
|
8412
8760
|
scored = filterByTermCoverage(
|
|
8413
8761
|
scored,
|
|
8414
8762
|
queryTokens,
|
|
@@ -8417,7 +8765,8 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
8417
8765
|
params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
8418
8766
|
params.exact ?? false,
|
|
8419
8767
|
termGroups,
|
|
8420
|
-
partialFields ?? []
|
|
8768
|
+
partialFields ?? [],
|
|
8769
|
+
phrases
|
|
8421
8770
|
);
|
|
8422
8771
|
}
|
|
8423
8772
|
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
@@ -8459,8 +8808,9 @@ function deduplicateTokens(tokens) {
|
|
|
8459
8808
|
}
|
|
8460
8809
|
return result;
|
|
8461
8810
|
}
|
|
8462
|
-
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
|
|
8811
|
+
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields, phrases) {
|
|
8463
8812
|
const requiredCount = policy === "all" ? queryTokens.length : policy;
|
|
8813
|
+
const phraseKeys = phrases === void 0 ? null : new Set(phrases.map((phrase) => phrase.key));
|
|
8464
8814
|
if (requiredCount <= 0) return scored;
|
|
8465
8815
|
if (requiredCount > queryTokens.length) return [];
|
|
8466
8816
|
const groups = /* @__PURE__ */ new Map();
|
|
@@ -8476,12 +8826,22 @@ function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLeng
|
|
|
8476
8826
|
});
|
|
8477
8827
|
}
|
|
8478
8828
|
return scored.filter((doc) => {
|
|
8479
|
-
const matched = countDocTermMatches(
|
|
8829
|
+
const matched = countDocTermMatches(
|
|
8830
|
+
doc,
|
|
8831
|
+
queryTokens,
|
|
8832
|
+
tolerance,
|
|
8833
|
+
prefixLength,
|
|
8834
|
+
exact,
|
|
8835
|
+
groups,
|
|
8836
|
+
partialFields,
|
|
8837
|
+
phraseKeys
|
|
8838
|
+
);
|
|
8480
8839
|
return matched >= requiredCount;
|
|
8481
8840
|
});
|
|
8482
8841
|
}
|
|
8483
|
-
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
|
|
8484
|
-
const
|
|
8842
|
+
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields, phraseKeys) {
|
|
8843
|
+
const componentKeys = Object.keys(doc.idf);
|
|
8844
|
+
const indexTokens = phraseKeys === null ? componentKeys : componentKeys.filter((key) => !phraseKeys.has(key));
|
|
8485
8845
|
if (indexTokens.length === 0) return 0;
|
|
8486
8846
|
let count = 0;
|
|
8487
8847
|
for (const qt of queryTokens) {
|
|
@@ -8634,4 +8994,4 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
8634
8994
|
return count;
|
|
8635
8995
|
}
|
|
8636
8996
|
|
|
8637
|
-
export { FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FNV_OFFSET_BASIS, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, applyPagination, applyPartitionFiltersBitset, applyProjection, applySorting, availablePatternSearch, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUint32s, fulltextMatches, isCompositePartition, joinPatternWorkMeter, kWayMerge, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|
|
8997
|
+
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FNV_OFFSET_BASIS, MAX_BM25_B, MAX_BM25_K1, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, analyseQueryPhrases, applyPagination, applyPartitionFiltersBitset, applyProjection, applySorting, availablePatternSearch, bm25SetsValue, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUint32s, fulltextMatches, holdsQuotedText, isCompositePartition, joinPatternWorkMeter, kWayMerge, loadPhraseCodeFor, mergeBM25Levels, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireLoadedPhraseCode, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|