@delali/narsil 0.3.1-next.128 → 0.3.1-next.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/{chunk-LJS6PREA.mjs → chunk-3NVZD52E.mjs} +3 -1
- package/dist/browser/{chunk-4Y2EGA3U.mjs → chunk-5D6NJD6C.mjs} +554 -127
- package/dist/browser/{chunk-6POI2KNI.mjs → chunk-II5POT55.mjs} +3 -3
- package/dist/browser/{chunk-2CBQ2PCC.mjs → chunk-WLD6DDIX.mjs} +1 -1
- package/dist/browser/{chunk-V5DCQIOT.mjs → chunk-WZBTVTSU.mjs} +2 -2
- package/dist/browser/{chunk-B65QVTST.mjs → chunk-XTFJZHQP.mjs} +14 -9
- package/dist/browser/{chunk-P7C2S5JX.mjs → chunk-YAGXZU4W.mjs} +2 -2
- package/dist/browser/{durability-wiring-PUC4QSCR.mjs → durability-wiring-HCCC6BJB.mjs} +6 -6
- package/dist/browser/{module-ACWDC7YI.mjs → module-S2VDU2JE.mjs} +5 -5
- package/dist/browser/{restore-queue-MLUEUEAN.mjs → restore-queue-BZ7WWEC4.mjs} +1 -1
- package/dist/browser/{runtime-BX6XRFYR.mjs → runtime-PL4TTFJE.mjs} +1 -1
- package/dist/browser/{snapshot-U5Y346XG.mjs → snapshot-RTUCHI2L.mjs} +7 -7
- package/dist/browser/{write-GZHR6AVG.mjs → write-IMU7G5NU.mjs} +5 -5
- package/dist/{capabilities-9YwwCcqP.d.ts → capabilities-BTjv-tK8.d.ts} +10 -3
- package/dist/{chunk-N57XXGPG.mjs → chunk-2HQQ7Y5D.mjs} +23 -4
- package/dist/{chunk-ENBN5VQ6.mjs → chunk-2YOAGAGF.mjs} +115 -46
- package/dist/{chunk-QSCVJJW5.mjs → chunk-37RTD6KD.mjs} +8 -8
- package/dist/{chunk-N6J6EJW3.mjs → chunk-3LJVM6XM.mjs} +2 -2
- package/dist/{chunk-VK45J47R.mjs → chunk-3XHFVYIQ.mjs} +1 -1
- package/dist/{chunk-LOBYK2FG.mjs → chunk-43Q7HAI2.mjs} +20 -9
- package/dist/{chunk-6PN3U6XW.mjs → chunk-44TB6AIZ.mjs} +1 -1
- package/dist/{chunk-6ODIY6ZB.mjs → chunk-4XEVROSP.mjs} +1 -1
- package/dist/{chunk-4IHMMDRF.mjs → chunk-5LC5VMDQ.mjs} +4 -4
- package/dist/{chunk-MFPJVITN.mjs → chunk-5VSTAJET.mjs} +2 -2
- package/dist/{chunk-DG62H7OD.mjs → chunk-AFOZR4ZQ.mjs} +2 -2
- package/dist/{chunk-P34VZEXB.mjs → chunk-AN3CUMPZ.mjs} +2 -2
- package/dist/{chunk-4PQNDHSA.mjs → chunk-CSSCLRGY.mjs} +2 -2
- package/dist/{chunk-HWQDZB2U.mjs → chunk-CTF6WFTX.mjs} +3 -3
- package/dist/{chunk-E74YLAIG.mjs → chunk-DX2FI4LU.mjs} +2 -1
- package/dist/{chunk-22CK2LQO.mjs → chunk-EZIGAOUJ.mjs} +5 -5
- package/dist/{chunk-KCQG2PJT.mjs → chunk-F4UKMTKM.mjs} +4 -4
- package/dist/{chunk-542TD4BL.mjs → chunk-FRSWYMXH.mjs} +1 -1
- package/dist/{chunk-ANFNYCQG.mjs → chunk-J4LRE5Q7.mjs} +6 -6
- package/dist/{chunk-OGFSUXYY.mjs → chunk-K73PZTBR.mjs} +4 -4
- package/dist/{chunk-IBGMFR5Y.mjs → chunk-K76F5YYX.mjs} +274 -75
- package/dist/{chunk-FUUP45IS.mjs → chunk-M6MBSVIJ.mjs} +1 -1
- package/dist/{chunk-IR4KQ4WB.mjs → chunk-MGD7XETB.mjs} +32 -30
- package/dist/{chunk-7ZZQNOIY.mjs → chunk-OPO5SE4J.mjs} +1 -1
- package/dist/{chunk-UKXMSO5H.mjs → chunk-PCRWCOZC.mjs} +155 -31
- package/dist/{chunk-HFJXEFFV.mjs → chunk-PTIJAZWL.mjs} +3 -3
- package/dist/chunk-Q2AP2FSN.mjs +15 -0
- package/dist/{chunk-SEG4CL6W.mjs → chunk-QQS4QZQA.mjs} +4 -4
- package/dist/{chunk-OFYKBKKV.mjs → chunk-RC27KF7F.mjs} +1 -1
- package/dist/{chunk-JEXJDCQH.mjs → chunk-RYH3PASD.mjs} +17 -12
- package/dist/{chunk-IE43SRL3.mjs → chunk-TB7YMDD5.mjs} +10 -6
- package/dist/{chunk-ZR63OGT2.mjs → chunk-TY64A3ZW.mjs} +2 -2
- package/dist/{chunk-Z4DZ4KK2.mjs → chunk-UJWGQIXU.mjs} +1 -0
- package/dist/{chunk-OBWUWZRA.mjs → chunk-WI5Q3L43.mjs} +1 -0
- package/dist/{chunk-SJEIIR56.mjs → chunk-WQYB4TNR.mjs} +2 -2
- package/dist/{chunk-JIXO442E.mjs → chunk-WUIDPBSY.mjs} +16 -10
- package/dist/{chunk-62IPM62F.mjs → chunk-XRXBCOA4.mjs} +2 -2
- package/dist/{chunk-U6NGTWI3.mjs → chunk-XTER67YM.mjs} +1 -1
- package/dist/{chunk-LXWDQLTW.mjs → chunk-Y4NOHICF.mjs} +1 -1
- package/dist/{chunk-OCB7TMUQ.mjs → chunk-Y4US5EGH.mjs} +1 -1
- package/dist/{client-CsuhZ13c.d.ts → client-BOcveV7A.d.ts} +19 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +8 -2
- package/dist/{config-DLVnlByZ.d.ts → config-C3YXk5Rn.d.ts} +2 -2
- package/dist/{config-CtTfxbpV.d.ts → config-CPmIY3aH.d.ts} +2 -2
- package/dist/{direct-executor-6KCVVHYT.mjs → direct-executor-ULA6V5HC.mjs} +12 -12
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +18 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +33 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +71 -43
- package/dist/distribution.mjs +359 -115
- package/dist/{durability-wiring-UGGUNATA.mjs → durability-wiring-6R26AJDI.mjs} +26 -26
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-Bq5wz6c1.d.ts → engine-JjRAXpT3.d.ts} +38 -8
- package/dist/{events-x2aVu0_o.d.ts → events-pM0lna7M.d.ts} +12 -1
- package/dist/{index-Cl5qQauj.d.ts → index-DkQ2DlgR.d.ts} +37 -3
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +19 -19
- package/dist/{long-query-thread-5BIPZVS6.mjs → long-query-thread-6VLA6YJI.mjs} +15 -15
- package/dist/{module-7OHZ2OPI.mjs → module-S4WGMFNW.mjs} +11 -11
- package/dist/partial-words.mjs +6 -6
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +22 -22
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +1 -0
- package/dist/{rerank-NYX5S7OG.mjs → rerank-V4J4MIWR.mjs} +6 -6
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-NVMWJMHI.mjs → restore-queue-PUGWCPCX.mjs} +1 -1
- package/dist/{runtime-XNYW6AWX.mjs → runtime-TBFBQQQO.mjs} +7 -7
- package/dist/{schema-4W_7tk1f.d.ts → schema-DGAP_8HJ.d.ts} +50 -24
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +27 -15
- package/dist/{snapshot-P7IQGNNQ.mjs → snapshot-PUVSMV7E.mjs} +27 -27
- package/dist/synonyms.mjs +3 -3
- package/dist/{tasks-DLQIfXqy.d.ts → tasks-txGvNz6h.d.ts} +4 -6
- package/dist/{thread-QPETQEPK.mjs → thread-TA7QQBWP.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-WZVM7ZL6.mjs → write-TGKYT3C4.mjs} +21 -21
- package/package.json +7 -7
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { flattenSchema } from './chunk-
|
|
2
|
-
import { PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-
|
|
1
|
+
import { flattenSchema } from './chunk-AFOZR4ZQ.mjs';
|
|
2
|
+
import { PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-DX2FI4LU.mjs';
|
|
3
3
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
4
|
-
import { isWordIndexedBase } from './chunk-
|
|
5
|
-
import { encodeStringBlob, tokenize, resolveStopWords, transformToken, visitTokens, visitSurfaceSpellings } from './chunk-
|
|
4
|
+
import { isWordIndexedBase } from './chunk-4XEVROSP.mjs';
|
|
5
|
+
import { encodeStringBlob, tokenize, resolveStopWords, transformToken, visitTokens, visitSurfaceSpellings } from './chunk-RC27KF7F.mjs';
|
|
6
6
|
import { compareCodePoints, codePointCount } from './chunk-YTJCGKM4.mjs';
|
|
7
7
|
|
|
8
8
|
// src/core/partition/suggestions.ts
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { vectorsPerFile } from './chunk-
|
|
2
|
-
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-
|
|
3
|
-
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-
|
|
4
|
-
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-
|
|
5
|
-
import { HEADER_SIZE } from './chunk-
|
|
1
|
+
import { vectorsPerFile } from './chunk-QQS4QZQA.mjs';
|
|
2
|
+
import { writeLiveVectors, decodeSegmentManifest, openManifestPayload, listedKeysOf, decodeManifestPayload, SINGLE_NODE_PRIMARY_TERM } from './chunk-WUIDPBSY.mjs';
|
|
3
|
+
import { vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-Y4US5EGH.mjs';
|
|
4
|
+
import { writeCheckpointSegments, DEFAULT_COMPACTION_THRESHOLD, commitCheckpointManifest, manifestKey, manifestFileRefs, partitionContentsOf } from './chunk-37RTD6KD.mjs';
|
|
5
|
+
import { HEADER_SIZE } from './chunk-2YOAGAGF.mjs';
|
|
6
6
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
7
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
7
|
+
import { NarsilError, ErrorCodes } from './chunk-WI5Q3L43.mjs';
|
|
8
8
|
|
|
9
9
|
// src/persistence/archive/memory-store.ts
|
|
10
10
|
function joined(chunks) {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { rerankQueryTextOf } from './chunk-KZ3NX2P7.mjs';
|
|
2
|
-
import { fixedHybridDepthOf, pageByPosition, requireWithinHybridDepth } from './chunk-
|
|
2
|
+
import { fixedHybridDepthOf, pageByPosition, requireWithinHybridDepth } from './chunk-TY64A3ZW.mjs';
|
|
3
3
|
import { DEFAULT_RERANK_WINDOW, MIN_RERANK_WINDOW } from './chunk-HJSN7UYG.mjs';
|
|
4
|
-
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-
|
|
5
|
-
import { RESULT_WINDOW } from './chunk-
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { DEFAULT_LIMIT, DEFAULT_OFFSET } from './chunk-Y4NOHICF.mjs';
|
|
5
|
+
import { RESULT_WINDOW } from './chunk-DX2FI4LU.mjs';
|
|
6
|
+
import { NarsilError, ErrorCodes } from './chunk-WI5Q3L43.mjs';
|
|
7
7
|
|
|
8
8
|
// src/rerank/params.ts
|
|
9
9
|
function invalid(message, details = {}) {
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, bitsetHas, createBitSet, bitsetSet, bitsetIsEmpty, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, TYPO_CLAIM_SLOTS_PER_WORD, bitsetFromSet, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-
|
|
1
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, bitsetHas, createBitSet, bitsetSet, bitsetIsEmpty, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, TYPO_CLAIM_SLOTS_PER_WORD, bitsetFromSet, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-AN3CUMPZ.mjs';
|
|
2
2
|
import { GEO_UNIT_METRES, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
3
|
-
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-
|
|
4
|
-
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-
|
|
5
|
-
import { MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PREFIX_MAX_EXPANSIONS } from './chunk-
|
|
3
|
+
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-F4UKMTKM.mjs';
|
|
4
|
+
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-AFOZR4ZQ.mjs';
|
|
5
|
+
import { MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PREFIX_MAX_EXPANSIONS } from './chunk-DX2FI4LU.mjs';
|
|
6
6
|
import { fuzzyPrefixOf, boundedLevenshtein, fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-
|
|
8
|
-
import { tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, visitTokens, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-
|
|
7
|
+
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-4XEVROSP.mjs';
|
|
8
|
+
import { tokenize, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, visitTokens, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-RC27KF7F.mjs';
|
|
9
9
|
import { singleFoldTable, appendFoldedCodePoint, sortModeOf, defaultSortMode, SORT_VALUE_MAX_CODE_POINTS, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, isSortMode, SORT_MODES, compareSortValues, compareComparableValues, truncateSortString } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { compareCodePoints, exceedsCodePoints, codePointCount, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
11
|
import { isGeopointOrVectorType, PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-WI5Q3L43.mjs';
|
|
13
13
|
|
|
14
14
|
// src/core/partition/partial-words/platform.ts
|
|
15
15
|
function platformPartialWordSearch() {
|
|
@@ -597,6 +597,10 @@ function createDocumentStore() {
|
|
|
597
597
|
// src/core/constants.ts
|
|
598
598
|
var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
599
599
|
var MAX_TERM_FREQUENCY = 65535;
|
|
600
|
+
var DEFAULT_BM25_K1 = 1.2;
|
|
601
|
+
var DEFAULT_BM25_B = 0.75;
|
|
602
|
+
var MAX_BM25_K1 = 1e3;
|
|
603
|
+
var MAX_BM25_B = 1;
|
|
600
604
|
var PENDING_REMOVAL_MERGE_FLOOR = 64;
|
|
601
605
|
var PENDING_REMOVAL_MERGE_DIVISOR = 256;
|
|
602
606
|
|
|
@@ -4732,19 +4736,63 @@ function partitionFilterMatches(state, filters, schema, meter) {
|
|
|
4732
4736
|
}
|
|
4733
4737
|
|
|
4734
4738
|
// src/core/scorer.ts
|
|
4735
|
-
var DEFAULT_K1 = 1.2;
|
|
4736
|
-
var DEFAULT_B = 0.75;
|
|
4737
4739
|
function computeIDF(docFrequency, totalDocs) {
|
|
4738
4740
|
if (totalDocs === 0) return 0;
|
|
4739
4741
|
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
4740
4742
|
}
|
|
4743
|
+
function pairKeepsBoundSound(k1, b) {
|
|
4744
|
+
return k1 >= 0 && k1 <= MAX_BM25_K1 && b >= 0 && b <= MAX_BM25_B;
|
|
4745
|
+
}
|
|
4741
4746
|
function bm25PruningSound(params) {
|
|
4742
|
-
const k1 = params?.k1 ??
|
|
4743
|
-
const b = params?.b ??
|
|
4744
|
-
|
|
4747
|
+
const k1 = params?.k1 ?? DEFAULT_BM25_K1;
|
|
4748
|
+
const b = params?.b ?? DEFAULT_BM25_B;
|
|
4749
|
+
if (!pairKeepsBoundSound(k1, b)) return false;
|
|
4750
|
+
const fields = params?.fields;
|
|
4751
|
+
if (fields === void 0) return true;
|
|
4752
|
+
for (const name of Object.keys(fields)) {
|
|
4753
|
+
if (!pairKeepsBoundSound(fields[name].k1 ?? k1, fields[name].b ?? b)) return false;
|
|
4754
|
+
}
|
|
4755
|
+
return true;
|
|
4745
4756
|
}
|
|
4746
|
-
function
|
|
4747
|
-
|
|
4757
|
+
function bm25SetsValue(params) {
|
|
4758
|
+
if (typeof params !== "object" || params === null) return params !== void 0;
|
|
4759
|
+
if (params.k1 !== void 0 || params.b !== void 0) return true;
|
|
4760
|
+
const fields = params.fields;
|
|
4761
|
+
if (fields === void 0) return false;
|
|
4762
|
+
if (typeof fields !== "object" || fields === null) return true;
|
|
4763
|
+
for (const name of Object.keys(fields)) {
|
|
4764
|
+
const pair = fields[name];
|
|
4765
|
+
if (typeof pair !== "object" || pair === null) return true;
|
|
4766
|
+
if (pair.k1 !== void 0 || pair.b !== void 0) return true;
|
|
4767
|
+
}
|
|
4768
|
+
return false;
|
|
4769
|
+
}
|
|
4770
|
+
function ownField(fields, name) {
|
|
4771
|
+
return fields !== void 0 && Object.hasOwn(fields, name) ? fields[name] : void 0;
|
|
4772
|
+
}
|
|
4773
|
+
function mergeBM25Levels(search, index) {
|
|
4774
|
+
if (search === void 0 || !bm25SetsValue(search)) return index;
|
|
4775
|
+
const k1 = search.k1 ?? index?.k1;
|
|
4776
|
+
const b = search.b ?? index?.b;
|
|
4777
|
+
const merged = {
|
|
4778
|
+
...k1 !== void 0 ? { k1 } : {},
|
|
4779
|
+
...b !== void 0 ? { b } : {}
|
|
4780
|
+
};
|
|
4781
|
+
const names = [...Object.keys(search.fields ?? {}), ...Object.keys(index?.fields ?? {})];
|
|
4782
|
+
if (names.length === 0) return merged;
|
|
4783
|
+
const fields = /* @__PURE__ */ Object.create(null);
|
|
4784
|
+
for (const name of names) {
|
|
4785
|
+
const fromSearch = ownField(search.fields, name);
|
|
4786
|
+
const fromIndex = ownField(index?.fields, name);
|
|
4787
|
+
const fieldK1 = fromSearch?.k1 ?? search.k1 ?? fromIndex?.k1;
|
|
4788
|
+
const fieldB = fromSearch?.b ?? search.b ?? fromIndex?.b;
|
|
4789
|
+
fields[name] = {
|
|
4790
|
+
...fieldK1 !== void 0 ? { k1: fieldK1 } : {},
|
|
4791
|
+
...fieldB !== void 0 ? { b: fieldB } : {}
|
|
4792
|
+
};
|
|
4793
|
+
}
|
|
4794
|
+
merged.fields = fields;
|
|
4795
|
+
return merged;
|
|
4748
4796
|
}
|
|
4749
4797
|
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
4750
4798
|
if (avgFieldLength === 0) return 0;
|
|
@@ -4753,29 +4801,49 @@ function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1,
|
|
|
4753
4801
|
if (denominator === 0) return 0;
|
|
4754
4802
|
return idf * (numerator / denominator);
|
|
4755
4803
|
}
|
|
4756
|
-
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
4757
|
-
if (totalDocs === 0) return 0;
|
|
4758
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
4759
|
-
const b = params?.b ?? DEFAULT_B;
|
|
4760
|
-
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
4761
|
-
}
|
|
4762
4804
|
|
|
4763
4805
|
// src/core/partition/field-scoring.ts
|
|
4764
|
-
|
|
4806
|
+
var NO_FIELD_VALUES = new Float64Array(0);
|
|
4807
|
+
function differentPairsOf(searchable, k1s, bs) {
|
|
4808
|
+
const pairs = [];
|
|
4809
|
+
for (let fieldIndex = 0; fieldIndex < searchable.length; fieldIndex++) {
|
|
4810
|
+
if (searchable[fieldIndex] !== 1) continue;
|
|
4811
|
+
let seen = false;
|
|
4812
|
+
for (let at = 0; at < pairs.length && !seen; at += 2) {
|
|
4813
|
+
seen = pairs[at] === k1s[fieldIndex] && pairs[at + 1] === bs[fieldIndex];
|
|
4814
|
+
}
|
|
4815
|
+
if (!seen) pairs.push(k1s[fieldIndex], bs[fieldIndex]);
|
|
4816
|
+
}
|
|
4817
|
+
return pairs;
|
|
4818
|
+
}
|
|
4819
|
+
function loadFieldScoring(state, fields, boost, averageFieldLengths, bm25) {
|
|
4765
4820
|
const fieldNames = state.fieldNameTable.names;
|
|
4821
|
+
const fieldCount = fieldNames.length;
|
|
4822
|
+
const k1 = bm25?.k1 ?? DEFAULT_BM25_K1;
|
|
4823
|
+
const b = bm25?.b ?? DEFAULT_BM25_B;
|
|
4824
|
+
const perField = bm25?.fields;
|
|
4766
4825
|
const scoring = {
|
|
4767
|
-
searchable: new Uint8Array(
|
|
4768
|
-
boosts: new Float64Array(
|
|
4769
|
-
averageLengths: new Float64Array(
|
|
4770
|
-
lengthColumns: new Array(
|
|
4826
|
+
searchable: new Uint8Array(fieldCount),
|
|
4827
|
+
boosts: new Float64Array(fieldCount),
|
|
4828
|
+
averageLengths: new Float64Array(fieldCount),
|
|
4829
|
+
lengthColumns: new Array(fieldCount).fill(null),
|
|
4830
|
+
k1s: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
4831
|
+
bs: perField === void 0 ? NO_FIELD_VALUES : new Float64Array(fieldCount),
|
|
4832
|
+
boundPairs: [k1, b]
|
|
4771
4833
|
};
|
|
4772
|
-
for (let fieldIndex = 0; fieldIndex <
|
|
4834
|
+
for (let fieldIndex = 0; fieldIndex < fieldCount; fieldIndex++) {
|
|
4773
4835
|
const fieldName = fieldNames[fieldIndex];
|
|
4774
4836
|
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
4775
4837
|
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
4776
4838
|
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
4777
4839
|
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
4840
|
+
if (perField !== void 0) {
|
|
4841
|
+
const fieldPair = Object.hasOwn(perField, fieldName) ? perField[fieldName] : void 0;
|
|
4842
|
+
scoring.k1s[fieldIndex] = fieldPair?.k1 ?? k1;
|
|
4843
|
+
scoring.bs[fieldIndex] = fieldPair?.b ?? b;
|
|
4844
|
+
}
|
|
4778
4845
|
}
|
|
4846
|
+
if (perField !== void 0) scoring.boundPairs = differentPairsOf(scoring.searchable, scoring.k1s, scoring.bs);
|
|
4779
4847
|
return scoring;
|
|
4780
4848
|
}
|
|
4781
4849
|
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
@@ -4791,6 +4859,15 @@ function bestSearchable(searchable, values) {
|
|
|
4791
4859
|
}
|
|
4792
4860
|
return best;
|
|
4793
4861
|
}
|
|
4862
|
+
function blockEntryBound(fields, maxTermFrequency, idf, minFieldLength, maxAverageLength) {
|
|
4863
|
+
const pairs = fields.boundPairs;
|
|
4864
|
+
let best = 0;
|
|
4865
|
+
for (let at = 0; at < pairs.length; at += 2) {
|
|
4866
|
+
const bound = computeBM25WithIDF(maxTermFrequency, idf, minFieldLength, maxAverageLength, pairs[at], pairs[at + 1]);
|
|
4867
|
+
if (bound > best) best = bound;
|
|
4868
|
+
}
|
|
4869
|
+
return best;
|
|
4870
|
+
}
|
|
4794
4871
|
|
|
4795
4872
|
// src/core/partition/posting-columns.ts
|
|
4796
4873
|
function postingColumns(list) {
|
|
@@ -4884,8 +4961,9 @@ function groupPostingCount(matches) {
|
|
|
4884
4961
|
return total;
|
|
4885
4962
|
}
|
|
4886
4963
|
function computeGroupContributions(context, matches, collect) {
|
|
4887
|
-
const { fields, fieldNames, filterBitset, partialMarks
|
|
4964
|
+
const { fields, fieldNames, filterBitset, partialMarks } = context;
|
|
4888
4965
|
const scoresAreZero = context.totalDocs === 0;
|
|
4966
|
+
const onePair = fields.boundPairs.length === 2;
|
|
4889
4967
|
const best = /* @__PURE__ */ new Map();
|
|
4890
4968
|
for (const match of matches) {
|
|
4891
4969
|
const perTerm = /* @__PURE__ */ new Map();
|
|
@@ -4905,7 +4983,14 @@ function computeGroupContributions(context, matches, collect) {
|
|
|
4905
4983
|
const fieldBoost = fields.boosts[fieldIndex];
|
|
4906
4984
|
const avgLen = fields.averageLengths[fieldIndex];
|
|
4907
4985
|
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
4908
|
-
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
4986
|
+
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
4987
|
+
termFrequency,
|
|
4988
|
+
match.idf,
|
|
4989
|
+
actualFieldLength,
|
|
4990
|
+
avgLen,
|
|
4991
|
+
onePair ? fields.boundPairs[0] : fields.k1s[fieldIndex],
|
|
4992
|
+
onePair ? fields.boundPairs[1] : fields.bs[fieldIndex]
|
|
4993
|
+
)) * fieldBoost * factor;
|
|
4909
4994
|
const existing = perTerm.get(internalId);
|
|
4910
4995
|
if (existing) {
|
|
4911
4996
|
existing.score += termScore;
|
|
@@ -5469,7 +5554,7 @@ function boostsKeepScoresNonNegative(fields) {
|
|
|
5469
5554
|
}
|
|
5470
5555
|
return true;
|
|
5471
5556
|
}
|
|
5472
|
-
function
|
|
5557
|
+
function samePairUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
5473
5558
|
if (scan2.scoresAreZero) return 0;
|
|
5474
5559
|
const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
|
|
5475
5560
|
let best = 0;
|
|
@@ -5487,7 +5572,24 @@ function termUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
|
5487
5572
|
}
|
|
5488
5573
|
return best;
|
|
5489
5574
|
}
|
|
5490
|
-
function
|
|
5575
|
+
function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
5576
|
+
if (scan2.scoresAreZero) return 0;
|
|
5577
|
+
const bounds = blockBoundsFor(list, scan2.fields.lengthColumns);
|
|
5578
|
+
let best = 0;
|
|
5579
|
+
for (let block = 0; block < bounds.blockCount; block++) {
|
|
5580
|
+
const bestEntry = blockEntryBound(
|
|
5581
|
+
scan2.fields,
|
|
5582
|
+
bounds.maxTermFrequency[block],
|
|
5583
|
+
scan2.idf,
|
|
5584
|
+
bounds.minFieldLength[block],
|
|
5585
|
+
maxAverageLength
|
|
5586
|
+
);
|
|
5587
|
+
const blockBound = bestEntry * maxBoost * bounds.maxEntriesPerDocument[block];
|
|
5588
|
+
if (blockBound > best) best = blockBound;
|
|
5589
|
+
}
|
|
5590
|
+
return best;
|
|
5591
|
+
}
|
|
5592
|
+
function samePairEntryScore(scan2, entry, internalId) {
|
|
5491
5593
|
const { fields, columns } = scan2;
|
|
5492
5594
|
const fieldIndex = columns.fieldNameIndices[entry];
|
|
5493
5595
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
@@ -5495,17 +5597,55 @@ function entryScore(scan2, entry, internalId) {
|
|
|
5495
5597
|
const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan2.idf, fieldLength, averageLength, scan2.k1, scan2.b);
|
|
5496
5598
|
return bm25 * fields.boosts[fieldIndex];
|
|
5497
5599
|
}
|
|
5498
|
-
function
|
|
5600
|
+
function fieldPairEntryScore(scan2, entry, internalId) {
|
|
5601
|
+
const { fields, columns } = scan2;
|
|
5602
|
+
const fieldIndex = columns.fieldNameIndices[entry];
|
|
5603
|
+
const averageLength = fields.averageLengths[fieldIndex];
|
|
5604
|
+
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
5605
|
+
const bm25 = scan2.scoresAreZero ? 0 : computeBM25WithIDF(
|
|
5606
|
+
columns.termFrequencies[entry],
|
|
5607
|
+
scan2.idf,
|
|
5608
|
+
fieldLength,
|
|
5609
|
+
averageLength,
|
|
5610
|
+
fields.k1s[fieldIndex],
|
|
5611
|
+
fields.bs[fieldIndex]
|
|
5612
|
+
);
|
|
5613
|
+
return bm25 * fields.boosts[fieldIndex];
|
|
5614
|
+
}
|
|
5615
|
+
function samePairScoreEveryDocument(scan2, buffer) {
|
|
5499
5616
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5500
5617
|
const searchable = scan2.fields.searchable;
|
|
5501
5618
|
for (let entry = 0; entry < count; entry++) {
|
|
5502
5619
|
const internalId = docIds[entry];
|
|
5503
5620
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5504
5621
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5505
|
-
addScore(buffer, internalId,
|
|
5622
|
+
addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
|
|
5506
5623
|
}
|
|
5507
5624
|
}
|
|
5508
|
-
function
|
|
5625
|
+
function fieldPairScoreEveryDocument(scan2, buffer) {
|
|
5626
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5627
|
+
const searchable = scan2.fields.searchable;
|
|
5628
|
+
for (let entry = 0; entry < count; entry++) {
|
|
5629
|
+
const internalId = docIds[entry];
|
|
5630
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5631
|
+
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5632
|
+
addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
|
|
5633
|
+
}
|
|
5634
|
+
}
|
|
5635
|
+
function samePairScoreKnownDocuments(scan2, buffer) {
|
|
5636
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5637
|
+
const searchable = scan2.fields.searchable;
|
|
5638
|
+
let counted = 0;
|
|
5639
|
+
for (let entry = 0; entry < count; entry++) {
|
|
5640
|
+
const internalId = docIds[entry];
|
|
5641
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5642
|
+
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5643
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan2, entry, internalId));
|
|
5644
|
+
else if (markCounted(buffer, internalId)) counted++;
|
|
5645
|
+
}
|
|
5646
|
+
return counted;
|
|
5647
|
+
}
|
|
5648
|
+
function fieldPairScoreKnownDocuments(scan2, buffer) {
|
|
5509
5649
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5510
5650
|
const searchable = scan2.fields.searchable;
|
|
5511
5651
|
let counted = 0;
|
|
@@ -5513,31 +5653,58 @@ function scoreKnownDocumentsAndCountTheRest(scan2, buffer) {
|
|
|
5513
5653
|
const internalId = docIds[entry];
|
|
5514
5654
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5515
5655
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5516
|
-
if (hasScore(buffer, internalId)) addScore(buffer, internalId,
|
|
5656
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan2, entry, internalId));
|
|
5517
5657
|
else if (markCounted(buffer, internalId)) counted++;
|
|
5518
5658
|
}
|
|
5519
5659
|
return counted;
|
|
5520
5660
|
}
|
|
5521
|
-
function
|
|
5661
|
+
function firstEntryOf(docIds, count, internalId) {
|
|
5662
|
+
let low = 0;
|
|
5663
|
+
let high = count;
|
|
5664
|
+
while (low < high) {
|
|
5665
|
+
const middle = low + high >>> 1;
|
|
5666
|
+
if (docIds[middle] < internalId) low = middle + 1;
|
|
5667
|
+
else high = middle;
|
|
5668
|
+
}
|
|
5669
|
+
return low;
|
|
5670
|
+
}
|
|
5671
|
+
function samePairQueryOrderScore(scans, internalId) {
|
|
5522
5672
|
let score = 0;
|
|
5523
5673
|
for (const scan2 of scans) {
|
|
5524
5674
|
const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5525
5675
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5526
|
-
let
|
|
5527
|
-
|
|
5528
|
-
|
|
5529
|
-
const middle = low + high >>> 1;
|
|
5530
|
-
if (docIds[middle] < internalId) low = middle + 1;
|
|
5531
|
-
else high = middle;
|
|
5676
|
+
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
5677
|
+
if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
5678
|
+
score += samePairEntryScore(scan2, entry, internalId);
|
|
5532
5679
|
}
|
|
5533
|
-
|
|
5680
|
+
}
|
|
5681
|
+
return score;
|
|
5682
|
+
}
|
|
5683
|
+
function fieldPairQueryOrderScore(scans, internalId) {
|
|
5684
|
+
let score = 0;
|
|
5685
|
+
for (const scan2 of scans) {
|
|
5686
|
+
const { docIds, deletedDocs, hasDeleted, count } = scan2.columns;
|
|
5687
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5688
|
+
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
5534
5689
|
if (scan2.fields.searchable[scan2.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
5535
|
-
score +=
|
|
5690
|
+
score += fieldPairEntryScore(scan2, entry, internalId);
|
|
5536
5691
|
}
|
|
5537
5692
|
}
|
|
5538
5693
|
return score;
|
|
5539
5694
|
}
|
|
5540
|
-
|
|
5695
|
+
var SAME_PAIR_SCORER = {
|
|
5696
|
+
upperBound: samePairUpperBound,
|
|
5697
|
+
scoreEveryDocument: samePairScoreEveryDocument,
|
|
5698
|
+
scoreKnownDocumentsAndCountTheRest: samePairScoreKnownDocuments,
|
|
5699
|
+
queryOrderScore: samePairQueryOrderScore
|
|
5700
|
+
};
|
|
5701
|
+
var FIELD_PAIR_SCORER = {
|
|
5702
|
+
upperBound: fieldPairUpperBound,
|
|
5703
|
+
scoreEveryDocument: fieldPairScoreEveryDocument,
|
|
5704
|
+
scoreKnownDocumentsAndCountTheRest: fieldPairScoreKnownDocuments,
|
|
5705
|
+
queryOrderScore: fieldPairQueryOrderScore
|
|
5706
|
+
};
|
|
5707
|
+
function pageFromBuffer(scorer, scans, buffer, selection, resolver) {
|
|
5541
5708
|
const approximateKth = kthBestScore(buffer, selection, resolver);
|
|
5542
5709
|
const gatherFrom = approximateKth - Math.abs(approximateKth) * PRUNING_REJECTION_SLACK;
|
|
5543
5710
|
const heap = [];
|
|
@@ -5549,7 +5716,7 @@ function pageFromBuffer(scans, buffer, selection, resolver) {
|
|
|
5549
5716
|
if (scores[internalId] < gatherFrom) continue;
|
|
5550
5717
|
const externalId = resolver.toExternal(internalId);
|
|
5551
5718
|
if (externalId === void 0) continue;
|
|
5552
|
-
const score = queryOrderScore(scans, internalId);
|
|
5719
|
+
const score = scorer.queryOrderScore(scans, internalId);
|
|
5553
5720
|
if (full && score < threshold) continue;
|
|
5554
5721
|
const candidate = { internalId, externalId, score };
|
|
5555
5722
|
if (!full) {
|
|
@@ -5579,18 +5746,28 @@ function pageFromBuffer(scans, buffer, selection, resolver) {
|
|
|
5579
5746
|
return scored;
|
|
5580
5747
|
}
|
|
5581
5748
|
function multiTermTopK(request) {
|
|
5582
|
-
const { terms, docFrequencies, totalDocs,
|
|
5749
|
+
const { terms, docFrequencies, totalDocs, limit, fields, resolver, buffer } = request;
|
|
5583
5750
|
if (!boostsKeepScoresNonNegative(fields)) return null;
|
|
5584
|
-
const
|
|
5751
|
+
const onePair = fields.boundPairs.length === 2;
|
|
5752
|
+
const scorer = onePair ? SAME_PAIR_SCORER : FIELD_PAIR_SCORER;
|
|
5753
|
+
const k1 = onePair ? fields.boundPairs[0] : 0;
|
|
5754
|
+
const b = onePair ? fields.boundPairs[1] : 0;
|
|
5585
5755
|
const scans = new Array(terms.length);
|
|
5586
5756
|
for (let term = 0; term < terms.length; term++) {
|
|
5587
5757
|
const idf = computeIDF(docFrequencies[term], totalDocs);
|
|
5588
5758
|
if (!(idf >= 0)) return null;
|
|
5589
|
-
scans[term] = {
|
|
5759
|
+
scans[term] = {
|
|
5760
|
+
columns: postingColumns(terms[term].list),
|
|
5761
|
+
idf,
|
|
5762
|
+
scoresAreZero: totalDocs === 0,
|
|
5763
|
+
fields,
|
|
5764
|
+
k1,
|
|
5765
|
+
b
|
|
5766
|
+
};
|
|
5590
5767
|
}
|
|
5591
5768
|
const maxBoost = bestSearchable(fields.searchable, fields.boosts);
|
|
5592
5769
|
const maxAverageLength = bestSearchable(fields.searchable, fields.averageLengths);
|
|
5593
|
-
const upperBounds = scans.map((scan2, term) =>
|
|
5770
|
+
const upperBounds = scans.map((scan2, term) => scorer.upperBound(scan2, terms[term].list, maxBoost, maxAverageLength));
|
|
5594
5771
|
const order = scans.map((_, term) => term).sort((left, right) => upperBounds[right] - upperBounds[left]);
|
|
5595
5772
|
const remainingBound = new Float64Array(order.length + 1);
|
|
5596
5773
|
for (let position2 = order.length - 1; position2 >= 0; position2--) {
|
|
@@ -5605,15 +5782,15 @@ function multiTermTopK(request) {
|
|
|
5605
5782
|
const threshold = kthBestScore(buffer, selection, resolver);
|
|
5606
5783
|
if (remaining < threshold - threshold * PRUNING_REJECTION_SLACK) break;
|
|
5607
5784
|
}
|
|
5608
|
-
scoreEveryDocument(scans[order[position]], buffer);
|
|
5785
|
+
scorer.scoreEveryDocument(scans[order[position]], buffer);
|
|
5609
5786
|
}
|
|
5610
5787
|
let countedOnly = 0;
|
|
5611
5788
|
for (let rest = position; rest < order.length; rest++) {
|
|
5612
|
-
countedOnly += scoreKnownDocumentsAndCountTheRest(scans[order[rest]], buffer);
|
|
5789
|
+
countedOnly += scorer.scoreKnownDocumentsAndCountTheRest(scans[order[rest]], buffer);
|
|
5613
5790
|
}
|
|
5614
5791
|
const totalMatched = buffer.touchedCount + countedOnly;
|
|
5615
5792
|
if (wanted === 0) return { scored: [], totalMatched };
|
|
5616
|
-
return { scored: pageFromBuffer(scans, buffer, selection, resolver), totalMatched };
|
|
5793
|
+
return { scored: pageFromBuffer(scorer, scans, buffer, selection, resolver), totalMatched };
|
|
5617
5794
|
}
|
|
5618
5795
|
|
|
5619
5796
|
// src/core/partition/single-term-topk.ts
|
|
@@ -5641,20 +5818,25 @@ function prunableSingleTermList(params, index, fieldNames) {
|
|
|
5641
5818
|
return list;
|
|
5642
5819
|
}
|
|
5643
5820
|
function singleTermTopK(request) {
|
|
5644
|
-
const { list, docFrequency, totalDocs,
|
|
5821
|
+
const { list, docFrequency, totalDocs, limit, resolver } = request;
|
|
5645
5822
|
const {
|
|
5646
5823
|
searchable: fieldSearchable,
|
|
5647
5824
|
boosts: fieldBoosts,
|
|
5648
5825
|
averageLengths: fieldAvgLengths,
|
|
5649
|
-
lengthColumns: fieldLengthColumns
|
|
5826
|
+
lengthColumns: fieldLengthColumns,
|
|
5827
|
+
k1s,
|
|
5828
|
+
bs,
|
|
5829
|
+
boundPairs
|
|
5650
5830
|
} = request.fields;
|
|
5831
|
+
const onePair = boundPairs.length === 2;
|
|
5832
|
+
const onlyK1 = boundPairs[0];
|
|
5833
|
+
const onlyB = boundPairs[1];
|
|
5651
5834
|
const wanted = Number.isFinite(limit) ? Math.max(0, Math.floor(limit)) : 0;
|
|
5652
5835
|
const { docIds, termFrequencies, fieldNameIndices, deletedDocs: deleted, hasDeleted } = postingColumns(list);
|
|
5653
5836
|
const bounds = blockBoundsFor(list, fieldLengthColumns);
|
|
5654
5837
|
const maxBoost = bestSearchable(fieldSearchable, fieldBoosts);
|
|
5655
5838
|
const maxAverageLength = bestSearchable(fieldSearchable, fieldAvgLengths);
|
|
5656
5839
|
const idf = computeIDF(docFrequency, totalDocs);
|
|
5657
|
-
const { k1, b } = resolveBM25Params(bm25Params);
|
|
5658
5840
|
const scoresAreZero = totalDocs === 0;
|
|
5659
5841
|
const heap = [];
|
|
5660
5842
|
let full = false;
|
|
@@ -5664,14 +5846,12 @@ function singleTermTopK(request) {
|
|
|
5664
5846
|
const start = block === 0 ? 0 : bounds.entryEnd[block - 1];
|
|
5665
5847
|
const end = bounds.entryEnd[block];
|
|
5666
5848
|
if (full) {
|
|
5667
|
-
|
|
5668
|
-
|
|
5669
|
-
|
|
5670
|
-
|
|
5671
|
-
|
|
5672
|
-
|
|
5673
|
-
bm25Params
|
|
5674
|
-
);
|
|
5849
|
+
let bestEntry = 0;
|
|
5850
|
+
if (!scoresAreZero) {
|
|
5851
|
+
const blockMaxTf = bounds.maxTermFrequency[block];
|
|
5852
|
+
const blockMinLength = bounds.minFieldLength[block];
|
|
5853
|
+
bestEntry = onePair ? computeBM25WithIDF(blockMaxTf, idf, blockMinLength, maxAverageLength, onlyK1, onlyB) : blockEntryBound(request.fields, blockMaxTf, idf, blockMinLength, maxAverageLength);
|
|
5854
|
+
}
|
|
5675
5855
|
if (bestEntry * maxBoost * bounds.maxEntriesPerDocument[block] < threshold) {
|
|
5676
5856
|
totalMatched += bounds.documentCount[block];
|
|
5677
5857
|
continue;
|
|
@@ -5690,7 +5870,14 @@ function singleTermTopK(request) {
|
|
|
5690
5870
|
const fieldIndex = fieldNameIndices[entry];
|
|
5691
5871
|
if (fieldSearchable[fieldIndex] === 1) {
|
|
5692
5872
|
const fieldLength = fieldLengthOf(fieldLengthColumns, fieldIndex, internalId, fieldAvgLengths[fieldIndex]);
|
|
5693
|
-
score += (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
5873
|
+
score += (scoresAreZero ? 0 : computeBM25WithIDF(
|
|
5874
|
+
termFrequencies[entry],
|
|
5875
|
+
idf,
|
|
5876
|
+
fieldLength,
|
|
5877
|
+
fieldAvgLengths[fieldIndex],
|
|
5878
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
5879
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
5880
|
+
)) * fieldBoosts[fieldIndex];
|
|
5694
5881
|
scored2 = true;
|
|
5695
5882
|
}
|
|
5696
5883
|
entry++;
|
|
@@ -5758,7 +5945,6 @@ function searchFulltext(state, params) {
|
|
|
5758
5945
|
}
|
|
5759
5946
|
const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
|
|
5760
5947
|
const avgFieldLengths = globalStats?.averageFieldLengths ?? state.stats.averageFieldLengths;
|
|
5761
|
-
const { k1, b } = resolveBM25Params(bm25Params);
|
|
5762
5948
|
const scoresAreZero = totalDocs === 0;
|
|
5763
5949
|
if (state.scoreBuffer === null) state.scoreBuffer = createScoreBuffer(state.docStore.internalIdCapacity());
|
|
5764
5950
|
const scoreBuffer = state.scoreBuffer;
|
|
@@ -5767,8 +5953,11 @@ function searchFulltext(state, params) {
|
|
|
5767
5953
|
const useIntersection = termMatch === "all" && queryTokens.length > 1;
|
|
5768
5954
|
const fieldNames = state.fieldNameTable.names;
|
|
5769
5955
|
const resolver = state.docStore.resolver();
|
|
5770
|
-
const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths);
|
|
5771
|
-
const { searchable, boosts, averageLengths, lengthColumns } = fieldScoring;
|
|
5956
|
+
const fieldScoring = loadFieldScoring(state, fields, boost, avgFieldLengths, bm25Params);
|
|
5957
|
+
const { searchable, boosts, averageLengths, lengthColumns, k1s, bs, boundPairs } = fieldScoring;
|
|
5958
|
+
const onePair = boundPairs.length === 2;
|
|
5959
|
+
const onlyK1 = boundPairs[0];
|
|
5960
|
+
const onlyB = boundPairs[1];
|
|
5772
5961
|
function documentFrequency(token, list) {
|
|
5773
5962
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
5774
5963
|
}
|
|
@@ -5778,8 +5967,6 @@ function searchFulltext(state, params) {
|
|
|
5778
5967
|
partialMarks: partialFieldMarks(fieldNames, partialFields),
|
|
5779
5968
|
fieldNames,
|
|
5780
5969
|
totalDocs,
|
|
5781
|
-
k1,
|
|
5782
|
-
b,
|
|
5783
5970
|
filterBitset,
|
|
5784
5971
|
globalFrequency(token) {
|
|
5785
5972
|
if (globalStats === void 0 || !Object.hasOwn(globalStats.docFrequencies, token)) return void 0;
|
|
@@ -5805,7 +5992,14 @@ function searchFulltext(state, params) {
|
|
|
5805
5992
|
if (searchable[fieldIndex] === 0) continue;
|
|
5806
5993
|
const avgLen = averageLengths[fieldIndex];
|
|
5807
5994
|
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
5808
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
5995
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
5996
|
+
termFrequencies[pi],
|
|
5997
|
+
match.idf,
|
|
5998
|
+
actualFieldLength,
|
|
5999
|
+
avgLen,
|
|
6000
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
6001
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
6002
|
+
);
|
|
5809
6003
|
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
5810
6004
|
}
|
|
5811
6005
|
return;
|
|
@@ -5820,7 +6014,14 @@ function searchFulltext(state, params) {
|
|
|
5820
6014
|
const termFrequency = termFrequencies[pi];
|
|
5821
6015
|
const avgLen = averageLengths[fieldIndex];
|
|
5822
6016
|
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
5823
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
6017
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
6018
|
+
termFrequency,
|
|
6019
|
+
match.idf,
|
|
6020
|
+
actualFieldLength,
|
|
6021
|
+
avgLen,
|
|
6022
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
6023
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
6024
|
+
);
|
|
5824
6025
|
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
5825
6026
|
if (components !== null) {
|
|
5826
6027
|
recordComponents(
|
|
@@ -5848,7 +6049,6 @@ function searchFulltext(state, params) {
|
|
|
5848
6049
|
list: prunableList,
|
|
5849
6050
|
docFrequency: documentFrequency(queryTokens[0].token, prunableList),
|
|
5850
6051
|
totalDocs,
|
|
5851
|
-
bm25Params,
|
|
5852
6052
|
limit: maxResults,
|
|
5853
6053
|
fields: fieldScoring,
|
|
5854
6054
|
resolver
|
|
@@ -5860,7 +6060,6 @@ function searchFulltext(state, params) {
|
|
|
5860
6060
|
terms: prunableLists,
|
|
5861
6061
|
docFrequencies: prunableLists.map((term) => documentFrequency(term.token, term.list)),
|
|
5862
6062
|
totalDocs,
|
|
5863
|
-
bm25Params,
|
|
5864
6063
|
limit: maxResults,
|
|
5865
6064
|
fields: fieldScoring,
|
|
5866
6065
|
resolver,
|
|
@@ -5881,7 +6080,7 @@ function searchFulltext(state, params) {
|
|
|
5881
6080
|
}
|
|
5882
6081
|
resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
|
|
5883
6082
|
}
|
|
5884
|
-
resolved.sort((a,
|
|
6083
|
+
resolved.sort((a, b) => a.totalPostings - b.totalPostings);
|
|
5885
6084
|
for (let tokenIndex = 0; tokenIndex < resolved.length; tokenIndex++) {
|
|
5886
6085
|
const onlyScored = tokenIndex > 0;
|
|
5887
6086
|
for (const match of resolved[tokenIndex].matches) scorePostings(match, onlyScored);
|
|
@@ -8634,4 +8833,4 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
8634
8833
|
return count;
|
|
8635
8834
|
}
|
|
8636
8835
|
|
|
8637
|
-
export { FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FNV_OFFSET_BASIS, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, applyPagination, applyPartitionFiltersBitset, applyProjection, applySorting, availablePatternSearch, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUint32s, fulltextMatches, isCompositePartition, joinPatternWorkMeter, kWayMerge, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|
|
8836
|
+
export { DEFAULT_BM25_B, DEFAULT_BM25_K1, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FNV_OFFSET_BASIS, MAX_BM25_B, MAX_BM25_K1, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, applyPagination, applyPartitionFiltersBitset, applyProjection, applySorting, availablePatternSearch, bm25SetsValue, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUint32s, fulltextMatches, isCompositePartition, joinPatternWorkMeter, kWayMerge, mergeBM25Levels, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|