@delali/narsil 0.3.1-next.129 → 0.3.1-next.130
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/browser/chunk-26A5JFT2.mjs +89 -0
- package/dist/browser/{chunk-5D6NJD6C.mjs → chunk-2Y3CO4EQ.mjs} +342 -201
- package/dist/browser/{chunk-WLD6DDIX.mjs → chunk-AIIJDWMG.mjs} +1 -1
- package/dist/browser/{chunk-YAGXZU4W.mjs → chunk-D7WQF6DV.mjs} +2 -2
- package/dist/browser/{chunk-WZBTVTSU.mjs → chunk-M5UAAXGU.mjs} +2 -2
- package/dist/browser/{chunk-XTFJZHQP.mjs → chunk-NSCSR6MA.mjs} +15 -5
- package/dist/browser/{chunk-3NVZD52E.mjs → chunk-UDGB2MWT.mjs} +5 -28
- package/dist/browser/{chunk-II5POT55.mjs → chunk-VBBLTD7O.mjs} +4 -3
- package/dist/browser/chunk-VEGHFP2P.mjs +30 -0
- package/dist/browser/{durability-wiring-HCCC6BJB.mjs → durability-wiring-FNZSRH4N.mjs} +8 -6
- package/dist/browser/{module-S2VDU2JE.mjs → module-4HIMOQM5.mjs} +6 -5
- package/dist/browser/phrase-module-WESYVU7K.mjs +327 -0
- package/dist/browser/{restore-queue-BZ7WWEC4.mjs → restore-queue-GEZ54CA5.mjs} +1 -1
- package/dist/browser/{runtime-PL4TTFJE.mjs → runtime-3X7ZJ5T3.mjs} +2 -1
- package/dist/browser/{snapshot-RTUCHI2L.mjs → snapshot-V7VF6BN6.mjs} +25 -10
- package/dist/browser/{write-IMU7G5NU.mjs → write-YIC2MKRY.mjs} +7 -5
- package/dist/{capabilities-BTjv-tK8.d.ts → capabilities-Bl8G9taj.d.ts} +2 -2
- package/dist/{chunk-3XHFVYIQ.mjs → chunk-2C3WXBQS.mjs} +1 -1
- package/dist/{chunk-37RTD6KD.mjs → chunk-2JWNJSNE.mjs} +8 -8
- package/dist/{chunk-K76F5YYX.mjs → chunk-2MKX2SCO.mjs} +219 -58
- package/dist/{chunk-AN3CUMPZ.mjs → chunk-2SEL6SBS.mjs} +4 -95
- package/dist/{chunk-OPO5SE4J.mjs → chunk-2WBY4Y4M.mjs} +1 -1
- package/dist/{chunk-PTIJAZWL.mjs → chunk-4GDLFDPZ.mjs} +1 -1
- package/dist/chunk-4UMZ7KQL.mjs +17 -0
- package/dist/{chunk-TY64A3ZW.mjs → chunk-5LX6MOIB.mjs} +2 -2
- package/dist/{chunk-PCRWCOZC.mjs → chunk-6M6SDH3S.mjs} +43 -23
- package/dist/{chunk-MGD7XETB.mjs → chunk-6SSVIW7Y.mjs} +19 -3
- package/dist/{chunk-5LC5VMDQ.mjs → chunk-7OIQ2SIO.mjs} +4 -4
- package/dist/{chunk-4XEVROSP.mjs → chunk-7XY6VD7B.mjs} +1 -1
- package/dist/{chunk-WI5Q3L43.mjs → chunk-CZ7ISLXA.mjs} +2 -0
- package/dist/{chunk-RC27KF7F.mjs → chunk-D5TFRB5G.mjs} +3 -15
- package/dist/{chunk-F4UKMTKM.mjs → chunk-EUCX5K74.mjs} +4 -4
- package/dist/{chunk-CTF6WFTX.mjs → chunk-F6AAXI7T.mjs} +3 -3
- package/dist/{chunk-DX2FI4LU.mjs → chunk-FQBFZAWG.mjs} +3 -1
- package/dist/{chunk-XTER67YM.mjs → chunk-HCGJKDZV.mjs} +1 -1
- package/dist/{chunk-WQYB4TNR.mjs → chunk-JJ6EUMXB.mjs} +2 -2
- package/dist/{chunk-2HQQ7Y5D.mjs → chunk-JT3YZGCE.mjs} +2 -2
- package/dist/chunk-JUKYYXSP.mjs +93 -0
- package/dist/{chunk-WUIDPBSY.mjs → chunk-K77FJXKS.mjs} +17 -7
- package/dist/{chunk-CSSCLRGY.mjs → chunk-NMXRXFGZ.mjs} +2 -2
- package/dist/{chunk-44TB6AIZ.mjs → chunk-NNO7CO3A.mjs} +1 -1
- package/dist/{chunk-FRSWYMXH.mjs → chunk-PATMDIMN.mjs} +1 -1
- package/dist/{chunk-AFOZR4ZQ.mjs → chunk-PU6SKH7O.mjs} +2 -2
- package/dist/{chunk-EZIGAOUJ.mjs → chunk-QQTXT65X.mjs} +6 -5
- package/dist/{chunk-Y4NOHICF.mjs → chunk-RNSC4NFI.mjs} +1 -1
- package/dist/{chunk-RYH3PASD.mjs → chunk-RWESFSJI.mjs} +21 -11
- package/dist/{chunk-QQS4QZQA.mjs → chunk-SOHMMNCH.mjs} +4 -4
- package/dist/{chunk-K73PZTBR.mjs → chunk-SZGRYEIC.mjs} +4 -4
- package/dist/{chunk-43Q7HAI2.mjs → chunk-T4OFG4IQ.mjs} +4 -4
- package/dist/{chunk-2YOAGAGF.mjs → chunk-TGHNXN7W.mjs} +59 -30
- package/dist/{chunk-J4LRE5Q7.mjs → chunk-WVRDRLIC.mjs} +6 -6
- package/dist/{chunk-XRXBCOA4.mjs → chunk-X5CNUKD5.mjs} +2 -2
- package/dist/{chunk-TB7YMDD5.mjs → chunk-XABMCBHA.mjs} +6 -4
- package/dist/{chunk-Y4US5EGH.mjs → chunk-Y2U5VVWX.mjs} +1 -1
- package/dist/{chunk-5VSTAJET.mjs → chunk-Y4BVTQ53.mjs} +2 -2
- package/dist/{chunk-3LJVM6XM.mjs → chunk-YBWSV5DV.mjs} +2 -2
- package/dist/{chunk-M6MBSVIJ.mjs → chunk-ZR4VM4A6.mjs} +1 -1
- package/dist/{client-BOcveV7A.d.ts → client-Cx-TiqPE.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +2 -0
- package/dist/{config-CPmIY3aH.d.ts → config-CzB9YvF_.d.ts} +2 -2
- package/dist/{config-C3YXk5Rn.d.ts → config-hOWGBeC6.d.ts} +7 -5
- package/dist/direct-executor-HQ4QVDX7.mjs +28 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +85 -45
- package/dist/{durability-wiring-6R26AJDI.mjs → durability-wiring-S7OX2MAI.mjs} +27 -25
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-JjRAXpT3.d.ts → engine-B_W9sdZK.d.ts} +2 -2
- package/dist/{events-pM0lna7M.d.ts → events-DErop1LM.d.ts} +12 -2
- package/dist/{index-DkQ2DlgR.d.ts → index-CKI1fDEl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +20 -18
- package/dist/{long-query-thread-6VLA6YJI.mjs → long-query-thread-WVSMIBYP.mjs} +17 -15
- package/dist/{module-S4WGMFNW.mjs → module-2E4XYSGT.mjs} +11 -11
- package/dist/partial-words.mjs +7 -6
- package/dist/pattern.mjs +4 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -21
- package/dist/phrase-module-JERF2IDG.mjs +329 -0
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +2 -0
- package/dist/{rerank-V4J4MIWR.mjs → rerank-DMMT4FZA.mjs} +6 -6
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{restore-queue-PUGWCPCX.mjs → restore-queue-JAMNRC6T.mjs} +1 -1
- package/dist/{runtime-TBFBQQQO.mjs → runtime-7MLQ7J64.mjs} +7 -7
- package/dist/{schema-DGAP_8HJ.d.ts → schema-CS-clLun.d.ts} +4 -4
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +15 -15
- package/dist/{snapshot-PUVSMV7E.mjs → snapshot-DOQLGGEX.mjs} +44 -29
- package/dist/synonyms.mjs +4 -3
- package/dist/{tasks-txGvNz6h.d.ts → tasks-BfF-VRhm.d.ts} +2 -2
- package/dist/{thread-TA7QQBWP.mjs → thread-DNK5XWXG.mjs} +16 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-TGKYT3C4.mjs → write-ZKR7SNRT.mjs} +22 -20
- package/package.json +7 -7
- package/dist/direct-executor-ULA6V5HC.mjs +0 -26
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
2
|
-
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-
|
|
3
|
-
import
|
|
4
|
-
import './chunk-
|
|
1
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-M5UAAXGU.mjs';
|
|
2
|
+
import { decodeStoredDocument, openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-2Y3CO4EQ.mjs';
|
|
3
|
+
import './chunk-26A5JFT2.mjs';
|
|
4
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-D7WQF6DV.mjs';
|
|
5
|
+
import './chunk-AIIJDWMG.mjs';
|
|
5
6
|
import './chunk-JPTQZWJQ.mjs';
|
|
6
|
-
import { EXPORT_CHUNK_BYTES
|
|
7
|
+
import { EXPORT_CHUNK_BYTES } from './chunk-VEGHFP2P.mjs';
|
|
8
|
+
import { NarsilError, ErrorCodes } from './chunk-UDGB2MWT.mjs';
|
|
7
9
|
import { decode } from '@msgpack/msgpack';
|
|
8
10
|
|
|
9
11
|
// src/engine/export/json-line.ts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-BQFQ4PRz.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { h as TaskStore } from './tasks-
|
|
2
|
+
import { C as ClusterNamespace } from './config-CzB9YvF_.js';
|
|
3
|
+
import { h as TaskStore } from './tasks-BfF-VRhm.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-
|
|
1
|
+
import { KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment, restoreVectorFields, KEYWORD_SEGMENT_PAYLOAD_VERSION, littleEndianBytes, uint64Bytes } from './chunk-7OIQ2SIO.mjs';
|
|
2
2
|
import { WIRE_BATCH_BUDGET } from './chunk-P36MUFEJ.mjs';
|
|
3
3
|
import { ReplicationMessageTypes } from './chunk-YH5OIL7B.mjs';
|
|
4
|
-
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-
|
|
5
|
-
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-
|
|
6
|
-
import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-
|
|
7
|
-
import { generateId, getLanguage } from './chunk-
|
|
8
|
-
import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-
|
|
4
|
+
import { packChecksummedEnvelope, crc32, openChecksummedEnvelope, reconstructSchemaFromMetadata, HEADER_SIZE, prepareDocumentVectors } from './chunk-TGHNXN7W.mjs';
|
|
5
|
+
import { freezeSegmentShared, createSharedFrozenSegment, createFrozenSegment, createPartitionManager, createPartitionRouter, buildSegmentPayload, resolvePartitionInsertOptions, selectionKeeps, decodeMapWithoutFields, shareSegmentArrays, segmentArraysOf, mergeSegmentContents } from './chunk-RWESFSJI.mjs';
|
|
6
|
+
import { fnv1a, isCompositePartition, survivorDocumentsOf, partitionStateOf } from './chunk-2MKX2SCO.mjs';
|
|
7
|
+
import { generateId, getLanguage } from './chunk-ZR4VM4A6.mjs';
|
|
8
|
+
import { extractVectorFieldsFromSchema, partialFieldsOf, messagePackUpperBoundBytes } from './chunk-PU6SKH7O.mjs';
|
|
9
9
|
import { MAX_FILE_BYTES } from './chunk-U476IXKQ.mjs';
|
|
10
|
-
import { resolveIndexAnalysis } from './chunk-
|
|
11
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
10
|
+
import { resolveIndexAnalysis } from './chunk-6SSVIW7Y.mjs';
|
|
11
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
12
12
|
import { encode, decode } from '@msgpack/msgpack';
|
|
13
13
|
|
|
14
14
|
// src/persistence/durability/constants.ts
|
|
@@ -1,15 +1,17 @@
|
|
|
1
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP,
|
|
1
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, TYPO_CLAIM_SLOTS_PER_WORD, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-2SEL6SBS.mjs';
|
|
2
2
|
import { GEO_UNIT_METRES, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
3
|
+
import { bitsetHas, createBitSet, bitsetSet, bitsetIsEmpty, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet } from './chunk-JUKYYXSP.mjs';
|
|
4
|
+
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-EUCX5K74.mjs';
|
|
5
|
+
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-PU6SKH7O.mjs';
|
|
6
|
+
import { MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PREFIX_MAX_EXPANSIONS } from './chunk-FQBFZAWG.mjs';
|
|
6
7
|
import { fuzzyPrefixOf, boundedLevenshtein, fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-
|
|
8
|
-
import { tokenize,
|
|
8
|
+
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-7XY6VD7B.mjs';
|
|
9
|
+
import { tokenize, visitTokens } from './chunk-D5TFRB5G.mjs';
|
|
10
|
+
import { SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, TARGET_BLOCK_ENTRIES, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-4UMZ7KQL.mjs';
|
|
9
11
|
import { singleFoldTable, appendFoldedCodePoint, sortModeOf, defaultSortMode, SORT_VALUE_MAX_CODE_POINTS, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, isSortMode, SORT_MODES, compareSortValues, compareComparableValues, truncateSortString } from './chunk-R6A773WK.mjs';
|
|
10
12
|
import { compareCodePoints, exceedsCodePoints, codePointCount, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
13
|
import { isGeopointOrVectorType, PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
14
|
+
import { NarsilError, ErrorCodes } from './chunk-CZ7ISLXA.mjs';
|
|
13
15
|
|
|
14
16
|
// src/core/partition/partial-words/platform.ts
|
|
15
17
|
function platformPartialWordSearch() {
|
|
@@ -165,6 +167,17 @@ function requirePatternSearch(fieldPath) {
|
|
|
165
167
|
);
|
|
166
168
|
}
|
|
167
169
|
|
|
170
|
+
// src/core/constants.ts
|
|
171
|
+
var POSTING_LIST_COMPACTION_THRESHOLD = 0.3;
|
|
172
|
+
var MAX_TERM_FREQUENCY = 65535;
|
|
173
|
+
var ARRAY_POSITION_GAP = 100;
|
|
174
|
+
var DEFAULT_BM25_K1 = 1.2;
|
|
175
|
+
var DEFAULT_BM25_B = 0.75;
|
|
176
|
+
var MAX_BM25_K1 = 1e3;
|
|
177
|
+
var MAX_BM25_B = 1;
|
|
178
|
+
var PENDING_REMOVAL_MERGE_FLOOR = 64;
|
|
179
|
+
var PENDING_REMOVAL_MERGE_DIVISOR = 256;
|
|
180
|
+
|
|
168
181
|
// src/core/projection.ts
|
|
169
182
|
function requireFieldPathList(paths, option) {
|
|
170
183
|
if (paths === void 0) return;
|
|
@@ -594,15 +607,64 @@ function createDocumentStore() {
|
|
|
594
607
|
};
|
|
595
608
|
}
|
|
596
609
|
|
|
597
|
-
// src/
|
|
598
|
-
var
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
610
|
+
// src/search/quotes.ts
|
|
611
|
+
var QUOTE_CHARACTER = /["“”„]/;
|
|
612
|
+
function holdsQuoteCharacter(term) {
|
|
613
|
+
return QUOTE_CHARACTER.test(term);
|
|
614
|
+
}
|
|
615
|
+
function holdsQuotedText(term) {
|
|
616
|
+
if (!QUOTE_CHARACTER.test(term)) return false;
|
|
617
|
+
const parts = term.split(QUOTE_CHARACTER);
|
|
618
|
+
for (let index = 1; index < parts.length; index += 2) {
|
|
619
|
+
if (parts[index].trim().length > 0) return true;
|
|
620
|
+
}
|
|
621
|
+
return false;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
// src/search/phrase-code.ts
|
|
625
|
+
var loaded = null;
|
|
626
|
+
var loading = null;
|
|
627
|
+
function requireLoadedPhraseCode() {
|
|
628
|
+
if (loaded === null) {
|
|
629
|
+
throw new NarsilError(ErrorCodes.SEARCH_PHRASE_NOT_READY, "The phrase code has not loaded. Try the query again");
|
|
630
|
+
}
|
|
631
|
+
return loaded;
|
|
632
|
+
}
|
|
633
|
+
function loadPhraseCode() {
|
|
634
|
+
if (loading === null) {
|
|
635
|
+
loading = import('./phrase-module-JERF2IDG.mjs').then(
|
|
636
|
+
(code) => {
|
|
637
|
+
loaded = code;
|
|
638
|
+
return code;
|
|
639
|
+
},
|
|
640
|
+
(err) => {
|
|
641
|
+
loading = null;
|
|
642
|
+
throw new NarsilError(
|
|
643
|
+
ErrorCodes.SEARCH_PHRASE_NOT_READY,
|
|
644
|
+
"The phrase code failed to load. Remove the quotes, or try again once it can load",
|
|
645
|
+
{ cause: err instanceof Error ? err.message : String(err) }
|
|
646
|
+
);
|
|
647
|
+
}
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
return loading;
|
|
651
|
+
}
|
|
652
|
+
function loadPhraseCodeFor(term) {
|
|
653
|
+
if (loaded !== null || term === void 0 || !holdsQuoteCharacter(term)) return null;
|
|
654
|
+
return loadPhraseCode();
|
|
655
|
+
}
|
|
656
|
+
function analyseQueryPhrases(term, language, analysis) {
|
|
657
|
+
if (!holdsQuoteCharacter(term)) return null;
|
|
658
|
+
return requireLoadedPhraseCode().analyseQueryPhrases(
|
|
659
|
+
term.split(QUOTE_CHARACTER),
|
|
660
|
+
(text) => tokenize(text, language, {
|
|
661
|
+
stem: true,
|
|
662
|
+
removeStopWords: true,
|
|
663
|
+
customTokenizer: analysis.customTokenizer,
|
|
664
|
+
stopWordOverride: analysis.stopWords
|
|
665
|
+
}).tokens
|
|
666
|
+
);
|
|
667
|
+
}
|
|
606
668
|
|
|
607
669
|
// src/core/inverted-index/constants.ts
|
|
608
670
|
var TERM_TABLE_START_CAPACITY = 1 << 10;
|
|
@@ -1034,6 +1096,13 @@ var LivePostingView = class {
|
|
|
1034
1096
|
const starts = this.store.pool.positionStarts;
|
|
1035
1097
|
return starts === null ? 0 : this.store.positions.countAt(starts[this.syncedStart + row]);
|
|
1036
1098
|
}
|
|
1099
|
+
get positionStore() {
|
|
1100
|
+
return this.holdsPositions ? this.store.positions.values : null;
|
|
1101
|
+
}
|
|
1102
|
+
positionStartAt(row) {
|
|
1103
|
+
const starts = this.store.pool.positionStarts;
|
|
1104
|
+
return starts === null ? 0 : starts[this.syncedStart + row] + 1;
|
|
1105
|
+
}
|
|
1037
1106
|
copyPositionsAt(row, target, offset) {
|
|
1038
1107
|
if (!this.holdsPositions) return 0;
|
|
1039
1108
|
const starts = this.store.pool.positionStarts;
|
|
@@ -3026,7 +3095,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
3026
3095
|
if (state.trackPositions) {
|
|
3027
3096
|
const tokenFreqs = /* @__PURE__ */ new Map();
|
|
3028
3097
|
let positionOffset = 0;
|
|
3029
|
-
let
|
|
3098
|
+
let lastItemPosition = -1;
|
|
3030
3099
|
const addToken = (token, position, surface) => {
|
|
3031
3100
|
const existing = tokenFreqs.get(token);
|
|
3032
3101
|
if (existing) {
|
|
@@ -3037,12 +3106,12 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
3037
3106
|
}
|
|
3038
3107
|
fieldTokenList.push(token);
|
|
3039
3108
|
countOccurrence(surfaceCounts, spellings, surface, token);
|
|
3040
|
-
|
|
3109
|
+
lastItemPosition = position;
|
|
3041
3110
|
};
|
|
3042
3111
|
for (const item of arr) {
|
|
3043
|
-
|
|
3112
|
+
lastItemPosition = -1;
|
|
3044
3113
|
visitTokens(item, language, opts, addToken);
|
|
3045
|
-
positionOffset +=
|
|
3114
|
+
if (lastItemPosition >= 0) positionOffset += lastItemPosition + 1 + ARRAY_POSITION_GAP;
|
|
3046
3115
|
}
|
|
3047
3116
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
3048
3117
|
for (const [token, freq] of tokenFreqs) {
|
|
@@ -5047,11 +5116,20 @@ function searchedFieldMarks(fieldNames, fields) {
|
|
|
5047
5116
|
function searchFulltextMatches(state, params) {
|
|
5048
5117
|
const { queryTokens, termGroups, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
|
|
5049
5118
|
const fieldNames = state.fieldNameTable.names;
|
|
5050
|
-
const matched = createBitSet(state.docStore.internalIdCapacity());
|
|
5051
5119
|
const searched = searchedFieldMarks(fieldNames, fields);
|
|
5052
5120
|
const partial = partialFieldMarks(fieldNames, params.partialFields);
|
|
5053
5121
|
const searchedPartial = searched.map((mark, fieldIndex) => mark & partial[fieldIndex]);
|
|
5054
|
-
|
|
5122
|
+
const capacity = state.docStore.internalIdCapacity();
|
|
5123
|
+
const matched = params.phrases === void 0 ? createBitSet(capacity) : requireLoadedPhraseCode().matchPhrases(
|
|
5124
|
+
state.invertedIdx,
|
|
5125
|
+
params.phrases,
|
|
5126
|
+
searched,
|
|
5127
|
+
filterBitset,
|
|
5128
|
+
capacity,
|
|
5129
|
+
params.phraseWork,
|
|
5130
|
+
false
|
|
5131
|
+
).matched;
|
|
5132
|
+
for (const qt of params.phrases === void 0 ? queryTokens : []) {
|
|
5055
5133
|
const group = termGroupOf(termGroups, qt.token);
|
|
5056
5134
|
for (const match of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
5057
5135
|
markPostingList(matched, match.postingList, searched, filterBitset);
|
|
@@ -5937,10 +6015,10 @@ function searchFulltext(state, params) {
|
|
|
5937
6015
|
globalStats,
|
|
5938
6016
|
maxResults,
|
|
5939
6017
|
termMatch,
|
|
5940
|
-
|
|
6018
|
+
phrases
|
|
5941
6019
|
} = params;
|
|
5942
6020
|
const collectComponents = params.collectComponents !== false;
|
|
5943
|
-
if (queryTokens.length === 0) {
|
|
6021
|
+
if (queryTokens.length === 0 && phrases === void 0) {
|
|
5944
6022
|
return { scored: [], totalMatched: 0 };
|
|
5945
6023
|
}
|
|
5946
6024
|
const totalDocs = globalStats?.totalDocuments ?? state.stats.totalDocuments;
|
|
@@ -5958,9 +6036,48 @@ function searchFulltext(state, params) {
|
|
|
5958
6036
|
const onePair = boundPairs.length === 2;
|
|
5959
6037
|
const onlyK1 = boundPairs[0];
|
|
5960
6038
|
const onlyB = boundPairs[1];
|
|
6039
|
+
const phraseMatches = phrases === void 0 ? null : requireLoadedPhraseCode().matchPhrases(
|
|
6040
|
+
state.invertedIdx,
|
|
6041
|
+
phrases,
|
|
6042
|
+
searchable,
|
|
6043
|
+
params.filterBitset,
|
|
6044
|
+
state.docStore.internalIdCapacity(),
|
|
6045
|
+
params.phraseWork,
|
|
6046
|
+
true
|
|
6047
|
+
);
|
|
6048
|
+
if (phraseMatches !== null && bitsetIsEmpty(phraseMatches.matched)) {
|
|
6049
|
+
return { scored: [], totalMatched: 0 };
|
|
6050
|
+
}
|
|
6051
|
+
const filterBitset = phraseMatches?.matched ?? params.filterBitset;
|
|
5961
6052
|
function documentFrequency(token, list) {
|
|
5962
6053
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
5963
6054
|
}
|
|
6055
|
+
function scorePhraseRows(rows, matched) {
|
|
6056
|
+
let idf = 0;
|
|
6057
|
+
for (let entry = 0; entry < rows.phrase.tokens.length; entry++) {
|
|
6058
|
+
idf += computeIDF(documentFrequency(rows.phrase.tokens[entry], rows.lists[entry]), totalDocs);
|
|
6059
|
+
}
|
|
6060
|
+
for (let row = 0; row < rows.docIds.length; row++) {
|
|
6061
|
+
const internalId = rows.docIds.values[row];
|
|
6062
|
+
if (!bitsetHas(matched, internalId)) continue;
|
|
6063
|
+
const fieldIndex = rows.fieldIndices.values[row];
|
|
6064
|
+
const frequency = rows.frequencies.values[row];
|
|
6065
|
+
const avgLen = averageLengths[fieldIndex];
|
|
6066
|
+
const fieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
6067
|
+
const score = scoresAreZero ? 0 : computeBM25WithIDF(
|
|
6068
|
+
frequency,
|
|
6069
|
+
idf,
|
|
6070
|
+
fieldLength,
|
|
6071
|
+
avgLen,
|
|
6072
|
+
onePair ? onlyK1 : k1s[fieldIndex],
|
|
6073
|
+
onePair ? onlyB : bs[fieldIndex]
|
|
6074
|
+
);
|
|
6075
|
+
addScore(scoreBuffer, internalId, score * boosts[fieldIndex]);
|
|
6076
|
+
if (components !== null) {
|
|
6077
|
+
recordComponents(components, internalId, fieldNames[fieldIndex], rows.phrase.key, frequency, fieldLength, idf);
|
|
6078
|
+
}
|
|
6079
|
+
}
|
|
6080
|
+
}
|
|
5964
6081
|
const groupContext = {
|
|
5965
6082
|
index: state.invertedIdx,
|
|
5966
6083
|
fields: fieldScoring,
|
|
@@ -6043,7 +6160,7 @@ function searchFulltext(state, params) {
|
|
|
6043
6160
|
mergeGroupContribution(internalId, contribution);
|
|
6044
6161
|
}
|
|
6045
6162
|
}
|
|
6046
|
-
const prunableList = prunableSingleTermList(params, state.invertedIdx, fieldNames);
|
|
6163
|
+
const prunableList = phrases === void 0 ? prunableSingleTermList(params, state.invertedIdx, fieldNames) : null;
|
|
6047
6164
|
if (prunableList !== null && maxResults !== void 0) {
|
|
6048
6165
|
return singleTermTopK({
|
|
6049
6166
|
list: prunableList,
|
|
@@ -6054,7 +6171,7 @@ function searchFulltext(state, params) {
|
|
|
6054
6171
|
resolver
|
|
6055
6172
|
});
|
|
6056
6173
|
}
|
|
6057
|
-
const prunableLists = prunableMultiTermLists(params, state.invertedIdx);
|
|
6174
|
+
const prunableLists = phrases === void 0 ? prunableMultiTermLists(params, state.invertedIdx) : null;
|
|
6058
6175
|
if (prunableLists !== null && maxResults !== void 0) {
|
|
6059
6176
|
const pruned = multiTermTopK({
|
|
6060
6177
|
terms: prunableLists,
|
|
@@ -6095,6 +6212,9 @@ function searchFulltext(state, params) {
|
|
|
6095
6212
|
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), false);
|
|
6096
6213
|
}
|
|
6097
6214
|
}
|
|
6215
|
+
if (phraseMatches !== null) {
|
|
6216
|
+
for (const rows of phraseMatches.rows) scorePhraseRows(rows, phraseMatches.matched);
|
|
6217
|
+
}
|
|
6098
6218
|
const totalMatched = scoreBuffer.touchedCount;
|
|
6099
6219
|
const k = maxResults === void 0 ? totalMatched : Math.max(0, Math.min(maxResults, totalMatched));
|
|
6100
6220
|
const scored = topKFromBuffer(scoreBuffer, k, resolver, components);
|
|
@@ -7330,6 +7450,9 @@ function heapPop(heap) {
|
|
|
7330
7450
|
function collectCandidateTerms(subs, params) {
|
|
7331
7451
|
const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
|
|
7332
7452
|
const terms = /* @__PURE__ */ new Set();
|
|
7453
|
+
for (const phrase of params.phrases ?? []) {
|
|
7454
|
+
for (const token of phrase.tokens) terms.add(token);
|
|
7455
|
+
}
|
|
7333
7456
|
for (const qt of queryTokens) {
|
|
7334
7457
|
const group = termGroupOf(termGroups, qt.token);
|
|
7335
7458
|
if (group !== void 0) {
|
|
@@ -7431,7 +7554,7 @@ function searchSingleSub(subs, layout, params, index) {
|
|
|
7431
7554
|
return { ...result, matchedOrdinalBitset };
|
|
7432
7555
|
}
|
|
7433
7556
|
function compositeSearchFulltext(subs, layout, params) {
|
|
7434
|
-
if (params.queryTokens.length === 0) {
|
|
7557
|
+
if (params.queryTokens.length === 0 && params.phrases === void 0) {
|
|
7435
7558
|
return { scored: [], totalMatched: 0 };
|
|
7436
7559
|
}
|
|
7437
7560
|
const single = onlyPopulatedSub(subs);
|
|
@@ -7860,7 +7983,7 @@ var PREPARED_TYPO_TEST_BYTES = Math.ceil(MAX_FILTER_TESTS / TYPO_CLAIM_SLOTS_PER
|
|
|
7860
7983
|
function capExceeded(cap) {
|
|
7861
7984
|
return new NarsilError(
|
|
7862
7985
|
ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
|
|
7863
|
-
`The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
|
|
7986
|
+
`The engine stopped this search before it checked every candidate, because the work on its pattern tests and phrases passed the cap of ${cap} units. Narrow the tests or the phrases, or raise the patternWorkCap engine setting`,
|
|
7864
7987
|
{ cap }
|
|
7865
7988
|
);
|
|
7866
7989
|
}
|
|
@@ -8091,15 +8214,19 @@ function addPartitionTokens(tokens, addedMembers, partition, term, analysed, exp
|
|
|
8091
8214
|
}
|
|
8092
8215
|
}
|
|
8093
8216
|
}
|
|
8094
|
-
function addReachedTokens(tokens, partitions, term,
|
|
8217
|
+
function addReachedTokens(tokens, partitions, term, requested, language, analysis) {
|
|
8218
|
+
if (requested.options.exact === true) return [];
|
|
8219
|
+
const quoted = analyseQueryPhrases(term, language, analysis);
|
|
8220
|
+
const words = quoted?.words ?? term;
|
|
8221
|
+
if (quoted !== null && words.trim().length === 0) return [];
|
|
8222
|
+
const expansion = quoted?.endsWithPhrase === true ? { schema: requested.schema, options: { ...requested.options, prefix: false } } : requested;
|
|
8095
8223
|
const { options } = expansion;
|
|
8096
|
-
if (options.exact === true) return [];
|
|
8097
8224
|
if (!((options.tolerance ?? 0) > 0 || options.prefix === true || options.partialWords === true)) return [];
|
|
8098
|
-
const analysed = queryTokensOf(
|
|
8225
|
+
const analysed = queryTokensOf(words, language, analysis);
|
|
8099
8226
|
if (analysed === null) return [];
|
|
8100
8227
|
const addedMembers = /* @__PURE__ */ new Set();
|
|
8101
8228
|
for (const partition of partitions) {
|
|
8102
|
-
addPartitionTokens(tokens, addedMembers, partition,
|
|
8229
|
+
addPartitionTokens(tokens, addedMembers, partition, words, analysed, expansion, language, analysis);
|
|
8103
8230
|
}
|
|
8104
8231
|
for (const token of addedMembers) tokens.add(token);
|
|
8105
8232
|
return [...addedMembers];
|
|
@@ -8122,17 +8249,24 @@ function partitionsIn(manager, partitionIds) {
|
|
|
8122
8249
|
|
|
8123
8250
|
// src/partitioning/distributed-scoring.ts
|
|
8124
8251
|
function analysedQueryTokens(term, language, analysis) {
|
|
8125
|
-
const
|
|
8126
|
-
|
|
8127
|
-
removeStopWords: true,
|
|
8128
|
-
customTokenizer: analysis.customTokenizer,
|
|
8129
|
-
stopWordOverride: analysis.stopWords
|
|
8130
|
-
});
|
|
8252
|
+
const quoted = analyseQueryPhrases(term, language, analysis);
|
|
8253
|
+
const words = quoted?.words ?? term;
|
|
8131
8254
|
const unique = /* @__PURE__ */ new Set();
|
|
8132
|
-
|
|
8133
|
-
|
|
8255
|
+
if (quoted === null || words.trim().length > 0) {
|
|
8256
|
+
const { tokens } = tokenize(words, language, {
|
|
8257
|
+
stem: true,
|
|
8258
|
+
removeStopWords: true,
|
|
8259
|
+
customTokenizer: analysis.customTokenizer,
|
|
8260
|
+
stopWordOverride: analysis.stopWords
|
|
8261
|
+
});
|
|
8262
|
+
for (const { token } of tokens) {
|
|
8263
|
+
unique.add(token);
|
|
8264
|
+
}
|
|
8134
8265
|
}
|
|
8135
8266
|
for (const synonym of analysis.synonyms?.expand(unique)?.tokens ?? []) unique.add(synonym);
|
|
8267
|
+
for (const phrase of quoted?.phrases ?? []) {
|
|
8268
|
+
for (const token of phrase.tokens) unique.add(token);
|
|
8269
|
+
}
|
|
8136
8270
|
return unique;
|
|
8137
8271
|
}
|
|
8138
8272
|
function copyNumericRecord(source) {
|
|
@@ -8535,6 +8669,14 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
|
|
|
8535
8669
|
}
|
|
8536
8670
|
|
|
8537
8671
|
// src/search/fulltext.ts
|
|
8672
|
+
function analysedWords(term, language, options) {
|
|
8673
|
+
return tokenize(term, language, {
|
|
8674
|
+
stem: true,
|
|
8675
|
+
removeStopWords: true,
|
|
8676
|
+
customTokenizer: options?.customTokenizer,
|
|
8677
|
+
stopWordOverride: options?.stopWords
|
|
8678
|
+
}).tokens;
|
|
8679
|
+
}
|
|
8538
8680
|
function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
8539
8681
|
if (!params.term || params.term.trim().length === 0) return null;
|
|
8540
8682
|
if (params.fields && params.fields.length === 0) return null;
|
|
@@ -8542,14 +8684,16 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
8542
8684
|
if (params.fields) {
|
|
8543
8685
|
validateSearchFields(params.fields, flatSchema);
|
|
8544
8686
|
}
|
|
8545
|
-
const
|
|
8546
|
-
|
|
8547
|
-
|
|
8548
|
-
customTokenizer: options?.customTokenizer,
|
|
8549
|
-
stopWordOverride: options?.stopWords
|
|
8687
|
+
const quoted = analyseQueryPhrases(params.term, language, {
|
|
8688
|
+
stopWords: options?.stopWords,
|
|
8689
|
+
customTokenizer: options?.customTokenizer
|
|
8550
8690
|
});
|
|
8551
|
-
|
|
8552
|
-
const
|
|
8691
|
+
const wordParams = quoted === null ? params : requireLoadedPhraseCode().withWordsOnly(params, quoted);
|
|
8692
|
+
const phrases = quoted !== null && quoted.phrases.length > 0 ? quoted.phrases : void 0;
|
|
8693
|
+
const words = wordParams.term ?? "";
|
|
8694
|
+
const analysedTokens = quoted !== null && words.trim().length === 0 ? [] : analysedWords(words, language, options);
|
|
8695
|
+
if (analysedTokens.length === 0 && phrases === void 0) return null;
|
|
8696
|
+
const queryTokens = deduplicateTokens(analysedTokens);
|
|
8553
8697
|
let filterBitset;
|
|
8554
8698
|
if (params.filters) {
|
|
8555
8699
|
filterBitset = partition.applyFiltersBitset(
|
|
@@ -8559,16 +8703,19 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
8559
8703
|
);
|
|
8560
8704
|
if (bitsetIsEmpty(filterBitset)) return null;
|
|
8561
8705
|
}
|
|
8562
|
-
|
|
8706
|
+
if (analysedTokens.length === 0) return { queryTokens, phrases, filterBitset };
|
|
8707
|
+
const lastToken = analysedTokens[analysedTokens.length - 1];
|
|
8563
8708
|
const synonyms = params.exact === true ? void 0 : options?.synonyms?.expand(queryTokens.map((qt) => qt.token));
|
|
8564
|
-
const groups = resolveTermGroups(partition,
|
|
8565
|
-
return { queryTokens, ...groups, filterBitset };
|
|
8709
|
+
const groups = resolveTermGroups(partition, wordParams, language, schema, queryTokens, lastToken, options, synonyms);
|
|
8710
|
+
return { queryTokens, phrases, ...groups, filterBitset };
|
|
8566
8711
|
}
|
|
8567
8712
|
function fulltextMatches(partition, params, language, schema, options) {
|
|
8568
8713
|
const prepared = prepareFulltextQuery(partition, params, language, schema, options);
|
|
8569
8714
|
if (prepared === null) return null;
|
|
8570
8715
|
return partition.searchFulltextMatches({
|
|
8571
8716
|
queryTokens: prepared.queryTokens,
|
|
8717
|
+
phrases: prepared.phrases,
|
|
8718
|
+
phraseWork: options?.patternWork,
|
|
8572
8719
|
termGroups: prepared.termGroups,
|
|
8573
8720
|
partialFields: prepared.partialFields,
|
|
8574
8721
|
fields: params.fields,
|
|
@@ -8583,13 +8730,15 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
8583
8730
|
if (prepared === null) {
|
|
8584
8731
|
return { scored: [], totalMatched: 0 };
|
|
8585
8732
|
}
|
|
8586
|
-
const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
|
|
8733
|
+
const { queryTokens, phrases, termGroups, partialFields, filterBitset } = prepared;
|
|
8587
8734
|
const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
8588
8735
|
const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
|
|
8589
8736
|
const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
|
|
8590
8737
|
const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
|
|
8591
8738
|
const rawResult = partition.searchFulltext({
|
|
8592
8739
|
queryTokens,
|
|
8740
|
+
phrases,
|
|
8741
|
+
phraseWork: options?.patternWork,
|
|
8593
8742
|
termGroups,
|
|
8594
8743
|
partialFields,
|
|
8595
8744
|
fields: params.fields,
|
|
@@ -8607,7 +8756,7 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
8607
8756
|
});
|
|
8608
8757
|
let scored = rawResult.scored;
|
|
8609
8758
|
const termMatch = params.termMatch ?? "any";
|
|
8610
|
-
if (termMatch !== "any") {
|
|
8759
|
+
if (termMatch !== "any" && queryTokens.length > 0) {
|
|
8611
8760
|
scored = filterByTermCoverage(
|
|
8612
8761
|
scored,
|
|
8613
8762
|
queryTokens,
|
|
@@ -8616,7 +8765,8 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
8616
8765
|
params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
8617
8766
|
params.exact ?? false,
|
|
8618
8767
|
termGroups,
|
|
8619
|
-
partialFields ?? []
|
|
8768
|
+
partialFields ?? [],
|
|
8769
|
+
phrases
|
|
8620
8770
|
);
|
|
8621
8771
|
}
|
|
8622
8772
|
if (params.minScore !== void 0 && params.minScore > 0) {
|
|
@@ -8658,8 +8808,9 @@ function deduplicateTokens(tokens) {
|
|
|
8658
8808
|
}
|
|
8659
8809
|
return result;
|
|
8660
8810
|
}
|
|
8661
|
-
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
|
|
8811
|
+
function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields, phrases) {
|
|
8662
8812
|
const requiredCount = policy === "all" ? queryTokens.length : policy;
|
|
8813
|
+
const phraseKeys = phrases === void 0 ? null : new Set(phrases.map((phrase) => phrase.key));
|
|
8663
8814
|
if (requiredCount <= 0) return scored;
|
|
8664
8815
|
if (requiredCount > queryTokens.length) return [];
|
|
8665
8816
|
const groups = /* @__PURE__ */ new Map();
|
|
@@ -8675,12 +8826,22 @@ function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLeng
|
|
|
8675
8826
|
});
|
|
8676
8827
|
}
|
|
8677
8828
|
return scored.filter((doc) => {
|
|
8678
|
-
const matched = countDocTermMatches(
|
|
8829
|
+
const matched = countDocTermMatches(
|
|
8830
|
+
doc,
|
|
8831
|
+
queryTokens,
|
|
8832
|
+
tolerance,
|
|
8833
|
+
prefixLength,
|
|
8834
|
+
exact,
|
|
8835
|
+
groups,
|
|
8836
|
+
partialFields,
|
|
8837
|
+
phraseKeys
|
|
8838
|
+
);
|
|
8679
8839
|
return matched >= requiredCount;
|
|
8680
8840
|
});
|
|
8681
8841
|
}
|
|
8682
|
-
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
|
|
8683
|
-
const
|
|
8842
|
+
function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields, phraseKeys) {
|
|
8843
|
+
const componentKeys = Object.keys(doc.idf);
|
|
8844
|
+
const indexTokens = phraseKeys === null ? componentKeys : componentKeys.filter((key) => !phraseKeys.has(key));
|
|
8684
8845
|
if (indexTokens.length === 0) return 0;
|
|
8685
8846
|
let count = 0;
|
|
8686
8847
|
for (const qt of queryTokens) {
|
|
@@ -8833,4 +8994,4 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
8833
8994
|
return count;
|
|
8834
8995
|
}
|
|
8835
8996
|
|
|
8836
|
-
export { DEFAULT_BM25_B, DEFAULT_BM25_K1, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FNV_OFFSET_BASIS, MAX_BM25_B, MAX_BM25_K1, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, applyPagination, applyPartitionFiltersBitset, applyProjection, applySorting, availablePatternSearch, bm25SetsValue, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUint32s, fulltextMatches, isCompositePartition, joinPatternWorkMeter, kWayMerge, mergeBM25Levels, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|
|
8997
|
+
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FNV_OFFSET_BASIS, MAX_BM25_B, MAX_BM25_K1, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, analyseQueryPhrases, applyPagination, applyPartitionFiltersBitset, applyProjection, applySorting, availablePatternSearch, bm25SetsValue, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUint32s, fulltextMatches, holdsQuotedText, isCompositePartition, joinPatternWorkMeter, kWayMerge, loadPhraseCodeFor, mergeBM25Levels, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireLoadedPhraseCode, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|