@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, TYPO_CLAIM_SLOTS_PER_WORD, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-
|
|
1
|
+
import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, TYPO_CLAIM_SLOTS_PER_WORD, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-RKAVHNRW.mjs';
|
|
2
2
|
import { GEO_UNIT_METRES, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH } from './chunk-WQBCBKY3.mjs';
|
|
3
3
|
import { bitsetHas, createBitSet, bitsetSet, bitsetIsEmpty, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, bitsetFromSet } from './chunk-JUKYYXSP.mjs';
|
|
4
|
-
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-
|
|
5
|
-
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-
|
|
4
|
+
import { expandTermPrefix, suggestDisplayTerms, getNestedValue, textFieldsChanged, getFlatSchema, tokenizeOptions, partialWordSearchModule, getOrCreateFieldNameIndex, partitionDocumentFrequency } from './chunk-DQNKVOJ2.mjs';
|
|
5
|
+
import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, validateDocument, validateDocumentStrict, ANTIMERIDIAN_SPAN_DEGREES, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, DEG_TO_RAD, VINCENTY_MAX_ITERATIONS } from './chunk-KGOXHLLZ.mjs';
|
|
6
6
|
import { MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PREFIX_MAX_EXPANSIONS } from './chunk-FQBFZAWG.mjs';
|
|
7
7
|
import { fuzzyPrefixOf, boundedLevenshtein, fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
8
|
-
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-
|
|
9
|
-
import { tokenize, visitTokens } from './chunk-
|
|
10
|
-
import { SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD,
|
|
8
|
+
import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedBase, isWordIndexedFieldType, isListBase } from './chunk-ZSWHPKGM.mjs';
|
|
9
|
+
import { tokenize, visitTokens } from './chunk-NPNK57CA.mjs';
|
|
10
|
+
import { TARGET_BLOCK_ENTRIES, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, PRUNING_REJECTION_SLACK, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT } from './chunk-4UMZ7KQL.mjs';
|
|
11
11
|
import { singleFoldTable, appendFoldedCodePoint, sortModeOf, defaultSortMode, SORT_VALUE_MAX_CODE_POINTS, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, isSortMode, SORT_MODES, compareSortValues, compareComparableValues, truncateSortString } from './chunk-R6A773WK.mjs';
|
|
12
12
|
import { compareCodePoints, exceedsCodePoints, codePointCount, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
13
13
|
import { isGeopointOrVectorType, PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
|
|
14
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
14
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
15
15
|
|
|
16
16
|
// src/core/partition/partial-words/platform.ts
|
|
17
17
|
function platformPartialWordSearch() {
|
|
@@ -178,6 +178,31 @@ var MAX_BM25_B = 1;
|
|
|
178
178
|
var PENDING_REMOVAL_MERGE_FLOOR = 64;
|
|
179
179
|
var PENDING_REMOVAL_MERGE_DIVISOR = 256;
|
|
180
180
|
|
|
181
|
+
// src/core/partition/frozen/memory.ts
|
|
182
|
+
function frozenSegmentBytes(segment) {
|
|
183
|
+
if (segment.paged !== null) return segment.paged.heldBytes(segment);
|
|
184
|
+
const arrays = segment.arrays;
|
|
185
|
+
let bytes = arrays.tokenTable.bytes + arrays.idTable.bytes;
|
|
186
|
+
bytes += arrays.postingOffsets.byteLength;
|
|
187
|
+
bytes += arrays.postingDocIds.byteLength;
|
|
188
|
+
bytes += arrays.postingFrequencies.byteLength;
|
|
189
|
+
bytes += arrays.postingFieldIndices.byteLength;
|
|
190
|
+
if (arrays.positionOffsets !== null) bytes += arrays.positionOffsets.byteLength;
|
|
191
|
+
if (arrays.positionValues !== null) bytes += arrays.positionValues.byteLength;
|
|
192
|
+
if (arrays.documentTable !== null) {
|
|
193
|
+
bytes += arrays.documentTable.blob.byteLength + arrays.documentTable.offsets.byteLength;
|
|
194
|
+
}
|
|
195
|
+
for (const column of arrays.fieldLengthColumns) bytes += column.byteLength;
|
|
196
|
+
for (const entry of [...arrays.numeric, ...arrays.sizes]) bytes += entry.docIds.byteLength + entry.values.byteLength;
|
|
197
|
+
for (const entry of arrays.boolean) bytes += entry.trueDocs.byteLength + entry.falseDocs.byteLength;
|
|
198
|
+
for (const entry of arrays.enums) bytes += entry.offsets.byteLength + entry.docIds.byteLength;
|
|
199
|
+
for (const entry of arrays.geo) {
|
|
200
|
+
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
201
|
+
}
|
|
202
|
+
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
203
|
+
return bytes + segment.partialWords.bytes() + segment.removalBytes() + (segment.vectorOrdinalMaps?.bytes ?? 0);
|
|
204
|
+
}
|
|
205
|
+
|
|
181
206
|
// src/core/projection.ts
|
|
182
207
|
function requireFieldPathList(paths, option) {
|
|
183
208
|
if (paths === void 0) return;
|
|
@@ -607,65 +632,6 @@ function createDocumentStore() {
|
|
|
607
632
|
};
|
|
608
633
|
}
|
|
609
634
|
|
|
610
|
-
// src/search/quotes.ts
|
|
611
|
-
var QUOTE_CHARACTER = /["“”„]/;
|
|
612
|
-
function holdsQuoteCharacter(term) {
|
|
613
|
-
return QUOTE_CHARACTER.test(term);
|
|
614
|
-
}
|
|
615
|
-
function holdsQuotedText(term) {
|
|
616
|
-
if (!QUOTE_CHARACTER.test(term)) return false;
|
|
617
|
-
const parts = term.split(QUOTE_CHARACTER);
|
|
618
|
-
for (let index = 1; index < parts.length; index += 2) {
|
|
619
|
-
if (parts[index].trim().length > 0) return true;
|
|
620
|
-
}
|
|
621
|
-
return false;
|
|
622
|
-
}
|
|
623
|
-
|
|
624
|
-
// src/search/phrase-code.ts
|
|
625
|
-
var loaded = null;
|
|
626
|
-
var loading = null;
|
|
627
|
-
function requireLoadedPhraseCode() {
|
|
628
|
-
if (loaded === null) {
|
|
629
|
-
throw new NarsilError(ErrorCodes.SEARCH_PHRASE_NOT_READY, "The phrase code has not loaded. Try the query again");
|
|
630
|
-
}
|
|
631
|
-
return loaded;
|
|
632
|
-
}
|
|
633
|
-
function loadPhraseCode() {
|
|
634
|
-
if (loading === null) {
|
|
635
|
-
loading = import('./phrase-module-JERF2IDG.mjs').then(
|
|
636
|
-
(code) => {
|
|
637
|
-
loaded = code;
|
|
638
|
-
return code;
|
|
639
|
-
},
|
|
640
|
-
(err) => {
|
|
641
|
-
loading = null;
|
|
642
|
-
throw new NarsilError(
|
|
643
|
-
ErrorCodes.SEARCH_PHRASE_NOT_READY,
|
|
644
|
-
"The phrase code failed to load. Remove the quotes, or try again once it can load",
|
|
645
|
-
{ cause: err instanceof Error ? err.message : String(err) }
|
|
646
|
-
);
|
|
647
|
-
}
|
|
648
|
-
);
|
|
649
|
-
}
|
|
650
|
-
return loading;
|
|
651
|
-
}
|
|
652
|
-
function loadPhraseCodeFor(term) {
|
|
653
|
-
if (loaded !== null || term === void 0 || !holdsQuoteCharacter(term)) return null;
|
|
654
|
-
return loadPhraseCode();
|
|
655
|
-
}
|
|
656
|
-
function analyseQueryPhrases(term, language, analysis) {
|
|
657
|
-
if (!holdsQuoteCharacter(term)) return null;
|
|
658
|
-
return requireLoadedPhraseCode().analyseQueryPhrases(
|
|
659
|
-
term.split(QUOTE_CHARACTER),
|
|
660
|
-
(text) => tokenize(text, language, {
|
|
661
|
-
stem: true,
|
|
662
|
-
removeStopWords: true,
|
|
663
|
-
customTokenizer: analysis.customTokenizer,
|
|
664
|
-
stopWordOverride: analysis.stopWords
|
|
665
|
-
}).tokens
|
|
666
|
-
);
|
|
667
|
-
}
|
|
668
|
-
|
|
669
635
|
// src/core/inverted-index/constants.ts
|
|
670
636
|
var TERM_TABLE_START_CAPACITY = 1 << 10;
|
|
671
637
|
var POSTING_POOL_START_ENTRIES = 1 << 12;
|
|
@@ -4804,6 +4770,65 @@ function partitionFilterMatches(state, filters, schema, meter) {
|
|
|
4804
4770
|
};
|
|
4805
4771
|
}
|
|
4806
4772
|
|
|
4773
|
+
// src/search/quotes.ts
|
|
4774
|
+
var QUOTE_CHARACTER = /["“”„]/;
|
|
4775
|
+
function holdsQuoteCharacter(term) {
|
|
4776
|
+
return QUOTE_CHARACTER.test(term);
|
|
4777
|
+
}
|
|
4778
|
+
function holdsQuotedText(term) {
|
|
4779
|
+
if (!QUOTE_CHARACTER.test(term)) return false;
|
|
4780
|
+
const parts = term.split(QUOTE_CHARACTER);
|
|
4781
|
+
for (let index = 1; index < parts.length; index += 2) {
|
|
4782
|
+
if (parts[index].trim().length > 0) return true;
|
|
4783
|
+
}
|
|
4784
|
+
return false;
|
|
4785
|
+
}
|
|
4786
|
+
|
|
4787
|
+
// src/search/phrase-code.ts
|
|
4788
|
+
var loaded = null;
|
|
4789
|
+
var loading = null;
|
|
4790
|
+
function requireLoadedPhraseCode() {
|
|
4791
|
+
if (loaded === null) {
|
|
4792
|
+
throw new NarsilError(ErrorCodes.SEARCH_PHRASE_NOT_READY, "The phrase code has not loaded. Try the query again");
|
|
4793
|
+
}
|
|
4794
|
+
return loaded;
|
|
4795
|
+
}
|
|
4796
|
+
function loadPhraseCode() {
|
|
4797
|
+
if (loading === null) {
|
|
4798
|
+
loading = import('./phrase-module-5QHBD2VJ.mjs').then(
|
|
4799
|
+
(code) => {
|
|
4800
|
+
loaded = code;
|
|
4801
|
+
return code;
|
|
4802
|
+
},
|
|
4803
|
+
(err) => {
|
|
4804
|
+
loading = null;
|
|
4805
|
+
throw new NarsilError(
|
|
4806
|
+
ErrorCodes.SEARCH_PHRASE_NOT_READY,
|
|
4807
|
+
"The phrase code failed to load. Remove the quotes, or try again once it can load",
|
|
4808
|
+
{ cause: err instanceof Error ? err.message : String(err) }
|
|
4809
|
+
);
|
|
4810
|
+
}
|
|
4811
|
+
);
|
|
4812
|
+
}
|
|
4813
|
+
return loading;
|
|
4814
|
+
}
|
|
4815
|
+
function loadPhraseCodeFor(term) {
|
|
4816
|
+
if (loaded !== null || term === void 0 || !holdsQuoteCharacter(term)) return null;
|
|
4817
|
+
return loadPhraseCode();
|
|
4818
|
+
}
|
|
4819
|
+
function analyseQueryPhrases(term, language, analysis) {
|
|
4820
|
+
if (!holdsQuoteCharacter(term)) return null;
|
|
4821
|
+
return requireLoadedPhraseCode().analyseQueryPhrases(
|
|
4822
|
+
term.split(QUOTE_CHARACTER),
|
|
4823
|
+
(text) => tokenize(text, language, {
|
|
4824
|
+
stem: true,
|
|
4825
|
+
removeStopWords: true,
|
|
4826
|
+
customTokenizer: analysis.customTokenizer,
|
|
4827
|
+
stopWordOverride: analysis.stopWords
|
|
4828
|
+
}).tokens
|
|
4829
|
+
);
|
|
4830
|
+
}
|
|
4831
|
+
|
|
4807
4832
|
// src/core/scorer.ts
|
|
4808
4833
|
function computeIDF(docFrequency, totalDocs) {
|
|
4809
4834
|
if (totalDocs === 0) return 0;
|
|
@@ -5360,7 +5385,7 @@ function fieldLengthBound(columns, fieldIndex, internalId) {
|
|
|
5360
5385
|
function entriesCovered(bounds) {
|
|
5361
5386
|
return bounds.blockCount === 0 ? 0 : bounds.entryEnd[bounds.blockCount - 1];
|
|
5362
5387
|
}
|
|
5363
|
-
function
|
|
5388
|
+
function scanPostingBlocks(list, columns, previous, keepBlocks) {
|
|
5364
5389
|
const { docIds, termFrequencies, fieldNameIndices, length } = list;
|
|
5365
5390
|
const startEntry = previous !== null && keepBlocks > 0 ? previous.entryEnd[keepBlocks - 1] : 0;
|
|
5366
5391
|
const capacity = keepBlocks + Math.ceil((length - startEntry) / TARGET_BLOCK_ENTRIES) + 1;
|
|
@@ -5434,12 +5459,13 @@ function blockBoundsFor(list, columns) {
|
|
|
5434
5459
|
const covered = entriesCovered(existing);
|
|
5435
5460
|
if (covered === list.length) return existing;
|
|
5436
5461
|
if (list.ordered && covered < list.length) {
|
|
5437
|
-
const extended =
|
|
5462
|
+
const extended = scanPostingBlocks(list, columns, existing, Math.max(0, existing.blockCount - 1));
|
|
5438
5463
|
boundsByList.set(list, extended);
|
|
5439
5464
|
return extended;
|
|
5440
5465
|
}
|
|
5441
5466
|
}
|
|
5442
|
-
const
|
|
5467
|
+
const stored = list.storedBlockBounds?.() ?? null;
|
|
5468
|
+
const built = stored === null ? scanPostingBlocks(list, columns, null, 0) : { ...stored, structureRevision: list.structureRevision };
|
|
5443
5469
|
boundsByList.set(list, built);
|
|
5444
5470
|
return built;
|
|
5445
5471
|
}
|
|
@@ -5632,33 +5658,33 @@ function boostsKeepScoresNonNegative(fields) {
|
|
|
5632
5658
|
}
|
|
5633
5659
|
return true;
|
|
5634
5660
|
}
|
|
5635
|
-
function samePairUpperBound(
|
|
5636
|
-
if (
|
|
5637
|
-
const bounds = blockBoundsFor(list,
|
|
5661
|
+
function samePairUpperBound(scan, list, maxBoost, maxAverageLength) {
|
|
5662
|
+
if (scan.scoresAreZero) return 0;
|
|
5663
|
+
const bounds = blockBoundsFor(list, scan.fields.lengthColumns);
|
|
5638
5664
|
let best = 0;
|
|
5639
5665
|
for (let block = 0; block < bounds.blockCount; block++) {
|
|
5640
5666
|
const bestEntry = computeBM25WithIDF(
|
|
5641
5667
|
bounds.maxTermFrequency[block],
|
|
5642
|
-
|
|
5668
|
+
scan.idf,
|
|
5643
5669
|
bounds.minFieldLength[block],
|
|
5644
5670
|
maxAverageLength,
|
|
5645
|
-
|
|
5646
|
-
|
|
5671
|
+
scan.k1,
|
|
5672
|
+
scan.b
|
|
5647
5673
|
);
|
|
5648
5674
|
const blockBound = bestEntry * maxBoost * bounds.maxEntriesPerDocument[block];
|
|
5649
5675
|
if (blockBound > best) best = blockBound;
|
|
5650
5676
|
}
|
|
5651
5677
|
return best;
|
|
5652
5678
|
}
|
|
5653
|
-
function fieldPairUpperBound(
|
|
5654
|
-
if (
|
|
5655
|
-
const bounds = blockBoundsFor(list,
|
|
5679
|
+
function fieldPairUpperBound(scan, list, maxBoost, maxAverageLength) {
|
|
5680
|
+
if (scan.scoresAreZero) return 0;
|
|
5681
|
+
const bounds = blockBoundsFor(list, scan.fields.lengthColumns);
|
|
5656
5682
|
let best = 0;
|
|
5657
5683
|
for (let block = 0; block < bounds.blockCount; block++) {
|
|
5658
5684
|
const bestEntry = blockEntryBound(
|
|
5659
|
-
|
|
5685
|
+
scan.fields,
|
|
5660
5686
|
bounds.maxTermFrequency[block],
|
|
5661
|
-
|
|
5687
|
+
scan.idf,
|
|
5662
5688
|
bounds.minFieldLength[block],
|
|
5663
5689
|
maxAverageLength
|
|
5664
5690
|
);
|
|
@@ -5667,22 +5693,22 @@ function fieldPairUpperBound(scan2, list, maxBoost, maxAverageLength) {
|
|
|
5667
5693
|
}
|
|
5668
5694
|
return best;
|
|
5669
5695
|
}
|
|
5670
|
-
function samePairEntryScore(
|
|
5671
|
-
const { fields, columns } =
|
|
5696
|
+
function samePairEntryScore(scan, entry, internalId) {
|
|
5697
|
+
const { fields, columns } = scan;
|
|
5672
5698
|
const fieldIndex = columns.fieldNameIndices[entry];
|
|
5673
5699
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
5674
5700
|
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
5675
|
-
const bm25 =
|
|
5701
|
+
const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(columns.termFrequencies[entry], scan.idf, fieldLength, averageLength, scan.k1, scan.b);
|
|
5676
5702
|
return bm25 * fields.boosts[fieldIndex];
|
|
5677
5703
|
}
|
|
5678
|
-
function fieldPairEntryScore(
|
|
5679
|
-
const { fields, columns } =
|
|
5704
|
+
function fieldPairEntryScore(scan, entry, internalId) {
|
|
5705
|
+
const { fields, columns } = scan;
|
|
5680
5706
|
const fieldIndex = columns.fieldNameIndices[entry];
|
|
5681
5707
|
const averageLength = fields.averageLengths[fieldIndex];
|
|
5682
5708
|
const fieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, averageLength);
|
|
5683
|
-
const bm25 =
|
|
5709
|
+
const bm25 = scan.scoresAreZero ? 0 : computeBM25WithIDF(
|
|
5684
5710
|
columns.termFrequencies[entry],
|
|
5685
|
-
|
|
5711
|
+
scan.idf,
|
|
5686
5712
|
fieldLength,
|
|
5687
5713
|
averageLength,
|
|
5688
5714
|
fields.k1s[fieldIndex],
|
|
@@ -5690,48 +5716,48 @@ function fieldPairEntryScore(scan2, entry, internalId) {
|
|
|
5690
5716
|
);
|
|
5691
5717
|
return bm25 * fields.boosts[fieldIndex];
|
|
5692
5718
|
}
|
|
5693
|
-
function samePairScoreEveryDocument(
|
|
5694
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
5695
|
-
const searchable =
|
|
5719
|
+
function samePairScoreEveryDocument(scan, buffer) {
|
|
5720
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
5721
|
+
const searchable = scan.fields.searchable;
|
|
5696
5722
|
for (let entry = 0; entry < count; entry++) {
|
|
5697
5723
|
const internalId = docIds[entry];
|
|
5698
5724
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5699
5725
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5700
|
-
addScore(buffer, internalId, samePairEntryScore(
|
|
5726
|
+
addScore(buffer, internalId, samePairEntryScore(scan, entry, internalId));
|
|
5701
5727
|
}
|
|
5702
5728
|
}
|
|
5703
|
-
function fieldPairScoreEveryDocument(
|
|
5704
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
5705
|
-
const searchable =
|
|
5729
|
+
function fieldPairScoreEveryDocument(scan, buffer) {
|
|
5730
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
5731
|
+
const searchable = scan.fields.searchable;
|
|
5706
5732
|
for (let entry = 0; entry < count; entry++) {
|
|
5707
5733
|
const internalId = docIds[entry];
|
|
5708
5734
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5709
5735
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5710
|
-
addScore(buffer, internalId, fieldPairEntryScore(
|
|
5736
|
+
addScore(buffer, internalId, fieldPairEntryScore(scan, entry, internalId));
|
|
5711
5737
|
}
|
|
5712
5738
|
}
|
|
5713
|
-
function samePairScoreKnownDocuments(
|
|
5714
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
5715
|
-
const searchable =
|
|
5739
|
+
function samePairScoreKnownDocuments(scan, buffer) {
|
|
5740
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
5741
|
+
const searchable = scan.fields.searchable;
|
|
5716
5742
|
let counted = 0;
|
|
5717
5743
|
for (let entry = 0; entry < count; entry++) {
|
|
5718
5744
|
const internalId = docIds[entry];
|
|
5719
5745
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5720
5746
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5721
|
-
if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(
|
|
5747
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, samePairEntryScore(scan, entry, internalId));
|
|
5722
5748
|
else if (markCounted(buffer, internalId)) counted++;
|
|
5723
5749
|
}
|
|
5724
5750
|
return counted;
|
|
5725
5751
|
}
|
|
5726
|
-
function fieldPairScoreKnownDocuments(
|
|
5727
|
-
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } =
|
|
5728
|
-
const searchable =
|
|
5752
|
+
function fieldPairScoreKnownDocuments(scan, buffer) {
|
|
5753
|
+
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = scan.columns;
|
|
5754
|
+
const searchable = scan.fields.searchable;
|
|
5729
5755
|
let counted = 0;
|
|
5730
5756
|
for (let entry = 0; entry < count; entry++) {
|
|
5731
5757
|
const internalId = docIds[entry];
|
|
5732
5758
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5733
5759
|
if (searchable[fieldNameIndices[entry]] !== 1) continue;
|
|
5734
|
-
if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(
|
|
5760
|
+
if (hasScore(buffer, internalId)) addScore(buffer, internalId, fieldPairEntryScore(scan, entry, internalId));
|
|
5735
5761
|
else if (markCounted(buffer, internalId)) counted++;
|
|
5736
5762
|
}
|
|
5737
5763
|
return counted;
|
|
@@ -5748,24 +5774,24 @@ function firstEntryOf(docIds, count, internalId) {
|
|
|
5748
5774
|
}
|
|
5749
5775
|
function samePairQueryOrderScore(scans, internalId) {
|
|
5750
5776
|
let score = 0;
|
|
5751
|
-
for (const
|
|
5752
|
-
const { docIds, deletedDocs, hasDeleted, count } =
|
|
5777
|
+
for (const scan of scans) {
|
|
5778
|
+
const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
|
|
5753
5779
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5754
5780
|
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
5755
|
-
if (
|
|
5756
|
-
score += samePairEntryScore(
|
|
5781
|
+
if (scan.fields.searchable[scan.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
5782
|
+
score += samePairEntryScore(scan, entry, internalId);
|
|
5757
5783
|
}
|
|
5758
5784
|
}
|
|
5759
5785
|
return score;
|
|
5760
5786
|
}
|
|
5761
5787
|
function fieldPairQueryOrderScore(scans, internalId) {
|
|
5762
5788
|
let score = 0;
|
|
5763
|
-
for (const
|
|
5764
|
-
const { docIds, deletedDocs, hasDeleted, count } =
|
|
5789
|
+
for (const scan of scans) {
|
|
5790
|
+
const { docIds, deletedDocs, hasDeleted, count } = scan.columns;
|
|
5765
5791
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
5766
5792
|
for (let entry = firstEntryOf(docIds, count, internalId); entry < count && docIds[entry] === internalId; entry++) {
|
|
5767
|
-
if (
|
|
5768
|
-
score += fieldPairEntryScore(
|
|
5793
|
+
if (scan.fields.searchable[scan.columns.fieldNameIndices[entry]] !== 1) continue;
|
|
5794
|
+
score += fieldPairEntryScore(scan, entry, internalId);
|
|
5769
5795
|
}
|
|
5770
5796
|
}
|
|
5771
5797
|
return score;
|
|
@@ -5845,7 +5871,7 @@ function multiTermTopK(request) {
|
|
|
5845
5871
|
}
|
|
5846
5872
|
const maxBoost = bestSearchable(fields.searchable, fields.boosts);
|
|
5847
5873
|
const maxAverageLength = bestSearchable(fields.searchable, fields.averageLengths);
|
|
5848
|
-
const upperBounds = scans.map((
|
|
5874
|
+
const upperBounds = scans.map((scan, term) => scorer.upperBound(scan, terms[term].list, maxBoost, maxAverageLength));
|
|
5849
5875
|
const order = scans.map((_, term) => term).sort((left, right) => upperBounds[right] - upperBounds[left]);
|
|
5850
5876
|
const remainingBound = new Float64Array(order.length + 1);
|
|
5851
5877
|
for (let position2 = order.length - 1; position2 >= 0; position2--) {
|
|
@@ -7019,147 +7045,6 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
7019
7045
|
return partition;
|
|
7020
7046
|
}
|
|
7021
7047
|
|
|
7022
|
-
// src/core/hash.ts
|
|
7023
|
-
var FNV_OFFSET_BASIS = 2166136261;
|
|
7024
|
-
var FNV_PRIME = 16777619;
|
|
7025
|
-
var encoder = new TextEncoder();
|
|
7026
|
-
function fnv1aBytes(bytes) {
|
|
7027
|
-
return fnv1aContinue(FNV_OFFSET_BASIS, bytes, bytes.length) >>> 0;
|
|
7028
|
-
}
|
|
7029
|
-
function fnv1aContinue(start, bytes, length) {
|
|
7030
|
-
let hash = start;
|
|
7031
|
-
for (let i = 0; i < length; i++) {
|
|
7032
|
-
hash = Math.imul(hash ^ bytes[i], FNV_PRIME);
|
|
7033
|
-
}
|
|
7034
|
-
return hash;
|
|
7035
|
-
}
|
|
7036
|
-
var scratchNumber = new Float64Array(1);
|
|
7037
|
-
var scratchBytes = new Uint8Array(scratchNumber.buffer);
|
|
7038
|
-
var HOST_IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
7039
|
-
function fnv1aContinueTaggedNumbers(start, tag, numbers) {
|
|
7040
|
-
let hash = start;
|
|
7041
|
-
const b = scratchBytes;
|
|
7042
|
-
for (let i = 0; i < numbers.length; i++) {
|
|
7043
|
-
scratchNumber[0] = Math.fround(numbers[i]);
|
|
7044
|
-
hash = Math.imul(hash ^ tag, FNV_PRIME);
|
|
7045
|
-
if (HOST_IS_LITTLE_ENDIAN) {
|
|
7046
|
-
hash = Math.imul(hash ^ b[7], FNV_PRIME);
|
|
7047
|
-
hash = Math.imul(hash ^ b[6], FNV_PRIME);
|
|
7048
|
-
hash = Math.imul(hash ^ b[5], FNV_PRIME);
|
|
7049
|
-
hash = Math.imul(hash ^ b[4], FNV_PRIME);
|
|
7050
|
-
hash = Math.imul(hash ^ b[3], FNV_PRIME);
|
|
7051
|
-
hash = Math.imul(hash ^ b[2], FNV_PRIME);
|
|
7052
|
-
hash = Math.imul(hash ^ b[1], FNV_PRIME);
|
|
7053
|
-
hash = Math.imul(hash ^ b[0], FNV_PRIME);
|
|
7054
|
-
} else {
|
|
7055
|
-
for (let at = 0; at < 8; at++) hash = Math.imul(hash ^ b[at], FNV_PRIME);
|
|
7056
|
-
}
|
|
7057
|
-
}
|
|
7058
|
-
return hash;
|
|
7059
|
-
}
|
|
7060
|
-
function fnv1aUint32s(values) {
|
|
7061
|
-
let hash = FNV_OFFSET_BASIS;
|
|
7062
|
-
for (const value of values) {
|
|
7063
|
-
for (let shift = 24; shift >= 0; shift -= 8) {
|
|
7064
|
-
hash ^= value >>> shift & 255;
|
|
7065
|
-
hash = Math.imul(hash, FNV_PRIME);
|
|
7066
|
-
}
|
|
7067
|
-
}
|
|
7068
|
-
return hash >>> 0;
|
|
7069
|
-
}
|
|
7070
|
-
function fnv1a(input) {
|
|
7071
|
-
return fnv1aBytes(encoder.encode(input));
|
|
7072
|
-
}
|
|
7073
|
-
|
|
7074
|
-
// src/search/oversample.ts
|
|
7075
|
-
function oversampledShardSize(size) {
|
|
7076
|
-
return Math.ceil(size * 1.5) + 10;
|
|
7077
|
-
}
|
|
7078
|
-
|
|
7079
|
-
// src/search/facets.ts
|
|
7080
|
-
function keptValueCount(fieldConfig) {
|
|
7081
|
-
return keptFacetValues(fieldConfig?.limit);
|
|
7082
|
-
}
|
|
7083
|
-
function everyValueFacetConfig(config) {
|
|
7084
|
-
const widened = {};
|
|
7085
|
-
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
7086
|
-
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
7087
|
-
}
|
|
7088
|
-
return widened;
|
|
7089
|
-
}
|
|
7090
|
-
function oversampledFacetConfig(config) {
|
|
7091
|
-
const widened = {};
|
|
7092
|
-
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
7093
|
-
if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
|
|
7094
|
-
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
7095
|
-
continue;
|
|
7096
|
-
}
|
|
7097
|
-
const kept = keptValueCount(fieldConfig);
|
|
7098
|
-
widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
|
|
7099
|
-
}
|
|
7100
|
-
return widened;
|
|
7101
|
-
}
|
|
7102
|
-
function mergeFacets(partitionFacets, config) {
|
|
7103
|
-
const merged = /* @__PURE__ */ new Map();
|
|
7104
|
-
const bounds = /* @__PURE__ */ new Map();
|
|
7105
|
-
for (const partition of partitionFacets) {
|
|
7106
|
-
for (const [field, facetResult] of Object.entries(partition)) {
|
|
7107
|
-
let fieldMap = merged.get(field);
|
|
7108
|
-
if (!fieldMap) {
|
|
7109
|
-
fieldMap = /* @__PURE__ */ new Map();
|
|
7110
|
-
merged.set(field, fieldMap);
|
|
7111
|
-
}
|
|
7112
|
-
bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
|
|
7113
|
-
for (const [value, count] of Object.entries(facetResult.values)) {
|
|
7114
|
-
fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
|
|
7115
|
-
}
|
|
7116
|
-
}
|
|
7117
|
-
}
|
|
7118
|
-
const result = {};
|
|
7119
|
-
for (const [field, valueMap] of merged) {
|
|
7120
|
-
const fieldConfig = config[field];
|
|
7121
|
-
const ascending = fieldConfig?.sort === "asc";
|
|
7122
|
-
const ordered = Array.from(valueMap.entries());
|
|
7123
|
-
ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
7124
|
-
const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
|
|
7125
|
-
let largestLeftOut = 0;
|
|
7126
|
-
for (let index = kept; index < ordered.length; index++) {
|
|
7127
|
-
if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
|
|
7128
|
-
}
|
|
7129
|
-
const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
|
|
7130
|
-
const values = {};
|
|
7131
|
-
for (let index = 0; index < kept; index++) {
|
|
7132
|
-
values[ordered[index][0]] = ordered[index][1];
|
|
7133
|
-
}
|
|
7134
|
-
result[field] = { values, count: kept, errorBound };
|
|
7135
|
-
}
|
|
7136
|
-
return result;
|
|
7137
|
-
}
|
|
7138
|
-
|
|
7139
|
-
// src/core/partition/frozen/memory.ts
|
|
7140
|
-
function frozenSegmentBytes(segment) {
|
|
7141
|
-
const arrays = segment.arrays;
|
|
7142
|
-
let bytes = arrays.tokenTable.bytes + arrays.idTable.bytes;
|
|
7143
|
-
bytes += arrays.postingOffsets.byteLength;
|
|
7144
|
-
bytes += arrays.postingDocIds.byteLength;
|
|
7145
|
-
bytes += arrays.postingFrequencies.byteLength;
|
|
7146
|
-
bytes += arrays.postingFieldIndices.byteLength;
|
|
7147
|
-
if (arrays.positionOffsets !== null) bytes += arrays.positionOffsets.byteLength;
|
|
7148
|
-
if (arrays.positionValues !== null) bytes += arrays.positionValues.byteLength;
|
|
7149
|
-
if (arrays.documentTable !== null) {
|
|
7150
|
-
bytes += arrays.documentTable.blob.byteLength + arrays.documentTable.offsets.byteLength;
|
|
7151
|
-
}
|
|
7152
|
-
for (const column of arrays.fieldLengthColumns) bytes += column.byteLength;
|
|
7153
|
-
for (const entry of [...arrays.numeric, ...arrays.sizes]) bytes += entry.docIds.byteLength + entry.values.byteLength;
|
|
7154
|
-
for (const entry of arrays.boolean) bytes += entry.trueDocs.byteLength + entry.falseDocs.byteLength;
|
|
7155
|
-
for (const entry of arrays.enums) bytes += entry.offsets.byteLength + entry.docIds.byteLength;
|
|
7156
|
-
for (const entry of arrays.geo) {
|
|
7157
|
-
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
7158
|
-
}
|
|
7159
|
-
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
7160
|
-
return bytes + segment.partialWords.bytes() + segment.removalBytes() + (segment.vectorOrdinalMaps?.bytes ?? 0);
|
|
7161
|
-
}
|
|
7162
|
-
|
|
7163
7048
|
// src/core/partition/composite/compaction.ts
|
|
7164
7049
|
function resolveFrozenSegments(frozen, segmentIds, partitionId) {
|
|
7165
7050
|
const byId = new Map(frozen.map((segment) => [segment.segmentId, segment]));
|
|
@@ -7249,6 +7134,71 @@ function compositeSortedDocIds(frozen, live) {
|
|
|
7249
7134
|
return merged;
|
|
7250
7135
|
}
|
|
7251
7136
|
|
|
7137
|
+
// src/search/oversample.ts
|
|
7138
|
+
function oversampledShardSize(size) {
|
|
7139
|
+
return Math.ceil(size * 1.5) + 10;
|
|
7140
|
+
}
|
|
7141
|
+
|
|
7142
|
+
// src/search/facets.ts
|
|
7143
|
+
function keptValueCount(fieldConfig) {
|
|
7144
|
+
return keptFacetValues(fieldConfig?.limit);
|
|
7145
|
+
}
|
|
7146
|
+
function everyValueFacetConfig(config) {
|
|
7147
|
+
const widened = {};
|
|
7148
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
7149
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
7150
|
+
}
|
|
7151
|
+
return widened;
|
|
7152
|
+
}
|
|
7153
|
+
function oversampledFacetConfig(config) {
|
|
7154
|
+
const widened = {};
|
|
7155
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
7156
|
+
if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
|
|
7157
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
7158
|
+
continue;
|
|
7159
|
+
}
|
|
7160
|
+
const kept = keptValueCount(fieldConfig);
|
|
7161
|
+
widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
|
|
7162
|
+
}
|
|
7163
|
+
return widened;
|
|
7164
|
+
}
|
|
7165
|
+
function mergeFacets(partitionFacets, config) {
|
|
7166
|
+
const merged = /* @__PURE__ */ new Map();
|
|
7167
|
+
const bounds = /* @__PURE__ */ new Map();
|
|
7168
|
+
for (const partition of partitionFacets) {
|
|
7169
|
+
for (const [field, facetResult] of Object.entries(partition)) {
|
|
7170
|
+
let fieldMap = merged.get(field);
|
|
7171
|
+
if (!fieldMap) {
|
|
7172
|
+
fieldMap = /* @__PURE__ */ new Map();
|
|
7173
|
+
merged.set(field, fieldMap);
|
|
7174
|
+
}
|
|
7175
|
+
bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
|
|
7176
|
+
for (const [value, count] of Object.entries(facetResult.values)) {
|
|
7177
|
+
fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
|
|
7178
|
+
}
|
|
7179
|
+
}
|
|
7180
|
+
}
|
|
7181
|
+
const result = {};
|
|
7182
|
+
for (const [field, valueMap] of merged) {
|
|
7183
|
+
const fieldConfig = config[field];
|
|
7184
|
+
const ascending = fieldConfig?.sort === "asc";
|
|
7185
|
+
const ordered = Array.from(valueMap.entries());
|
|
7186
|
+
ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
7187
|
+
const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
|
|
7188
|
+
let largestLeftOut = 0;
|
|
7189
|
+
for (let index = kept; index < ordered.length; index++) {
|
|
7190
|
+
if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
|
|
7191
|
+
}
|
|
7192
|
+
const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
|
|
7193
|
+
const values = {};
|
|
7194
|
+
for (let index = 0; index < kept; index++) {
|
|
7195
|
+
values[ordered[index][0]] = ordered[index][1];
|
|
7196
|
+
}
|
|
7197
|
+
result[field] = { values, count: kept, errorBound };
|
|
7198
|
+
}
|
|
7199
|
+
return result;
|
|
7200
|
+
}
|
|
7201
|
+
|
|
7252
7202
|
// src/core/partition/composite/filters.ts
|
|
7253
7203
|
var WORD_BITS = 32;
|
|
7254
7204
|
function alignToWord(value) {
|
|
@@ -7329,6 +7279,29 @@ function subMatchesView(matches, layout, subIndex) {
|
|
|
7329
7279
|
};
|
|
7330
7280
|
}
|
|
7331
7281
|
|
|
7282
|
+
// src/core/partition/composite/facets.ts
|
|
7283
|
+
function compositeFacets(subs, layout, matched, config, schema) {
|
|
7284
|
+
const everyValue = everyValueFacetConfig(config);
|
|
7285
|
+
if ("ordinalBitset" in matched) {
|
|
7286
|
+
const ordinalLayout = layout();
|
|
7287
|
+
return mergeFacets(
|
|
7288
|
+
subs.map(
|
|
7289
|
+
(sub, index) => computeFacets(
|
|
7290
|
+
sub,
|
|
7291
|
+
{ ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
|
|
7292
|
+
everyValue,
|
|
7293
|
+
schema
|
|
7294
|
+
)
|
|
7295
|
+
),
|
|
7296
|
+
config
|
|
7297
|
+
);
|
|
7298
|
+
}
|
|
7299
|
+
return mergeFacets(
|
|
7300
|
+
subs.map((sub) => computeFacets(sub, matched, everyValue, schema)),
|
|
7301
|
+
config
|
|
7302
|
+
);
|
|
7303
|
+
}
|
|
7304
|
+
|
|
7332
7305
|
// src/core/partition/scored-merge.ts
|
|
7333
7306
|
function kWayMerge(arrays) {
|
|
7334
7307
|
const nonEmpty = arrays.filter((a) => a.length > 0);
|
|
@@ -7708,7 +7681,8 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
7708
7681
|
}
|
|
7709
7682
|
function cachedDocFrequencies() {
|
|
7710
7683
|
if (docFrequenciesCache === null) {
|
|
7711
|
-
|
|
7684
|
+
const paged = frozen.find((segment) => segment.paged !== null)?.paged;
|
|
7685
|
+
docFrequenciesCache = paged ? paged.termFrequenciesAcross(subs) : mergeDocFrequencies(subs());
|
|
7712
7686
|
}
|
|
7713
7687
|
return docFrequenciesCache;
|
|
7714
7688
|
}
|
|
@@ -7779,6 +7753,20 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
7779
7753
|
frozen.push(segment);
|
|
7780
7754
|
bindSegments();
|
|
7781
7755
|
},
|
|
7756
|
+
frozenSegments() {
|
|
7757
|
+
return frozen;
|
|
7758
|
+
},
|
|
7759
|
+
replaceFrozenSegments(segments) {
|
|
7760
|
+
frozen.length = 0;
|
|
7761
|
+
frozen.push(...segments);
|
|
7762
|
+
bindSegments();
|
|
7763
|
+
},
|
|
7764
|
+
documentIdsHeldInMemory() {
|
|
7765
|
+
return compositeDocIds(
|
|
7766
|
+
frozen.filter((segment) => segment.paged === null),
|
|
7767
|
+
live
|
|
7768
|
+
);
|
|
7769
|
+
},
|
|
7782
7770
|
mergeSegment(segment) {
|
|
7783
7771
|
invalidateDocFrequencies();
|
|
7784
7772
|
live.mergeSegment(segment);
|
|
@@ -7801,6 +7789,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
7801
7789
|
live.deserialize(data, schema);
|
|
7802
7790
|
},
|
|
7803
7791
|
clear() {
|
|
7792
|
+
for (const segment of frozen) segment.paged?.close();
|
|
7804
7793
|
frozen.length = 0;
|
|
7805
7794
|
invalidateDocFrequencies();
|
|
7806
7795
|
live.clear();
|
|
@@ -7867,25 +7856,7 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
7867
7856
|
return compositeFilterMatches(subs(), layout(), filters, schema, meter);
|
|
7868
7857
|
},
|
|
7869
7858
|
computeFacets(matched, config, schema) {
|
|
7870
|
-
|
|
7871
|
-
if ("ordinalBitset" in matched) {
|
|
7872
|
-
const ordinalLayout = layout();
|
|
7873
|
-
return mergeFacets(
|
|
7874
|
-
subs().map(
|
|
7875
|
-
(sub, index) => computeFacets(
|
|
7876
|
-
sub,
|
|
7877
|
-
{ ordinalBitset: subBitsetView(matched.ordinalBitset, ordinalLayout, index) },
|
|
7878
|
-
everyValue,
|
|
7879
|
-
schema
|
|
7880
|
-
)
|
|
7881
|
-
),
|
|
7882
|
-
config
|
|
7883
|
-
);
|
|
7884
|
-
}
|
|
7885
|
-
return mergeFacets(
|
|
7886
|
-
subs().map((sub) => computeFacets(sub, matched, everyValue, schema)),
|
|
7887
|
-
config
|
|
7888
|
-
);
|
|
7859
|
+
return compositeFacets(subs(), layout, matched, config, schema);
|
|
7889
7860
|
},
|
|
7890
7861
|
sortedPage(request) {
|
|
7891
7862
|
return compositeSortedPage(subs(), layout(), request);
|
|
@@ -7951,6 +7922,58 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
7951
7922
|
};
|
|
7952
7923
|
}
|
|
7953
7924
|
|
|
7925
|
+
// src/core/hash.ts
|
|
7926
|
+
var FNV_OFFSET_BASIS = 2166136261;
|
|
7927
|
+
var FNV_PRIME = 16777619;
|
|
7928
|
+
var encoder = new TextEncoder();
|
|
7929
|
+
function fnv1aBytes(bytes) {
|
|
7930
|
+
return fnv1aContinue(FNV_OFFSET_BASIS, bytes, bytes.length) >>> 0;
|
|
7931
|
+
}
|
|
7932
|
+
function fnv1aContinue(start, bytes, length) {
|
|
7933
|
+
let hash = start;
|
|
7934
|
+
for (let i = 0; i < length; i++) {
|
|
7935
|
+
hash = Math.imul(hash ^ bytes[i], FNV_PRIME);
|
|
7936
|
+
}
|
|
7937
|
+
return hash;
|
|
7938
|
+
}
|
|
7939
|
+
var scratchNumber = new Float64Array(1);
|
|
7940
|
+
var scratchBytes = new Uint8Array(scratchNumber.buffer);
|
|
7941
|
+
var HOST_IS_LITTLE_ENDIAN = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
|
|
7942
|
+
function fnv1aContinueTaggedNumbers(start, tag, numbers) {
|
|
7943
|
+
let hash = start;
|
|
7944
|
+
const b = scratchBytes;
|
|
7945
|
+
for (let i = 0; i < numbers.length; i++) {
|
|
7946
|
+
scratchNumber[0] = Math.fround(numbers[i]);
|
|
7947
|
+
hash = Math.imul(hash ^ tag, FNV_PRIME);
|
|
7948
|
+
if (HOST_IS_LITTLE_ENDIAN) {
|
|
7949
|
+
hash = Math.imul(hash ^ b[7], FNV_PRIME);
|
|
7950
|
+
hash = Math.imul(hash ^ b[6], FNV_PRIME);
|
|
7951
|
+
hash = Math.imul(hash ^ b[5], FNV_PRIME);
|
|
7952
|
+
hash = Math.imul(hash ^ b[4], FNV_PRIME);
|
|
7953
|
+
hash = Math.imul(hash ^ b[3], FNV_PRIME);
|
|
7954
|
+
hash = Math.imul(hash ^ b[2], FNV_PRIME);
|
|
7955
|
+
hash = Math.imul(hash ^ b[1], FNV_PRIME);
|
|
7956
|
+
hash = Math.imul(hash ^ b[0], FNV_PRIME);
|
|
7957
|
+
} else {
|
|
7958
|
+
for (let at = 0; at < 8; at++) hash = Math.imul(hash ^ b[at], FNV_PRIME);
|
|
7959
|
+
}
|
|
7960
|
+
}
|
|
7961
|
+
return hash;
|
|
7962
|
+
}
|
|
7963
|
+
function fnv1aUint32s(values) {
|
|
7964
|
+
let hash = FNV_OFFSET_BASIS;
|
|
7965
|
+
for (const value of values) {
|
|
7966
|
+
for (let shift = 24; shift >= 0; shift -= 8) {
|
|
7967
|
+
hash ^= value >>> shift & 255;
|
|
7968
|
+
hash = Math.imul(hash, FNV_PRIME);
|
|
7969
|
+
}
|
|
7970
|
+
}
|
|
7971
|
+
return hash >>> 0;
|
|
7972
|
+
}
|
|
7973
|
+
function fnv1a(input) {
|
|
7974
|
+
return fnv1aBytes(encoder.encode(input));
|
|
7975
|
+
}
|
|
7976
|
+
|
|
7954
7977
|
// src/partitioning/manager/nested-values.ts
|
|
7955
7978
|
function setNestedValue(obj, path, value) {
|
|
7956
7979
|
if (!path.includes(".")) {
|
|
@@ -8994,4 +9017,4 @@ function fanOutMatchCount(manager, params, language, schema, options) {
|
|
|
8994
9017
|
return count;
|
|
8995
9018
|
}
|
|
8996
9019
|
|
|
8997
|
-
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FNV_OFFSET_BASIS, MAX_BM25_B, MAX_BM25_K1, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, analyseQueryPhrases, applyPagination, applyPartitionFiltersBitset, applyProjection, applySorting, availablePatternSearch, bm25SetsValue, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUint32s, fulltextMatches, holdsQuotedText, isCompositePartition, joinPatternWorkMeter, kWayMerge, loadPhraseCodeFor, mergeBM25Levels, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireLoadedPhraseCode, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|
|
9020
|
+
export { ARRAY_POSITION_GAP, DEFAULT_BM25_B, DEFAULT_BM25_K1, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, FNV_OFFSET_BASIS, MAX_BM25_B, MAX_BM25_K1, MAX_TERM_FREQUENCY, PATTERN_HOST, ValueAffixes, analyseQueryPhrases, applyPagination, applyPartitionFiltersBitset, applyProjection, applySorting, availablePatternSearch, bm25SetsValue, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, copyVectorsInto, countPartialSpelling, countsWithoutScores, createBooleanIndex, createCompositePartition, createEnumIndex, createFrozenPartialWords, createGeoIndex, createNumericIndex, createPartitionIndex, createPatternWorkMeter, createSurfaceRegistry, decodePageCursor, documentValueBytes, documentWithOwnBuffers, encodeFieldIndexes, encodePageCursor, encodePartialSpellingsOf, encodeSegmentState, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aContinue, fnv1aContinueTaggedNumbers, fnv1aUint32s, fulltextMatches, holdsQuotedText, isCompositePartition, joinPatternWorkMeter, kWayMerge, loadPhraseCodeFor, mergeBM25Levels, mergeDocFrequencies, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionStateOf, partitionsIn, patternIndexOf, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, readProjectedPath, readSortValues, recordValueSize, registerPartialWordSearch, registerPatternSearch, requireLoadedPhraseCode, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, scanPostingBlocks, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, survivorDocumentsOf, undeclaredFieldOnStrictIndex, validatedFilterOf, valueWithOwnBuffers };
|