@delali/narsil 0.3.1-next.73 → 0.3.1-next.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{capabilities-CYeAWYxQ.d.ts → capabilities-BHNksicw.d.ts} +16 -2
- package/dist/{chunk-PU43WDY4.mjs → chunk-2E35UM3G.mjs} +1 -13
- package/dist/{chunk-AK3BLGCI.mjs → chunk-356UOTK2.mjs} +15 -48
- package/dist/chunk-7ODNRREZ.mjs +2050 -0
- package/dist/chunk-AIUOAN7Y.mjs +658 -0
- package/dist/{chunk-WVDFG45M.mjs → chunk-AMGDG4ZJ.mjs} +3 -2
- package/dist/{chunk-AFKH4ZK3.mjs → chunk-BT62MUM5.mjs} +1 -1
- package/dist/{chunk-LPBWCHBE.mjs → chunk-BT6BI4GV.mjs} +42 -4
- package/dist/{chunk-LJYTW7GT.mjs → chunk-IBR6634K.mjs} +1 -1
- package/dist/chunk-KEHK2O6P.mjs +13 -0
- package/dist/{chunk-CVFAFZSN.mjs → chunk-LOARLB4F.mjs} +46 -34
- package/dist/{chunk-3YKWSO54.mjs → chunk-NAMGDK45.mjs} +228 -649
- package/dist/{chunk-2WLD2NFW.mjs → chunk-PXDY5VRT.mjs} +3 -3
- package/dist/chunk-UOJIV2NG.mjs +593 -0
- package/dist/chunk-V3B24XR7.mjs +54 -0
- package/dist/{chunk-ZULSEZCE.mjs → chunk-WL2GDSCV.mjs} +100 -19
- package/dist/{chunk-THMSFOJO.mjs → chunk-YOVS4EV5.mjs} +582 -598
- package/dist/{client-D698E5ge.d.ts → client-D4sXFS9f.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/{config-Bv8etwUZ.d.ts → config-B-GRonqw.d.ts} +2 -2
- package/dist/{config-QXU3HMpq.d.ts → config-C1kbq1X2.d.ts} +2 -2
- package/dist/direct-executor-AVNWB73C.mjs +16 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +23 -13
- package/dist/{engine-D_OzKZ-6.d.ts → engine-DcGSd5vh.d.ts} +2 -2
- package/dist/index.browser.mjs +1272 -834
- package/dist/index.d.ts +27 -8
- package/dist/index.mjs +14 -11
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-B4jMmygw.d.ts → memory-M9kPmYP8.d.ts} +1257 -1178
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +11 -0
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +33 -13
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -11
- package/dist/react.d.ts +7 -7
- package/dist/{schema-BBOCdhND.d.ts → schema-DgyDijhf.d.ts} +6 -2
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +36 -11
- package/dist/{tasks-DpgCpW1x.d.ts → tasks-A8se0J98.d.ts} +1 -1
- package/dist/{thread-3LNVA43V.mjs → thread-BSIKT6ED.mjs} +16 -9
- package/dist/{types-BiUmaiWT.d.ts → types-ChrVy4lH.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -3
- package/dist/workers/entry.mjs +3 -3
- package/package.json +16 -7
- package/dist/chunk-XW25X6AU.mjs +0 -614
- package/dist/chunk-ZKRV5M2P.mjs +0 -75
- package/dist/direct-executor-5SRDFZAS.mjs +0 -13
|
@@ -1,11 +1,14 @@
|
|
|
1
|
+
import { requirePatternSearchFor, patternIndexOf, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, requirePatternSearch, PATTERN_HOST, everyValueFacetConfig, mergeFacets, undeclaredFilterField, requireValidOperands, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, coveringRanges } from './chunk-NAMGDK45.mjs';
|
|
1
2
|
import { getLanguage, detectRuntime } from './chunk-4MQ5F4IL.mjs';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
3
|
+
import { MAX_DOC_ID_LENGTH, GEO_UNIT_METRES } from './chunk-V3B24XR7.mjs';
|
|
4
|
+
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, topLayerInParts, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-AMGDG4ZJ.mjs';
|
|
5
|
+
import { noteFallback } from './chunk-2E35UM3G.mjs';
|
|
6
|
+
import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, PATTERN_RUN_CODE_POINTS, bitsetFromSet, bitsetIsEmpty, bitsetAnd, bitsetNot } from './chunk-7ODNRREZ.mjs';
|
|
7
|
+
import { encodePartialSpellings, countSpelling, expandTermPrefix, suggestDisplayTerms, spellingCountsOf, codePointOrder, encodeStringBlob, partialWordSearchModule, partitionDocumentFrequency } from './chunk-AIUOAN7Y.mjs';
|
|
8
|
+
import { tokenizeOptions, getNestedValue, visitTokens, textFieldsChanged, fuzzyPrefixOf, boundedLevenshtein, getFlatSchema, tokenize, getOrCreateFieldNameIndex, tokenizeIterator, getFieldValueByInternalId } from './chunk-UOJIV2NG.mjs';
|
|
9
|
+
import { partialFieldsOf, schemaFieldsOf, isWordIndexedBase, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, parsedTypeOf, isListBase, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-356UOTK2.mjs';
|
|
10
|
+
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
11
|
+
import { compareCodePoints, toReducedSortValue, readSortField, defaultSortMode, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-IBR6634K.mjs';
|
|
9
12
|
import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
|
|
10
13
|
import { totalmem } from 'os';
|
|
11
14
|
import { getHeapStatistics } from 'v8';
|
|
@@ -411,8 +414,6 @@ function compactList(list) {
|
|
|
411
414
|
}
|
|
412
415
|
}
|
|
413
416
|
writeIdx++;
|
|
414
|
-
} else {
|
|
415
|
-
list.totalTermFrequency -= list.termFrequencies[i];
|
|
416
417
|
}
|
|
417
418
|
}
|
|
418
419
|
list.docIds.length = writeIdx;
|
|
@@ -437,8 +438,6 @@ function compactDocEntries(list, internalId) {
|
|
|
437
438
|
}
|
|
438
439
|
}
|
|
439
440
|
writeIdx++;
|
|
440
|
-
} else {
|
|
441
|
-
list.totalTermFrequency -= list.termFrequencies[i];
|
|
442
441
|
}
|
|
443
442
|
}
|
|
444
443
|
list.docIds.length = writeIdx;
|
|
@@ -524,12 +523,13 @@ function createInvertedIndex(fieldNameTable) {
|
|
|
524
523
|
}
|
|
525
524
|
list.length++;
|
|
526
525
|
},
|
|
527
|
-
remove(token, internalId) {
|
|
526
|
+
remove(token, internalId, occurrences) {
|
|
528
527
|
const list = index.get(token);
|
|
529
528
|
if (!list || !list.docIdSet.has(internalId)) return;
|
|
530
529
|
list.structureRevision++;
|
|
531
530
|
list.docIdSet.delete(internalId);
|
|
532
531
|
list.deletedDocs.add(internalId);
|
|
532
|
+
list.totalTermFrequency = Math.max(0, list.totalTermFrequency - occurrences);
|
|
533
533
|
if (list.docIdSet.size === 0) {
|
|
534
534
|
index.delete(token);
|
|
535
535
|
untrackToken(token);
|
|
@@ -562,6 +562,9 @@ function createInvertedIndex(fieldNameTable) {
|
|
|
562
562
|
lookup(token) {
|
|
563
563
|
return index.get(token);
|
|
564
564
|
},
|
|
565
|
+
documentFrequency(token) {
|
|
566
|
+
return index.get(token)?.docIdSet.size ?? 0;
|
|
567
|
+
},
|
|
565
568
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
566
569
|
if (tolerance === 0) {
|
|
567
570
|
const exact = index.get(token);
|
|
@@ -940,54 +943,6 @@ function createSurfaceRegistry() {
|
|
|
940
943
|
};
|
|
941
944
|
}
|
|
942
945
|
|
|
943
|
-
// src/core/pattern-index/platform.ts
|
|
944
|
-
function platformPatternSearch() {
|
|
945
|
-
return patternSearchModule;
|
|
946
|
-
}
|
|
947
|
-
|
|
948
|
-
// src/core/pattern-index/registry.ts
|
|
949
|
-
var registered = null;
|
|
950
|
-
function isPatternSearchModule(value) {
|
|
951
|
-
return value !== null && typeof value === "object" && value.name === "pattern" && "createIndex" in value && typeof value.createIndex === "function" && "readArrays" in value && typeof value.readArrays === "function" && "mergeArrays" in value && typeof value.mergeArrays === "function" && "matchInto" in value && typeof value.matchInto === "function";
|
|
952
|
-
}
|
|
953
|
-
function registerPatternSearch(module) {
|
|
954
|
-
if (!isPatternSearchModule(module)) {
|
|
955
|
-
throw new NarsilError(
|
|
956
|
-
ErrorCodes.CONFIG_INVALID,
|
|
957
|
-
'registerPatternSearch takes the patternSearch object that "@delali/narsil/pattern" exports'
|
|
958
|
-
);
|
|
959
|
-
}
|
|
960
|
-
registered = module;
|
|
961
|
-
}
|
|
962
|
-
function availablePatternSearch() {
|
|
963
|
-
return registered ?? platformPatternSearch();
|
|
964
|
-
}
|
|
965
|
-
function requirePatternSearchFor(schema) {
|
|
966
|
-
for (const { path, pattern } of schemaFieldsOf(schema)) {
|
|
967
|
-
if (pattern) requirePatternSearch(path);
|
|
968
|
-
}
|
|
969
|
-
}
|
|
970
|
-
function requirePatternSearch(fieldPath) {
|
|
971
|
-
const module = availablePatternSearch();
|
|
972
|
-
if (module !== null) return module;
|
|
973
|
-
throw new NarsilError(
|
|
974
|
-
ErrorCodes.CONFIG_INVALID,
|
|
975
|
-
`Field "${fieldPath}" is a pattern field, but this bundle contains no pattern search code. Add import { patternSearch } from '@delali/narsil/pattern', then call registerPatternSearch(patternSearch) before you create or load the index`,
|
|
976
|
-
{ field: fieldPath }
|
|
977
|
-
);
|
|
978
|
-
}
|
|
979
|
-
|
|
980
|
-
// src/core/pattern-index/host.ts
|
|
981
|
-
var PATTERN_HOST = { appendFoldedCodePoint };
|
|
982
|
-
function patternIndexOf(indexes, fieldPath) {
|
|
983
|
-
let index = indexes.get(fieldPath);
|
|
984
|
-
if (index === void 0) {
|
|
985
|
-
index = requirePatternSearch(fieldPath).createIndex(PATTERN_HOST);
|
|
986
|
-
indexes.set(fieldPath, index);
|
|
987
|
-
}
|
|
988
|
-
return index;
|
|
989
|
-
}
|
|
990
|
-
|
|
991
946
|
// src/geo/haversine.ts
|
|
992
947
|
function haversineDistance(lat1, lon1, lat2, lon2) {
|
|
993
948
|
const phi1 = lat1 * DEG_TO_RAD;
|
|
@@ -1667,6 +1622,13 @@ function resolveInternalId(state, docId) {
|
|
|
1667
1622
|
}
|
|
1668
1623
|
return internalId;
|
|
1669
1624
|
}
|
|
1625
|
+
function emptyIndexedText(fields) {
|
|
1626
|
+
const holdsPartialField = fields.some((field) => field.partial);
|
|
1627
|
+
return { fieldLengths: {}, tokensByField: {}, partialSpellings: holdsPartialField ? /* @__PURE__ */ new Map() : null };
|
|
1628
|
+
}
|
|
1629
|
+
function partialSpellingsFor(field, indexed) {
|
|
1630
|
+
return field.partial ? indexed.partialSpellings : null;
|
|
1631
|
+
}
|
|
1670
1632
|
function countSurface(counts, surface, token) {
|
|
1671
1633
|
if (surface === void 0 || surface.length === 0) return;
|
|
1672
1634
|
const existing = counts.get(surface);
|
|
@@ -1676,6 +1638,18 @@ function countSurface(counts, surface, token) {
|
|
|
1676
1638
|
counts.set(surface, { token, count: 1 });
|
|
1677
1639
|
}
|
|
1678
1640
|
}
|
|
1641
|
+
function countOccurrence(counts, spellings, surface, token) {
|
|
1642
|
+
countSurface(counts, surface, token);
|
|
1643
|
+
if (spellings !== null) countSpelling(spellings, token, surface ?? token, 1);
|
|
1644
|
+
}
|
|
1645
|
+
function addFieldOccurrences(occurrencesByToken, fieldTokens) {
|
|
1646
|
+
const fieldCounts = /* @__PURE__ */ new Map();
|
|
1647
|
+
for (const token of fieldTokens) fieldCounts.set(token, (fieldCounts.get(token) ?? 0) + 1);
|
|
1648
|
+
for (const [token, count] of fieldCounts) {
|
|
1649
|
+
const stored = count > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : count;
|
|
1650
|
+
occurrencesByToken.set(token, (occurrencesByToken.get(token) ?? 0) + stored);
|
|
1651
|
+
}
|
|
1652
|
+
}
|
|
1679
1653
|
function applySurfaceCounts(state, counts, direction) {
|
|
1680
1654
|
for (const [surface, entry] of counts) {
|
|
1681
1655
|
if (direction === 1) {
|
|
@@ -1685,7 +1659,7 @@ function applySurfaceCounts(state, counts, direction) {
|
|
|
1685
1659
|
}
|
|
1686
1660
|
}
|
|
1687
1661
|
}
|
|
1688
|
-
function indexStringField(state, internalId, fieldPath, text, language, options, fieldLengths, tokensByField) {
|
|
1662
|
+
function indexStringField(state, internalId, fieldPath, text, language, options, fieldLengths, tokensByField, spellings = null) {
|
|
1689
1663
|
const fieldTokenList = [];
|
|
1690
1664
|
const fieldNameIndex2 = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
|
|
1691
1665
|
const opts = tokenizeOptions(options);
|
|
@@ -1701,7 +1675,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
1701
1675
|
tokenFreqs.set(t.token, { count: 1, positions: [t.position] });
|
|
1702
1676
|
}
|
|
1703
1677
|
fieldTokenList.push(t.token);
|
|
1704
|
-
|
|
1678
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
1705
1679
|
}
|
|
1706
1680
|
for (const [token, freq] of tokenFreqs) {
|
|
1707
1681
|
state.invertedIdx.insert(token, internalId, freq.count, fieldNameIndex2, freq.positions);
|
|
@@ -1711,7 +1685,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
1711
1685
|
for (const t of tokenizeIterator(text, language, opts)) {
|
|
1712
1686
|
tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
|
|
1713
1687
|
fieldTokenList.push(t.token);
|
|
1714
|
-
|
|
1688
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
1715
1689
|
}
|
|
1716
1690
|
for (const [token, count] of tokenCounts) {
|
|
1717
1691
|
state.invertedIdx.insert(token, internalId, count, fieldNameIndex2, null);
|
|
@@ -1721,7 +1695,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
1721
1695
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
1722
1696
|
tokensByField[fieldPath] = fieldTokenList;
|
|
1723
1697
|
}
|
|
1724
|
-
function indexStringArrayField(state, internalId, fieldPath, arr, language, options, fieldLengths, tokensByField) {
|
|
1698
|
+
function indexStringArrayField(state, internalId, fieldPath, arr, language, options, fieldLengths, tokensByField, spellings = null) {
|
|
1725
1699
|
const fieldTokenList = [];
|
|
1726
1700
|
const fieldNameIndex2 = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
|
|
1727
1701
|
const opts = tokenizeOptions(options);
|
|
@@ -1740,7 +1714,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
1740
1714
|
tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
|
|
1741
1715
|
}
|
|
1742
1716
|
fieldTokenList.push(t.token);
|
|
1743
|
-
|
|
1717
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
1744
1718
|
itemTokenCount++;
|
|
1745
1719
|
}
|
|
1746
1720
|
positionOffset += itemTokenCount;
|
|
@@ -1755,7 +1729,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
1755
1729
|
for (const t of tokenizeIterator(item, language, opts)) {
|
|
1756
1730
|
tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
|
|
1757
1731
|
fieldTokenList.push(t.token);
|
|
1758
|
-
|
|
1732
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
1759
1733
|
}
|
|
1760
1734
|
}
|
|
1761
1735
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
@@ -1767,17 +1741,28 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
1767
1741
|
tokensByField[fieldPath] = fieldTokenList;
|
|
1768
1742
|
}
|
|
1769
1743
|
function indexDocument(state, docId, document, fields, language, options) {
|
|
1770
|
-
const
|
|
1771
|
-
const tokensByField =
|
|
1744
|
+
const indexed = emptyIndexedText(fields);
|
|
1745
|
+
const { fieldLengths, tokensByField } = indexed;
|
|
1772
1746
|
const internalId = resolveInternalId(state, docId);
|
|
1773
|
-
for (const
|
|
1747
|
+
for (const field of fields) {
|
|
1748
|
+
const { path: fieldPath, base, pattern } = field;
|
|
1774
1749
|
const value = getNestedValue(document, fieldPath);
|
|
1775
1750
|
if (value === void 0 || value === null) continue;
|
|
1776
1751
|
recordValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
1777
1752
|
ensureFieldIndex(state, fieldPath, base);
|
|
1778
1753
|
if (pattern) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
|
|
1779
1754
|
if (base === "string") {
|
|
1780
|
-
indexStringField(
|
|
1755
|
+
indexStringField(
|
|
1756
|
+
state,
|
|
1757
|
+
internalId,
|
|
1758
|
+
fieldPath,
|
|
1759
|
+
value,
|
|
1760
|
+
language,
|
|
1761
|
+
options,
|
|
1762
|
+
fieldLengths,
|
|
1763
|
+
tokensByField,
|
|
1764
|
+
partialSpellingsFor(field, indexed)
|
|
1765
|
+
);
|
|
1781
1766
|
} else if (base === "number") {
|
|
1782
1767
|
getOrCreateNumericIndex(state, fieldPath).insert(internalId, value);
|
|
1783
1768
|
} else if (base === "boolean") {
|
|
@@ -1796,7 +1781,8 @@ function indexDocument(state, docId, document, fields, language, options) {
|
|
|
1796
1781
|
language,
|
|
1797
1782
|
options,
|
|
1798
1783
|
fieldLengths,
|
|
1799
|
-
tokensByField
|
|
1784
|
+
tokensByField,
|
|
1785
|
+
partialSpellingsFor(field, indexed)
|
|
1800
1786
|
);
|
|
1801
1787
|
} else if (base === "number[]") {
|
|
1802
1788
|
const numIdx = getOrCreateNumericIndex(state, fieldPath);
|
|
@@ -1811,15 +1797,17 @@ function indexDocument(state, docId, document, fields, language, options) {
|
|
|
1811
1797
|
}
|
|
1812
1798
|
}
|
|
1813
1799
|
}
|
|
1814
|
-
return
|
|
1800
|
+
return indexed;
|
|
1815
1801
|
}
|
|
1816
1802
|
function removeFromIndexes(state, docId, storedDoc, schemaFields, language, options) {
|
|
1817
|
-
const
|
|
1818
|
-
const tokensByField =
|
|
1803
|
+
const removed = emptyIndexedText(schemaFields);
|
|
1804
|
+
const { fieldLengths, tokensByField } = removed;
|
|
1819
1805
|
const fields = storedDoc.fields;
|
|
1820
1806
|
const internalId = resolveInternalId(state, docId);
|
|
1821
1807
|
const opts = tokenizeOptions(options);
|
|
1822
|
-
|
|
1808
|
+
const occurrencesByToken = /* @__PURE__ */ new Map();
|
|
1809
|
+
for (const field of schemaFields) {
|
|
1810
|
+
const { path: fieldPath, base, pattern } = field;
|
|
1823
1811
|
const value = getNestedValue(fields, fieldPath);
|
|
1824
1812
|
if (value === void 0 || value === null) continue;
|
|
1825
1813
|
forgetValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
@@ -1827,16 +1815,15 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
1827
1815
|
if (base === "string") {
|
|
1828
1816
|
const result = tokenize(value, language, opts);
|
|
1829
1817
|
fieldLengths[fieldPath] = result.tokens.length;
|
|
1830
|
-
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
1831
1818
|
const fieldTokenList = [];
|
|
1832
1819
|
const surfaceCounts = /* @__PURE__ */ new Map();
|
|
1820
|
+
const spellings = partialSpellingsFor(field, removed);
|
|
1833
1821
|
for (let i = 0; i < result.tokens.length; i++) {
|
|
1834
1822
|
const t = result.tokens[i];
|
|
1835
|
-
uniqueTokens.add(t.token);
|
|
1836
1823
|
fieldTokenList.push(t.token);
|
|
1837
|
-
|
|
1824
|
+
countOccurrence(surfaceCounts, spellings, result.surfaces?.[i], t.token);
|
|
1838
1825
|
}
|
|
1839
|
-
|
|
1826
|
+
addFieldOccurrences(occurrencesByToken, fieldTokenList);
|
|
1840
1827
|
applySurfaceCounts(state, surfaceCounts, -1);
|
|
1841
1828
|
tokensByField[fieldPath] = fieldTokenList;
|
|
1842
1829
|
} else if (base === "number") {
|
|
@@ -1849,20 +1836,19 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
1849
1836
|
state.geoIndexes.get(fieldPath)?.remove(internalId);
|
|
1850
1837
|
} else if (base === "string[]") {
|
|
1851
1838
|
const arr = value;
|
|
1852
|
-
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
1853
1839
|
const fieldTokenList = [];
|
|
1854
1840
|
const surfaceCounts = /* @__PURE__ */ new Map();
|
|
1841
|
+
const spellings = partialSpellingsFor(field, removed);
|
|
1855
1842
|
for (const item of arr) {
|
|
1856
1843
|
const result = tokenize(item, language, opts);
|
|
1857
1844
|
for (let i = 0; i < result.tokens.length; i++) {
|
|
1858
1845
|
const t = result.tokens[i];
|
|
1859
|
-
uniqueTokens.add(t.token);
|
|
1860
1846
|
fieldTokenList.push(t.token);
|
|
1861
|
-
|
|
1847
|
+
countOccurrence(surfaceCounts, spellings, result.surfaces?.[i], t.token);
|
|
1862
1848
|
}
|
|
1863
1849
|
}
|
|
1864
1850
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
1865
|
-
|
|
1851
|
+
addFieldOccurrences(occurrencesByToken, fieldTokenList);
|
|
1866
1852
|
applySurfaceCounts(state, surfaceCounts, -1);
|
|
1867
1853
|
tokensByField[fieldPath] = fieldTokenList;
|
|
1868
1854
|
} else if (base === "number[]") {
|
|
@@ -1882,7 +1868,8 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
1882
1868
|
}
|
|
1883
1869
|
}
|
|
1884
1870
|
}
|
|
1885
|
-
|
|
1871
|
+
for (const [token, occurrences] of occurrencesByToken) state.invertedIdx.remove(token, internalId, occurrences);
|
|
1872
|
+
return removed;
|
|
1886
1873
|
}
|
|
1887
1874
|
|
|
1888
1875
|
// src/core/partition/field-updates.ts
|
|
@@ -1999,6 +1986,8 @@ function evaluatePatternTests(fieldPath, filter, context, bitsets) {
|
|
|
1999
1986
|
if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
|
|
2000
1987
|
if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
|
|
2001
1988
|
if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
|
|
1989
|
+
if (typeof filter.wildcard === "string") bitsets.push(matching("wildcard", [filter.wildcard]));
|
|
1990
|
+
if (typeof filter.regex === "string") bitsets.push(matching("regex", [filter.regex]));
|
|
2002
1991
|
}
|
|
2003
1992
|
|
|
2004
1993
|
// src/filters/field-tests.ts
|
|
@@ -2203,6 +2192,8 @@ var FIELD_FILTER_OPERATORS = [
|
|
|
2203
2192
|
"startsWith",
|
|
2204
2193
|
"endsWith",
|
|
2205
2194
|
"contains",
|
|
2195
|
+
"wildcard",
|
|
2196
|
+
"regex",
|
|
2206
2197
|
"caseFold",
|
|
2207
2198
|
"containsAll",
|
|
2208
2199
|
"matchesAny",
|
|
@@ -3017,6 +3008,67 @@ function partitionFilterMatches(state, filters, schema, meter) {
|
|
|
3017
3008
|
};
|
|
3018
3009
|
}
|
|
3019
3010
|
|
|
3011
|
+
// src/core/scorer.ts
|
|
3012
|
+
var DEFAULT_K1 = 1.2;
|
|
3013
|
+
var DEFAULT_B = 0.75;
|
|
3014
|
+
function computeIDF(docFrequency, totalDocs) {
|
|
3015
|
+
if (totalDocs === 0) return 0;
|
|
3016
|
+
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
3017
|
+
}
|
|
3018
|
+
function bm25PruningSound(params) {
|
|
3019
|
+
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
3020
|
+
const b = params?.b ?? DEFAULT_B;
|
|
3021
|
+
return Number.isFinite(k1) && k1 >= 0 && b >= 0 && b <= 1;
|
|
3022
|
+
}
|
|
3023
|
+
function resolveBM25Params(params) {
|
|
3024
|
+
return { k1: params?.k1 ?? DEFAULT_K1, b: params?.b ?? DEFAULT_B };
|
|
3025
|
+
}
|
|
3026
|
+
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
3027
|
+
if (avgFieldLength === 0) return 0;
|
|
3028
|
+
const numerator = termFrequency * (k1 + 1);
|
|
3029
|
+
const denominator = termFrequency + k1 * (1 - b + b * fieldLength / avgFieldLength);
|
|
3030
|
+
if (denominator === 0) return 0;
|
|
3031
|
+
return idf * (numerator / denominator);
|
|
3032
|
+
}
|
|
3033
|
+
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
3034
|
+
if (totalDocs === 0) return 0;
|
|
3035
|
+
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
3036
|
+
const b = params?.b ?? DEFAULT_B;
|
|
3037
|
+
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
3038
|
+
}
|
|
3039
|
+
|
|
3040
|
+
// src/core/partition/field-scoring.ts
|
|
3041
|
+
function loadFieldScoring(state, fields, boost, averageFieldLengths) {
|
|
3042
|
+
const fieldNames = state.fieldNameTable.names;
|
|
3043
|
+
const scoring = {
|
|
3044
|
+
searchable: new Uint8Array(fieldNames.length),
|
|
3045
|
+
boosts: new Float64Array(fieldNames.length),
|
|
3046
|
+
averageLengths: new Float64Array(fieldNames.length),
|
|
3047
|
+
lengthColumns: new Array(fieldNames.length).fill(null)
|
|
3048
|
+
};
|
|
3049
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
3050
|
+
const fieldName = fieldNames[fieldIndex];
|
|
3051
|
+
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
3052
|
+
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
3053
|
+
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
3054
|
+
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
3055
|
+
}
|
|
3056
|
+
return scoring;
|
|
3057
|
+
}
|
|
3058
|
+
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
3059
|
+
const column = fieldIndex < columns.length ? columns[fieldIndex] : null;
|
|
3060
|
+
if (column === null || internalId >= column.length) return averageLength;
|
|
3061
|
+
const stored = column[internalId];
|
|
3062
|
+
return stored > 0 ? stored : averageLength;
|
|
3063
|
+
}
|
|
3064
|
+
function bestSearchable(searchable, values) {
|
|
3065
|
+
let best = 0;
|
|
3066
|
+
for (let index = 0; index < searchable.length; index++) {
|
|
3067
|
+
if (searchable[index] === 1 && values[index] > best) best = values[index];
|
|
3068
|
+
}
|
|
3069
|
+
return best;
|
|
3070
|
+
}
|
|
3071
|
+
|
|
3020
3072
|
// src/core/partition/posting-columns.ts
|
|
3021
3073
|
function postingColumns(list) {
|
|
3022
3074
|
const deletedDocs = list.deletedDocs;
|
|
@@ -3030,39 +3082,174 @@ function postingColumns(list) {
|
|
|
3030
3082
|
};
|
|
3031
3083
|
}
|
|
3032
3084
|
|
|
3085
|
+
// src/core/partition/scoring.ts
|
|
3086
|
+
var EMPTY_COMPONENTS = Object.freeze({});
|
|
3087
|
+
function recordComponents(components, internalId, fieldName, token, termFrequency, fieldLength, idf) {
|
|
3088
|
+
const existing = components.get(internalId);
|
|
3089
|
+
if (existing) {
|
|
3090
|
+
existing.termFrequencies[`${fieldName}:${token}`] = termFrequency;
|
|
3091
|
+
existing.fieldLengths[fieldName] = fieldLength;
|
|
3092
|
+
existing.idf[token] = idf;
|
|
3093
|
+
return;
|
|
3094
|
+
}
|
|
3095
|
+
components.set(internalId, {
|
|
3096
|
+
termFrequencies: { [`${fieldName}:${token}`]: termFrequency },
|
|
3097
|
+
fieldLengths: { [fieldName]: fieldLength },
|
|
3098
|
+
idf: { [token]: idf }
|
|
3099
|
+
});
|
|
3100
|
+
}
|
|
3101
|
+
function mergeGroupComponents(components, internalId, contribution) {
|
|
3102
|
+
const existing = components.get(internalId);
|
|
3103
|
+
if (existing) {
|
|
3104
|
+
Object.assign(existing.termFrequencies, contribution.termFrequencies);
|
|
3105
|
+
Object.assign(existing.fieldLengths, contribution.fieldLengths);
|
|
3106
|
+
existing.idf[contribution.token] = contribution.idf;
|
|
3107
|
+
return;
|
|
3108
|
+
}
|
|
3109
|
+
components.set(internalId, {
|
|
3110
|
+
termFrequencies: contribution.termFrequencies,
|
|
3111
|
+
fieldLengths: contribution.fieldLengths,
|
|
3112
|
+
idf: { [contribution.token]: contribution.idf }
|
|
3113
|
+
});
|
|
3114
|
+
}
|
|
3115
|
+
|
|
3116
|
+
// src/core/partition/group-scoring.ts
|
|
3117
|
+
function partialFieldMarks(fieldNames, partialFields) {
|
|
3118
|
+
const marks = new Uint8Array(fieldNames.length);
|
|
3119
|
+
if (partialFields === void 0 || partialFields.length === 0) return marks;
|
|
3120
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
3121
|
+
if (partialFields.includes(fieldNames[fieldIndex])) marks[fieldIndex] = 1;
|
|
3122
|
+
}
|
|
3123
|
+
return marks;
|
|
3124
|
+
}
|
|
3125
|
+
function termGroupOf(groups, token) {
|
|
3126
|
+
if (groups === void 0) return void 0;
|
|
3127
|
+
for (const group of groups) {
|
|
3128
|
+
if (group.token === token) return group;
|
|
3129
|
+
}
|
|
3130
|
+
return void 0;
|
|
3131
|
+
}
|
|
3132
|
+
function typoMatchesFor(index, token, group, exact, tolerance, prefixLength) {
|
|
3133
|
+
if (group !== void 0) {
|
|
3134
|
+
if (!group.typoMatches || exact || tolerance === 0) return [];
|
|
3135
|
+
return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token);
|
|
3136
|
+
}
|
|
3137
|
+
if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
|
|
3138
|
+
const postingList = index.lookup(token);
|
|
3139
|
+
return postingList ? [{ token, postingList }] : [];
|
|
3140
|
+
}
|
|
3141
|
+
function resolveGroupMatches(context, group) {
|
|
3142
|
+
let groupDocumentFrequency = 0;
|
|
3143
|
+
const found = [];
|
|
3144
|
+
for (const member of group.members) {
|
|
3145
|
+
const documents = context.globalFrequency(member.token) ?? context.index.documentFrequency(member.token);
|
|
3146
|
+
if (documents > groupDocumentFrequency) groupDocumentFrequency = documents;
|
|
3147
|
+
const postingList = context.index.lookup(member.token);
|
|
3148
|
+
if (postingList === void 0) continue;
|
|
3149
|
+
found.push({ token: member.token, factor: member.factor, partialFactor: member.partialFactor, postingList });
|
|
3150
|
+
}
|
|
3151
|
+
if (found.length === 0) return [];
|
|
3152
|
+
const idf = computeIDF(groupDocumentFrequency, context.totalDocs);
|
|
3153
|
+
return found.map((entry) => ({ ...entry, idf }));
|
|
3154
|
+
}
|
|
3155
|
+
function groupPostingCount(matches) {
|
|
3156
|
+
let total = 0;
|
|
3157
|
+
for (const match of matches) total += match.postingList.length;
|
|
3158
|
+
return total;
|
|
3159
|
+
}
|
|
3160
|
+
function computeGroupContributions(context, matches, collect) {
|
|
3161
|
+
const { fields, fieldNames, filterBitset, partialMarks, k1, b } = context;
|
|
3162
|
+
const scoresAreZero = context.totalDocs === 0;
|
|
3163
|
+
const best = /* @__PURE__ */ new Map();
|
|
3164
|
+
for (const match of matches) {
|
|
3165
|
+
const perTerm = /* @__PURE__ */ new Map();
|
|
3166
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
3167
|
+
match.postingList
|
|
3168
|
+
);
|
|
3169
|
+
for (let pi = 0; pi < count; pi++) {
|
|
3170
|
+
const internalId = docIds[pi];
|
|
3171
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
3172
|
+
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
3173
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
3174
|
+
if (fields.searchable[fieldIndex] === 0) continue;
|
|
3175
|
+
const factor = partialMarks[fieldIndex] === 1 ? match.partialFactor : match.factor;
|
|
3176
|
+
if (factor === 0) continue;
|
|
3177
|
+
const fieldName = fieldNames[fieldIndex];
|
|
3178
|
+
const termFrequency = termFrequencies[pi];
|
|
3179
|
+
const fieldBoost = fields.boosts[fieldIndex];
|
|
3180
|
+
const avgLen = fields.averageLengths[fieldIndex];
|
|
3181
|
+
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
3182
|
+
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b)) * fieldBoost * factor;
|
|
3183
|
+
const existing = perTerm.get(internalId);
|
|
3184
|
+
if (existing) {
|
|
3185
|
+
existing.score += termScore;
|
|
3186
|
+
if (collect) {
|
|
3187
|
+
existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
|
|
3188
|
+
existing.fieldLengths[fieldName] = actualFieldLength;
|
|
3189
|
+
}
|
|
3190
|
+
} else if (collect) {
|
|
3191
|
+
perTerm.set(internalId, {
|
|
3192
|
+
score: termScore,
|
|
3193
|
+
token: match.token,
|
|
3194
|
+
idf: match.idf,
|
|
3195
|
+
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
3196
|
+
fieldLengths: { [fieldName]: actualFieldLength }
|
|
3197
|
+
});
|
|
3198
|
+
} else {
|
|
3199
|
+
perTerm.set(internalId, {
|
|
3200
|
+
score: termScore,
|
|
3201
|
+
token: match.token,
|
|
3202
|
+
idf: match.idf,
|
|
3203
|
+
termFrequencies: EMPTY_COMPONENTS,
|
|
3204
|
+
fieldLengths: EMPTY_COMPONENTS
|
|
3205
|
+
});
|
|
3206
|
+
}
|
|
3207
|
+
}
|
|
3208
|
+
for (const [internalId, contribution] of perTerm) {
|
|
3209
|
+
const current = best.get(internalId);
|
|
3210
|
+
if (!current || contribution.score > current.score) {
|
|
3211
|
+
best.set(internalId, contribution);
|
|
3212
|
+
}
|
|
3213
|
+
}
|
|
3214
|
+
}
|
|
3215
|
+
return best;
|
|
3216
|
+
}
|
|
3217
|
+
|
|
3033
3218
|
// src/core/partition/matches.ts
|
|
3034
|
-
function markPostingList(matched, list,
|
|
3219
|
+
function markPostingList(matched, list, markedFields, filterBitset) {
|
|
3035
3220
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(list);
|
|
3036
3221
|
for (let pi = 0; pi < count; pi++) {
|
|
3037
3222
|
const internalId = docIds[pi];
|
|
3038
3223
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
3039
3224
|
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
3040
|
-
if (
|
|
3225
|
+
if (markedFields[fieldNameIndices[pi]] === 0) continue;
|
|
3041
3226
|
bitsetSet(matched, internalId);
|
|
3042
3227
|
}
|
|
3043
3228
|
}
|
|
3229
|
+
function searchedFieldMarks(fieldNames, fields) {
|
|
3230
|
+
const marks = new Uint8Array(fieldNames.length);
|
|
3231
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
3232
|
+
if (fields === void 0 || fields.includes(fieldNames[fieldIndex])) marks[fieldIndex] = 1;
|
|
3233
|
+
}
|
|
3234
|
+
return marks;
|
|
3235
|
+
}
|
|
3044
3236
|
function searchFulltextMatches(state, params) {
|
|
3045
|
-
const { queryTokens,
|
|
3237
|
+
const { queryTokens, termGroups, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
|
|
3046
3238
|
const fieldNames = state.fieldNameTable.names;
|
|
3047
3239
|
const matched = createBitSet(state.docStore.internalIdCapacity());
|
|
3240
|
+
const searched = searchedFieldMarks(fieldNames, fields);
|
|
3241
|
+
const partial = partialFieldMarks(fieldNames, params.partialFields);
|
|
3242
|
+
const searchedPartial = searched.map((mark, fieldIndex) => mark & partial[fieldIndex]);
|
|
3048
3243
|
for (const qt of queryTokens) {
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
if (seen.has(term)) continue;
|
|
3053
|
-
seen.add(term);
|
|
3054
|
-
const postingList = state.invertedIdx.lookup(term);
|
|
3055
|
-
if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
|
|
3056
|
-
}
|
|
3057
|
-
continue;
|
|
3058
|
-
}
|
|
3059
|
-
if (exact || tolerance === 0) {
|
|
3060
|
-
const postingList = state.invertedIdx.lookup(qt.token);
|
|
3061
|
-
if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
|
|
3062
|
-
continue;
|
|
3244
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
3245
|
+
for (const match of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
3246
|
+
markPostingList(matched, match.postingList, searched, filterBitset);
|
|
3063
3247
|
}
|
|
3064
|
-
|
|
3065
|
-
|
|
3248
|
+
if (group === void 0) continue;
|
|
3249
|
+
for (const member of group.members) {
|
|
3250
|
+
const postingList = state.invertedIdx.lookup(member.token);
|
|
3251
|
+
if (postingList === void 0) continue;
|
|
3252
|
+
markPostingList(matched, postingList, member.factor > 0 ? searched : searchedPartial, filterBitset);
|
|
3066
3253
|
}
|
|
3067
3254
|
}
|
|
3068
3255
|
const docStore = state.docStore;
|
|
@@ -3150,6 +3337,7 @@ function estimatePartitionBytes(state) {
|
|
|
3150
3337
|
bytes += docCount * state.geoIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
3151
3338
|
bytes += docCount * state.sizeIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
3152
3339
|
for (const index of state.patternIndexes.values()) bytes += index.bytes();
|
|
3340
|
+
bytes += state.partialWords.bytes();
|
|
3153
3341
|
return bytes + sortColumnBytes(state);
|
|
3154
3342
|
}
|
|
3155
3343
|
|
|
@@ -3281,10 +3469,71 @@ function mergeSegmentState(target, source) {
|
|
|
3281
3469
|
mergeFieldIndexes(target, source, idMapping);
|
|
3282
3470
|
mergeStatistics(target, source);
|
|
3283
3471
|
mergeSurfaceForms(target, source);
|
|
3472
|
+
target.partialWords.afterMerge(source.partialWords.spellings(), [...idMapping.values()]);
|
|
3284
3473
|
target.sortColumns = null;
|
|
3285
3474
|
target.scoreBuffer = null;
|
|
3286
3475
|
}
|
|
3287
3476
|
|
|
3477
|
+
// src/core/partition/partial-words/platform.ts
|
|
3478
|
+
function platformPartialWordSearch() {
|
|
3479
|
+
return partialWordSearchModule;
|
|
3480
|
+
}
|
|
3481
|
+
|
|
3482
|
+
// src/core/partition/partial-words/registry.ts
|
|
3483
|
+
var registered = null;
|
|
3484
|
+
function isPartialWordSearchModule(value) {
|
|
3485
|
+
return value !== null && typeof value === "object" && value.name === "partialWords" && "createGrowingSlot" in value && typeof value.createGrowingSlot === "function" && "createFrozenSlot" in value && typeof value.createFrozenSlot === "function" && "buildTable" in value && typeof value.buildTable === "function" && "expand" in value && typeof value.expand === "function";
|
|
3486
|
+
}
|
|
3487
|
+
function registerPartialWordSearch(module) {
|
|
3488
|
+
if (!isPartialWordSearchModule(module)) {
|
|
3489
|
+
throw new NarsilError(
|
|
3490
|
+
ErrorCodes.CONFIG_INVALID,
|
|
3491
|
+
'Pass registerPartialWordSearch the partialWordSearch object that "@delali/narsil/partial-words" exports'
|
|
3492
|
+
);
|
|
3493
|
+
}
|
|
3494
|
+
registered = module;
|
|
3495
|
+
}
|
|
3496
|
+
function availablePartialWordSearch() {
|
|
3497
|
+
return registered ?? platformPartialWordSearch();
|
|
3498
|
+
}
|
|
3499
|
+
function requirePartialWordSearchFor(schema) {
|
|
3500
|
+
const fields = partialFieldsOf(schema);
|
|
3501
|
+
if (fields.length === 0 || availablePartialWordSearch() !== null) return;
|
|
3502
|
+
throw new NarsilError(
|
|
3503
|
+
ErrorCodes.CONFIG_INVALID,
|
|
3504
|
+
`The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load the index`,
|
|
3505
|
+
{ field: fields[0] }
|
|
3506
|
+
);
|
|
3507
|
+
}
|
|
3508
|
+
var EMPTY_GROWING_SLOT = {
|
|
3509
|
+
readerFor: () => null,
|
|
3510
|
+
spellings: () => null,
|
|
3511
|
+
bytes: () => 0,
|
|
3512
|
+
afterDocument: () => void 0,
|
|
3513
|
+
afterMerge: () => void 0,
|
|
3514
|
+
forgetSpellings: () => void 0,
|
|
3515
|
+
clear: () => void 0
|
|
3516
|
+
};
|
|
3517
|
+
var EMPTY_FROZEN_SLOT = {
|
|
3518
|
+
readerFor: () => null,
|
|
3519
|
+
spellings: () => null,
|
|
3520
|
+
bytes: () => 0,
|
|
3521
|
+
afterRemoval: () => void 0
|
|
3522
|
+
};
|
|
3523
|
+
function createGrowingPartialWords(view) {
|
|
3524
|
+
return availablePartialWordSearch()?.createGrowingSlot(view) ?? EMPTY_GROWING_SLOT;
|
|
3525
|
+
}
|
|
3526
|
+
function createFrozenPartialWords(initial, tokenTable, documents) {
|
|
3527
|
+
return availablePartialWordSearch()?.createFrozenSlot(initial, tokenTable, documents) ?? EMPTY_FROZEN_SLOT;
|
|
3528
|
+
}
|
|
3529
|
+
function buildPartialWordTableFrom(source, tokenTable) {
|
|
3530
|
+
return availablePartialWordSearch()?.buildTable(source, tokenTable) ?? null;
|
|
3531
|
+
}
|
|
3532
|
+
function expandPartialWords(subs, analysis, requests) {
|
|
3533
|
+
const module = availablePartialWordSearch();
|
|
3534
|
+
return module === null ? requests.map(() => []) : module.expand(subs, analysis, requests);
|
|
3535
|
+
}
|
|
3536
|
+
|
|
3288
3537
|
// src/core/partition/rebuild.ts
|
|
3289
3538
|
var EMPTY_STATS = {
|
|
3290
3539
|
totalDocuments: 0,
|
|
@@ -3340,70 +3589,10 @@ function rebuildTextIndex(state, schema, language, options) {
|
|
|
3340
3589
|
}
|
|
3341
3590
|
} finally {
|
|
3342
3591
|
state.invertedIdx.endBatch();
|
|
3592
|
+
state.partialWords.forgetSpellings();
|
|
3343
3593
|
}
|
|
3344
3594
|
}
|
|
3345
3595
|
|
|
3346
|
-
// src/core/scorer.ts
|
|
3347
|
-
var DEFAULT_K1 = 1.2;
|
|
3348
|
-
var DEFAULT_B = 0.75;
|
|
3349
|
-
function computeIDF(docFrequency, totalDocs) {
|
|
3350
|
-
if (totalDocs === 0) return 0;
|
|
3351
|
-
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
3352
|
-
}
|
|
3353
|
-
function bm25PruningSound(params) {
|
|
3354
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
3355
|
-
const b = params?.b ?? DEFAULT_B;
|
|
3356
|
-
return Number.isFinite(k1) && k1 >= 0 && b >= 0 && b <= 1;
|
|
3357
|
-
}
|
|
3358
|
-
function resolveBM25Params(params) {
|
|
3359
|
-
return { k1: params?.k1 ?? DEFAULT_K1, b: params?.b ?? DEFAULT_B };
|
|
3360
|
-
}
|
|
3361
|
-
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
3362
|
-
if (avgFieldLength === 0) return 0;
|
|
3363
|
-
const numerator = termFrequency * (k1 + 1);
|
|
3364
|
-
const denominator = termFrequency + k1 * (1 - b + b * fieldLength / avgFieldLength);
|
|
3365
|
-
if (denominator === 0) return 0;
|
|
3366
|
-
return idf * (numerator / denominator);
|
|
3367
|
-
}
|
|
3368
|
-
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
3369
|
-
if (totalDocs === 0) return 0;
|
|
3370
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
3371
|
-
const b = params?.b ?? DEFAULT_B;
|
|
3372
|
-
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
3373
|
-
}
|
|
3374
|
-
|
|
3375
|
-
// src/core/partition/field-scoring.ts
|
|
3376
|
-
function loadFieldScoring(state, fields, boost, averageFieldLengths) {
|
|
3377
|
-
const fieldNames = state.fieldNameTable.names;
|
|
3378
|
-
const scoring = {
|
|
3379
|
-
searchable: new Uint8Array(fieldNames.length),
|
|
3380
|
-
boosts: new Float64Array(fieldNames.length),
|
|
3381
|
-
averageLengths: new Float64Array(fieldNames.length),
|
|
3382
|
-
lengthColumns: new Array(fieldNames.length).fill(null)
|
|
3383
|
-
};
|
|
3384
|
-
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
3385
|
-
const fieldName = fieldNames[fieldIndex];
|
|
3386
|
-
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
3387
|
-
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
3388
|
-
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
3389
|
-
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
3390
|
-
}
|
|
3391
|
-
return scoring;
|
|
3392
|
-
}
|
|
3393
|
-
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
3394
|
-
const column = fieldIndex < columns.length ? columns[fieldIndex] : null;
|
|
3395
|
-
if (column === null || internalId >= column.length) return averageLength;
|
|
3396
|
-
const stored = column[internalId];
|
|
3397
|
-
return stored > 0 ? stored : averageLength;
|
|
3398
|
-
}
|
|
3399
|
-
function bestSearchable(searchable, values) {
|
|
3400
|
-
let best = 0;
|
|
3401
|
-
for (let index = 0; index < searchable.length; index++) {
|
|
3402
|
-
if (searchable[index] === 1 && values[index] > best) best = values[index];
|
|
3403
|
-
}
|
|
3404
|
-
return best;
|
|
3405
|
-
}
|
|
3406
|
-
|
|
3407
3596
|
// src/core/partition/block-bounds.ts
|
|
3408
3597
|
var ABSENT_FIELD_LENGTH_BOUND = 0;
|
|
3409
3598
|
var boundsByList = /* @__PURE__ */ new WeakMap();
|
|
@@ -3499,37 +3688,6 @@ function blockBoundsFor(list, columns) {
|
|
|
3499
3688
|
return built;
|
|
3500
3689
|
}
|
|
3501
3690
|
|
|
3502
|
-
// src/core/partition/scoring.ts
|
|
3503
|
-
var EMPTY_COMPONENTS = Object.freeze({});
|
|
3504
|
-
function recordComponents(components, internalId, fieldName, token, termFrequency, fieldLength, idf) {
|
|
3505
|
-
const existing = components.get(internalId);
|
|
3506
|
-
if (existing) {
|
|
3507
|
-
existing.termFrequencies[`${fieldName}:${token}`] = termFrequency;
|
|
3508
|
-
existing.fieldLengths[fieldName] = fieldLength;
|
|
3509
|
-
existing.idf[token] = idf;
|
|
3510
|
-
return;
|
|
3511
|
-
}
|
|
3512
|
-
components.set(internalId, {
|
|
3513
|
-
termFrequencies: { [`${fieldName}:${token}`]: termFrequency },
|
|
3514
|
-
fieldLengths: { [fieldName]: fieldLength },
|
|
3515
|
-
idf: { [token]: idf }
|
|
3516
|
-
});
|
|
3517
|
-
}
|
|
3518
|
-
function mergePrefixComponents(components, internalId, contribution) {
|
|
3519
|
-
const existing = components.get(internalId);
|
|
3520
|
-
if (existing) {
|
|
3521
|
-
Object.assign(existing.termFrequencies, contribution.termFrequencies);
|
|
3522
|
-
Object.assign(existing.fieldLengths, contribution.fieldLengths);
|
|
3523
|
-
existing.idf[contribution.token] = contribution.idf;
|
|
3524
|
-
return;
|
|
3525
|
-
}
|
|
3526
|
-
components.set(internalId, {
|
|
3527
|
-
termFrequencies: contribution.termFrequencies,
|
|
3528
|
-
fieldLengths: contribution.fieldLengths,
|
|
3529
|
-
idf: { [contribution.token]: contribution.idf }
|
|
3530
|
-
});
|
|
3531
|
-
}
|
|
3532
|
-
|
|
3533
3691
|
// src/core/partition/top-k-heap.ts
|
|
3534
3692
|
function candidateWorse(a, b) {
|
|
3535
3693
|
if (a.score !== b.score) return a.score < b.score;
|
|
@@ -3693,7 +3851,7 @@ function topKFromBuffer(buffer, k, resolver, components) {
|
|
|
3693
3851
|
// src/core/partition/multi-term-topk.ts
|
|
3694
3852
|
function prunableMultiTermLists(params, index) {
|
|
3695
3853
|
if (params.queryTokens.length < 2) return null;
|
|
3696
|
-
if (params.
|
|
3854
|
+
if (params.termGroups !== void 0 && params.termGroups.length > 0) return null;
|
|
3697
3855
|
if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
|
|
3698
3856
|
if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
|
|
3699
3857
|
if (params.collectComponents !== false) return null;
|
|
@@ -3865,87 +4023,6 @@ function multiTermTopK(request) {
|
|
|
3865
4023
|
return { scored: pageFromBuffer(scans, buffer, selection, resolver), totalMatched };
|
|
3866
4024
|
}
|
|
3867
4025
|
|
|
3868
|
-
// src/core/partition/prefix-scoring.ts
|
|
3869
|
-
function resolvePrefixMatches(context, token, expansionTerms) {
|
|
3870
|
-
const found = [];
|
|
3871
|
-
const seen = /* @__PURE__ */ new Set();
|
|
3872
|
-
for (const term of [token, ...expansionTerms]) {
|
|
3873
|
-
if (seen.has(term)) continue;
|
|
3874
|
-
seen.add(term);
|
|
3875
|
-
const postingList = context.index.lookup(term);
|
|
3876
|
-
if (!postingList) continue;
|
|
3877
|
-
found.push({ token: term, postingList, docFreq: context.documentFrequency(term, postingList) });
|
|
3878
|
-
}
|
|
3879
|
-
if (found.length === 0) return [];
|
|
3880
|
-
let blendedDf = 0;
|
|
3881
|
-
for (const f of found) {
|
|
3882
|
-
if (f.docFreq > blendedDf) blendedDf = f.docFreq;
|
|
3883
|
-
}
|
|
3884
|
-
const blendedIdf = computeIDF(blendedDf, context.totalDocs);
|
|
3885
|
-
return found.map((f) => ({
|
|
3886
|
-
token: f.token,
|
|
3887
|
-
factor: Math.min(1, token.length / f.token.length),
|
|
3888
|
-
postingList: f.postingList,
|
|
3889
|
-
docFreq: blendedDf,
|
|
3890
|
-
idf: blendedIdf
|
|
3891
|
-
}));
|
|
3892
|
-
}
|
|
3893
|
-
function computePrefixContributions(context, matches, collect) {
|
|
3894
|
-
const { fields, fieldNames, filterBitset, k1, b } = context;
|
|
3895
|
-
const scoresAreZero = context.totalDocs === 0;
|
|
3896
|
-
const best = /* @__PURE__ */ new Map();
|
|
3897
|
-
for (const match of matches) {
|
|
3898
|
-
const perTerm = /* @__PURE__ */ new Map();
|
|
3899
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
3900
|
-
match.postingList
|
|
3901
|
-
);
|
|
3902
|
-
for (let pi = 0; pi < count; pi++) {
|
|
3903
|
-
const internalId = docIds[pi];
|
|
3904
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
3905
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
3906
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
3907
|
-
if (fields.searchable[fieldIndex] === 0) continue;
|
|
3908
|
-
const fieldName = fieldNames[fieldIndex];
|
|
3909
|
-
const termFrequency = termFrequencies[pi];
|
|
3910
|
-
const fieldBoost = fields.boosts[fieldIndex];
|
|
3911
|
-
const avgLen = fields.averageLengths[fieldIndex];
|
|
3912
|
-
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
3913
|
-
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b)) * fieldBoost * match.factor;
|
|
3914
|
-
const existing = perTerm.get(internalId);
|
|
3915
|
-
if (existing) {
|
|
3916
|
-
existing.score += termScore;
|
|
3917
|
-
if (collect) {
|
|
3918
|
-
existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
|
|
3919
|
-
existing.fieldLengths[fieldName] = actualFieldLength;
|
|
3920
|
-
}
|
|
3921
|
-
} else if (collect) {
|
|
3922
|
-
perTerm.set(internalId, {
|
|
3923
|
-
score: termScore,
|
|
3924
|
-
token: match.token,
|
|
3925
|
-
idf: match.idf,
|
|
3926
|
-
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
3927
|
-
fieldLengths: { [fieldName]: actualFieldLength }
|
|
3928
|
-
});
|
|
3929
|
-
} else {
|
|
3930
|
-
perTerm.set(internalId, {
|
|
3931
|
-
score: termScore,
|
|
3932
|
-
token: match.token,
|
|
3933
|
-
idf: match.idf,
|
|
3934
|
-
termFrequencies: EMPTY_COMPONENTS,
|
|
3935
|
-
fieldLengths: EMPTY_COMPONENTS
|
|
3936
|
-
});
|
|
3937
|
-
}
|
|
3938
|
-
}
|
|
3939
|
-
for (const [internalId, contribution] of perTerm) {
|
|
3940
|
-
const current = best.get(internalId);
|
|
3941
|
-
if (!current || contribution.score > current.score) {
|
|
3942
|
-
best.set(internalId, contribution);
|
|
3943
|
-
}
|
|
3944
|
-
}
|
|
3945
|
-
}
|
|
3946
|
-
return best;
|
|
3947
|
-
}
|
|
3948
|
-
|
|
3949
4026
|
// src/core/partition/single-term-topk.ts
|
|
3950
4027
|
function fieldsCoverEvery(fields, fieldNames) {
|
|
3951
4028
|
if (fields === void 0) return true;
|
|
@@ -3956,7 +4033,7 @@ function fieldsCoverEvery(fields, fieldNames) {
|
|
|
3956
4033
|
}
|
|
3957
4034
|
function prunableSingleTermList(params, index, fieldNames) {
|
|
3958
4035
|
if (params.queryTokens.length !== 1) return null;
|
|
3959
|
-
if (params.
|
|
4036
|
+
if (params.termGroups !== void 0 && params.termGroups.length > 0) return null;
|
|
3960
4037
|
if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
|
|
3961
4038
|
if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
|
|
3962
4039
|
if (params.collectComponents !== false) return null;
|
|
@@ -4066,15 +4143,11 @@ function singleTermTopK(request) {
|
|
|
4066
4143
|
function globalDocFreqFor(docFreqs, term, fallback) {
|
|
4067
4144
|
return Object.hasOwn(docFreqs, term) ? docFreqs[term] : fallback;
|
|
4068
4145
|
}
|
|
4069
|
-
function matchesFor(index, token, exact, tolerance, prefixLength) {
|
|
4070
|
-
if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
|
|
4071
|
-
const postingList = index.lookup(token);
|
|
4072
|
-
return postingList ? [{ token, postingList }] : [];
|
|
4073
|
-
}
|
|
4074
4146
|
function searchFulltext(state, params) {
|
|
4075
4147
|
const {
|
|
4076
4148
|
queryTokens,
|
|
4077
|
-
|
|
4149
|
+
termGroups,
|
|
4150
|
+
partialFields,
|
|
4078
4151
|
fields,
|
|
4079
4152
|
boost,
|
|
4080
4153
|
tolerance = 0,
|
|
@@ -4106,19 +4179,75 @@ function searchFulltext(state, params) {
|
|
|
4106
4179
|
function documentFrequency(token, list) {
|
|
4107
4180
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
4108
4181
|
}
|
|
4109
|
-
const
|
|
4182
|
+
const groupContext = {
|
|
4110
4183
|
index: state.invertedIdx,
|
|
4111
4184
|
fields: fieldScoring,
|
|
4185
|
+
partialMarks: partialFieldMarks(fieldNames, partialFields),
|
|
4112
4186
|
fieldNames,
|
|
4113
4187
|
totalDocs,
|
|
4114
4188
|
k1,
|
|
4115
4189
|
b,
|
|
4116
4190
|
filterBitset,
|
|
4117
|
-
|
|
4191
|
+
globalFrequency(token) {
|
|
4192
|
+
if (globalStats === void 0 || !Object.hasOwn(globalStats.docFrequencies, token)) return void 0;
|
|
4193
|
+
return globalStats.docFrequencies[token];
|
|
4194
|
+
}
|
|
4118
4195
|
};
|
|
4119
|
-
function
|
|
4196
|
+
function mergeGroupContribution(internalId, contribution) {
|
|
4120
4197
|
addScore(scoreBuffer, internalId, contribution.score);
|
|
4121
|
-
if (components !== null)
|
|
4198
|
+
if (components !== null) mergeGroupComponents(components, internalId, contribution);
|
|
4199
|
+
}
|
|
4200
|
+
function resolveMatch(token, postingList) {
|
|
4201
|
+
const docFreq = documentFrequency(token, postingList);
|
|
4202
|
+
return { token, docFreq, idf: computeIDF(docFreq, totalDocs), postingList };
|
|
4203
|
+
}
|
|
4204
|
+
function scorePostings(match, onlyScored) {
|
|
4205
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
4206
|
+
match.postingList
|
|
4207
|
+
);
|
|
4208
|
+
if (!onlyScored && !hasDeleted && filterBitset === void 0 && components === null) {
|
|
4209
|
+
for (let pi = 0; pi < count; pi++) {
|
|
4210
|
+
const internalId = docIds[pi];
|
|
4211
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
4212
|
+
if (searchable[fieldIndex] === 0) continue;
|
|
4213
|
+
const avgLen = averageLengths[fieldIndex];
|
|
4214
|
+
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
4215
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[pi], match.idf, actualFieldLength, avgLen, k1, b);
|
|
4216
|
+
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
4217
|
+
}
|
|
4218
|
+
return;
|
|
4219
|
+
}
|
|
4220
|
+
for (let pi = 0; pi < count; pi++) {
|
|
4221
|
+
const internalId = docIds[pi];
|
|
4222
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
4223
|
+
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
4224
|
+
if (onlyScored && !hasScore(scoreBuffer, internalId)) continue;
|
|
4225
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
4226
|
+
if (searchable[fieldIndex] === 0) continue;
|
|
4227
|
+
const termFrequency = termFrequencies[pi];
|
|
4228
|
+
const avgLen = averageLengths[fieldIndex];
|
|
4229
|
+
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
4230
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b);
|
|
4231
|
+
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
4232
|
+
if (components !== null) {
|
|
4233
|
+
recordComponents(
|
|
4234
|
+
components,
|
|
4235
|
+
internalId,
|
|
4236
|
+
fieldNames[fieldIndex],
|
|
4237
|
+
match.token,
|
|
4238
|
+
termFrequency,
|
|
4239
|
+
actualFieldLength,
|
|
4240
|
+
match.idf
|
|
4241
|
+
);
|
|
4242
|
+
}
|
|
4243
|
+
}
|
|
4244
|
+
}
|
|
4245
|
+
function scoreGroup(group, onlyScored) {
|
|
4246
|
+
if (group === null) return;
|
|
4247
|
+
for (const [internalId, contribution] of computeGroupContributions(groupContext, group, collectComponents)) {
|
|
4248
|
+
if (onlyScored && !hasScore(scoreBuffer, internalId)) continue;
|
|
4249
|
+
mergeGroupContribution(internalId, contribution);
|
|
4250
|
+
}
|
|
4122
4251
|
}
|
|
4123
4252
|
const prunableList = prunableSingleTermList(params, state.invertedIdx, fieldNames);
|
|
4124
4253
|
if (prunableList !== null && maxResults !== void 0) {
|
|
@@ -4148,127 +4277,30 @@ function searchFulltext(state, params) {
|
|
|
4148
4277
|
}
|
|
4149
4278
|
if (useIntersection) {
|
|
4150
4279
|
const resolved = [];
|
|
4151
|
-
let prefixMatches = [];
|
|
4152
4280
|
for (const qt of queryTokens) {
|
|
4153
|
-
|
|
4154
|
-
|
|
4155
|
-
|
|
4156
|
-
for (const m of prefixMatches) {
|
|
4157
|
-
totalPostings2 += m.postingList.length;
|
|
4158
|
-
}
|
|
4159
|
-
resolved.push({ token: qt.token, matches: [], totalPostings: totalPostings2, isPrefix: true });
|
|
4160
|
-
continue;
|
|
4161
|
-
}
|
|
4162
|
-
const rawMatches = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
|
|
4163
|
-
let totalPostings = 0;
|
|
4281
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
4282
|
+
const groupMatches = group === void 0 ? null : resolveGroupMatches(groupContext, group);
|
|
4283
|
+
let totalPostings = groupMatches === null ? 0 : groupPostingCount(groupMatches);
|
|
4164
4284
|
const matches = [];
|
|
4165
|
-
for (const m of
|
|
4166
|
-
const docFreq = documentFrequency(m.token, m.postingList);
|
|
4167
|
-
const idf = computeIDF(docFreq, totalDocs);
|
|
4285
|
+
for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
4168
4286
|
totalPostings += m.postingList.length;
|
|
4169
|
-
matches.push(
|
|
4287
|
+
matches.push(resolveMatch(m.token, m.postingList));
|
|
4170
4288
|
}
|
|
4171
|
-
resolved.push({ token: qt.token, matches, totalPostings });
|
|
4289
|
+
resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
|
|
4172
4290
|
}
|
|
4173
4291
|
resolved.sort((a, b2) => a.totalPostings - b2.totalPostings);
|
|
4174
4292
|
for (let tokenIndex = 0; tokenIndex < resolved.length; tokenIndex++) {
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
if (tokenIndex > 0 && !hasScore(scoreBuffer, internalId)) continue;
|
|
4179
|
-
mergePrefixContribution(internalId, contribution);
|
|
4180
|
-
}
|
|
4181
|
-
continue;
|
|
4182
|
-
}
|
|
4183
|
-
for (const match of resolved[tokenIndex].matches) {
|
|
4184
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
4185
|
-
match.postingList
|
|
4186
|
-
);
|
|
4187
|
-
for (let pi = 0; pi < count; pi++) {
|
|
4188
|
-
const internalId = docIds[pi];
|
|
4189
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
4190
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
4191
|
-
if (tokenIndex > 0 && !hasScore(scoreBuffer, internalId)) continue;
|
|
4192
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
4193
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
4194
|
-
const termFrequency = termFrequencies[pi];
|
|
4195
|
-
const fieldBoost = boosts[fieldIndex];
|
|
4196
|
-
const avgLen = averageLengths[fieldIndex];
|
|
4197
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
4198
|
-
let termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b);
|
|
4199
|
-
termScore *= fieldBoost;
|
|
4200
|
-
addScore(scoreBuffer, internalId, termScore);
|
|
4201
|
-
if (components !== null) {
|
|
4202
|
-
recordComponents(
|
|
4203
|
-
components,
|
|
4204
|
-
internalId,
|
|
4205
|
-
fieldNames[fieldIndex],
|
|
4206
|
-
match.token,
|
|
4207
|
-
termFrequency,
|
|
4208
|
-
actualFieldLength,
|
|
4209
|
-
match.idf
|
|
4210
|
-
);
|
|
4211
|
-
}
|
|
4212
|
-
}
|
|
4213
|
-
}
|
|
4293
|
+
const onlyScored = tokenIndex > 0;
|
|
4294
|
+
for (const match of resolved[tokenIndex].matches) scorePostings(match, onlyScored);
|
|
4295
|
+
scoreGroup(resolved[tokenIndex].group, onlyScored);
|
|
4214
4296
|
}
|
|
4215
4297
|
} else {
|
|
4216
4298
|
for (const qt of queryTokens) {
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
resolvePrefixMatches(prefixContext, qt.token, prefixExpansion.terms),
|
|
4221
|
-
collectComponents
|
|
4222
|
-
);
|
|
4223
|
-
for (const [internalId, contribution] of contributions) {
|
|
4224
|
-
mergePrefixContribution(internalId, contribution);
|
|
4225
|
-
}
|
|
4226
|
-
continue;
|
|
4227
|
-
}
|
|
4228
|
-
const matchingPostings = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
|
|
4229
|
-
for (const match of matchingPostings) {
|
|
4230
|
-
const idf = computeIDF(documentFrequency(match.token, match.postingList), totalDocs);
|
|
4231
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
4232
|
-
match.postingList
|
|
4233
|
-
);
|
|
4234
|
-
if (!hasDeleted && filterBitset === void 0 && components === null) {
|
|
4235
|
-
for (let pi = 0; pi < count; pi++) {
|
|
4236
|
-
const internalId = docIds[pi];
|
|
4237
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
4238
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
4239
|
-
const avgLen = averageLengths[fieldIndex];
|
|
4240
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
4241
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[pi], idf, actualFieldLength, avgLen, k1, b);
|
|
4242
|
-
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
4243
|
-
}
|
|
4244
|
-
continue;
|
|
4245
|
-
}
|
|
4246
|
-
for (let pi = 0; pi < count; pi++) {
|
|
4247
|
-
const internalId = docIds[pi];
|
|
4248
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
4249
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
4250
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
4251
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
4252
|
-
const termFrequency = termFrequencies[pi];
|
|
4253
|
-
const fieldBoost = boosts[fieldIndex];
|
|
4254
|
-
const avgLen = averageLengths[fieldIndex];
|
|
4255
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
4256
|
-
let termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, idf, actualFieldLength, avgLen, k1, b);
|
|
4257
|
-
termScore *= fieldBoost;
|
|
4258
|
-
addScore(scoreBuffer, internalId, termScore);
|
|
4259
|
-
if (components !== null) {
|
|
4260
|
-
recordComponents(
|
|
4261
|
-
components,
|
|
4262
|
-
internalId,
|
|
4263
|
-
fieldNames[fieldIndex],
|
|
4264
|
-
match.token,
|
|
4265
|
-
termFrequency,
|
|
4266
|
-
actualFieldLength,
|
|
4267
|
-
idf
|
|
4268
|
-
);
|
|
4269
|
-
}
|
|
4270
|
-
}
|
|
4299
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
4300
|
+
for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
4301
|
+
scorePostings(resolveMatch(m.token, m.postingList), false);
|
|
4271
4302
|
}
|
|
4303
|
+
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), false);
|
|
4272
4304
|
}
|
|
4273
4305
|
}
|
|
4274
4306
|
const totalMatched = scoreBuffer.touchedCount;
|
|
@@ -4477,6 +4509,7 @@ function encodeSegmentState(state) {
|
|
|
4477
4509
|
totalFieldLengths: fieldLengths.totals,
|
|
4478
4510
|
docFrequencies,
|
|
4479
4511
|
surfaceForms: state.surfaceRegistry.size() === 0 ? null : state.surfaceRegistry.serialize(),
|
|
4512
|
+
partialSpellings: state.partialWords.spellings(),
|
|
4480
4513
|
...encodeFieldIndexes(state, remap)
|
|
4481
4514
|
};
|
|
4482
4515
|
}
|
|
@@ -4595,6 +4628,8 @@ function mergeSegmentPayload(target, payload, documents) {
|
|
|
4595
4628
|
target.surfaceRegistry.add(surface, value[1], value[0]);
|
|
4596
4629
|
}
|
|
4597
4630
|
}
|
|
4631
|
+
const mergedIds = Array.from({ length: payload.docIds.length }, (_, at) => ordinalBase + at);
|
|
4632
|
+
target.partialWords.afterMerge(payload.partialSpellings, mergedIds);
|
|
4598
4633
|
target.sortColumns = null;
|
|
4599
4634
|
target.scoreBuffer = null;
|
|
4600
4635
|
}
|
|
@@ -4884,72 +4919,7 @@ function deserializePartition(state, data, clearFn, schema) {
|
|
|
4884
4919
|
}
|
|
4885
4920
|
loadPatternIndexes(state, data.fieldIndexes.pattern, schema);
|
|
4886
4921
|
rebuildSizeIndexes(state, schema);
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
// src/core/partition/suggestions.ts
|
|
4890
|
-
function verbatimOccurrences(state, token) {
|
|
4891
|
-
const postingList = state.invertedIdx.lookup(token);
|
|
4892
|
-
if (!postingList) return 0;
|
|
4893
|
-
const derived = postingList.totalTermFrequency - state.surfaceRegistry.stemChangedTotalFor(token);
|
|
4894
|
-
return derived > 0 ? derived : 0;
|
|
4895
|
-
}
|
|
4896
|
-
function suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit) {
|
|
4897
|
-
if (limit <= 0 || surfacePrefix.length === 0) return [];
|
|
4898
|
-
const groups = /* @__PURE__ */ new Map();
|
|
4899
|
-
for (const candidate of state.surfaceRegistry.candidatesForPrefix(surfacePrefix)) {
|
|
4900
|
-
const postingList = state.invertedIdx.lookup(candidate.token);
|
|
4901
|
-
if (!postingList || postingList.docIdSet.size === 0) continue;
|
|
4902
|
-
let group = groups.get(candidate.token);
|
|
4903
|
-
if (!group) {
|
|
4904
|
-
group = { documentFrequency: postingList.docIdSet.size, surfaces: [] };
|
|
4905
|
-
groups.set(candidate.token, group);
|
|
4906
|
-
}
|
|
4907
|
-
group.surfaces.push({ surface: candidate.surface, occurrences: candidate.occurrences });
|
|
4908
|
-
}
|
|
4909
|
-
const termPrefixes = state.surfaceRegistry.size() === 0 && stemmedPrefix !== surfacePrefix && stemmedPrefix.length > 0 ? [surfacePrefix, stemmedPrefix] : [surfacePrefix];
|
|
4910
|
-
for (const prefix of termPrefixes) {
|
|
4911
|
-
for (const suggestion of state.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
|
|
4912
|
-
const occurrences = verbatimOccurrences(state, suggestion.term);
|
|
4913
|
-
if (occurrences === 0) continue;
|
|
4914
|
-
let group = groups.get(suggestion.term);
|
|
4915
|
-
if (!group) {
|
|
4916
|
-
group = { documentFrequency: suggestion.documentFrequency, surfaces: [] };
|
|
4917
|
-
groups.set(suggestion.term, group);
|
|
4918
|
-
} else if (group.surfaces.some((s) => s.surface === suggestion.term)) {
|
|
4919
|
-
continue;
|
|
4920
|
-
}
|
|
4921
|
-
group.surfaces.push({ surface: suggestion.term, occurrences });
|
|
4922
|
-
}
|
|
4923
|
-
}
|
|
4924
|
-
const results = [];
|
|
4925
|
-
for (const [token, group] of groups) {
|
|
4926
|
-
results.push({ token, documentFrequency: group.documentFrequency, surfaces: group.surfaces });
|
|
4927
|
-
}
|
|
4928
|
-
results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.token, b.token));
|
|
4929
|
-
if (results.length > limit) results.length = limit;
|
|
4930
|
-
return results;
|
|
4931
|
-
}
|
|
4932
|
-
function expandTermPrefix(state, surfacePrefix, stemmedToken, maxExpansions) {
|
|
4933
|
-
if (maxExpansions <= 0 || surfacePrefix.length === 0) return [];
|
|
4934
|
-
const dfByToken = /* @__PURE__ */ new Map();
|
|
4935
|
-
for (const candidate of state.surfaceRegistry.candidatesForPrefix(surfacePrefix)) {
|
|
4936
|
-
if (dfByToken.has(candidate.token)) continue;
|
|
4937
|
-
const postingList = state.invertedIdx.lookup(candidate.token);
|
|
4938
|
-
if (!postingList || postingList.docIdSet.size === 0) continue;
|
|
4939
|
-
dfByToken.set(candidate.token, postingList.docIdSet.size);
|
|
4940
|
-
}
|
|
4941
|
-
const termPrefixes = state.surfaceRegistry.size() === 0 && stemmedToken !== surfacePrefix && stemmedToken.length > 0 ? [surfacePrefix, stemmedToken] : [surfacePrefix];
|
|
4942
|
-
for (const prefix of termPrefixes) {
|
|
4943
|
-
for (const suggestion of state.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
|
|
4944
|
-
if (dfByToken.has(suggestion.term)) continue;
|
|
4945
|
-
if (verbatimOccurrences(state, suggestion.term) === 0) continue;
|
|
4946
|
-
dfByToken.set(suggestion.term, suggestion.documentFrequency);
|
|
4947
|
-
}
|
|
4948
|
-
}
|
|
4949
|
-
const entries = Array.from(dfByToken.entries());
|
|
4950
|
-
entries.sort((a, b) => b[1] - a[1] || compareCodePoints(a[0], b[0]));
|
|
4951
|
-
if (entries.length > maxExpansions) entries.length = maxExpansions;
|
|
4952
|
-
return entries.map((e) => e[0]);
|
|
4922
|
+
state.partialWords.forgetSpellings();
|
|
4953
4923
|
}
|
|
4954
4924
|
|
|
4955
4925
|
// src/core/partition/index.ts
|
|
@@ -4966,9 +4936,10 @@ function partitionStateOf(partition) {
|
|
|
4966
4936
|
}
|
|
4967
4937
|
function createPartitionIndex(partitionId, trackPositions = true) {
|
|
4968
4938
|
const fieldNameTable = { names: [], indexMap: /* @__PURE__ */ new Map() };
|
|
4939
|
+
const docStore = createDocumentStore();
|
|
4969
4940
|
const state = {
|
|
4970
4941
|
invertedIdx: createInvertedIndex(fieldNameTable),
|
|
4971
|
-
docStore
|
|
4942
|
+
docStore,
|
|
4972
4943
|
stats: createPartitionStats(),
|
|
4973
4944
|
surfaceRegistry: createSurfaceRegistry(),
|
|
4974
4945
|
numericIndexes: /* @__PURE__ */ new Map(),
|
|
@@ -4978,6 +4949,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
4978
4949
|
patternIndexes: /* @__PURE__ */ new Map(),
|
|
4979
4950
|
sizeIndexes: /* @__PURE__ */ new Map(),
|
|
4980
4951
|
fieldNameTable,
|
|
4952
|
+
partialWords: createGrowingPartialWords({ docStore }),
|
|
4981
4953
|
flatSchemaCache: null,
|
|
4982
4954
|
lastSchemaRef: null,
|
|
4983
4955
|
trackPositions,
|
|
@@ -4998,6 +4970,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
4998
4970
|
state.geoIndexes.clear();
|
|
4999
4971
|
state.patternIndexes.clear();
|
|
5000
4972
|
state.sizeIndexes.clear();
|
|
4973
|
+
state.partialWords.clear();
|
|
5001
4974
|
state.stats.deserialize({ totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} });
|
|
5002
4975
|
state.flatSchemaCache = null;
|
|
5003
4976
|
state.lastSchemaRef = null;
|
|
@@ -5026,7 +4999,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5026
4999
|
}
|
|
5027
5000
|
}
|
|
5028
5001
|
const internalId = state.docStore.ensureInternalId(docId);
|
|
5029
|
-
const { fieldLengths, tokensByField } = indexDocument(
|
|
5002
|
+
const { fieldLengths, tokensByField, partialSpellings } = indexDocument(
|
|
5030
5003
|
state,
|
|
5031
5004
|
docId,
|
|
5032
5005
|
document,
|
|
@@ -5040,6 +5013,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5040
5013
|
state.docStore.store(docId, document, fieldLengths);
|
|
5041
5014
|
}
|
|
5042
5015
|
state.stats.addDocument(fieldLengths, tokensByField);
|
|
5016
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, partialSpellings, 1);
|
|
5043
5017
|
recordSortValues(state, internalId, document);
|
|
5044
5018
|
},
|
|
5045
5019
|
remove(docId, schema, language, options) {
|
|
@@ -5050,7 +5024,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5050
5024
|
partitionId
|
|
5051
5025
|
});
|
|
5052
5026
|
}
|
|
5053
|
-
const { fieldLengths, tokensByField } = removeFromIndexes(
|
|
5027
|
+
const { fieldLengths, tokensByField, partialSpellings } = removeFromIndexes(
|
|
5054
5028
|
state,
|
|
5055
5029
|
docId,
|
|
5056
5030
|
stored,
|
|
@@ -5061,6 +5035,9 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5061
5035
|
const internalId = state.docStore.getInternalId(docId);
|
|
5062
5036
|
state.docStore.remove(docId);
|
|
5063
5037
|
state.stats.removeDocument(fieldLengths, tokensByField);
|
|
5038
|
+
if (internalId !== void 0) {
|
|
5039
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, partialSpellings, -1);
|
|
5040
|
+
}
|
|
5064
5041
|
forgetSortValues(state, internalId);
|
|
5065
5042
|
},
|
|
5066
5043
|
beginBatch() {
|
|
@@ -5096,29 +5073,19 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5096
5073
|
const fields = schemaFieldsOf(schema);
|
|
5097
5074
|
const needsTextReindex = textFieldsChanged(stored.fields, document, fields);
|
|
5098
5075
|
if (needsTextReindex) {
|
|
5099
|
-
const
|
|
5100
|
-
|
|
5101
|
-
docId,
|
|
5102
|
-
stored,
|
|
5103
|
-
fields,
|
|
5104
|
-
language,
|
|
5105
|
-
options
|
|
5106
|
-
);
|
|
5107
|
-
state.stats.removeDocument(oldFieldLengths, oldTokens);
|
|
5076
|
+
const previous = removeFromIndexes(state, docId, stored, fields, language, options);
|
|
5077
|
+
state.stats.removeDocument(previous.fieldLengths, previous.tokensByField);
|
|
5108
5078
|
const previousInternalId = state.docStore.getInternalId(docId);
|
|
5079
|
+
if (previousInternalId !== void 0) {
|
|
5080
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), previousInternalId, previous.partialSpellings, -1);
|
|
5081
|
+
}
|
|
5109
5082
|
state.docStore.remove(docId);
|
|
5110
5083
|
forgetSortValues(state, previousInternalId);
|
|
5111
5084
|
const internalId = state.docStore.ensureInternalId(docId);
|
|
5112
|
-
const
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
5116
|
-
fields,
|
|
5117
|
-
language,
|
|
5118
|
-
options
|
|
5119
|
-
);
|
|
5120
|
-
state.docStore.store(docId, document, newFieldLengths);
|
|
5121
|
-
state.stats.addDocument(newFieldLengths, newTokens);
|
|
5085
|
+
const next = indexDocument(state, docId, document, fields, language, options);
|
|
5086
|
+
state.docStore.store(docId, document, next.fieldLengths);
|
|
5087
|
+
state.stats.addDocument(next.fieldLengths, next.tokensByField);
|
|
5088
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, next.partialSpellings, 1);
|
|
5122
5089
|
recordSortValues(state, internalId, document);
|
|
5123
5090
|
} else {
|
|
5124
5091
|
updateFieldIndexOnly(state, docId, stored.fields, document, fields);
|
|
@@ -5189,7 +5156,13 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5189
5156
|
return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
|
|
5190
5157
|
},
|
|
5191
5158
|
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
5192
|
-
return expandTermPrefix(state, surfacePrefix, stemmedToken, maxExpansions);
|
|
5159
|
+
return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
|
|
5160
|
+
},
|
|
5161
|
+
expandPartialWords(analysis, requests) {
|
|
5162
|
+
return expandPartialWords([state], analysis, requests);
|
|
5163
|
+
},
|
|
5164
|
+
preparePartialWords(analysis) {
|
|
5165
|
+
expandPartialWords([state], analysis, []);
|
|
5193
5166
|
},
|
|
5194
5167
|
serialize(indexName, totalPartitions, language, schema) {
|
|
5195
5168
|
return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
|
|
@@ -5223,7 +5196,7 @@ function frozenSegmentBytes(segment) {
|
|
|
5223
5196
|
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
5224
5197
|
}
|
|
5225
5198
|
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
5226
|
-
return bytes;
|
|
5199
|
+
return bytes + segment.partialWords.bytes();
|
|
5227
5200
|
}
|
|
5228
5201
|
|
|
5229
5202
|
// src/core/partition/composite/compaction.ts
|
|
@@ -5358,20 +5331,19 @@ function subMatchesView(matches, layout, subIndex) {
|
|
|
5358
5331
|
|
|
5359
5332
|
// src/core/partition/composite/stats.ts
|
|
5360
5333
|
function collectCandidateTerms(subs, params) {
|
|
5361
|
-
const { queryTokens,
|
|
5334
|
+
const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
|
|
5362
5335
|
const terms = /* @__PURE__ */ new Set();
|
|
5363
5336
|
for (const qt of queryTokens) {
|
|
5364
|
-
|
|
5365
|
-
|
|
5366
|
-
for (const
|
|
5367
|
-
continue;
|
|
5337
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
5338
|
+
if (group !== void 0) {
|
|
5339
|
+
for (const member of group.members) terms.add(member.token);
|
|
5368
5340
|
}
|
|
5369
|
-
if (exact || tolerance === 0) {
|
|
5341
|
+
if (group === void 0 && (exact || tolerance === 0)) {
|
|
5370
5342
|
terms.add(qt.token);
|
|
5371
5343
|
continue;
|
|
5372
5344
|
}
|
|
5373
5345
|
for (const sub of subs) {
|
|
5374
|
-
for (const match of sub.invertedIdx
|
|
5346
|
+
for (const match of typoMatchesFor(sub.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
5375
5347
|
terms.add(match.token);
|
|
5376
5348
|
}
|
|
5377
5349
|
}
|
|
@@ -5381,11 +5353,7 @@ function collectCandidateTerms(subs, params) {
|
|
|
5381
5353
|
function resolveQueryTermFrequencies(subs, params) {
|
|
5382
5354
|
const frequencies = /* @__PURE__ */ Object.create(null);
|
|
5383
5355
|
for (const term of collectCandidateTerms(subs, params)) {
|
|
5384
|
-
|
|
5385
|
-
for (const sub of subs) {
|
|
5386
|
-
const list = sub.invertedIdx.lookup(term);
|
|
5387
|
-
if (list !== void 0) total += list.docIdSet.size;
|
|
5388
|
-
}
|
|
5356
|
+
const total = partitionDocumentFrequency(subs, term);
|
|
5389
5357
|
if (total > 0) frequencies[term] = total;
|
|
5390
5358
|
}
|
|
5391
5359
|
return frequencies;
|
|
@@ -5571,9 +5539,10 @@ function compositeSortValues(subs, docId, fields, fieldTypes, modes) {
|
|
|
5571
5539
|
|
|
5572
5540
|
// src/core/partition/composite/suggest.ts
|
|
5573
5541
|
function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
5542
|
+
if (limit <= 0) return [];
|
|
5574
5543
|
const merged = /* @__PURE__ */ new Map();
|
|
5575
5544
|
for (const sub of subs) {
|
|
5576
|
-
for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix,
|
|
5545
|
+
for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix, Number.MAX_SAFE_INTEGER)) {
|
|
5577
5546
|
let entry = merged.get(suggestion.token);
|
|
5578
5547
|
if (entry === void 0) {
|
|
5579
5548
|
entry = { documentFrequency: 0, surfaceOccurrences: /* @__PURE__ */ new Map() };
|
|
@@ -5598,19 +5567,6 @@ function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
|
5598
5567
|
if (suggestions.length > limit) suggestions.length = limit;
|
|
5599
5568
|
return suggestions;
|
|
5600
5569
|
}
|
|
5601
|
-
function compositeExpandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansions) {
|
|
5602
|
-
const expansions = [];
|
|
5603
|
-
const seen = /* @__PURE__ */ new Set();
|
|
5604
|
-
for (const sub of subs) {
|
|
5605
|
-
for (const term of expandTermPrefix(sub, surfacePrefix, stemmedToken, maxExpansions)) {
|
|
5606
|
-
if (seen.has(term)) continue;
|
|
5607
|
-
seen.add(term);
|
|
5608
|
-
expansions.push(term);
|
|
5609
|
-
if (expansions.length === maxExpansions) return expansions;
|
|
5610
|
-
}
|
|
5611
|
-
}
|
|
5612
|
-
return expansions;
|
|
5613
|
-
}
|
|
5614
5570
|
|
|
5615
5571
|
// src/core/partition/composite/index.ts
|
|
5616
5572
|
function isCompositePartition(partition) {
|
|
@@ -5814,7 +5770,13 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5814
5770
|
return compositeSuggestTerms(subs(), surfacePrefix, stemmedPrefix, limit);
|
|
5815
5771
|
},
|
|
5816
5772
|
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
5817
|
-
return
|
|
5773
|
+
return expandTermPrefix(subs(), surfacePrefix, stemmedToken, maxExpansions);
|
|
5774
|
+
},
|
|
5775
|
+
expandPartialWords(analysis, requests) {
|
|
5776
|
+
return expandPartialWords(subs(), analysis, requests);
|
|
5777
|
+
},
|
|
5778
|
+
preparePartialWords(analysis) {
|
|
5779
|
+
expandPartialWords(subs(), analysis, []);
|
|
5818
5780
|
},
|
|
5819
5781
|
get(docId, projection) {
|
|
5820
5782
|
const owner = frozenOwner(docId);
|
|
@@ -5875,6 +5837,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5875
5837
|
}
|
|
5876
5838
|
|
|
5877
5839
|
// src/partitioning/insert-options.ts
|
|
5840
|
+
function resolvePartialWordAnalysis(config, language, analysis) {
|
|
5841
|
+
const fields = partialFieldsOf(config.schema);
|
|
5842
|
+
if (fields.length === 0) return null;
|
|
5843
|
+
return { fields, language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
5844
|
+
}
|
|
5878
5845
|
function resolvePartitionInsertOptions(config, analysis, options) {
|
|
5879
5846
|
const applyStrict = config.strict === true;
|
|
5880
5847
|
const applyAnalyzer = analysis.stopWords !== void 0 || analysis.customTokenizer !== void 0;
|
|
@@ -5933,8 +5900,10 @@ function createCapacityChecks(source) {
|
|
|
5933
5900
|
// src/partitioning/manager/index.ts
|
|
5934
5901
|
function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
|
|
5935
5902
|
requirePatternSearchFor(config.schema);
|
|
5903
|
+
requirePartialWordSearchFor(config.schema);
|
|
5936
5904
|
const count = initialPartitionCount ?? 1;
|
|
5937
5905
|
const analysis = resolveIndexAnalysis(config);
|
|
5906
|
+
const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
5938
5907
|
const trackPositions = config.trackPositions ?? true;
|
|
5939
5908
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
5940
5909
|
let partitions = [];
|
|
@@ -5954,6 +5923,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5954
5923
|
function resolveInsertOptions(options) {
|
|
5955
5924
|
return resolvePartitionInsertOptions(config, analysis, options);
|
|
5956
5925
|
}
|
|
5926
|
+
function preparePartialWords(partitionId) {
|
|
5927
|
+
if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
|
|
5928
|
+
}
|
|
5957
5929
|
function locateDocument(docId) {
|
|
5958
5930
|
return docPartitionMap.get(docId);
|
|
5959
5931
|
}
|
|
@@ -6009,6 +5981,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
6009
5981
|
get analysis() {
|
|
6010
5982
|
return analysis;
|
|
6011
5983
|
},
|
|
5984
|
+
get partialWordAnalysis() {
|
|
5985
|
+
return partialWordAnalysis;
|
|
5986
|
+
},
|
|
6012
5987
|
getPartition(partitionId) {
|
|
6013
5988
|
validatePartitionId(partitionId);
|
|
6014
5989
|
return partitions[partitionId];
|
|
@@ -6085,6 +6060,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
6085
6060
|
rebuildTextIndex(partitionId) {
|
|
6086
6061
|
validatePartitionId(partitionId);
|
|
6087
6062
|
partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
|
|
6063
|
+
preparePartialWords(partitionId);
|
|
6088
6064
|
},
|
|
6089
6065
|
beginBatchRemove() {
|
|
6090
6066
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -6186,6 +6162,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
6186
6162
|
deserializePartition(partitionId, data) {
|
|
6187
6163
|
validatePartitionId(partitionId);
|
|
6188
6164
|
partitions[partitionId].deserialize(data, config.schema);
|
|
6165
|
+
preparePartialWords(partitionId);
|
|
6189
6166
|
rebuildDocPartitionMapForPartition(partitionId);
|
|
6190
6167
|
},
|
|
6191
6168
|
getAggregateStats() {
|
|
@@ -8681,6 +8658,15 @@ function decodeStoredDocument(bytes, discardedFields) {
|
|
|
8681
8658
|
}
|
|
8682
8659
|
return document;
|
|
8683
8660
|
}
|
|
8661
|
+
function* storedDocumentsOf(data) {
|
|
8662
|
+
const count = data.offsets.length - 1;
|
|
8663
|
+
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
8664
|
+
yield decodeStoredDocument(
|
|
8665
|
+
data.blob.subarray(data.offsets[ordinal], data.offsets[ordinal + 1]),
|
|
8666
|
+
data.discardedFields
|
|
8667
|
+
);
|
|
8668
|
+
}
|
|
8669
|
+
}
|
|
8684
8670
|
function decodeEveryStoredDocument(data) {
|
|
8685
8671
|
const count = data.offsets.length - 1;
|
|
8686
8672
|
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
@@ -8724,6 +8710,9 @@ function wrapEncodedDocumentTable(data) {
|
|
|
8724
8710
|
decodedBytes += documentValueBytes(document);
|
|
8725
8711
|
}
|
|
8726
8712
|
return document;
|
|
8713
|
+
},
|
|
8714
|
+
readAt(ordinal) {
|
|
8715
|
+
return decoded[ordinal] ?? decodeStoredDocument(blob.subarray(offsets[ordinal], offsets[ordinal + 1]), data.discardedFields);
|
|
8727
8716
|
}
|
|
8728
8717
|
};
|
|
8729
8718
|
}
|
|
@@ -8737,75 +8726,15 @@ function wrapDocumentArray(documents) {
|
|
|
8737
8726
|
byteLength: residentBytes,
|
|
8738
8727
|
docAt(ordinal) {
|
|
8739
8728
|
return documents[ordinal];
|
|
8729
|
+
},
|
|
8730
|
+
readAt(ordinal) {
|
|
8731
|
+
return documents[ordinal];
|
|
8740
8732
|
}
|
|
8741
8733
|
};
|
|
8742
8734
|
}
|
|
8743
8735
|
|
|
8744
|
-
// src/core/partition/frozen/string-blob.ts
|
|
8745
|
-
var SURROGATE = /[\ud800-\udfff]/;
|
|
8746
|
-
var encoder = new TextEncoder();
|
|
8747
|
-
function encodeOneByOne(strings) {
|
|
8748
|
-
const encoded = new Array(strings.length);
|
|
8749
|
-
let blobLength = 0;
|
|
8750
|
-
for (let i = 0; i < strings.length; i++) {
|
|
8751
|
-
encoded[i] = encoder.encode(strings[i]);
|
|
8752
|
-
blobLength += encoded[i].length;
|
|
8753
|
-
}
|
|
8754
|
-
const blob = new Uint8Array(blobLength);
|
|
8755
|
-
const offsets = new Uint32Array(strings.length + 1);
|
|
8756
|
-
let cursor = 0;
|
|
8757
|
-
for (let i = 0; i < strings.length; i++) {
|
|
8758
|
-
blob.set(encoded[i], cursor);
|
|
8759
|
-
cursor += encoded[i].length;
|
|
8760
|
-
offsets[i + 1] = cursor;
|
|
8761
|
-
}
|
|
8762
|
-
return { blob, offsets };
|
|
8763
|
-
}
|
|
8764
|
-
function basicPlaneUtf8Bytes(text) {
|
|
8765
|
-
let bytes = text.length;
|
|
8766
|
-
for (let i = 0; i < text.length; i++) {
|
|
8767
|
-
const unit = text.charCodeAt(i);
|
|
8768
|
-
if (unit >= 128) bytes += unit < 2048 ? 1 : 2;
|
|
8769
|
-
}
|
|
8770
|
-
return bytes;
|
|
8771
|
-
}
|
|
8772
|
-
function encodeStringBlob(strings) {
|
|
8773
|
-
const joined = strings.join("");
|
|
8774
|
-
if (SURROGATE.test(joined)) return encodeOneByOne(strings);
|
|
8775
|
-
const blob = encoder.encode(joined);
|
|
8776
|
-
const offsets = new Uint32Array(strings.length + 1);
|
|
8777
|
-
let cursor = 0;
|
|
8778
|
-
for (let i = 0; i < strings.length; i++) {
|
|
8779
|
-
cursor += basicPlaneUtf8Bytes(strings[i]);
|
|
8780
|
-
offsets[i + 1] = cursor;
|
|
8781
|
-
}
|
|
8782
|
-
return cursor === blob.length ? { blob, offsets } : encodeOneByOne(strings);
|
|
8783
|
-
}
|
|
8784
|
-
function orderByComparator(strings) {
|
|
8785
|
-
const order = new Array(strings.length);
|
|
8786
|
-
for (let i = 0; i < strings.length; i++) order[i] = i;
|
|
8787
|
-
return order.sort((a, b) => compareCodePoints(strings[a], strings[b]));
|
|
8788
|
-
}
|
|
8789
|
-
function alreadyInCodePointOrder(strings) {
|
|
8790
|
-
for (let i = 1; i < strings.length; i++) {
|
|
8791
|
-
if (compareCodePoints(strings[i - 1], strings[i]) > 0) return false;
|
|
8792
|
-
}
|
|
8793
|
-
return true;
|
|
8794
|
-
}
|
|
8795
|
-
function codePointOrder(strings) {
|
|
8796
|
-
if (alreadyInCodePointOrder(strings)) return strings.map((_, i) => i);
|
|
8797
|
-
if (SURROGATE.test(strings.join(""))) return orderByComparator(strings);
|
|
8798
|
-
const positions = /* @__PURE__ */ new Map();
|
|
8799
|
-
for (let i = 0; i < strings.length; i++) positions.set(strings[i], i);
|
|
8800
|
-
if (positions.size !== strings.length) return orderByComparator(strings);
|
|
8801
|
-
const order = new Array(strings.length);
|
|
8802
|
-
const sorted = [...strings].sort();
|
|
8803
|
-
for (let i = 0; i < sorted.length; i++) order[i] = positions.get(sorted[i]) ?? 0;
|
|
8804
|
-
return order;
|
|
8805
|
-
}
|
|
8806
|
-
|
|
8807
8736
|
// src/core/partition/frozen/external-ids.ts
|
|
8808
|
-
var
|
|
8737
|
+
var encoder = new TextEncoder();
|
|
8809
8738
|
var decoder = new TextDecoder();
|
|
8810
8739
|
function encodeExternalIdTableData(docIds) {
|
|
8811
8740
|
const { blob, offsets } = encodeStringBlob(docIds);
|
|
@@ -8837,7 +8766,7 @@ function wrapExternalIdTable(data) {
|
|
|
8837
8766
|
bytes: blob.byteLength + offsets.byteLength + sortedOrdinals.byteLength,
|
|
8838
8767
|
idAt,
|
|
8839
8768
|
ordinalOf(docId) {
|
|
8840
|
-
const query =
|
|
8769
|
+
const query = encoder.encode(docId);
|
|
8841
8770
|
let lo = 0;
|
|
8842
8771
|
let hi = count;
|
|
8843
8772
|
while (lo < hi) {
|
|
@@ -9123,6 +9052,10 @@ function createFrozenInvertedReader(table, views) {
|
|
|
9123
9052
|
if (at < 0) return void 0;
|
|
9124
9053
|
return views.viewReadOnce(table.payloadSlot(at), table.documentFrequencyAt(at));
|
|
9125
9054
|
},
|
|
9055
|
+
documentFrequency(token) {
|
|
9056
|
+
const at = table.find(token);
|
|
9057
|
+
return at < 0 ? 0 : table.documentFrequencyAt(at);
|
|
9058
|
+
},
|
|
9126
9059
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
9127
9060
|
if (tolerance === 0) {
|
|
9128
9061
|
const at = table.find(token);
|
|
@@ -9277,7 +9210,7 @@ function createLazySurfaceReader(load) {
|
|
|
9277
9210
|
}
|
|
9278
9211
|
|
|
9279
9212
|
// src/core/partition/frozen/token-table.ts
|
|
9280
|
-
var
|
|
9213
|
+
var encoder2 = new TextEncoder();
|
|
9281
9214
|
var decoder3 = new TextDecoder();
|
|
9282
9215
|
function compareBytes2(blob, start, end, query) {
|
|
9283
9216
|
const length = end - start;
|
|
@@ -9341,7 +9274,7 @@ function wrapFrozenTokenTable(data) {
|
|
|
9341
9274
|
size,
|
|
9342
9275
|
bytes: blob.byteLength + offsets.byteLength + payloadSlots.byteLength + documentFrequencies.byteLength,
|
|
9343
9276
|
find(token) {
|
|
9344
|
-
const query =
|
|
9277
|
+
const query = encoder2.encode(token);
|
|
9345
9278
|
const at = lowerBound3(query);
|
|
9346
9279
|
if (at >= size) return -1;
|
|
9347
9280
|
return compareBytes2(blob, offsets[at], offsets[at + 1], query) === 0 ? at : -1;
|
|
@@ -9364,7 +9297,7 @@ function wrapFrozenTokenTable(data) {
|
|
|
9364
9297
|
return documentFrequencies[sortedIndex];
|
|
9365
9298
|
},
|
|
9366
9299
|
prefixRange(prefix) {
|
|
9367
|
-
const bytes =
|
|
9300
|
+
const bytes = encoder2.encode(prefix);
|
|
9368
9301
|
const start = lowerBound3(bytes);
|
|
9369
9302
|
return { start, end: prefixEnd(start, bytes) };
|
|
9370
9303
|
},
|
|
@@ -9382,8 +9315,8 @@ function wrapFrozenTokenTable(data) {
|
|
|
9382
9315
|
fromCodePoint = unit;
|
|
9383
9316
|
toCodePoint = nextCodePoint(unit);
|
|
9384
9317
|
}
|
|
9385
|
-
const from =
|
|
9386
|
-
const to =
|
|
9318
|
+
const from = encoder2.encode(String.fromCodePoint(fromCodePoint));
|
|
9319
|
+
const to = encoder2.encode(String.fromCodePoint(toCodePoint));
|
|
9387
9320
|
return { start: lowerBound3(from), end: lowerBound3(to) };
|
|
9388
9321
|
}
|
|
9389
9322
|
};
|
|
@@ -9445,6 +9378,35 @@ function sharedSurfaceTable(payload) {
|
|
|
9445
9378
|
if (table.counts.length === 0) return null;
|
|
9446
9379
|
return { blob: sharedUint8(table.blob), offsets: sharedUint32(table.offsets), counts: sharedUint32(table.counts) };
|
|
9447
9380
|
}
|
|
9381
|
+
function sharePartialWordTable(table) {
|
|
9382
|
+
if (typeof SharedArrayBuffer !== "function") return table;
|
|
9383
|
+
return {
|
|
9384
|
+
fields: [...table.fields],
|
|
9385
|
+
spellings: sharedUint8(table.spellings),
|
|
9386
|
+
spellingOffsets: sharedUint32(table.spellingOffsets),
|
|
9387
|
+
spellingTokens: sharedUint32(table.spellingTokens),
|
|
9388
|
+
spellingCounts: sharedUint32(table.spellingCounts),
|
|
9389
|
+
runs: sharedUint32(table.runs),
|
|
9390
|
+
runOffsets: sharedUint32(table.runOffsets),
|
|
9391
|
+
runEntries: sharedUint32(table.runEntries)
|
|
9392
|
+
};
|
|
9393
|
+
}
|
|
9394
|
+
function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
|
|
9395
|
+
let source = spellings;
|
|
9396
|
+
if (source === null && analysis !== null && analysis.fields.length > 0) {
|
|
9397
|
+
source = encodePartialSpellings(analysis.fields, spellingCountsOf(storedDocumentsOf(documentData), analysis));
|
|
9398
|
+
}
|
|
9399
|
+
if (source === null || source.fields.length === 0) return null;
|
|
9400
|
+
const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
|
|
9401
|
+
return table === null ? null : sharePartialWordTable(table);
|
|
9402
|
+
}
|
|
9403
|
+
function shareSegmentArrays(arrays, analysis, segmentId) {
|
|
9404
|
+
return {
|
|
9405
|
+
...arrays,
|
|
9406
|
+
segmentId: segmentId ?? generateId(),
|
|
9407
|
+
partialWords: sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis)
|
|
9408
|
+
};
|
|
9409
|
+
}
|
|
9448
9410
|
function segmentArraysOf(payload, documentData) {
|
|
9449
9411
|
const tokenTable = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
|
|
9450
9412
|
const idTable = encodeExternalIdTableData(payload.docIds);
|
|
@@ -9473,11 +9435,11 @@ function segmentArraysOf(payload, documentData) {
|
|
|
9473
9435
|
surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable
|
|
9474
9436
|
};
|
|
9475
9437
|
}
|
|
9476
|
-
function freezeSegmentShared(payload, documents, segmentId) {
|
|
9438
|
+
function freezeSegmentShared(payload, documents, segmentId, analysis = null) {
|
|
9477
9439
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
9478
|
-
return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId);
|
|
9440
|
+
return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId, analysis);
|
|
9479
9441
|
}
|
|
9480
|
-
function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
9442
|
+
function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null) {
|
|
9481
9443
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
9482
9444
|
const tokenData = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
|
|
9483
9445
|
const idData = encodeExternalIdTableData(payload.docIds);
|
|
@@ -9544,7 +9506,8 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
|
9544
9506
|
offsets: sharedUint32(entry.offsets),
|
|
9545
9507
|
postings: sharedUint32(entry.postings)
|
|
9546
9508
|
})),
|
|
9547
|
-
surfaceTable: sharedSurfaceTable(payload)
|
|
9509
|
+
surfaceTable: sharedSurfaceTable(payload),
|
|
9510
|
+
partialWords: sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis)
|
|
9548
9511
|
};
|
|
9549
9512
|
}
|
|
9550
9513
|
|
|
@@ -9580,7 +9543,12 @@ function buildGeoReaders(entries) {
|
|
|
9580
9543
|
}
|
|
9581
9544
|
return readers;
|
|
9582
9545
|
}
|
|
9583
|
-
function
|
|
9546
|
+
function* survivingDocuments(documentSource, tombstones) {
|
|
9547
|
+
for (let ordinal = 0; ordinal < documentSource.count; ordinal++) {
|
|
9548
|
+
if (!tombstones.has(ordinal)) yield documentSource.readAt(ordinal);
|
|
9549
|
+
}
|
|
9550
|
+
}
|
|
9551
|
+
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
|
|
9584
9552
|
const tombstones = createFrozenTombstones();
|
|
9585
9553
|
const postingViews = createFrozenPostingViews(source, tombstones);
|
|
9586
9554
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
@@ -9588,6 +9556,10 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
9588
9556
|
lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
|
|
9589
9557
|
}
|
|
9590
9558
|
const docStore = createFrozenDocTable(source.idTable, lengthColumns, documentSource, tombstones);
|
|
9559
|
+
const partialWords = createFrozenPartialWords(partialWordTable, source.tokenTable, {
|
|
9560
|
+
documentAt: (ordinal) => documentSource.readAt(ordinal),
|
|
9561
|
+
survivors: () => survivingDocuments(documentSource, tombstones)
|
|
9562
|
+
});
|
|
9591
9563
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
9592
9564
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
9593
9565
|
const sizeIndexes = /* @__PURE__ */ new Map();
|
|
@@ -9620,6 +9592,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
9620
9592
|
patternIndexes,
|
|
9621
9593
|
sizeIndexes,
|
|
9622
9594
|
fieldNameTable,
|
|
9595
|
+
partialWords,
|
|
9623
9596
|
trackPositions: source.positionOffsets !== null,
|
|
9624
9597
|
flatSchemaCache: null,
|
|
9625
9598
|
lastSchemaRef: null,
|
|
@@ -9639,6 +9612,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
9639
9612
|
if (ordinal === void 0) return false;
|
|
9640
9613
|
const added = tombstones.add(ordinal);
|
|
9641
9614
|
if (added) {
|
|
9615
|
+
partialWords.afterRemoval(ordinal);
|
|
9642
9616
|
docStore.releaseSortedDocIds();
|
|
9643
9617
|
segment.sortColumns = null;
|
|
9644
9618
|
segment.scoreBuffer = null;
|
|
@@ -9654,18 +9628,20 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
9654
9628
|
return segment;
|
|
9655
9629
|
}
|
|
9656
9630
|
function createFrozenSegment(payload, documents, segmentId) {
|
|
9631
|
+
const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
|
|
9657
9632
|
return assembleFrozenSegment(
|
|
9658
9633
|
segmentId ?? generateId(),
|
|
9659
9634
|
{
|
|
9660
9635
|
...payload,
|
|
9661
|
-
tokenTable
|
|
9636
|
+
tokenTable,
|
|
9662
9637
|
idTable: buildExternalIdTable(payload.docIds),
|
|
9663
9638
|
documentTable: null,
|
|
9664
9639
|
readSurfaceForms: () => payload.surfaceForms,
|
|
9665
9640
|
docFrequencies: () => payload.docFrequencies
|
|
9666
9641
|
},
|
|
9667
9642
|
wrapDocumentArray(documents),
|
|
9668
|
-
null
|
|
9643
|
+
null,
|
|
9644
|
+
payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, tokenTable)
|
|
9669
9645
|
);
|
|
9670
9646
|
}
|
|
9671
9647
|
function createSharedFrozenSegment(snapshot) {
|
|
@@ -9687,7 +9663,8 @@ function createSharedFrozenSegment(snapshot) {
|
|
|
9687
9663
|
}
|
|
9688
9664
|
},
|
|
9689
9665
|
wrapEncodedDocumentTable(snapshot.documentTable),
|
|
9690
|
-
snapshot
|
|
9666
|
+
snapshot,
|
|
9667
|
+
snapshot.partialWords
|
|
9691
9668
|
);
|
|
9692
9669
|
}
|
|
9693
9670
|
var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
@@ -10269,10 +10246,10 @@ function mergeDocuments2(inputs, documentCount) {
|
|
|
10269
10246
|
}
|
|
10270
10247
|
return { blob, offsets };
|
|
10271
10248
|
}
|
|
10272
|
-
function mergeFrozenSegments(segments) {
|
|
10249
|
+
function mergeFrozenSegments(segments, analysis) {
|
|
10273
10250
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
10274
10251
|
const merged = mergeSegmentContents(segments.map((segment) => ({ segment })));
|
|
10275
|
-
return freezeEncodedSegmentShared(merged.payload, merged.documents);
|
|
10252
|
+
return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis);
|
|
10276
10253
|
}
|
|
10277
10254
|
function mergeSegmentContents(selections) {
|
|
10278
10255
|
const docIds = [];
|
|
@@ -10301,6 +10278,7 @@ function mergeSegmentContents(selections) {
|
|
|
10301
10278
|
totalFieldLengths: lengths.totalFieldLengths,
|
|
10302
10279
|
docFrequencies: postings.docFrequencies,
|
|
10303
10280
|
surfaceForms: mergeSurfaceForms2(inputs),
|
|
10281
|
+
partialSpellings: null,
|
|
10304
10282
|
...mergeFieldIndexes2(inputs)
|
|
10305
10283
|
};
|
|
10306
10284
|
return { payload, documents: mergeDocuments2(inputs, documentCount) };
|
|
@@ -10571,14 +10549,18 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
10571
10549
|
patternIndexes: /* @__PURE__ */ new Map(),
|
|
10572
10550
|
sizeIndexes: /* @__PURE__ */ new Map()
|
|
10573
10551
|
};
|
|
10552
|
+
const partialFields = partialFieldsOf(schema);
|
|
10553
|
+
const spellings = /* @__PURE__ */ new Map();
|
|
10554
|
+
let fieldSpellings = null;
|
|
10574
10555
|
let positionOffset = 0;
|
|
10575
10556
|
const addToken = (token, position, surface) => {
|
|
10576
10557
|
postings.addToken(token, positionOffset + position);
|
|
10577
10558
|
surfaces.count(surface, token);
|
|
10559
|
+
if (fieldSpellings !== null) countSpelling(fieldSpellings, token, surface ?? token, 1);
|
|
10578
10560
|
};
|
|
10579
10561
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
10580
10562
|
const document = documents[ordinal].document;
|
|
10581
|
-
for (const { path: fieldPath, base: fieldType, pattern } of fields) {
|
|
10563
|
+
for (const { path: fieldPath, base: fieldType, pattern, partial } of fields) {
|
|
10582
10564
|
const value = getNestedValue(document, fieldPath);
|
|
10583
10565
|
if (value === void 0 || value === null) continue;
|
|
10584
10566
|
recordValueSize(filters.sizeIndexes, ordinal, fieldPath, fieldType, value);
|
|
@@ -10587,6 +10569,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
10587
10569
|
indexFilterValue(filters, ordinal, fieldPath, fieldType, value);
|
|
10588
10570
|
continue;
|
|
10589
10571
|
}
|
|
10572
|
+
fieldSpellings = partial ? spellings : null;
|
|
10590
10573
|
postings.beginField();
|
|
10591
10574
|
positionOffset = 0;
|
|
10592
10575
|
for (const text of fieldType === "string" ? [value] : value) {
|
|
@@ -10607,6 +10590,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
10607
10590
|
fieldLengthColumns: lengths.columns,
|
|
10608
10591
|
totalFieldLengths: lengths.totalsByName(),
|
|
10609
10592
|
surfaceForms: surfaces.encode(),
|
|
10593
|
+
partialSpellings: partialFields.length === 0 ? null : encodePartialSpellings(partialFields, spellings),
|
|
10610
10594
|
...encodeFieldIndexes(filters, identity)
|
|
10611
10595
|
};
|
|
10612
10596
|
}
|
|
@@ -10673,7 +10657,7 @@ function runSegmentAction(entry, action) {
|
|
|
10673
10657
|
}
|
|
10674
10658
|
case "compactSegments": {
|
|
10675
10659
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
10676
|
-
return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds));
|
|
10660
|
+
return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds), entry.manager.partialWordAnalysis);
|
|
10677
10661
|
}
|
|
10678
10662
|
case "swapSegments": {
|
|
10679
10663
|
requireComposite(entry, action.indexName, action.partitionId, "swap");
|
|
@@ -11150,4 +11134,4 @@ function createDirectExecutor(options) {
|
|
|
11150
11134
|
};
|
|
11151
11135
|
}
|
|
11152
11136
|
|
|
11153
|
-
export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics,
|
|
11137
|
+
export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerPartialWordSearch, registerStopWords, registerTokenizer, requirePartialWordSearchFor, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
|