@delali/narsil 0.3.1-next.73 → 0.3.1-next.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{capabilities-CYeAWYxQ.d.ts → capabilities-CAMzmkAg.d.ts} +2 -2
- package/dist/{chunk-PU43WDY4.mjs → chunk-2E35UM3G.mjs} +1 -13
- package/dist/chunk-3DDUNI6N.mjs +48 -0
- package/dist/{chunk-THMSFOJO.mjs → chunk-BVBIQQKG.mjs} +576 -547
- package/dist/{chunk-ZULSEZCE.mjs → chunk-CTOSGRM7.mjs} +100 -19
- package/dist/{chunk-CVFAFZSN.mjs → chunk-CUZOX6CN.mjs} +23 -15
- package/dist/chunk-I23G3IZC.mjs +593 -0
- package/dist/chunk-KEHK2O6P.mjs +13 -0
- package/dist/{chunk-AFKH4ZK3.mjs → chunk-LX4JERXI.mjs} +1 -1
- package/dist/{chunk-LPBWCHBE.mjs → chunk-NOZUQM7J.mjs} +2 -2
- package/dist/{chunk-WVDFG45M.mjs → chunk-PO53FV4Q.mjs} +2 -1
- package/dist/chunk-T5ZWEWVD.mjs +658 -0
- package/dist/{chunk-2WLD2NFW.mjs → chunk-TVFLHM7M.mjs} +2 -2
- package/dist/{chunk-3YKWSO54.mjs → chunk-VKX2BPXD.mjs} +147 -647
- package/dist/{chunk-AK3BLGCI.mjs → chunk-Z3DQPOJQ.mjs} +15 -48
- package/dist/{client-D698E5ge.d.ts → client-C-I70ArH.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/{config-Bv8etwUZ.d.ts → config-C6Iw5rCX.d.ts} +2 -2
- package/dist/{config-QXU3HMpq.d.ts → config-DJPH3IQB.d.ts} +2 -2
- package/dist/direct-executor-KFAGILJC.mjs +17 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +19 -9
- package/dist/{engine-D_OzKZ-6.d.ts → engine-B8eRPO7m.d.ts} +2 -2
- package/dist/index.browser.mjs +1200 -808
- package/dist/index.d.ts +27 -8
- package/dist/index.mjs +12 -8
- package/dist/{memory-B4jMmygw.d.ts → memory-DJEwWoCi.d.ts} +1205 -1180
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +11 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +33 -13
- package/dist/persistence/durability/checkpoint-worker.mjs +12 -8
- package/dist/react.d.ts +7 -7
- package/dist/{schema-BBOCdhND.d.ts → schema-DgyDijhf.d.ts} +6 -2
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -6
- package/dist/{tasks-DpgCpW1x.d.ts → tasks-BVmAKYBh.d.ts} +1 -1
- package/dist/{thread-3LNVA43V.mjs → thread-W6DPKEF6.mjs} +8 -5
- package/dist/{types-BiUmaiWT.d.ts → types-ChrVy4lH.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +3 -3
- package/package.json +15 -7
- package/dist/direct-executor-5SRDFZAS.mjs +0 -13
|
@@ -1,10 +1,14 @@
|
|
|
1
|
+
import { joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, everyValueFacetConfig, mergeFacets, undeclaredFilterField, requireValidOperands, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, coveringRanges, GEO_UNIT_METRES } from './chunk-VKX2BPXD.mjs';
|
|
1
2
|
import { getLanguage, detectRuntime } from './chunk-4MQ5F4IL.mjs';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { MAX_M, noteFallback, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-PU43WDY4.mjs';
|
|
3
|
+
import { MAX_DOC_ID_LENGTH } from './chunk-3DDUNI6N.mjs';
|
|
4
|
+
import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, topLayerInParts, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-PO53FV4Q.mjs';
|
|
5
|
+
import { noteFallback } from './chunk-2E35UM3G.mjs';
|
|
6
6
|
import { GrowableUint32, patternSearchModule } from './chunk-XW25X6AU.mjs';
|
|
7
7
|
import { bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, PATTERN_RUN_CODE_POINTS, bitsetFromSet, bitsetIsEmpty, bitsetAnd, bitsetNot } from './chunk-ZKRV5M2P.mjs';
|
|
8
|
+
import { encodePartialSpellings, countSpelling, expandTermPrefix, suggestDisplayTerms, spellingCountsOf, codePointOrder, encodeStringBlob, partialWordSearchModule, partitionDocumentFrequency } from './chunk-T5ZWEWVD.mjs';
|
|
9
|
+
import { tokenizeOptions, getNestedValue, visitTokens, textFieldsChanged, fuzzyPrefixOf, boundedLevenshtein, getFlatSchema, tokenize, getOrCreateFieldNameIndex, tokenizeIterator, getFieldValueByInternalId } from './chunk-I23G3IZC.mjs';
|
|
10
|
+
import { schemaFieldsOf, partialFieldsOf, isWordIndexedBase, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, parsedTypeOf, isListBase, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-Z3DQPOJQ.mjs';
|
|
11
|
+
import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
8
12
|
import { compareCodePoints, appendFoldedCodePoint, toReducedSortValue, readSortField, defaultSortMode, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-LJYTW7GT.mjs';
|
|
9
13
|
import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
|
|
10
14
|
import { totalmem } from 'os';
|
|
@@ -411,8 +415,6 @@ function compactList(list) {
|
|
|
411
415
|
}
|
|
412
416
|
}
|
|
413
417
|
writeIdx++;
|
|
414
|
-
} else {
|
|
415
|
-
list.totalTermFrequency -= list.termFrequencies[i];
|
|
416
418
|
}
|
|
417
419
|
}
|
|
418
420
|
list.docIds.length = writeIdx;
|
|
@@ -437,8 +439,6 @@ function compactDocEntries(list, internalId) {
|
|
|
437
439
|
}
|
|
438
440
|
}
|
|
439
441
|
writeIdx++;
|
|
440
|
-
} else {
|
|
441
|
-
list.totalTermFrequency -= list.termFrequencies[i];
|
|
442
442
|
}
|
|
443
443
|
}
|
|
444
444
|
list.docIds.length = writeIdx;
|
|
@@ -524,12 +524,13 @@ function createInvertedIndex(fieldNameTable) {
|
|
|
524
524
|
}
|
|
525
525
|
list.length++;
|
|
526
526
|
},
|
|
527
|
-
remove(token, internalId) {
|
|
527
|
+
remove(token, internalId, occurrences) {
|
|
528
528
|
const list = index.get(token);
|
|
529
529
|
if (!list || !list.docIdSet.has(internalId)) return;
|
|
530
530
|
list.structureRevision++;
|
|
531
531
|
list.docIdSet.delete(internalId);
|
|
532
532
|
list.deletedDocs.add(internalId);
|
|
533
|
+
list.totalTermFrequency = Math.max(0, list.totalTermFrequency - occurrences);
|
|
533
534
|
if (list.docIdSet.size === 0) {
|
|
534
535
|
index.delete(token);
|
|
535
536
|
untrackToken(token);
|
|
@@ -562,6 +563,9 @@ function createInvertedIndex(fieldNameTable) {
|
|
|
562
563
|
lookup(token) {
|
|
563
564
|
return index.get(token);
|
|
564
565
|
},
|
|
566
|
+
documentFrequency(token) {
|
|
567
|
+
return index.get(token)?.docIdSet.size ?? 0;
|
|
568
|
+
},
|
|
565
569
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
566
570
|
if (tolerance === 0) {
|
|
567
571
|
const exact = index.get(token);
|
|
@@ -1667,6 +1671,13 @@ function resolveInternalId(state, docId) {
|
|
|
1667
1671
|
}
|
|
1668
1672
|
return internalId;
|
|
1669
1673
|
}
|
|
1674
|
+
function emptyIndexedText(fields) {
|
|
1675
|
+
const holdsPartialField = fields.some((field) => field.partial);
|
|
1676
|
+
return { fieldLengths: {}, tokensByField: {}, partialSpellings: holdsPartialField ? /* @__PURE__ */ new Map() : null };
|
|
1677
|
+
}
|
|
1678
|
+
function partialSpellingsFor(field, indexed) {
|
|
1679
|
+
return field.partial ? indexed.partialSpellings : null;
|
|
1680
|
+
}
|
|
1670
1681
|
function countSurface(counts, surface, token) {
|
|
1671
1682
|
if (surface === void 0 || surface.length === 0) return;
|
|
1672
1683
|
const existing = counts.get(surface);
|
|
@@ -1676,6 +1687,18 @@ function countSurface(counts, surface, token) {
|
|
|
1676
1687
|
counts.set(surface, { token, count: 1 });
|
|
1677
1688
|
}
|
|
1678
1689
|
}
|
|
1690
|
+
function countOccurrence(counts, spellings, surface, token) {
|
|
1691
|
+
countSurface(counts, surface, token);
|
|
1692
|
+
if (spellings !== null) countSpelling(spellings, token, surface ?? token, 1);
|
|
1693
|
+
}
|
|
1694
|
+
function addFieldOccurrences(occurrencesByToken, fieldTokens) {
|
|
1695
|
+
const fieldCounts = /* @__PURE__ */ new Map();
|
|
1696
|
+
for (const token of fieldTokens) fieldCounts.set(token, (fieldCounts.get(token) ?? 0) + 1);
|
|
1697
|
+
for (const [token, count] of fieldCounts) {
|
|
1698
|
+
const stored = count > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : count;
|
|
1699
|
+
occurrencesByToken.set(token, (occurrencesByToken.get(token) ?? 0) + stored);
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1679
1702
|
function applySurfaceCounts(state, counts, direction) {
|
|
1680
1703
|
for (const [surface, entry] of counts) {
|
|
1681
1704
|
if (direction === 1) {
|
|
@@ -1685,7 +1708,7 @@ function applySurfaceCounts(state, counts, direction) {
|
|
|
1685
1708
|
}
|
|
1686
1709
|
}
|
|
1687
1710
|
}
|
|
1688
|
-
function indexStringField(state, internalId, fieldPath, text, language, options, fieldLengths, tokensByField) {
|
|
1711
|
+
function indexStringField(state, internalId, fieldPath, text, language, options, fieldLengths, tokensByField, spellings = null) {
|
|
1689
1712
|
const fieldTokenList = [];
|
|
1690
1713
|
const fieldNameIndex2 = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
|
|
1691
1714
|
const opts = tokenizeOptions(options);
|
|
@@ -1701,7 +1724,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
1701
1724
|
tokenFreqs.set(t.token, { count: 1, positions: [t.position] });
|
|
1702
1725
|
}
|
|
1703
1726
|
fieldTokenList.push(t.token);
|
|
1704
|
-
|
|
1727
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
1705
1728
|
}
|
|
1706
1729
|
for (const [token, freq] of tokenFreqs) {
|
|
1707
1730
|
state.invertedIdx.insert(token, internalId, freq.count, fieldNameIndex2, freq.positions);
|
|
@@ -1711,7 +1734,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
1711
1734
|
for (const t of tokenizeIterator(text, language, opts)) {
|
|
1712
1735
|
tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
|
|
1713
1736
|
fieldTokenList.push(t.token);
|
|
1714
|
-
|
|
1737
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
1715
1738
|
}
|
|
1716
1739
|
for (const [token, count] of tokenCounts) {
|
|
1717
1740
|
state.invertedIdx.insert(token, internalId, count, fieldNameIndex2, null);
|
|
@@ -1721,7 +1744,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
|
|
|
1721
1744
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
1722
1745
|
tokensByField[fieldPath] = fieldTokenList;
|
|
1723
1746
|
}
|
|
1724
|
-
function indexStringArrayField(state, internalId, fieldPath, arr, language, options, fieldLengths, tokensByField) {
|
|
1747
|
+
function indexStringArrayField(state, internalId, fieldPath, arr, language, options, fieldLengths, tokensByField, spellings = null) {
|
|
1725
1748
|
const fieldTokenList = [];
|
|
1726
1749
|
const fieldNameIndex2 = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
|
|
1727
1750
|
const opts = tokenizeOptions(options);
|
|
@@ -1740,7 +1763,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
1740
1763
|
tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
|
|
1741
1764
|
}
|
|
1742
1765
|
fieldTokenList.push(t.token);
|
|
1743
|
-
|
|
1766
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
1744
1767
|
itemTokenCount++;
|
|
1745
1768
|
}
|
|
1746
1769
|
positionOffset += itemTokenCount;
|
|
@@ -1755,7 +1778,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
1755
1778
|
for (const t of tokenizeIterator(item, language, opts)) {
|
|
1756
1779
|
tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
|
|
1757
1780
|
fieldTokenList.push(t.token);
|
|
1758
|
-
|
|
1781
|
+
countOccurrence(surfaceCounts, spellings, t.surface, t.token);
|
|
1759
1782
|
}
|
|
1760
1783
|
}
|
|
1761
1784
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
@@ -1767,17 +1790,28 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
|
|
|
1767
1790
|
tokensByField[fieldPath] = fieldTokenList;
|
|
1768
1791
|
}
|
|
1769
1792
|
function indexDocument(state, docId, document, fields, language, options) {
|
|
1770
|
-
const
|
|
1771
|
-
const tokensByField =
|
|
1793
|
+
const indexed = emptyIndexedText(fields);
|
|
1794
|
+
const { fieldLengths, tokensByField } = indexed;
|
|
1772
1795
|
const internalId = resolveInternalId(state, docId);
|
|
1773
|
-
for (const
|
|
1796
|
+
for (const field of fields) {
|
|
1797
|
+
const { path: fieldPath, base, pattern } = field;
|
|
1774
1798
|
const value = getNestedValue(document, fieldPath);
|
|
1775
1799
|
if (value === void 0 || value === null) continue;
|
|
1776
1800
|
recordValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
1777
1801
|
ensureFieldIndex(state, fieldPath, base);
|
|
1778
1802
|
if (pattern) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
|
|
1779
1803
|
if (base === "string") {
|
|
1780
|
-
indexStringField(
|
|
1804
|
+
indexStringField(
|
|
1805
|
+
state,
|
|
1806
|
+
internalId,
|
|
1807
|
+
fieldPath,
|
|
1808
|
+
value,
|
|
1809
|
+
language,
|
|
1810
|
+
options,
|
|
1811
|
+
fieldLengths,
|
|
1812
|
+
tokensByField,
|
|
1813
|
+
partialSpellingsFor(field, indexed)
|
|
1814
|
+
);
|
|
1781
1815
|
} else if (base === "number") {
|
|
1782
1816
|
getOrCreateNumericIndex(state, fieldPath).insert(internalId, value);
|
|
1783
1817
|
} else if (base === "boolean") {
|
|
@@ -1796,7 +1830,8 @@ function indexDocument(state, docId, document, fields, language, options) {
|
|
|
1796
1830
|
language,
|
|
1797
1831
|
options,
|
|
1798
1832
|
fieldLengths,
|
|
1799
|
-
tokensByField
|
|
1833
|
+
tokensByField,
|
|
1834
|
+
partialSpellingsFor(field, indexed)
|
|
1800
1835
|
);
|
|
1801
1836
|
} else if (base === "number[]") {
|
|
1802
1837
|
const numIdx = getOrCreateNumericIndex(state, fieldPath);
|
|
@@ -1811,15 +1846,17 @@ function indexDocument(state, docId, document, fields, language, options) {
|
|
|
1811
1846
|
}
|
|
1812
1847
|
}
|
|
1813
1848
|
}
|
|
1814
|
-
return
|
|
1849
|
+
return indexed;
|
|
1815
1850
|
}
|
|
1816
1851
|
function removeFromIndexes(state, docId, storedDoc, schemaFields, language, options) {
|
|
1817
|
-
const
|
|
1818
|
-
const tokensByField =
|
|
1852
|
+
const removed = emptyIndexedText(schemaFields);
|
|
1853
|
+
const { fieldLengths, tokensByField } = removed;
|
|
1819
1854
|
const fields = storedDoc.fields;
|
|
1820
1855
|
const internalId = resolveInternalId(state, docId);
|
|
1821
1856
|
const opts = tokenizeOptions(options);
|
|
1822
|
-
|
|
1857
|
+
const occurrencesByToken = /* @__PURE__ */ new Map();
|
|
1858
|
+
for (const field of schemaFields) {
|
|
1859
|
+
const { path: fieldPath, base, pattern } = field;
|
|
1823
1860
|
const value = getNestedValue(fields, fieldPath);
|
|
1824
1861
|
if (value === void 0 || value === null) continue;
|
|
1825
1862
|
forgetValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
@@ -1827,16 +1864,15 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
1827
1864
|
if (base === "string") {
|
|
1828
1865
|
const result = tokenize(value, language, opts);
|
|
1829
1866
|
fieldLengths[fieldPath] = result.tokens.length;
|
|
1830
|
-
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
1831
1867
|
const fieldTokenList = [];
|
|
1832
1868
|
const surfaceCounts = /* @__PURE__ */ new Map();
|
|
1869
|
+
const spellings = partialSpellingsFor(field, removed);
|
|
1833
1870
|
for (let i = 0; i < result.tokens.length; i++) {
|
|
1834
1871
|
const t = result.tokens[i];
|
|
1835
|
-
uniqueTokens.add(t.token);
|
|
1836
1872
|
fieldTokenList.push(t.token);
|
|
1837
|
-
|
|
1873
|
+
countOccurrence(surfaceCounts, spellings, result.surfaces?.[i], t.token);
|
|
1838
1874
|
}
|
|
1839
|
-
|
|
1875
|
+
addFieldOccurrences(occurrencesByToken, fieldTokenList);
|
|
1840
1876
|
applySurfaceCounts(state, surfaceCounts, -1);
|
|
1841
1877
|
tokensByField[fieldPath] = fieldTokenList;
|
|
1842
1878
|
} else if (base === "number") {
|
|
@@ -1849,20 +1885,19 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
1849
1885
|
state.geoIndexes.get(fieldPath)?.remove(internalId);
|
|
1850
1886
|
} else if (base === "string[]") {
|
|
1851
1887
|
const arr = value;
|
|
1852
|
-
const uniqueTokens = /* @__PURE__ */ new Set();
|
|
1853
1888
|
const fieldTokenList = [];
|
|
1854
1889
|
const surfaceCounts = /* @__PURE__ */ new Map();
|
|
1890
|
+
const spellings = partialSpellingsFor(field, removed);
|
|
1855
1891
|
for (const item of arr) {
|
|
1856
1892
|
const result = tokenize(item, language, opts);
|
|
1857
1893
|
for (let i = 0; i < result.tokens.length; i++) {
|
|
1858
1894
|
const t = result.tokens[i];
|
|
1859
|
-
uniqueTokens.add(t.token);
|
|
1860
1895
|
fieldTokenList.push(t.token);
|
|
1861
|
-
|
|
1896
|
+
countOccurrence(surfaceCounts, spellings, result.surfaces?.[i], t.token);
|
|
1862
1897
|
}
|
|
1863
1898
|
}
|
|
1864
1899
|
fieldLengths[fieldPath] = fieldTokenList.length;
|
|
1865
|
-
|
|
1900
|
+
addFieldOccurrences(occurrencesByToken, fieldTokenList);
|
|
1866
1901
|
applySurfaceCounts(state, surfaceCounts, -1);
|
|
1867
1902
|
tokensByField[fieldPath] = fieldTokenList;
|
|
1868
1903
|
} else if (base === "number[]") {
|
|
@@ -1882,7 +1917,8 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
1882
1917
|
}
|
|
1883
1918
|
}
|
|
1884
1919
|
}
|
|
1885
|
-
|
|
1920
|
+
for (const [token, occurrences] of occurrencesByToken) state.invertedIdx.remove(token, internalId, occurrences);
|
|
1921
|
+
return removed;
|
|
1886
1922
|
}
|
|
1887
1923
|
|
|
1888
1924
|
// src/core/partition/field-updates.ts
|
|
@@ -3017,6 +3053,67 @@ function partitionFilterMatches(state, filters, schema, meter) {
|
|
|
3017
3053
|
};
|
|
3018
3054
|
}
|
|
3019
3055
|
|
|
3056
|
+
// src/core/scorer.ts
|
|
3057
|
+
var DEFAULT_K1 = 1.2;
|
|
3058
|
+
var DEFAULT_B = 0.75;
|
|
3059
|
+
function computeIDF(docFrequency, totalDocs) {
|
|
3060
|
+
if (totalDocs === 0) return 0;
|
|
3061
|
+
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
3062
|
+
}
|
|
3063
|
+
function bm25PruningSound(params) {
|
|
3064
|
+
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
3065
|
+
const b = params?.b ?? DEFAULT_B;
|
|
3066
|
+
return Number.isFinite(k1) && k1 >= 0 && b >= 0 && b <= 1;
|
|
3067
|
+
}
|
|
3068
|
+
function resolveBM25Params(params) {
|
|
3069
|
+
return { k1: params?.k1 ?? DEFAULT_K1, b: params?.b ?? DEFAULT_B };
|
|
3070
|
+
}
|
|
3071
|
+
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
3072
|
+
if (avgFieldLength === 0) return 0;
|
|
3073
|
+
const numerator = termFrequency * (k1 + 1);
|
|
3074
|
+
const denominator = termFrequency + k1 * (1 - b + b * fieldLength / avgFieldLength);
|
|
3075
|
+
if (denominator === 0) return 0;
|
|
3076
|
+
return idf * (numerator / denominator);
|
|
3077
|
+
}
|
|
3078
|
+
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
3079
|
+
if (totalDocs === 0) return 0;
|
|
3080
|
+
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
3081
|
+
const b = params?.b ?? DEFAULT_B;
|
|
3082
|
+
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
3083
|
+
}
|
|
3084
|
+
|
|
3085
|
+
// src/core/partition/field-scoring.ts
|
|
3086
|
+
function loadFieldScoring(state, fields, boost, averageFieldLengths) {
|
|
3087
|
+
const fieldNames = state.fieldNameTable.names;
|
|
3088
|
+
const scoring = {
|
|
3089
|
+
searchable: new Uint8Array(fieldNames.length),
|
|
3090
|
+
boosts: new Float64Array(fieldNames.length),
|
|
3091
|
+
averageLengths: new Float64Array(fieldNames.length),
|
|
3092
|
+
lengthColumns: new Array(fieldNames.length).fill(null)
|
|
3093
|
+
};
|
|
3094
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
3095
|
+
const fieldName = fieldNames[fieldIndex];
|
|
3096
|
+
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
3097
|
+
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
3098
|
+
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
3099
|
+
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
3100
|
+
}
|
|
3101
|
+
return scoring;
|
|
3102
|
+
}
|
|
3103
|
+
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
3104
|
+
const column = fieldIndex < columns.length ? columns[fieldIndex] : null;
|
|
3105
|
+
if (column === null || internalId >= column.length) return averageLength;
|
|
3106
|
+
const stored = column[internalId];
|
|
3107
|
+
return stored > 0 ? stored : averageLength;
|
|
3108
|
+
}
|
|
3109
|
+
function bestSearchable(searchable, values) {
|
|
3110
|
+
let best = 0;
|
|
3111
|
+
for (let index = 0; index < searchable.length; index++) {
|
|
3112
|
+
if (searchable[index] === 1 && values[index] > best) best = values[index];
|
|
3113
|
+
}
|
|
3114
|
+
return best;
|
|
3115
|
+
}
|
|
3116
|
+
|
|
3020
3117
|
// src/core/partition/posting-columns.ts
|
|
3021
3118
|
function postingColumns(list) {
|
|
3022
3119
|
const deletedDocs = list.deletedDocs;
|
|
@@ -3030,39 +3127,174 @@ function postingColumns(list) {
|
|
|
3030
3127
|
};
|
|
3031
3128
|
}
|
|
3032
3129
|
|
|
3130
|
+
// src/core/partition/scoring.ts
|
|
3131
|
+
var EMPTY_COMPONENTS = Object.freeze({});
|
|
3132
|
+
function recordComponents(components, internalId, fieldName, token, termFrequency, fieldLength, idf) {
|
|
3133
|
+
const existing = components.get(internalId);
|
|
3134
|
+
if (existing) {
|
|
3135
|
+
existing.termFrequencies[`${fieldName}:${token}`] = termFrequency;
|
|
3136
|
+
existing.fieldLengths[fieldName] = fieldLength;
|
|
3137
|
+
existing.idf[token] = idf;
|
|
3138
|
+
return;
|
|
3139
|
+
}
|
|
3140
|
+
components.set(internalId, {
|
|
3141
|
+
termFrequencies: { [`${fieldName}:${token}`]: termFrequency },
|
|
3142
|
+
fieldLengths: { [fieldName]: fieldLength },
|
|
3143
|
+
idf: { [token]: idf }
|
|
3144
|
+
});
|
|
3145
|
+
}
|
|
3146
|
+
function mergeGroupComponents(components, internalId, contribution) {
|
|
3147
|
+
const existing = components.get(internalId);
|
|
3148
|
+
if (existing) {
|
|
3149
|
+
Object.assign(existing.termFrequencies, contribution.termFrequencies);
|
|
3150
|
+
Object.assign(existing.fieldLengths, contribution.fieldLengths);
|
|
3151
|
+
existing.idf[contribution.token] = contribution.idf;
|
|
3152
|
+
return;
|
|
3153
|
+
}
|
|
3154
|
+
components.set(internalId, {
|
|
3155
|
+
termFrequencies: contribution.termFrequencies,
|
|
3156
|
+
fieldLengths: contribution.fieldLengths,
|
|
3157
|
+
idf: { [contribution.token]: contribution.idf }
|
|
3158
|
+
});
|
|
3159
|
+
}
|
|
3160
|
+
|
|
3161
|
+
// src/core/partition/group-scoring.ts
|
|
3162
|
+
function partialFieldMarks(fieldNames, partialFields) {
|
|
3163
|
+
const marks = new Uint8Array(fieldNames.length);
|
|
3164
|
+
if (partialFields === void 0 || partialFields.length === 0) return marks;
|
|
3165
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
3166
|
+
if (partialFields.includes(fieldNames[fieldIndex])) marks[fieldIndex] = 1;
|
|
3167
|
+
}
|
|
3168
|
+
return marks;
|
|
3169
|
+
}
|
|
3170
|
+
function termGroupOf(groups, token) {
|
|
3171
|
+
if (groups === void 0) return void 0;
|
|
3172
|
+
for (const group of groups) {
|
|
3173
|
+
if (group.token === token) return group;
|
|
3174
|
+
}
|
|
3175
|
+
return void 0;
|
|
3176
|
+
}
|
|
3177
|
+
function typoMatchesFor(index, token, group, exact, tolerance, prefixLength) {
|
|
3178
|
+
if (group !== void 0) {
|
|
3179
|
+
if (!group.typoMatches || exact || tolerance === 0) return [];
|
|
3180
|
+
return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token);
|
|
3181
|
+
}
|
|
3182
|
+
if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
|
|
3183
|
+
const postingList = index.lookup(token);
|
|
3184
|
+
return postingList ? [{ token, postingList }] : [];
|
|
3185
|
+
}
|
|
3186
|
+
function resolveGroupMatches(context, group) {
|
|
3187
|
+
let groupDocumentFrequency = 0;
|
|
3188
|
+
const found = [];
|
|
3189
|
+
for (const member of group.members) {
|
|
3190
|
+
const documents = context.globalFrequency(member.token) ?? context.index.documentFrequency(member.token);
|
|
3191
|
+
if (documents > groupDocumentFrequency) groupDocumentFrequency = documents;
|
|
3192
|
+
const postingList = context.index.lookup(member.token);
|
|
3193
|
+
if (postingList === void 0) continue;
|
|
3194
|
+
found.push({ token: member.token, factor: member.factor, partialFactor: member.partialFactor, postingList });
|
|
3195
|
+
}
|
|
3196
|
+
if (found.length === 0) return [];
|
|
3197
|
+
const idf = computeIDF(groupDocumentFrequency, context.totalDocs);
|
|
3198
|
+
return found.map((entry) => ({ ...entry, idf }));
|
|
3199
|
+
}
|
|
3200
|
+
function groupPostingCount(matches) {
|
|
3201
|
+
let total = 0;
|
|
3202
|
+
for (const match of matches) total += match.postingList.length;
|
|
3203
|
+
return total;
|
|
3204
|
+
}
|
|
3205
|
+
function computeGroupContributions(context, matches, collect) {
|
|
3206
|
+
const { fields, fieldNames, filterBitset, partialMarks, k1, b } = context;
|
|
3207
|
+
const scoresAreZero = context.totalDocs === 0;
|
|
3208
|
+
const best = /* @__PURE__ */ new Map();
|
|
3209
|
+
for (const match of matches) {
|
|
3210
|
+
const perTerm = /* @__PURE__ */ new Map();
|
|
3211
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
3212
|
+
match.postingList
|
|
3213
|
+
);
|
|
3214
|
+
for (let pi = 0; pi < count; pi++) {
|
|
3215
|
+
const internalId = docIds[pi];
|
|
3216
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
3217
|
+
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
3218
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
3219
|
+
if (fields.searchable[fieldIndex] === 0) continue;
|
|
3220
|
+
const factor = partialMarks[fieldIndex] === 1 ? match.partialFactor : match.factor;
|
|
3221
|
+
if (factor === 0) continue;
|
|
3222
|
+
const fieldName = fieldNames[fieldIndex];
|
|
3223
|
+
const termFrequency = termFrequencies[pi];
|
|
3224
|
+
const fieldBoost = fields.boosts[fieldIndex];
|
|
3225
|
+
const avgLen = fields.averageLengths[fieldIndex];
|
|
3226
|
+
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
3227
|
+
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b)) * fieldBoost * factor;
|
|
3228
|
+
const existing = perTerm.get(internalId);
|
|
3229
|
+
if (existing) {
|
|
3230
|
+
existing.score += termScore;
|
|
3231
|
+
if (collect) {
|
|
3232
|
+
existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
|
|
3233
|
+
existing.fieldLengths[fieldName] = actualFieldLength;
|
|
3234
|
+
}
|
|
3235
|
+
} else if (collect) {
|
|
3236
|
+
perTerm.set(internalId, {
|
|
3237
|
+
score: termScore,
|
|
3238
|
+
token: match.token,
|
|
3239
|
+
idf: match.idf,
|
|
3240
|
+
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
3241
|
+
fieldLengths: { [fieldName]: actualFieldLength }
|
|
3242
|
+
});
|
|
3243
|
+
} else {
|
|
3244
|
+
perTerm.set(internalId, {
|
|
3245
|
+
score: termScore,
|
|
3246
|
+
token: match.token,
|
|
3247
|
+
idf: match.idf,
|
|
3248
|
+
termFrequencies: EMPTY_COMPONENTS,
|
|
3249
|
+
fieldLengths: EMPTY_COMPONENTS
|
|
3250
|
+
});
|
|
3251
|
+
}
|
|
3252
|
+
}
|
|
3253
|
+
for (const [internalId, contribution] of perTerm) {
|
|
3254
|
+
const current = best.get(internalId);
|
|
3255
|
+
if (!current || contribution.score > current.score) {
|
|
3256
|
+
best.set(internalId, contribution);
|
|
3257
|
+
}
|
|
3258
|
+
}
|
|
3259
|
+
}
|
|
3260
|
+
return best;
|
|
3261
|
+
}
|
|
3262
|
+
|
|
3033
3263
|
// src/core/partition/matches.ts
|
|
3034
|
-
function markPostingList(matched, list,
|
|
3264
|
+
function markPostingList(matched, list, markedFields, filterBitset) {
|
|
3035
3265
|
const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(list);
|
|
3036
3266
|
for (let pi = 0; pi < count; pi++) {
|
|
3037
3267
|
const internalId = docIds[pi];
|
|
3038
3268
|
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
3039
3269
|
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
3040
|
-
if (
|
|
3270
|
+
if (markedFields[fieldNameIndices[pi]] === 0) continue;
|
|
3041
3271
|
bitsetSet(matched, internalId);
|
|
3042
3272
|
}
|
|
3043
3273
|
}
|
|
3274
|
+
function searchedFieldMarks(fieldNames, fields) {
|
|
3275
|
+
const marks = new Uint8Array(fieldNames.length);
|
|
3276
|
+
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
3277
|
+
if (fields === void 0 || fields.includes(fieldNames[fieldIndex])) marks[fieldIndex] = 1;
|
|
3278
|
+
}
|
|
3279
|
+
return marks;
|
|
3280
|
+
}
|
|
3044
3281
|
function searchFulltextMatches(state, params) {
|
|
3045
|
-
const { queryTokens,
|
|
3282
|
+
const { queryTokens, termGroups, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
|
|
3046
3283
|
const fieldNames = state.fieldNameTable.names;
|
|
3047
3284
|
const matched = createBitSet(state.docStore.internalIdCapacity());
|
|
3285
|
+
const searched = searchedFieldMarks(fieldNames, fields);
|
|
3286
|
+
const partial = partialFieldMarks(fieldNames, params.partialFields);
|
|
3287
|
+
const searchedPartial = searched.map((mark, fieldIndex) => mark & partial[fieldIndex]);
|
|
3048
3288
|
for (const qt of queryTokens) {
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
if (seen.has(term)) continue;
|
|
3053
|
-
seen.add(term);
|
|
3054
|
-
const postingList = state.invertedIdx.lookup(term);
|
|
3055
|
-
if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
|
|
3056
|
-
}
|
|
3057
|
-
continue;
|
|
3058
|
-
}
|
|
3059
|
-
if (exact || tolerance === 0) {
|
|
3060
|
-
const postingList = state.invertedIdx.lookup(qt.token);
|
|
3061
|
-
if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
|
|
3062
|
-
continue;
|
|
3289
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
3290
|
+
for (const match of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
3291
|
+
markPostingList(matched, match.postingList, searched, filterBitset);
|
|
3063
3292
|
}
|
|
3064
|
-
|
|
3065
|
-
|
|
3293
|
+
if (group === void 0) continue;
|
|
3294
|
+
for (const member of group.members) {
|
|
3295
|
+
const postingList = state.invertedIdx.lookup(member.token);
|
|
3296
|
+
if (postingList === void 0) continue;
|
|
3297
|
+
markPostingList(matched, postingList, member.factor > 0 ? searched : searchedPartial, filterBitset);
|
|
3066
3298
|
}
|
|
3067
3299
|
}
|
|
3068
3300
|
const docStore = state.docStore;
|
|
@@ -3150,6 +3382,7 @@ function estimatePartitionBytes(state) {
|
|
|
3150
3382
|
bytes += docCount * state.geoIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
3151
3383
|
bytes += docCount * state.sizeIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
3152
3384
|
for (const index of state.patternIndexes.values()) bytes += index.bytes();
|
|
3385
|
+
bytes += state.partialWords.bytes();
|
|
3153
3386
|
return bytes + sortColumnBytes(state);
|
|
3154
3387
|
}
|
|
3155
3388
|
|
|
@@ -3281,10 +3514,71 @@ function mergeSegmentState(target, source) {
|
|
|
3281
3514
|
mergeFieldIndexes(target, source, idMapping);
|
|
3282
3515
|
mergeStatistics(target, source);
|
|
3283
3516
|
mergeSurfaceForms(target, source);
|
|
3517
|
+
target.partialWords.afterMerge(source.partialWords.spellings(), [...idMapping.values()]);
|
|
3284
3518
|
target.sortColumns = null;
|
|
3285
3519
|
target.scoreBuffer = null;
|
|
3286
3520
|
}
|
|
3287
3521
|
|
|
3522
|
+
// src/core/partition/partial-words/platform.ts
|
|
3523
|
+
function platformPartialWordSearch() {
|
|
3524
|
+
return partialWordSearchModule;
|
|
3525
|
+
}
|
|
3526
|
+
|
|
3527
|
+
// src/core/partition/partial-words/registry.ts
|
|
3528
|
+
var registered2 = null;
|
|
3529
|
+
function isPartialWordSearchModule(value) {
|
|
3530
|
+
return value !== null && typeof value === "object" && value.name === "partialWords" && "createGrowingSlot" in value && typeof value.createGrowingSlot === "function" && "createFrozenSlot" in value && typeof value.createFrozenSlot === "function" && "buildTable" in value && typeof value.buildTable === "function" && "expand" in value && typeof value.expand === "function";
|
|
3531
|
+
}
|
|
3532
|
+
function registerPartialWordSearch(module) {
|
|
3533
|
+
if (!isPartialWordSearchModule(module)) {
|
|
3534
|
+
throw new NarsilError(
|
|
3535
|
+
ErrorCodes.CONFIG_INVALID,
|
|
3536
|
+
'Pass registerPartialWordSearch the partialWordSearch object that "@delali/narsil/partial-words" exports'
|
|
3537
|
+
);
|
|
3538
|
+
}
|
|
3539
|
+
registered2 = module;
|
|
3540
|
+
}
|
|
3541
|
+
function availablePartialWordSearch() {
|
|
3542
|
+
return registered2 ?? platformPartialWordSearch();
|
|
3543
|
+
}
|
|
3544
|
+
function requirePartialWordSearchFor(schema) {
|
|
3545
|
+
const fields = partialFieldsOf(schema);
|
|
3546
|
+
if (fields.length === 0 || availablePartialWordSearch() !== null) return;
|
|
3547
|
+
throw new NarsilError(
|
|
3548
|
+
ErrorCodes.CONFIG_INVALID,
|
|
3549
|
+
`The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load the index`,
|
|
3550
|
+
{ field: fields[0] }
|
|
3551
|
+
);
|
|
3552
|
+
}
|
|
3553
|
+
var EMPTY_GROWING_SLOT = {
|
|
3554
|
+
readerFor: () => null,
|
|
3555
|
+
spellings: () => null,
|
|
3556
|
+
bytes: () => 0,
|
|
3557
|
+
afterDocument: () => void 0,
|
|
3558
|
+
afterMerge: () => void 0,
|
|
3559
|
+
forgetSpellings: () => void 0,
|
|
3560
|
+
clear: () => void 0
|
|
3561
|
+
};
|
|
3562
|
+
var EMPTY_FROZEN_SLOT = {
|
|
3563
|
+
readerFor: () => null,
|
|
3564
|
+
spellings: () => null,
|
|
3565
|
+
bytes: () => 0,
|
|
3566
|
+
afterRemoval: () => void 0
|
|
3567
|
+
};
|
|
3568
|
+
function createGrowingPartialWords(view) {
|
|
3569
|
+
return availablePartialWordSearch()?.createGrowingSlot(view) ?? EMPTY_GROWING_SLOT;
|
|
3570
|
+
}
|
|
3571
|
+
function createFrozenPartialWords(initial, tokenTable, documents) {
|
|
3572
|
+
return availablePartialWordSearch()?.createFrozenSlot(initial, tokenTable, documents) ?? EMPTY_FROZEN_SLOT;
|
|
3573
|
+
}
|
|
3574
|
+
function buildPartialWordTableFrom(source, tokenTable) {
|
|
3575
|
+
return availablePartialWordSearch()?.buildTable(source, tokenTable) ?? null;
|
|
3576
|
+
}
|
|
3577
|
+
function expandPartialWords(subs, analysis, requests) {
|
|
3578
|
+
const module = availablePartialWordSearch();
|
|
3579
|
+
return module === null ? requests.map(() => []) : module.expand(subs, analysis, requests);
|
|
3580
|
+
}
|
|
3581
|
+
|
|
3288
3582
|
// src/core/partition/rebuild.ts
|
|
3289
3583
|
var EMPTY_STATS = {
|
|
3290
3584
|
totalDocuments: 0,
|
|
@@ -3340,70 +3634,10 @@ function rebuildTextIndex(state, schema, language, options) {
|
|
|
3340
3634
|
}
|
|
3341
3635
|
} finally {
|
|
3342
3636
|
state.invertedIdx.endBatch();
|
|
3637
|
+
state.partialWords.forgetSpellings();
|
|
3343
3638
|
}
|
|
3344
3639
|
}
|
|
3345
3640
|
|
|
3346
|
-
// src/core/scorer.ts
|
|
3347
|
-
var DEFAULT_K1 = 1.2;
|
|
3348
|
-
var DEFAULT_B = 0.75;
|
|
3349
|
-
function computeIDF(docFrequency, totalDocs) {
|
|
3350
|
-
if (totalDocs === 0) return 0;
|
|
3351
|
-
return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
|
|
3352
|
-
}
|
|
3353
|
-
function bm25PruningSound(params) {
|
|
3354
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
3355
|
-
const b = params?.b ?? DEFAULT_B;
|
|
3356
|
-
return Number.isFinite(k1) && k1 >= 0 && b >= 0 && b <= 1;
|
|
3357
|
-
}
|
|
3358
|
-
function resolveBM25Params(params) {
|
|
3359
|
-
return { k1: params?.k1 ?? DEFAULT_K1, b: params?.b ?? DEFAULT_B };
|
|
3360
|
-
}
|
|
3361
|
-
function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
|
|
3362
|
-
if (avgFieldLength === 0) return 0;
|
|
3363
|
-
const numerator = termFrequency * (k1 + 1);
|
|
3364
|
-
const denominator = termFrequency + k1 * (1 - b + b * fieldLength / avgFieldLength);
|
|
3365
|
-
if (denominator === 0) return 0;
|
|
3366
|
-
return idf * (numerator / denominator);
|
|
3367
|
-
}
|
|
3368
|
-
function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
|
|
3369
|
-
if (totalDocs === 0) return 0;
|
|
3370
|
-
const k1 = params?.k1 ?? DEFAULT_K1;
|
|
3371
|
-
const b = params?.b ?? DEFAULT_B;
|
|
3372
|
-
return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
|
|
3373
|
-
}
|
|
3374
|
-
|
|
3375
|
-
// src/core/partition/field-scoring.ts
|
|
3376
|
-
function loadFieldScoring(state, fields, boost, averageFieldLengths) {
|
|
3377
|
-
const fieldNames = state.fieldNameTable.names;
|
|
3378
|
-
const scoring = {
|
|
3379
|
-
searchable: new Uint8Array(fieldNames.length),
|
|
3380
|
-
boosts: new Float64Array(fieldNames.length),
|
|
3381
|
-
averageLengths: new Float64Array(fieldNames.length),
|
|
3382
|
-
lengthColumns: new Array(fieldNames.length).fill(null)
|
|
3383
|
-
};
|
|
3384
|
-
for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
|
|
3385
|
-
const fieldName = fieldNames[fieldIndex];
|
|
3386
|
-
scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
|
|
3387
|
-
scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
|
|
3388
|
-
scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
|
|
3389
|
-
scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
|
|
3390
|
-
}
|
|
3391
|
-
return scoring;
|
|
3392
|
-
}
|
|
3393
|
-
function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
|
|
3394
|
-
const column = fieldIndex < columns.length ? columns[fieldIndex] : null;
|
|
3395
|
-
if (column === null || internalId >= column.length) return averageLength;
|
|
3396
|
-
const stored = column[internalId];
|
|
3397
|
-
return stored > 0 ? stored : averageLength;
|
|
3398
|
-
}
|
|
3399
|
-
function bestSearchable(searchable, values) {
|
|
3400
|
-
let best = 0;
|
|
3401
|
-
for (let index = 0; index < searchable.length; index++) {
|
|
3402
|
-
if (searchable[index] === 1 && values[index] > best) best = values[index];
|
|
3403
|
-
}
|
|
3404
|
-
return best;
|
|
3405
|
-
}
|
|
3406
|
-
|
|
3407
3641
|
// src/core/partition/block-bounds.ts
|
|
3408
3642
|
var ABSENT_FIELD_LENGTH_BOUND = 0;
|
|
3409
3643
|
var boundsByList = /* @__PURE__ */ new WeakMap();
|
|
@@ -3499,37 +3733,6 @@ function blockBoundsFor(list, columns) {
|
|
|
3499
3733
|
return built;
|
|
3500
3734
|
}
|
|
3501
3735
|
|
|
3502
|
-
// src/core/partition/scoring.ts
|
|
3503
|
-
var EMPTY_COMPONENTS = Object.freeze({});
|
|
3504
|
-
function recordComponents(components, internalId, fieldName, token, termFrequency, fieldLength, idf) {
|
|
3505
|
-
const existing = components.get(internalId);
|
|
3506
|
-
if (existing) {
|
|
3507
|
-
existing.termFrequencies[`${fieldName}:${token}`] = termFrequency;
|
|
3508
|
-
existing.fieldLengths[fieldName] = fieldLength;
|
|
3509
|
-
existing.idf[token] = idf;
|
|
3510
|
-
return;
|
|
3511
|
-
}
|
|
3512
|
-
components.set(internalId, {
|
|
3513
|
-
termFrequencies: { [`${fieldName}:${token}`]: termFrequency },
|
|
3514
|
-
fieldLengths: { [fieldName]: fieldLength },
|
|
3515
|
-
idf: { [token]: idf }
|
|
3516
|
-
});
|
|
3517
|
-
}
|
|
3518
|
-
function mergePrefixComponents(components, internalId, contribution) {
|
|
3519
|
-
const existing = components.get(internalId);
|
|
3520
|
-
if (existing) {
|
|
3521
|
-
Object.assign(existing.termFrequencies, contribution.termFrequencies);
|
|
3522
|
-
Object.assign(existing.fieldLengths, contribution.fieldLengths);
|
|
3523
|
-
existing.idf[contribution.token] = contribution.idf;
|
|
3524
|
-
return;
|
|
3525
|
-
}
|
|
3526
|
-
components.set(internalId, {
|
|
3527
|
-
termFrequencies: contribution.termFrequencies,
|
|
3528
|
-
fieldLengths: contribution.fieldLengths,
|
|
3529
|
-
idf: { [contribution.token]: contribution.idf }
|
|
3530
|
-
});
|
|
3531
|
-
}
|
|
3532
|
-
|
|
3533
3736
|
// src/core/partition/top-k-heap.ts
|
|
3534
3737
|
function candidateWorse(a, b) {
|
|
3535
3738
|
if (a.score !== b.score) return a.score < b.score;
|
|
@@ -3693,7 +3896,7 @@ function topKFromBuffer(buffer, k, resolver, components) {
|
|
|
3693
3896
|
// src/core/partition/multi-term-topk.ts
|
|
3694
3897
|
function prunableMultiTermLists(params, index) {
|
|
3695
3898
|
if (params.queryTokens.length < 2) return null;
|
|
3696
|
-
if (params.
|
|
3899
|
+
if (params.termGroups !== void 0 && params.termGroups.length > 0) return null;
|
|
3697
3900
|
if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
|
|
3698
3901
|
if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
|
|
3699
3902
|
if (params.collectComponents !== false) return null;
|
|
@@ -3865,87 +4068,6 @@ function multiTermTopK(request) {
|
|
|
3865
4068
|
return { scored: pageFromBuffer(scans, buffer, selection, resolver), totalMatched };
|
|
3866
4069
|
}
|
|
3867
4070
|
|
|
3868
|
-
// src/core/partition/prefix-scoring.ts
|
|
3869
|
-
function resolvePrefixMatches(context, token, expansionTerms) {
|
|
3870
|
-
const found = [];
|
|
3871
|
-
const seen = /* @__PURE__ */ new Set();
|
|
3872
|
-
for (const term of [token, ...expansionTerms]) {
|
|
3873
|
-
if (seen.has(term)) continue;
|
|
3874
|
-
seen.add(term);
|
|
3875
|
-
const postingList = context.index.lookup(term);
|
|
3876
|
-
if (!postingList) continue;
|
|
3877
|
-
found.push({ token: term, postingList, docFreq: context.documentFrequency(term, postingList) });
|
|
3878
|
-
}
|
|
3879
|
-
if (found.length === 0) return [];
|
|
3880
|
-
let blendedDf = 0;
|
|
3881
|
-
for (const f of found) {
|
|
3882
|
-
if (f.docFreq > blendedDf) blendedDf = f.docFreq;
|
|
3883
|
-
}
|
|
3884
|
-
const blendedIdf = computeIDF(blendedDf, context.totalDocs);
|
|
3885
|
-
return found.map((f) => ({
|
|
3886
|
-
token: f.token,
|
|
3887
|
-
factor: Math.min(1, token.length / f.token.length),
|
|
3888
|
-
postingList: f.postingList,
|
|
3889
|
-
docFreq: blendedDf,
|
|
3890
|
-
idf: blendedIdf
|
|
3891
|
-
}));
|
|
3892
|
-
}
|
|
3893
|
-
function computePrefixContributions(context, matches, collect) {
|
|
3894
|
-
const { fields, fieldNames, filterBitset, k1, b } = context;
|
|
3895
|
-
const scoresAreZero = context.totalDocs === 0;
|
|
3896
|
-
const best = /* @__PURE__ */ new Map();
|
|
3897
|
-
for (const match of matches) {
|
|
3898
|
-
const perTerm = /* @__PURE__ */ new Map();
|
|
3899
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
3900
|
-
match.postingList
|
|
3901
|
-
);
|
|
3902
|
-
for (let pi = 0; pi < count; pi++) {
|
|
3903
|
-
const internalId = docIds[pi];
|
|
3904
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
3905
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
3906
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
3907
|
-
if (fields.searchable[fieldIndex] === 0) continue;
|
|
3908
|
-
const fieldName = fieldNames[fieldIndex];
|
|
3909
|
-
const termFrequency = termFrequencies[pi];
|
|
3910
|
-
const fieldBoost = fields.boosts[fieldIndex];
|
|
3911
|
-
const avgLen = fields.averageLengths[fieldIndex];
|
|
3912
|
-
const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
|
|
3913
|
-
const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b)) * fieldBoost * match.factor;
|
|
3914
|
-
const existing = perTerm.get(internalId);
|
|
3915
|
-
if (existing) {
|
|
3916
|
-
existing.score += termScore;
|
|
3917
|
-
if (collect) {
|
|
3918
|
-
existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
|
|
3919
|
-
existing.fieldLengths[fieldName] = actualFieldLength;
|
|
3920
|
-
}
|
|
3921
|
-
} else if (collect) {
|
|
3922
|
-
perTerm.set(internalId, {
|
|
3923
|
-
score: termScore,
|
|
3924
|
-
token: match.token,
|
|
3925
|
-
idf: match.idf,
|
|
3926
|
-
termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
|
|
3927
|
-
fieldLengths: { [fieldName]: actualFieldLength }
|
|
3928
|
-
});
|
|
3929
|
-
} else {
|
|
3930
|
-
perTerm.set(internalId, {
|
|
3931
|
-
score: termScore,
|
|
3932
|
-
token: match.token,
|
|
3933
|
-
idf: match.idf,
|
|
3934
|
-
termFrequencies: EMPTY_COMPONENTS,
|
|
3935
|
-
fieldLengths: EMPTY_COMPONENTS
|
|
3936
|
-
});
|
|
3937
|
-
}
|
|
3938
|
-
}
|
|
3939
|
-
for (const [internalId, contribution] of perTerm) {
|
|
3940
|
-
const current = best.get(internalId);
|
|
3941
|
-
if (!current || contribution.score > current.score) {
|
|
3942
|
-
best.set(internalId, contribution);
|
|
3943
|
-
}
|
|
3944
|
-
}
|
|
3945
|
-
}
|
|
3946
|
-
return best;
|
|
3947
|
-
}
|
|
3948
|
-
|
|
3949
4071
|
// src/core/partition/single-term-topk.ts
|
|
3950
4072
|
function fieldsCoverEvery(fields, fieldNames) {
|
|
3951
4073
|
if (fields === void 0) return true;
|
|
@@ -3956,7 +4078,7 @@ function fieldsCoverEvery(fields, fieldNames) {
|
|
|
3956
4078
|
}
|
|
3957
4079
|
function prunableSingleTermList(params, index, fieldNames) {
|
|
3958
4080
|
if (params.queryTokens.length !== 1) return null;
|
|
3959
|
-
if (params.
|
|
4081
|
+
if (params.termGroups !== void 0 && params.termGroups.length > 0) return null;
|
|
3960
4082
|
if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
|
|
3961
4083
|
if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
|
|
3962
4084
|
if (params.collectComponents !== false) return null;
|
|
@@ -4066,15 +4188,11 @@ function singleTermTopK(request) {
|
|
|
4066
4188
|
function globalDocFreqFor(docFreqs, term, fallback) {
|
|
4067
4189
|
return Object.hasOwn(docFreqs, term) ? docFreqs[term] : fallback;
|
|
4068
4190
|
}
|
|
4069
|
-
function matchesFor(index, token, exact, tolerance, prefixLength) {
|
|
4070
|
-
if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
|
|
4071
|
-
const postingList = index.lookup(token);
|
|
4072
|
-
return postingList ? [{ token, postingList }] : [];
|
|
4073
|
-
}
|
|
4074
4191
|
function searchFulltext(state, params) {
|
|
4075
4192
|
const {
|
|
4076
4193
|
queryTokens,
|
|
4077
|
-
|
|
4194
|
+
termGroups,
|
|
4195
|
+
partialFields,
|
|
4078
4196
|
fields,
|
|
4079
4197
|
boost,
|
|
4080
4198
|
tolerance = 0,
|
|
@@ -4106,19 +4224,75 @@ function searchFulltext(state, params) {
|
|
|
4106
4224
|
function documentFrequency(token, list) {
|
|
4107
4225
|
return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
|
|
4108
4226
|
}
|
|
4109
|
-
const
|
|
4227
|
+
const groupContext = {
|
|
4110
4228
|
index: state.invertedIdx,
|
|
4111
4229
|
fields: fieldScoring,
|
|
4230
|
+
partialMarks: partialFieldMarks(fieldNames, partialFields),
|
|
4112
4231
|
fieldNames,
|
|
4113
4232
|
totalDocs,
|
|
4114
4233
|
k1,
|
|
4115
4234
|
b,
|
|
4116
4235
|
filterBitset,
|
|
4117
|
-
|
|
4236
|
+
globalFrequency(token) {
|
|
4237
|
+
if (globalStats === void 0 || !Object.hasOwn(globalStats.docFrequencies, token)) return void 0;
|
|
4238
|
+
return globalStats.docFrequencies[token];
|
|
4239
|
+
}
|
|
4118
4240
|
};
|
|
4119
|
-
function
|
|
4241
|
+
function mergeGroupContribution(internalId, contribution) {
|
|
4120
4242
|
addScore(scoreBuffer, internalId, contribution.score);
|
|
4121
|
-
if (components !== null)
|
|
4243
|
+
if (components !== null) mergeGroupComponents(components, internalId, contribution);
|
|
4244
|
+
}
|
|
4245
|
+
function resolveMatch(token, postingList) {
|
|
4246
|
+
const docFreq = documentFrequency(token, postingList);
|
|
4247
|
+
return { token, docFreq, idf: computeIDF(docFreq, totalDocs), postingList };
|
|
4248
|
+
}
|
|
4249
|
+
function scorePostings(match, onlyScored) {
|
|
4250
|
+
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
4251
|
+
match.postingList
|
|
4252
|
+
);
|
|
4253
|
+
if (!onlyScored && !hasDeleted && filterBitset === void 0 && components === null) {
|
|
4254
|
+
for (let pi = 0; pi < count; pi++) {
|
|
4255
|
+
const internalId = docIds[pi];
|
|
4256
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
4257
|
+
if (searchable[fieldIndex] === 0) continue;
|
|
4258
|
+
const avgLen = averageLengths[fieldIndex];
|
|
4259
|
+
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
4260
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[pi], match.idf, actualFieldLength, avgLen, k1, b);
|
|
4261
|
+
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
4262
|
+
}
|
|
4263
|
+
return;
|
|
4264
|
+
}
|
|
4265
|
+
for (let pi = 0; pi < count; pi++) {
|
|
4266
|
+
const internalId = docIds[pi];
|
|
4267
|
+
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
4268
|
+
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
4269
|
+
if (onlyScored && !hasScore(scoreBuffer, internalId)) continue;
|
|
4270
|
+
const fieldIndex = fieldNameIndices[pi];
|
|
4271
|
+
if (searchable[fieldIndex] === 0) continue;
|
|
4272
|
+
const termFrequency = termFrequencies[pi];
|
|
4273
|
+
const avgLen = averageLengths[fieldIndex];
|
|
4274
|
+
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
4275
|
+
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b);
|
|
4276
|
+
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
4277
|
+
if (components !== null) {
|
|
4278
|
+
recordComponents(
|
|
4279
|
+
components,
|
|
4280
|
+
internalId,
|
|
4281
|
+
fieldNames[fieldIndex],
|
|
4282
|
+
match.token,
|
|
4283
|
+
termFrequency,
|
|
4284
|
+
actualFieldLength,
|
|
4285
|
+
match.idf
|
|
4286
|
+
);
|
|
4287
|
+
}
|
|
4288
|
+
}
|
|
4289
|
+
}
|
|
4290
|
+
function scoreGroup(group, onlyScored) {
|
|
4291
|
+
if (group === null) return;
|
|
4292
|
+
for (const [internalId, contribution] of computeGroupContributions(groupContext, group, collectComponents)) {
|
|
4293
|
+
if (onlyScored && !hasScore(scoreBuffer, internalId)) continue;
|
|
4294
|
+
mergeGroupContribution(internalId, contribution);
|
|
4295
|
+
}
|
|
4122
4296
|
}
|
|
4123
4297
|
const prunableList = prunableSingleTermList(params, state.invertedIdx, fieldNames);
|
|
4124
4298
|
if (prunableList !== null && maxResults !== void 0) {
|
|
@@ -4148,127 +4322,30 @@ function searchFulltext(state, params) {
|
|
|
4148
4322
|
}
|
|
4149
4323
|
if (useIntersection) {
|
|
4150
4324
|
const resolved = [];
|
|
4151
|
-
let prefixMatches = [];
|
|
4152
4325
|
for (const qt of queryTokens) {
|
|
4153
|
-
|
|
4154
|
-
|
|
4155
|
-
|
|
4156
|
-
for (const m of prefixMatches) {
|
|
4157
|
-
totalPostings2 += m.postingList.length;
|
|
4158
|
-
}
|
|
4159
|
-
resolved.push({ token: qt.token, matches: [], totalPostings: totalPostings2, isPrefix: true });
|
|
4160
|
-
continue;
|
|
4161
|
-
}
|
|
4162
|
-
const rawMatches = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
|
|
4163
|
-
let totalPostings = 0;
|
|
4326
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
4327
|
+
const groupMatches = group === void 0 ? null : resolveGroupMatches(groupContext, group);
|
|
4328
|
+
let totalPostings = groupMatches === null ? 0 : groupPostingCount(groupMatches);
|
|
4164
4329
|
const matches = [];
|
|
4165
|
-
for (const m of
|
|
4166
|
-
const docFreq = documentFrequency(m.token, m.postingList);
|
|
4167
|
-
const idf = computeIDF(docFreq, totalDocs);
|
|
4330
|
+
for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
4168
4331
|
totalPostings += m.postingList.length;
|
|
4169
|
-
matches.push(
|
|
4332
|
+
matches.push(resolveMatch(m.token, m.postingList));
|
|
4170
4333
|
}
|
|
4171
|
-
resolved.push({ token: qt.token, matches, totalPostings });
|
|
4334
|
+
resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
|
|
4172
4335
|
}
|
|
4173
4336
|
resolved.sort((a, b2) => a.totalPostings - b2.totalPostings);
|
|
4174
4337
|
for (let tokenIndex = 0; tokenIndex < resolved.length; tokenIndex++) {
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
if (tokenIndex > 0 && !hasScore(scoreBuffer, internalId)) continue;
|
|
4179
|
-
mergePrefixContribution(internalId, contribution);
|
|
4180
|
-
}
|
|
4181
|
-
continue;
|
|
4182
|
-
}
|
|
4183
|
-
for (const match of resolved[tokenIndex].matches) {
|
|
4184
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
4185
|
-
match.postingList
|
|
4186
|
-
);
|
|
4187
|
-
for (let pi = 0; pi < count; pi++) {
|
|
4188
|
-
const internalId = docIds[pi];
|
|
4189
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
4190
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
4191
|
-
if (tokenIndex > 0 && !hasScore(scoreBuffer, internalId)) continue;
|
|
4192
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
4193
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
4194
|
-
const termFrequency = termFrequencies[pi];
|
|
4195
|
-
const fieldBoost = boosts[fieldIndex];
|
|
4196
|
-
const avgLen = averageLengths[fieldIndex];
|
|
4197
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
4198
|
-
let termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b);
|
|
4199
|
-
termScore *= fieldBoost;
|
|
4200
|
-
addScore(scoreBuffer, internalId, termScore);
|
|
4201
|
-
if (components !== null) {
|
|
4202
|
-
recordComponents(
|
|
4203
|
-
components,
|
|
4204
|
-
internalId,
|
|
4205
|
-
fieldNames[fieldIndex],
|
|
4206
|
-
match.token,
|
|
4207
|
-
termFrequency,
|
|
4208
|
-
actualFieldLength,
|
|
4209
|
-
match.idf
|
|
4210
|
-
);
|
|
4211
|
-
}
|
|
4212
|
-
}
|
|
4213
|
-
}
|
|
4338
|
+
const onlyScored = tokenIndex > 0;
|
|
4339
|
+
for (const match of resolved[tokenIndex].matches) scorePostings(match, onlyScored);
|
|
4340
|
+
scoreGroup(resolved[tokenIndex].group, onlyScored);
|
|
4214
4341
|
}
|
|
4215
4342
|
} else {
|
|
4216
4343
|
for (const qt of queryTokens) {
|
|
4217
|
-
|
|
4218
|
-
|
|
4219
|
-
|
|
4220
|
-
resolvePrefixMatches(prefixContext, qt.token, prefixExpansion.terms),
|
|
4221
|
-
collectComponents
|
|
4222
|
-
);
|
|
4223
|
-
for (const [internalId, contribution] of contributions) {
|
|
4224
|
-
mergePrefixContribution(internalId, contribution);
|
|
4225
|
-
}
|
|
4226
|
-
continue;
|
|
4227
|
-
}
|
|
4228
|
-
const matchingPostings = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
|
|
4229
|
-
for (const match of matchingPostings) {
|
|
4230
|
-
const idf = computeIDF(documentFrequency(match.token, match.postingList), totalDocs);
|
|
4231
|
-
const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
|
|
4232
|
-
match.postingList
|
|
4233
|
-
);
|
|
4234
|
-
if (!hasDeleted && filterBitset === void 0 && components === null) {
|
|
4235
|
-
for (let pi = 0; pi < count; pi++) {
|
|
4236
|
-
const internalId = docIds[pi];
|
|
4237
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
4238
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
4239
|
-
const avgLen = averageLengths[fieldIndex];
|
|
4240
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
4241
|
-
const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[pi], idf, actualFieldLength, avgLen, k1, b);
|
|
4242
|
-
addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
|
|
4243
|
-
}
|
|
4244
|
-
continue;
|
|
4245
|
-
}
|
|
4246
|
-
for (let pi = 0; pi < count; pi++) {
|
|
4247
|
-
const internalId = docIds[pi];
|
|
4248
|
-
if (hasDeleted && deletedDocs.has(internalId)) continue;
|
|
4249
|
-
if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
|
|
4250
|
-
const fieldIndex = fieldNameIndices[pi];
|
|
4251
|
-
if (searchable[fieldIndex] === 0) continue;
|
|
4252
|
-
const termFrequency = termFrequencies[pi];
|
|
4253
|
-
const fieldBoost = boosts[fieldIndex];
|
|
4254
|
-
const avgLen = averageLengths[fieldIndex];
|
|
4255
|
-
const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
|
|
4256
|
-
let termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, idf, actualFieldLength, avgLen, k1, b);
|
|
4257
|
-
termScore *= fieldBoost;
|
|
4258
|
-
addScore(scoreBuffer, internalId, termScore);
|
|
4259
|
-
if (components !== null) {
|
|
4260
|
-
recordComponents(
|
|
4261
|
-
components,
|
|
4262
|
-
internalId,
|
|
4263
|
-
fieldNames[fieldIndex],
|
|
4264
|
-
match.token,
|
|
4265
|
-
termFrequency,
|
|
4266
|
-
actualFieldLength,
|
|
4267
|
-
idf
|
|
4268
|
-
);
|
|
4269
|
-
}
|
|
4270
|
-
}
|
|
4344
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
4345
|
+
for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
4346
|
+
scorePostings(resolveMatch(m.token, m.postingList), false);
|
|
4271
4347
|
}
|
|
4348
|
+
if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), false);
|
|
4272
4349
|
}
|
|
4273
4350
|
}
|
|
4274
4351
|
const totalMatched = scoreBuffer.touchedCount;
|
|
@@ -4477,6 +4554,7 @@ function encodeSegmentState(state) {
|
|
|
4477
4554
|
totalFieldLengths: fieldLengths.totals,
|
|
4478
4555
|
docFrequencies,
|
|
4479
4556
|
surfaceForms: state.surfaceRegistry.size() === 0 ? null : state.surfaceRegistry.serialize(),
|
|
4557
|
+
partialSpellings: state.partialWords.spellings(),
|
|
4480
4558
|
...encodeFieldIndexes(state, remap)
|
|
4481
4559
|
};
|
|
4482
4560
|
}
|
|
@@ -4595,6 +4673,8 @@ function mergeSegmentPayload(target, payload, documents) {
|
|
|
4595
4673
|
target.surfaceRegistry.add(surface, value[1], value[0]);
|
|
4596
4674
|
}
|
|
4597
4675
|
}
|
|
4676
|
+
const mergedIds = Array.from({ length: payload.docIds.length }, (_, at) => ordinalBase + at);
|
|
4677
|
+
target.partialWords.afterMerge(payload.partialSpellings, mergedIds);
|
|
4598
4678
|
target.sortColumns = null;
|
|
4599
4679
|
target.scoreBuffer = null;
|
|
4600
4680
|
}
|
|
@@ -4884,72 +4964,7 @@ function deserializePartition(state, data, clearFn, schema) {
|
|
|
4884
4964
|
}
|
|
4885
4965
|
loadPatternIndexes(state, data.fieldIndexes.pattern, schema);
|
|
4886
4966
|
rebuildSizeIndexes(state, schema);
|
|
4887
|
-
|
|
4888
|
-
|
|
4889
|
-
// src/core/partition/suggestions.ts
|
|
4890
|
-
function verbatimOccurrences(state, token) {
|
|
4891
|
-
const postingList = state.invertedIdx.lookup(token);
|
|
4892
|
-
if (!postingList) return 0;
|
|
4893
|
-
const derived = postingList.totalTermFrequency - state.surfaceRegistry.stemChangedTotalFor(token);
|
|
4894
|
-
return derived > 0 ? derived : 0;
|
|
4895
|
-
}
|
|
4896
|
-
function suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit) {
|
|
4897
|
-
if (limit <= 0 || surfacePrefix.length === 0) return [];
|
|
4898
|
-
const groups = /* @__PURE__ */ new Map();
|
|
4899
|
-
for (const candidate of state.surfaceRegistry.candidatesForPrefix(surfacePrefix)) {
|
|
4900
|
-
const postingList = state.invertedIdx.lookup(candidate.token);
|
|
4901
|
-
if (!postingList || postingList.docIdSet.size === 0) continue;
|
|
4902
|
-
let group = groups.get(candidate.token);
|
|
4903
|
-
if (!group) {
|
|
4904
|
-
group = { documentFrequency: postingList.docIdSet.size, surfaces: [] };
|
|
4905
|
-
groups.set(candidate.token, group);
|
|
4906
|
-
}
|
|
4907
|
-
group.surfaces.push({ surface: candidate.surface, occurrences: candidate.occurrences });
|
|
4908
|
-
}
|
|
4909
|
-
const termPrefixes = state.surfaceRegistry.size() === 0 && stemmedPrefix !== surfacePrefix && stemmedPrefix.length > 0 ? [surfacePrefix, stemmedPrefix] : [surfacePrefix];
|
|
4910
|
-
for (const prefix of termPrefixes) {
|
|
4911
|
-
for (const suggestion of state.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
|
|
4912
|
-
const occurrences = verbatimOccurrences(state, suggestion.term);
|
|
4913
|
-
if (occurrences === 0) continue;
|
|
4914
|
-
let group = groups.get(suggestion.term);
|
|
4915
|
-
if (!group) {
|
|
4916
|
-
group = { documentFrequency: suggestion.documentFrequency, surfaces: [] };
|
|
4917
|
-
groups.set(suggestion.term, group);
|
|
4918
|
-
} else if (group.surfaces.some((s) => s.surface === suggestion.term)) {
|
|
4919
|
-
continue;
|
|
4920
|
-
}
|
|
4921
|
-
group.surfaces.push({ surface: suggestion.term, occurrences });
|
|
4922
|
-
}
|
|
4923
|
-
}
|
|
4924
|
-
const results = [];
|
|
4925
|
-
for (const [token, group] of groups) {
|
|
4926
|
-
results.push({ token, documentFrequency: group.documentFrequency, surfaces: group.surfaces });
|
|
4927
|
-
}
|
|
4928
|
-
results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.token, b.token));
|
|
4929
|
-
if (results.length > limit) results.length = limit;
|
|
4930
|
-
return results;
|
|
4931
|
-
}
|
|
4932
|
-
function expandTermPrefix(state, surfacePrefix, stemmedToken, maxExpansions) {
|
|
4933
|
-
if (maxExpansions <= 0 || surfacePrefix.length === 0) return [];
|
|
4934
|
-
const dfByToken = /* @__PURE__ */ new Map();
|
|
4935
|
-
for (const candidate of state.surfaceRegistry.candidatesForPrefix(surfacePrefix)) {
|
|
4936
|
-
if (dfByToken.has(candidate.token)) continue;
|
|
4937
|
-
const postingList = state.invertedIdx.lookup(candidate.token);
|
|
4938
|
-
if (!postingList || postingList.docIdSet.size === 0) continue;
|
|
4939
|
-
dfByToken.set(candidate.token, postingList.docIdSet.size);
|
|
4940
|
-
}
|
|
4941
|
-
const termPrefixes = state.surfaceRegistry.size() === 0 && stemmedToken !== surfacePrefix && stemmedToken.length > 0 ? [surfacePrefix, stemmedToken] : [surfacePrefix];
|
|
4942
|
-
for (const prefix of termPrefixes) {
|
|
4943
|
-
for (const suggestion of state.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
|
|
4944
|
-
if (dfByToken.has(suggestion.term)) continue;
|
|
4945
|
-
if (verbatimOccurrences(state, suggestion.term) === 0) continue;
|
|
4946
|
-
dfByToken.set(suggestion.term, suggestion.documentFrequency);
|
|
4947
|
-
}
|
|
4948
|
-
}
|
|
4949
|
-
const entries = Array.from(dfByToken.entries());
|
|
4950
|
-
entries.sort((a, b) => b[1] - a[1] || compareCodePoints(a[0], b[0]));
|
|
4951
|
-
if (entries.length > maxExpansions) entries.length = maxExpansions;
|
|
4952
|
-
return entries.map((e) => e[0]);
|
|
4967
|
+
state.partialWords.forgetSpellings();
|
|
4953
4968
|
}
|
|
4954
4969
|
|
|
4955
4970
|
// src/core/partition/index.ts
|
|
@@ -4966,9 +4981,10 @@ function partitionStateOf(partition) {
|
|
|
4966
4981
|
}
|
|
4967
4982
|
function createPartitionIndex(partitionId, trackPositions = true) {
|
|
4968
4983
|
const fieldNameTable = { names: [], indexMap: /* @__PURE__ */ new Map() };
|
|
4984
|
+
const docStore = createDocumentStore();
|
|
4969
4985
|
const state = {
|
|
4970
4986
|
invertedIdx: createInvertedIndex(fieldNameTable),
|
|
4971
|
-
docStore
|
|
4987
|
+
docStore,
|
|
4972
4988
|
stats: createPartitionStats(),
|
|
4973
4989
|
surfaceRegistry: createSurfaceRegistry(),
|
|
4974
4990
|
numericIndexes: /* @__PURE__ */ new Map(),
|
|
@@ -4978,6 +4994,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
4978
4994
|
patternIndexes: /* @__PURE__ */ new Map(),
|
|
4979
4995
|
sizeIndexes: /* @__PURE__ */ new Map(),
|
|
4980
4996
|
fieldNameTable,
|
|
4997
|
+
partialWords: createGrowingPartialWords({ docStore }),
|
|
4981
4998
|
flatSchemaCache: null,
|
|
4982
4999
|
lastSchemaRef: null,
|
|
4983
5000
|
trackPositions,
|
|
@@ -4998,6 +5015,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
4998
5015
|
state.geoIndexes.clear();
|
|
4999
5016
|
state.patternIndexes.clear();
|
|
5000
5017
|
state.sizeIndexes.clear();
|
|
5018
|
+
state.partialWords.clear();
|
|
5001
5019
|
state.stats.deserialize({ totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} });
|
|
5002
5020
|
state.flatSchemaCache = null;
|
|
5003
5021
|
state.lastSchemaRef = null;
|
|
@@ -5026,7 +5044,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5026
5044
|
}
|
|
5027
5045
|
}
|
|
5028
5046
|
const internalId = state.docStore.ensureInternalId(docId);
|
|
5029
|
-
const { fieldLengths, tokensByField } = indexDocument(
|
|
5047
|
+
const { fieldLengths, tokensByField, partialSpellings } = indexDocument(
|
|
5030
5048
|
state,
|
|
5031
5049
|
docId,
|
|
5032
5050
|
document,
|
|
@@ -5040,6 +5058,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5040
5058
|
state.docStore.store(docId, document, fieldLengths);
|
|
5041
5059
|
}
|
|
5042
5060
|
state.stats.addDocument(fieldLengths, tokensByField);
|
|
5061
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, partialSpellings, 1);
|
|
5043
5062
|
recordSortValues(state, internalId, document);
|
|
5044
5063
|
},
|
|
5045
5064
|
remove(docId, schema, language, options) {
|
|
@@ -5050,7 +5069,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5050
5069
|
partitionId
|
|
5051
5070
|
});
|
|
5052
5071
|
}
|
|
5053
|
-
const { fieldLengths, tokensByField } = removeFromIndexes(
|
|
5072
|
+
const { fieldLengths, tokensByField, partialSpellings } = removeFromIndexes(
|
|
5054
5073
|
state,
|
|
5055
5074
|
docId,
|
|
5056
5075
|
stored,
|
|
@@ -5061,6 +5080,9 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5061
5080
|
const internalId = state.docStore.getInternalId(docId);
|
|
5062
5081
|
state.docStore.remove(docId);
|
|
5063
5082
|
state.stats.removeDocument(fieldLengths, tokensByField);
|
|
5083
|
+
if (internalId !== void 0) {
|
|
5084
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, partialSpellings, -1);
|
|
5085
|
+
}
|
|
5064
5086
|
forgetSortValues(state, internalId);
|
|
5065
5087
|
},
|
|
5066
5088
|
beginBatch() {
|
|
@@ -5096,29 +5118,19 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5096
5118
|
const fields = schemaFieldsOf(schema);
|
|
5097
5119
|
const needsTextReindex = textFieldsChanged(stored.fields, document, fields);
|
|
5098
5120
|
if (needsTextReindex) {
|
|
5099
|
-
const
|
|
5100
|
-
|
|
5101
|
-
docId,
|
|
5102
|
-
stored,
|
|
5103
|
-
fields,
|
|
5104
|
-
language,
|
|
5105
|
-
options
|
|
5106
|
-
);
|
|
5107
|
-
state.stats.removeDocument(oldFieldLengths, oldTokens);
|
|
5121
|
+
const previous = removeFromIndexes(state, docId, stored, fields, language, options);
|
|
5122
|
+
state.stats.removeDocument(previous.fieldLengths, previous.tokensByField);
|
|
5108
5123
|
const previousInternalId = state.docStore.getInternalId(docId);
|
|
5124
|
+
if (previousInternalId !== void 0) {
|
|
5125
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), previousInternalId, previous.partialSpellings, -1);
|
|
5126
|
+
}
|
|
5109
5127
|
state.docStore.remove(docId);
|
|
5110
5128
|
forgetSortValues(state, previousInternalId);
|
|
5111
5129
|
const internalId = state.docStore.ensureInternalId(docId);
|
|
5112
|
-
const
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
5116
|
-
fields,
|
|
5117
|
-
language,
|
|
5118
|
-
options
|
|
5119
|
-
);
|
|
5120
|
-
state.docStore.store(docId, document, newFieldLengths);
|
|
5121
|
-
state.stats.addDocument(newFieldLengths, newTokens);
|
|
5130
|
+
const next = indexDocument(state, docId, document, fields, language, options);
|
|
5131
|
+
state.docStore.store(docId, document, next.fieldLengths);
|
|
5132
|
+
state.stats.addDocument(next.fieldLengths, next.tokensByField);
|
|
5133
|
+
state.partialWords.afterDocument(partialFieldsOf(schema), internalId, next.partialSpellings, 1);
|
|
5122
5134
|
recordSortValues(state, internalId, document);
|
|
5123
5135
|
} else {
|
|
5124
5136
|
updateFieldIndexOnly(state, docId, stored.fields, document, fields);
|
|
@@ -5189,7 +5201,13 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
5189
5201
|
return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
|
|
5190
5202
|
},
|
|
5191
5203
|
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
5192
|
-
return expandTermPrefix(state, surfacePrefix, stemmedToken, maxExpansions);
|
|
5204
|
+
return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
|
|
5205
|
+
},
|
|
5206
|
+
expandPartialWords(analysis, requests) {
|
|
5207
|
+
return expandPartialWords([state], analysis, requests);
|
|
5208
|
+
},
|
|
5209
|
+
preparePartialWords(analysis) {
|
|
5210
|
+
expandPartialWords([state], analysis, []);
|
|
5193
5211
|
},
|
|
5194
5212
|
serialize(indexName, totalPartitions, language, schema) {
|
|
5195
5213
|
return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
|
|
@@ -5223,7 +5241,7 @@ function frozenSegmentBytes(segment) {
|
|
|
5223
5241
|
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
5224
5242
|
}
|
|
5225
5243
|
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
5226
|
-
return bytes;
|
|
5244
|
+
return bytes + segment.partialWords.bytes();
|
|
5227
5245
|
}
|
|
5228
5246
|
|
|
5229
5247
|
// src/core/partition/composite/compaction.ts
|
|
@@ -5358,20 +5376,19 @@ function subMatchesView(matches, layout, subIndex) {
|
|
|
5358
5376
|
|
|
5359
5377
|
// src/core/partition/composite/stats.ts
|
|
5360
5378
|
function collectCandidateTerms(subs, params) {
|
|
5361
|
-
const { queryTokens,
|
|
5379
|
+
const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
|
|
5362
5380
|
const terms = /* @__PURE__ */ new Set();
|
|
5363
5381
|
for (const qt of queryTokens) {
|
|
5364
|
-
|
|
5365
|
-
|
|
5366
|
-
for (const
|
|
5367
|
-
continue;
|
|
5382
|
+
const group = termGroupOf(termGroups, qt.token);
|
|
5383
|
+
if (group !== void 0) {
|
|
5384
|
+
for (const member of group.members) terms.add(member.token);
|
|
5368
5385
|
}
|
|
5369
|
-
if (exact || tolerance === 0) {
|
|
5386
|
+
if (group === void 0 && (exact || tolerance === 0)) {
|
|
5370
5387
|
terms.add(qt.token);
|
|
5371
5388
|
continue;
|
|
5372
5389
|
}
|
|
5373
5390
|
for (const sub of subs) {
|
|
5374
|
-
for (const match of sub.invertedIdx
|
|
5391
|
+
for (const match of typoMatchesFor(sub.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
|
|
5375
5392
|
terms.add(match.token);
|
|
5376
5393
|
}
|
|
5377
5394
|
}
|
|
@@ -5381,11 +5398,7 @@ function collectCandidateTerms(subs, params) {
|
|
|
5381
5398
|
function resolveQueryTermFrequencies(subs, params) {
|
|
5382
5399
|
const frequencies = /* @__PURE__ */ Object.create(null);
|
|
5383
5400
|
for (const term of collectCandidateTerms(subs, params)) {
|
|
5384
|
-
|
|
5385
|
-
for (const sub of subs) {
|
|
5386
|
-
const list = sub.invertedIdx.lookup(term);
|
|
5387
|
-
if (list !== void 0) total += list.docIdSet.size;
|
|
5388
|
-
}
|
|
5401
|
+
const total = partitionDocumentFrequency(subs, term);
|
|
5389
5402
|
if (total > 0) frequencies[term] = total;
|
|
5390
5403
|
}
|
|
5391
5404
|
return frequencies;
|
|
@@ -5571,9 +5584,10 @@ function compositeSortValues(subs, docId, fields, fieldTypes, modes) {
|
|
|
5571
5584
|
|
|
5572
5585
|
// src/core/partition/composite/suggest.ts
|
|
5573
5586
|
function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
5587
|
+
if (limit <= 0) return [];
|
|
5574
5588
|
const merged = /* @__PURE__ */ new Map();
|
|
5575
5589
|
for (const sub of subs) {
|
|
5576
|
-
for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix,
|
|
5590
|
+
for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix, Number.MAX_SAFE_INTEGER)) {
|
|
5577
5591
|
let entry = merged.get(suggestion.token);
|
|
5578
5592
|
if (entry === void 0) {
|
|
5579
5593
|
entry = { documentFrequency: 0, surfaceOccurrences: /* @__PURE__ */ new Map() };
|
|
@@ -5598,19 +5612,6 @@ function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
|
|
|
5598
5612
|
if (suggestions.length > limit) suggestions.length = limit;
|
|
5599
5613
|
return suggestions;
|
|
5600
5614
|
}
|
|
5601
|
-
function compositeExpandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansions) {
|
|
5602
|
-
const expansions = [];
|
|
5603
|
-
const seen = /* @__PURE__ */ new Set();
|
|
5604
|
-
for (const sub of subs) {
|
|
5605
|
-
for (const term of expandTermPrefix(sub, surfacePrefix, stemmedToken, maxExpansions)) {
|
|
5606
|
-
if (seen.has(term)) continue;
|
|
5607
|
-
seen.add(term);
|
|
5608
|
-
expansions.push(term);
|
|
5609
|
-
if (expansions.length === maxExpansions) return expansions;
|
|
5610
|
-
}
|
|
5611
|
-
}
|
|
5612
|
-
return expansions;
|
|
5613
|
-
}
|
|
5614
5615
|
|
|
5615
5616
|
// src/core/partition/composite/index.ts
|
|
5616
5617
|
function isCompositePartition(partition) {
|
|
@@ -5814,7 +5815,13 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5814
5815
|
return compositeSuggestTerms(subs(), surfacePrefix, stemmedPrefix, limit);
|
|
5815
5816
|
},
|
|
5816
5817
|
expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
|
|
5817
|
-
return
|
|
5818
|
+
return expandTermPrefix(subs(), surfacePrefix, stemmedToken, maxExpansions);
|
|
5819
|
+
},
|
|
5820
|
+
expandPartialWords(analysis, requests) {
|
|
5821
|
+
return expandPartialWords(subs(), analysis, requests);
|
|
5822
|
+
},
|
|
5823
|
+
preparePartialWords(analysis) {
|
|
5824
|
+
expandPartialWords(subs(), analysis, []);
|
|
5818
5825
|
},
|
|
5819
5826
|
get(docId, projection) {
|
|
5820
5827
|
const owner = frozenOwner(docId);
|
|
@@ -5875,6 +5882,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
5875
5882
|
}
|
|
5876
5883
|
|
|
5877
5884
|
// src/partitioning/insert-options.ts
|
|
5885
|
+
function resolvePartialWordAnalysis(config, language, analysis) {
|
|
5886
|
+
const fields = partialFieldsOf(config.schema);
|
|
5887
|
+
if (fields.length === 0) return null;
|
|
5888
|
+
return { fields, language, options: resolvePartitionInsertOptions(config, analysis) };
|
|
5889
|
+
}
|
|
5878
5890
|
function resolvePartitionInsertOptions(config, analysis, options) {
|
|
5879
5891
|
const applyStrict = config.strict === true;
|
|
5880
5892
|
const applyAnalyzer = analysis.stopWords !== void 0 || analysis.customTokenizer !== void 0;
|
|
@@ -5933,8 +5945,10 @@ function createCapacityChecks(source) {
|
|
|
5933
5945
|
// src/partitioning/manager/index.ts
|
|
5934
5946
|
function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
|
|
5935
5947
|
requirePatternSearchFor(config.schema);
|
|
5948
|
+
requirePartialWordSearchFor(config.schema);
|
|
5936
5949
|
const count = initialPartitionCount ?? 1;
|
|
5937
5950
|
const analysis = resolveIndexAnalysis(config);
|
|
5951
|
+
const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
|
|
5938
5952
|
const trackPositions = config.trackPositions ?? true;
|
|
5939
5953
|
const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
|
|
5940
5954
|
let partitions = [];
|
|
@@ -5954,6 +5968,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
5954
5968
|
function resolveInsertOptions(options) {
|
|
5955
5969
|
return resolvePartitionInsertOptions(config, analysis, options);
|
|
5956
5970
|
}
|
|
5971
|
+
function preparePartialWords(partitionId) {
|
|
5972
|
+
if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
|
|
5973
|
+
}
|
|
5957
5974
|
function locateDocument(docId) {
|
|
5958
5975
|
return docPartitionMap.get(docId);
|
|
5959
5976
|
}
|
|
@@ -6009,6 +6026,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
6009
6026
|
get analysis() {
|
|
6010
6027
|
return analysis;
|
|
6011
6028
|
},
|
|
6029
|
+
get partialWordAnalysis() {
|
|
6030
|
+
return partialWordAnalysis;
|
|
6031
|
+
},
|
|
6012
6032
|
getPartition(partitionId) {
|
|
6013
6033
|
validatePartitionId(partitionId);
|
|
6014
6034
|
return partitions[partitionId];
|
|
@@ -6085,6 +6105,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
6085
6105
|
rebuildTextIndex(partitionId) {
|
|
6086
6106
|
validatePartitionId(partitionId);
|
|
6087
6107
|
partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
|
|
6108
|
+
preparePartialWords(partitionId);
|
|
6088
6109
|
},
|
|
6089
6110
|
beginBatchRemove() {
|
|
6090
6111
|
for (let i = 0; i < partitions.length; i++) {
|
|
@@ -6186,6 +6207,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
|
|
|
6186
6207
|
deserializePartition(partitionId, data) {
|
|
6187
6208
|
validatePartitionId(partitionId);
|
|
6188
6209
|
partitions[partitionId].deserialize(data, config.schema);
|
|
6210
|
+
preparePartialWords(partitionId);
|
|
6189
6211
|
rebuildDocPartitionMapForPartition(partitionId);
|
|
6190
6212
|
},
|
|
6191
6213
|
getAggregateStats() {
|
|
@@ -8681,6 +8703,15 @@ function decodeStoredDocument(bytes, discardedFields) {
|
|
|
8681
8703
|
}
|
|
8682
8704
|
return document;
|
|
8683
8705
|
}
|
|
8706
|
+
function* storedDocumentsOf(data) {
|
|
8707
|
+
const count = data.offsets.length - 1;
|
|
8708
|
+
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
8709
|
+
yield decodeStoredDocument(
|
|
8710
|
+
data.blob.subarray(data.offsets[ordinal], data.offsets[ordinal + 1]),
|
|
8711
|
+
data.discardedFields
|
|
8712
|
+
);
|
|
8713
|
+
}
|
|
8714
|
+
}
|
|
8684
8715
|
function decodeEveryStoredDocument(data) {
|
|
8685
8716
|
const count = data.offsets.length - 1;
|
|
8686
8717
|
for (let ordinal = 0; ordinal < count; ordinal++) {
|
|
@@ -8724,6 +8755,9 @@ function wrapEncodedDocumentTable(data) {
|
|
|
8724
8755
|
decodedBytes += documentValueBytes(document);
|
|
8725
8756
|
}
|
|
8726
8757
|
return document;
|
|
8758
|
+
},
|
|
8759
|
+
readAt(ordinal) {
|
|
8760
|
+
return decoded[ordinal] ?? decodeStoredDocument(blob.subarray(offsets[ordinal], offsets[ordinal + 1]), data.discardedFields);
|
|
8727
8761
|
}
|
|
8728
8762
|
};
|
|
8729
8763
|
}
|
|
@@ -8737,75 +8771,15 @@ function wrapDocumentArray(documents) {
|
|
|
8737
8771
|
byteLength: residentBytes,
|
|
8738
8772
|
docAt(ordinal) {
|
|
8739
8773
|
return documents[ordinal];
|
|
8774
|
+
},
|
|
8775
|
+
readAt(ordinal) {
|
|
8776
|
+
return documents[ordinal];
|
|
8740
8777
|
}
|
|
8741
8778
|
};
|
|
8742
8779
|
}
|
|
8743
8780
|
|
|
8744
|
-
// src/core/partition/frozen/string-blob.ts
|
|
8745
|
-
var SURROGATE = /[\ud800-\udfff]/;
|
|
8746
|
-
var encoder = new TextEncoder();
|
|
8747
|
-
function encodeOneByOne(strings) {
|
|
8748
|
-
const encoded = new Array(strings.length);
|
|
8749
|
-
let blobLength = 0;
|
|
8750
|
-
for (let i = 0; i < strings.length; i++) {
|
|
8751
|
-
encoded[i] = encoder.encode(strings[i]);
|
|
8752
|
-
blobLength += encoded[i].length;
|
|
8753
|
-
}
|
|
8754
|
-
const blob = new Uint8Array(blobLength);
|
|
8755
|
-
const offsets = new Uint32Array(strings.length + 1);
|
|
8756
|
-
let cursor = 0;
|
|
8757
|
-
for (let i = 0; i < strings.length; i++) {
|
|
8758
|
-
blob.set(encoded[i], cursor);
|
|
8759
|
-
cursor += encoded[i].length;
|
|
8760
|
-
offsets[i + 1] = cursor;
|
|
8761
|
-
}
|
|
8762
|
-
return { blob, offsets };
|
|
8763
|
-
}
|
|
8764
|
-
function basicPlaneUtf8Bytes(text) {
|
|
8765
|
-
let bytes = text.length;
|
|
8766
|
-
for (let i = 0; i < text.length; i++) {
|
|
8767
|
-
const unit = text.charCodeAt(i);
|
|
8768
|
-
if (unit >= 128) bytes += unit < 2048 ? 1 : 2;
|
|
8769
|
-
}
|
|
8770
|
-
return bytes;
|
|
8771
|
-
}
|
|
8772
|
-
function encodeStringBlob(strings) {
|
|
8773
|
-
const joined = strings.join("");
|
|
8774
|
-
if (SURROGATE.test(joined)) return encodeOneByOne(strings);
|
|
8775
|
-
const blob = encoder.encode(joined);
|
|
8776
|
-
const offsets = new Uint32Array(strings.length + 1);
|
|
8777
|
-
let cursor = 0;
|
|
8778
|
-
for (let i = 0; i < strings.length; i++) {
|
|
8779
|
-
cursor += basicPlaneUtf8Bytes(strings[i]);
|
|
8780
|
-
offsets[i + 1] = cursor;
|
|
8781
|
-
}
|
|
8782
|
-
return cursor === blob.length ? { blob, offsets } : encodeOneByOne(strings);
|
|
8783
|
-
}
|
|
8784
|
-
function orderByComparator(strings) {
|
|
8785
|
-
const order = new Array(strings.length);
|
|
8786
|
-
for (let i = 0; i < strings.length; i++) order[i] = i;
|
|
8787
|
-
return order.sort((a, b) => compareCodePoints(strings[a], strings[b]));
|
|
8788
|
-
}
|
|
8789
|
-
function alreadyInCodePointOrder(strings) {
|
|
8790
|
-
for (let i = 1; i < strings.length; i++) {
|
|
8791
|
-
if (compareCodePoints(strings[i - 1], strings[i]) > 0) return false;
|
|
8792
|
-
}
|
|
8793
|
-
return true;
|
|
8794
|
-
}
|
|
8795
|
-
function codePointOrder(strings) {
|
|
8796
|
-
if (alreadyInCodePointOrder(strings)) return strings.map((_, i) => i);
|
|
8797
|
-
if (SURROGATE.test(strings.join(""))) return orderByComparator(strings);
|
|
8798
|
-
const positions = /* @__PURE__ */ new Map();
|
|
8799
|
-
for (let i = 0; i < strings.length; i++) positions.set(strings[i], i);
|
|
8800
|
-
if (positions.size !== strings.length) return orderByComparator(strings);
|
|
8801
|
-
const order = new Array(strings.length);
|
|
8802
|
-
const sorted = [...strings].sort();
|
|
8803
|
-
for (let i = 0; i < sorted.length; i++) order[i] = positions.get(sorted[i]) ?? 0;
|
|
8804
|
-
return order;
|
|
8805
|
-
}
|
|
8806
|
-
|
|
8807
8781
|
// src/core/partition/frozen/external-ids.ts
|
|
8808
|
-
var
|
|
8782
|
+
var encoder = new TextEncoder();
|
|
8809
8783
|
var decoder = new TextDecoder();
|
|
8810
8784
|
function encodeExternalIdTableData(docIds) {
|
|
8811
8785
|
const { blob, offsets } = encodeStringBlob(docIds);
|
|
@@ -8837,7 +8811,7 @@ function wrapExternalIdTable(data) {
|
|
|
8837
8811
|
bytes: blob.byteLength + offsets.byteLength + sortedOrdinals.byteLength,
|
|
8838
8812
|
idAt,
|
|
8839
8813
|
ordinalOf(docId) {
|
|
8840
|
-
const query =
|
|
8814
|
+
const query = encoder.encode(docId);
|
|
8841
8815
|
let lo = 0;
|
|
8842
8816
|
let hi = count;
|
|
8843
8817
|
while (lo < hi) {
|
|
@@ -9123,6 +9097,10 @@ function createFrozenInvertedReader(table, views) {
|
|
|
9123
9097
|
if (at < 0) return void 0;
|
|
9124
9098
|
return views.viewReadOnce(table.payloadSlot(at), table.documentFrequencyAt(at));
|
|
9125
9099
|
},
|
|
9100
|
+
documentFrequency(token) {
|
|
9101
|
+
const at = table.find(token);
|
|
9102
|
+
return at < 0 ? 0 : table.documentFrequencyAt(at);
|
|
9103
|
+
},
|
|
9126
9104
|
fuzzyLookup(token, tolerance, prefixLength) {
|
|
9127
9105
|
if (tolerance === 0) {
|
|
9128
9106
|
const at = table.find(token);
|
|
@@ -9277,7 +9255,7 @@ function createLazySurfaceReader(load) {
|
|
|
9277
9255
|
}
|
|
9278
9256
|
|
|
9279
9257
|
// src/core/partition/frozen/token-table.ts
|
|
9280
|
-
var
|
|
9258
|
+
var encoder2 = new TextEncoder();
|
|
9281
9259
|
var decoder3 = new TextDecoder();
|
|
9282
9260
|
function compareBytes2(blob, start, end, query) {
|
|
9283
9261
|
const length = end - start;
|
|
@@ -9341,7 +9319,7 @@ function wrapFrozenTokenTable(data) {
|
|
|
9341
9319
|
size,
|
|
9342
9320
|
bytes: blob.byteLength + offsets.byteLength + payloadSlots.byteLength + documentFrequencies.byteLength,
|
|
9343
9321
|
find(token) {
|
|
9344
|
-
const query =
|
|
9322
|
+
const query = encoder2.encode(token);
|
|
9345
9323
|
const at = lowerBound3(query);
|
|
9346
9324
|
if (at >= size) return -1;
|
|
9347
9325
|
return compareBytes2(blob, offsets[at], offsets[at + 1], query) === 0 ? at : -1;
|
|
@@ -9364,7 +9342,7 @@ function wrapFrozenTokenTable(data) {
|
|
|
9364
9342
|
return documentFrequencies[sortedIndex];
|
|
9365
9343
|
},
|
|
9366
9344
|
prefixRange(prefix) {
|
|
9367
|
-
const bytes =
|
|
9345
|
+
const bytes = encoder2.encode(prefix);
|
|
9368
9346
|
const start = lowerBound3(bytes);
|
|
9369
9347
|
return { start, end: prefixEnd(start, bytes) };
|
|
9370
9348
|
},
|
|
@@ -9382,8 +9360,8 @@ function wrapFrozenTokenTable(data) {
|
|
|
9382
9360
|
fromCodePoint = unit;
|
|
9383
9361
|
toCodePoint = nextCodePoint(unit);
|
|
9384
9362
|
}
|
|
9385
|
-
const from =
|
|
9386
|
-
const to =
|
|
9363
|
+
const from = encoder2.encode(String.fromCodePoint(fromCodePoint));
|
|
9364
|
+
const to = encoder2.encode(String.fromCodePoint(toCodePoint));
|
|
9387
9365
|
return { start: lowerBound3(from), end: lowerBound3(to) };
|
|
9388
9366
|
}
|
|
9389
9367
|
};
|
|
@@ -9445,6 +9423,35 @@ function sharedSurfaceTable(payload) {
|
|
|
9445
9423
|
if (table.counts.length === 0) return null;
|
|
9446
9424
|
return { blob: sharedUint8(table.blob), offsets: sharedUint32(table.offsets), counts: sharedUint32(table.counts) };
|
|
9447
9425
|
}
|
|
9426
|
+
function sharePartialWordTable(table) {
|
|
9427
|
+
if (typeof SharedArrayBuffer !== "function") return table;
|
|
9428
|
+
return {
|
|
9429
|
+
fields: [...table.fields],
|
|
9430
|
+
spellings: sharedUint8(table.spellings),
|
|
9431
|
+
spellingOffsets: sharedUint32(table.spellingOffsets),
|
|
9432
|
+
spellingTokens: sharedUint32(table.spellingTokens),
|
|
9433
|
+
spellingCounts: sharedUint32(table.spellingCounts),
|
|
9434
|
+
runs: sharedUint32(table.runs),
|
|
9435
|
+
runOffsets: sharedUint32(table.runOffsets),
|
|
9436
|
+
runEntries: sharedUint32(table.runEntries)
|
|
9437
|
+
};
|
|
9438
|
+
}
|
|
9439
|
+
function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
|
|
9440
|
+
let source = spellings;
|
|
9441
|
+
if (source === null && analysis !== null && analysis.fields.length > 0) {
|
|
9442
|
+
source = encodePartialSpellings(analysis.fields, spellingCountsOf(storedDocumentsOf(documentData), analysis));
|
|
9443
|
+
}
|
|
9444
|
+
if (source === null || source.fields.length === 0) return null;
|
|
9445
|
+
const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
|
|
9446
|
+
return table === null ? null : sharePartialWordTable(table);
|
|
9447
|
+
}
|
|
9448
|
+
function shareSegmentArrays(arrays, analysis, segmentId) {
|
|
9449
|
+
return {
|
|
9450
|
+
...arrays,
|
|
9451
|
+
segmentId: segmentId ?? generateId(),
|
|
9452
|
+
partialWords: sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis)
|
|
9453
|
+
};
|
|
9454
|
+
}
|
|
9448
9455
|
function segmentArraysOf(payload, documentData) {
|
|
9449
9456
|
const tokenTable = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
|
|
9450
9457
|
const idTable = encodeExternalIdTableData(payload.docIds);
|
|
@@ -9473,11 +9480,11 @@ function segmentArraysOf(payload, documentData) {
|
|
|
9473
9480
|
surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable
|
|
9474
9481
|
};
|
|
9475
9482
|
}
|
|
9476
|
-
function freezeSegmentShared(payload, documents, segmentId) {
|
|
9483
|
+
function freezeSegmentShared(payload, documents, segmentId, analysis = null) {
|
|
9477
9484
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
9478
|
-
return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId);
|
|
9485
|
+
return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId, analysis);
|
|
9479
9486
|
}
|
|
9480
|
-
function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
9487
|
+
function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null) {
|
|
9481
9488
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
9482
9489
|
const tokenData = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
|
|
9483
9490
|
const idData = encodeExternalIdTableData(payload.docIds);
|
|
@@ -9544,7 +9551,8 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
|
9544
9551
|
offsets: sharedUint32(entry.offsets),
|
|
9545
9552
|
postings: sharedUint32(entry.postings)
|
|
9546
9553
|
})),
|
|
9547
|
-
surfaceTable: sharedSurfaceTable(payload)
|
|
9554
|
+
surfaceTable: sharedSurfaceTable(payload),
|
|
9555
|
+
partialWords: sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis)
|
|
9548
9556
|
};
|
|
9549
9557
|
}
|
|
9550
9558
|
|
|
@@ -9580,7 +9588,12 @@ function buildGeoReaders(entries) {
|
|
|
9580
9588
|
}
|
|
9581
9589
|
return readers;
|
|
9582
9590
|
}
|
|
9583
|
-
function
|
|
9591
|
+
function* survivingDocuments(documentSource, tombstones) {
|
|
9592
|
+
for (let ordinal = 0; ordinal < documentSource.count; ordinal++) {
|
|
9593
|
+
if (!tombstones.has(ordinal)) yield documentSource.readAt(ordinal);
|
|
9594
|
+
}
|
|
9595
|
+
}
|
|
9596
|
+
function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
|
|
9584
9597
|
const tombstones = createFrozenTombstones();
|
|
9585
9598
|
const postingViews = createFrozenPostingViews(source, tombstones);
|
|
9586
9599
|
const lengthColumns = /* @__PURE__ */ new Map();
|
|
@@ -9588,6 +9601,10 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
9588
9601
|
lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
|
|
9589
9602
|
}
|
|
9590
9603
|
const docStore = createFrozenDocTable(source.idTable, lengthColumns, documentSource, tombstones);
|
|
9604
|
+
const partialWords = createFrozenPartialWords(partialWordTable, source.tokenTable, {
|
|
9605
|
+
documentAt: (ordinal) => documentSource.readAt(ordinal),
|
|
9606
|
+
survivors: () => survivingDocuments(documentSource, tombstones)
|
|
9607
|
+
});
|
|
9591
9608
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
9592
9609
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
9593
9610
|
const sizeIndexes = /* @__PURE__ */ new Map();
|
|
@@ -9620,6 +9637,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
9620
9637
|
patternIndexes,
|
|
9621
9638
|
sizeIndexes,
|
|
9622
9639
|
fieldNameTable,
|
|
9640
|
+
partialWords,
|
|
9623
9641
|
trackPositions: source.positionOffsets !== null,
|
|
9624
9642
|
flatSchemaCache: null,
|
|
9625
9643
|
lastSchemaRef: null,
|
|
@@ -9639,6 +9657,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
9639
9657
|
if (ordinal === void 0) return false;
|
|
9640
9658
|
const added = tombstones.add(ordinal);
|
|
9641
9659
|
if (added) {
|
|
9660
|
+
partialWords.afterRemoval(ordinal);
|
|
9642
9661
|
docStore.releaseSortedDocIds();
|
|
9643
9662
|
segment.sortColumns = null;
|
|
9644
9663
|
segment.scoreBuffer = null;
|
|
@@ -9654,18 +9673,20 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
9654
9673
|
return segment;
|
|
9655
9674
|
}
|
|
9656
9675
|
function createFrozenSegment(payload, documents, segmentId) {
|
|
9676
|
+
const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
|
|
9657
9677
|
return assembleFrozenSegment(
|
|
9658
9678
|
segmentId ?? generateId(),
|
|
9659
9679
|
{
|
|
9660
9680
|
...payload,
|
|
9661
|
-
tokenTable
|
|
9681
|
+
tokenTable,
|
|
9662
9682
|
idTable: buildExternalIdTable(payload.docIds),
|
|
9663
9683
|
documentTable: null,
|
|
9664
9684
|
readSurfaceForms: () => payload.surfaceForms,
|
|
9665
9685
|
docFrequencies: () => payload.docFrequencies
|
|
9666
9686
|
},
|
|
9667
9687
|
wrapDocumentArray(documents),
|
|
9668
|
-
null
|
|
9688
|
+
null,
|
|
9689
|
+
payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, tokenTable)
|
|
9669
9690
|
);
|
|
9670
9691
|
}
|
|
9671
9692
|
function createSharedFrozenSegment(snapshot) {
|
|
@@ -9687,7 +9708,8 @@ function createSharedFrozenSegment(snapshot) {
|
|
|
9687
9708
|
}
|
|
9688
9709
|
},
|
|
9689
9710
|
wrapEncodedDocumentTable(snapshot.documentTable),
|
|
9690
|
-
snapshot
|
|
9711
|
+
snapshot,
|
|
9712
|
+
snapshot.partialWords
|
|
9691
9713
|
);
|
|
9692
9714
|
}
|
|
9693
9715
|
var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
@@ -10269,10 +10291,10 @@ function mergeDocuments2(inputs, documentCount) {
|
|
|
10269
10291
|
}
|
|
10270
10292
|
return { blob, offsets };
|
|
10271
10293
|
}
|
|
10272
|
-
function mergeFrozenSegments(segments) {
|
|
10294
|
+
function mergeFrozenSegments(segments, analysis) {
|
|
10273
10295
|
if (typeof SharedArrayBuffer !== "function") return null;
|
|
10274
10296
|
const merged = mergeSegmentContents(segments.map((segment) => ({ segment })));
|
|
10275
|
-
return freezeEncodedSegmentShared(merged.payload, merged.documents);
|
|
10297
|
+
return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis);
|
|
10276
10298
|
}
|
|
10277
10299
|
function mergeSegmentContents(selections) {
|
|
10278
10300
|
const docIds = [];
|
|
@@ -10301,6 +10323,7 @@ function mergeSegmentContents(selections) {
|
|
|
10301
10323
|
totalFieldLengths: lengths.totalFieldLengths,
|
|
10302
10324
|
docFrequencies: postings.docFrequencies,
|
|
10303
10325
|
surfaceForms: mergeSurfaceForms2(inputs),
|
|
10326
|
+
partialSpellings: null,
|
|
10304
10327
|
...mergeFieldIndexes2(inputs)
|
|
10305
10328
|
};
|
|
10306
10329
|
return { payload, documents: mergeDocuments2(inputs, documentCount) };
|
|
@@ -10571,14 +10594,18 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
10571
10594
|
patternIndexes: /* @__PURE__ */ new Map(),
|
|
10572
10595
|
sizeIndexes: /* @__PURE__ */ new Map()
|
|
10573
10596
|
};
|
|
10597
|
+
const partialFields = partialFieldsOf(schema);
|
|
10598
|
+
const spellings = /* @__PURE__ */ new Map();
|
|
10599
|
+
let fieldSpellings = null;
|
|
10574
10600
|
let positionOffset = 0;
|
|
10575
10601
|
const addToken = (token, position, surface) => {
|
|
10576
10602
|
postings.addToken(token, positionOffset + position);
|
|
10577
10603
|
surfaces.count(surface, token);
|
|
10604
|
+
if (fieldSpellings !== null) countSpelling(fieldSpellings, token, surface ?? token, 1);
|
|
10578
10605
|
};
|
|
10579
10606
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
10580
10607
|
const document = documents[ordinal].document;
|
|
10581
|
-
for (const { path: fieldPath, base: fieldType, pattern } of fields) {
|
|
10608
|
+
for (const { path: fieldPath, base: fieldType, pattern, partial } of fields) {
|
|
10582
10609
|
const value = getNestedValue(document, fieldPath);
|
|
10583
10610
|
if (value === void 0 || value === null) continue;
|
|
10584
10611
|
recordValueSize(filters.sizeIndexes, ordinal, fieldPath, fieldType, value);
|
|
@@ -10587,6 +10614,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
10587
10614
|
indexFilterValue(filters, ordinal, fieldPath, fieldType, value);
|
|
10588
10615
|
continue;
|
|
10589
10616
|
}
|
|
10617
|
+
fieldSpellings = partial ? spellings : null;
|
|
10590
10618
|
postings.beginField();
|
|
10591
10619
|
positionOffset = 0;
|
|
10592
10620
|
for (const text of fieldType === "string" ? [value] : value) {
|
|
@@ -10607,6 +10635,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
10607
10635
|
fieldLengthColumns: lengths.columns,
|
|
10608
10636
|
totalFieldLengths: lengths.totalsByName(),
|
|
10609
10637
|
surfaceForms: surfaces.encode(),
|
|
10638
|
+
partialSpellings: partialFields.length === 0 ? null : encodePartialSpellings(partialFields, spellings),
|
|
10610
10639
|
...encodeFieldIndexes(filters, identity)
|
|
10611
10640
|
};
|
|
10612
10641
|
}
|
|
@@ -10673,7 +10702,7 @@ function runSegmentAction(entry, action) {
|
|
|
10673
10702
|
}
|
|
10674
10703
|
case "compactSegments": {
|
|
10675
10704
|
const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
|
|
10676
|
-
return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds));
|
|
10705
|
+
return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds), entry.manager.partialWordAnalysis);
|
|
10677
10706
|
}
|
|
10678
10707
|
case "swapSegments": {
|
|
10679
10708
|
requireComposite(entry, action.indexName, action.partitionId, "swap");
|
|
@@ -11150,4 +11179,4 @@ function createDirectExecutor(options) {
|
|
|
11150
11179
|
};
|
|
11151
11180
|
}
|
|
11152
11181
|
|
|
11153
|
-
export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerPatternSearch, registerStopWords, registerTokenizer, requirePatternSearchFor, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
|
|
11182
|
+
export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerTokenizer, requirePartialWordSearchFor, requirePatternSearchFor, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
|