@delali/narsil 0.3.1-next.73 → 0.3.1-next.75

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/dist/{capabilities-CYeAWYxQ.d.ts → capabilities-BHNksicw.d.ts} +16 -2
  2. package/dist/{chunk-PU43WDY4.mjs → chunk-2E35UM3G.mjs} +1 -13
  3. package/dist/{chunk-AK3BLGCI.mjs → chunk-356UOTK2.mjs} +15 -48
  4. package/dist/chunk-7ODNRREZ.mjs +2050 -0
  5. package/dist/chunk-AIUOAN7Y.mjs +658 -0
  6. package/dist/{chunk-WVDFG45M.mjs → chunk-AMGDG4ZJ.mjs} +3 -2
  7. package/dist/{chunk-AFKH4ZK3.mjs → chunk-BT62MUM5.mjs} +1 -1
  8. package/dist/{chunk-LPBWCHBE.mjs → chunk-BT6BI4GV.mjs} +42 -4
  9. package/dist/{chunk-LJYTW7GT.mjs → chunk-IBR6634K.mjs} +1 -1
  10. package/dist/chunk-KEHK2O6P.mjs +13 -0
  11. package/dist/{chunk-CVFAFZSN.mjs → chunk-LOARLB4F.mjs} +46 -34
  12. package/dist/{chunk-3YKWSO54.mjs → chunk-NAMGDK45.mjs} +228 -649
  13. package/dist/{chunk-2WLD2NFW.mjs → chunk-PXDY5VRT.mjs} +3 -3
  14. package/dist/chunk-UOJIV2NG.mjs +593 -0
  15. package/dist/chunk-V3B24XR7.mjs +54 -0
  16. package/dist/{chunk-ZULSEZCE.mjs → chunk-WL2GDSCV.mjs} +100 -19
  17. package/dist/{chunk-THMSFOJO.mjs → chunk-YOVS4EV5.mjs} +582 -598
  18. package/dist/{client-D698E5ge.d.ts → client-D4sXFS9f.d.ts} +3 -3
  19. package/dist/client.d.ts +8 -8
  20. package/dist/{config-Bv8etwUZ.d.ts → config-B-GRonqw.d.ts} +2 -2
  21. package/dist/{config-QXU3HMpq.d.ts → config-C1kbq1X2.d.ts} +2 -2
  22. package/dist/direct-executor-AVNWB73C.mjs +16 -0
  23. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  24. package/dist/distribution/coordinator/etcd.mjs +1 -1
  25. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  26. package/dist/distribution/coordinator/in-memory.mjs +1 -1
  27. package/dist/distribution.d.ts +8 -8
  28. package/dist/distribution.mjs +23 -13
  29. package/dist/{engine-D_OzKZ-6.d.ts → engine-DcGSd5vh.d.ts} +2 -2
  30. package/dist/index.browser.mjs +1272 -834
  31. package/dist/index.d.ts +27 -8
  32. package/dist/index.mjs +14 -11
  33. package/dist/invalidation/filesystem.mjs +1 -1
  34. package/dist/{memory-B4jMmygw.d.ts → memory-M9kPmYP8.d.ts} +1257 -1178
  35. package/dist/partial-words-DTIHO8G4.d.ts +19 -0
  36. package/dist/partial-words.d.ts +22 -0
  37. package/dist/partial-words.mjs +11 -0
  38. package/dist/pattern.mjs +2 -2
  39. package/dist/persistence/durability/checkpoint-worker.d.ts +33 -13
  40. package/dist/persistence/durability/checkpoint-worker.mjs +14 -11
  41. package/dist/react.d.ts +7 -7
  42. package/dist/{schema-BBOCdhND.d.ts → schema-DgyDijhf.d.ts} +6 -2
  43. package/dist/server.d.ts +10 -10
  44. package/dist/server.mjs +36 -11
  45. package/dist/{tasks-DpgCpW1x.d.ts → tasks-A8se0J98.d.ts} +1 -1
  46. package/dist/{thread-3LNVA43V.mjs → thread-BSIKT6ED.mjs} +16 -9
  47. package/dist/{types-BiUmaiWT.d.ts → types-ChrVy4lH.d.ts} +1 -1
  48. package/dist/vector/search-worker.d.ts +1 -1
  49. package/dist/vector/search-worker.mjs +4 -3
  50. package/dist/workers/entry.mjs +3 -3
  51. package/package.json +16 -7
  52. package/dist/chunk-XW25X6AU.mjs +0 -614
  53. package/dist/chunk-ZKRV5M2P.mjs +0 -75
  54. package/dist/direct-executor-5SRDFZAS.mjs +0 -13
@@ -1,11 +1,14 @@
1
+ import { requirePatternSearchFor, patternIndexOf, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, computeFacets, cloneProjected, projectionKeepsField, setNestedValue, fnv1a, requirePatternSearch, PATTERN_HOST, everyValueFacetConfig, mergeFacets, undeclaredFilterField, requireValidOperands, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, coveringRanges } from './chunk-NAMGDK45.mjs';
1
2
  import { getLanguage, detectRuntime } from './chunk-4MQ5F4IL.mjs';
2
- import { tokenizeOptions, getNestedValue, visitTokens, joinPatternWorkMeter, countsWithoutScores, fanOutMatchCount, fanOutQuery, sanitizeGlobalStats, computeFacets, cloneProjected, textFieldsChanged, projectionKeepsField, setNestedValue, fnv1a, fuzzyPrefixOf, boundedLevenshtein, getFlatSchema, tokenize, getOrCreateFieldNameIndex, tokenizeIterator, everyValueFacetConfig, mergeFacets, getFieldValueByInternalId, undeclaredFilterField, requireValidOperands, kWayMerge, polygonContainment, polygonBoxes, radiusBoxes, locationCode, coveringRanges, GEO_UNIT_METRES } from './chunk-3YKWSO54.mjs';
3
- import { schemaFieldsOf, isWordIndexedBase, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, MAX_DOC_ID_LENGTH, parsedTypeOf, isListBase, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-AK3BLGCI.mjs';
4
- import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, topLayerInParts, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-WVDFG45M.mjs';
5
- import { MAX_M, noteFallback, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-PU43WDY4.mjs';
6
- import { GrowableUint32, patternSearchModule } from './chunk-XW25X6AU.mjs';
7
- import { bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, PATTERN_RUN_CODE_POINTS, bitsetFromSet, bitsetIsEmpty, bitsetAnd, bitsetNot } from './chunk-ZKRV5M2P.mjs';
8
- import { compareCodePoints, appendFoldedCodePoint, toReducedSortValue, readSortField, defaultSortMode, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-LJYTW7GT.mjs';
3
+ import { MAX_DOC_ID_LENGTH, GEO_UNIT_METRES } from './chunk-V3B24XR7.mjs';
4
+ import { SCRATCH_SLOTS_PER_THREAD_POOL, osqRecordBytes, openSharedVectorField, osqBitsOf, createVectorStore, DISK_STORAGE_BLOCK_BYTES, createOsqQuantizer, magnitude, createHNSWIndex, nativeStoreScratchBytes, createOrdinalFilter, isTombstoned, addToOrdinalFilter, ordinalFilterValues, nativeScoresOf, toScore, removeFromOrdinalFilter, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, ordinalFilterHas, STORE_CALIBRATION_GENERATION, entryForOrd, toDistance, sharedMemoryAvailable, requireWholeNeighbourLists, topLayerInParts, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, releaseLocksHeldBy, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS } from './chunk-AMGDG4ZJ.mjs';
5
+ import { noteFallback } from './chunk-2E35UM3G.mjs';
6
+ import { GrowableUint32, bitsetHas, createBitSet, bitsetSet, bitsetOrInto, bitsetAndNotInto, bitsetAndInto, PATTERN_RUN_CODE_POINTS, bitsetFromSet, bitsetIsEmpty, bitsetAnd, bitsetNot } from './chunk-7ODNRREZ.mjs';
7
+ import { encodePartialSpellings, countSpelling, expandTermPrefix, suggestDisplayTerms, spellingCountsOf, codePointOrder, encodeStringBlob, partialWordSearchModule, partitionDocumentFrequency } from './chunk-AIUOAN7Y.mjs';
8
+ import { tokenizeOptions, getNestedValue, visitTokens, textFieldsChanged, fuzzyPrefixOf, boundedLevenshtein, getFlatSchema, tokenize, getOrCreateFieldNameIndex, tokenizeIterator, getFieldValueByInternalId } from './chunk-UOJIV2NG.mjs';
9
+ import { partialFieldsOf, schemaFieldsOf, isWordIndexedBase, validateDocument, validateDocumentStrict, encodeDocument, extractVectorFieldsFromSchema, parsedTypeOf, isListBase, VINCENTY_CONVERGENCE_THRESHOLD, EARTH_RADIUS_METERS, VINCENTY_MAX_ITERATIONS, DEG_TO_RAD } from './chunk-356UOTK2.mjs';
10
+ import { MAX_M, EARLIER_RELEASE_MIN_M, MIN_M } from './chunk-KEHK2O6P.mjs';
11
+ import { compareCodePoints, toReducedSortValue, readSortField, defaultSortMode, compareComparableValues, truncateSortString, SORT_VALUE_MAX_CODE_POINTS } from './chunk-IBR6634K.mjs';
9
12
  import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
10
13
  import { totalmem } from 'os';
11
14
  import { getHeapStatistics } from 'v8';
@@ -411,8 +414,6 @@ function compactList(list) {
411
414
  }
412
415
  }
413
416
  writeIdx++;
414
- } else {
415
- list.totalTermFrequency -= list.termFrequencies[i];
416
417
  }
417
418
  }
418
419
  list.docIds.length = writeIdx;
@@ -437,8 +438,6 @@ function compactDocEntries(list, internalId) {
437
438
  }
438
439
  }
439
440
  writeIdx++;
440
- } else {
441
- list.totalTermFrequency -= list.termFrequencies[i];
442
441
  }
443
442
  }
444
443
  list.docIds.length = writeIdx;
@@ -524,12 +523,13 @@ function createInvertedIndex(fieldNameTable) {
524
523
  }
525
524
  list.length++;
526
525
  },
527
- remove(token, internalId) {
526
+ remove(token, internalId, occurrences) {
528
527
  const list = index.get(token);
529
528
  if (!list || !list.docIdSet.has(internalId)) return;
530
529
  list.structureRevision++;
531
530
  list.docIdSet.delete(internalId);
532
531
  list.deletedDocs.add(internalId);
532
+ list.totalTermFrequency = Math.max(0, list.totalTermFrequency - occurrences);
533
533
  if (list.docIdSet.size === 0) {
534
534
  index.delete(token);
535
535
  untrackToken(token);
@@ -562,6 +562,9 @@ function createInvertedIndex(fieldNameTable) {
562
562
  lookup(token) {
563
563
  return index.get(token);
564
564
  },
565
+ documentFrequency(token) {
566
+ return index.get(token)?.docIdSet.size ?? 0;
567
+ },
565
568
  fuzzyLookup(token, tolerance, prefixLength) {
566
569
  if (tolerance === 0) {
567
570
  const exact = index.get(token);
@@ -940,54 +943,6 @@ function createSurfaceRegistry() {
940
943
  };
941
944
  }
942
945
 
943
- // src/core/pattern-index/platform.ts
944
- function platformPatternSearch() {
945
- return patternSearchModule;
946
- }
947
-
948
- // src/core/pattern-index/registry.ts
949
- var registered = null;
950
- function isPatternSearchModule(value) {
951
- return value !== null && typeof value === "object" && value.name === "pattern" && "createIndex" in value && typeof value.createIndex === "function" && "readArrays" in value && typeof value.readArrays === "function" && "mergeArrays" in value && typeof value.mergeArrays === "function" && "matchInto" in value && typeof value.matchInto === "function";
952
- }
953
- function registerPatternSearch(module) {
954
- if (!isPatternSearchModule(module)) {
955
- throw new NarsilError(
956
- ErrorCodes.CONFIG_INVALID,
957
- 'registerPatternSearch takes the patternSearch object that "@delali/narsil/pattern" exports'
958
- );
959
- }
960
- registered = module;
961
- }
962
- function availablePatternSearch() {
963
- return registered ?? platformPatternSearch();
964
- }
965
- function requirePatternSearchFor(schema) {
966
- for (const { path, pattern } of schemaFieldsOf(schema)) {
967
- if (pattern) requirePatternSearch(path);
968
- }
969
- }
970
- function requirePatternSearch(fieldPath) {
971
- const module = availablePatternSearch();
972
- if (module !== null) return module;
973
- throw new NarsilError(
974
- ErrorCodes.CONFIG_INVALID,
975
- `Field "${fieldPath}" is a pattern field, but this bundle contains no pattern search code. Add import { patternSearch } from '@delali/narsil/pattern', then call registerPatternSearch(patternSearch) before you create or load the index`,
976
- { field: fieldPath }
977
- );
978
- }
979
-
980
- // src/core/pattern-index/host.ts
981
- var PATTERN_HOST = { appendFoldedCodePoint };
982
- function patternIndexOf(indexes, fieldPath) {
983
- let index = indexes.get(fieldPath);
984
- if (index === void 0) {
985
- index = requirePatternSearch(fieldPath).createIndex(PATTERN_HOST);
986
- indexes.set(fieldPath, index);
987
- }
988
- return index;
989
- }
990
-
991
946
  // src/geo/haversine.ts
992
947
  function haversineDistance(lat1, lon1, lat2, lon2) {
993
948
  const phi1 = lat1 * DEG_TO_RAD;
@@ -1667,6 +1622,13 @@ function resolveInternalId(state, docId) {
1667
1622
  }
1668
1623
  return internalId;
1669
1624
  }
1625
+ function emptyIndexedText(fields) {
1626
+ const holdsPartialField = fields.some((field) => field.partial);
1627
+ return { fieldLengths: {}, tokensByField: {}, partialSpellings: holdsPartialField ? /* @__PURE__ */ new Map() : null };
1628
+ }
1629
+ function partialSpellingsFor(field, indexed) {
1630
+ return field.partial ? indexed.partialSpellings : null;
1631
+ }
1670
1632
  function countSurface(counts, surface, token) {
1671
1633
  if (surface === void 0 || surface.length === 0) return;
1672
1634
  const existing = counts.get(surface);
@@ -1676,6 +1638,18 @@ function countSurface(counts, surface, token) {
1676
1638
  counts.set(surface, { token, count: 1 });
1677
1639
  }
1678
1640
  }
1641
+ function countOccurrence(counts, spellings, surface, token) {
1642
+ countSurface(counts, surface, token);
1643
+ if (spellings !== null) countSpelling(spellings, token, surface ?? token, 1);
1644
+ }
1645
+ function addFieldOccurrences(occurrencesByToken, fieldTokens) {
1646
+ const fieldCounts = /* @__PURE__ */ new Map();
1647
+ for (const token of fieldTokens) fieldCounts.set(token, (fieldCounts.get(token) ?? 0) + 1);
1648
+ for (const [token, count] of fieldCounts) {
1649
+ const stored = count > MAX_TERM_FREQUENCY ? MAX_TERM_FREQUENCY : count;
1650
+ occurrencesByToken.set(token, (occurrencesByToken.get(token) ?? 0) + stored);
1651
+ }
1652
+ }
1679
1653
  function applySurfaceCounts(state, counts, direction) {
1680
1654
  for (const [surface, entry] of counts) {
1681
1655
  if (direction === 1) {
@@ -1685,7 +1659,7 @@ function applySurfaceCounts(state, counts, direction) {
1685
1659
  }
1686
1660
  }
1687
1661
  }
1688
- function indexStringField(state, internalId, fieldPath, text, language, options, fieldLengths, tokensByField) {
1662
+ function indexStringField(state, internalId, fieldPath, text, language, options, fieldLengths, tokensByField, spellings = null) {
1689
1663
  const fieldTokenList = [];
1690
1664
  const fieldNameIndex2 = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
1691
1665
  const opts = tokenizeOptions(options);
@@ -1701,7 +1675,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
1701
1675
  tokenFreqs.set(t.token, { count: 1, positions: [t.position] });
1702
1676
  }
1703
1677
  fieldTokenList.push(t.token);
1704
- countSurface(surfaceCounts, t.surface, t.token);
1678
+ countOccurrence(surfaceCounts, spellings, t.surface, t.token);
1705
1679
  }
1706
1680
  for (const [token, freq] of tokenFreqs) {
1707
1681
  state.invertedIdx.insert(token, internalId, freq.count, fieldNameIndex2, freq.positions);
@@ -1711,7 +1685,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
1711
1685
  for (const t of tokenizeIterator(text, language, opts)) {
1712
1686
  tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
1713
1687
  fieldTokenList.push(t.token);
1714
- countSurface(surfaceCounts, t.surface, t.token);
1688
+ countOccurrence(surfaceCounts, spellings, t.surface, t.token);
1715
1689
  }
1716
1690
  for (const [token, count] of tokenCounts) {
1717
1691
  state.invertedIdx.insert(token, internalId, count, fieldNameIndex2, null);
@@ -1721,7 +1695,7 @@ function indexStringField(state, internalId, fieldPath, text, language, options,
1721
1695
  fieldLengths[fieldPath] = fieldTokenList.length;
1722
1696
  tokensByField[fieldPath] = fieldTokenList;
1723
1697
  }
1724
- function indexStringArrayField(state, internalId, fieldPath, arr, language, options, fieldLengths, tokensByField) {
1698
+ function indexStringArrayField(state, internalId, fieldPath, arr, language, options, fieldLengths, tokensByField, spellings = null) {
1725
1699
  const fieldTokenList = [];
1726
1700
  const fieldNameIndex2 = getOrCreateFieldNameIndex(state.fieldNameTable, fieldPath);
1727
1701
  const opts = tokenizeOptions(options);
@@ -1740,7 +1714,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
1740
1714
  tokenFreqs.set(t.token, { count: 1, positions: [positionOffset + t.position] });
1741
1715
  }
1742
1716
  fieldTokenList.push(t.token);
1743
- countSurface(surfaceCounts, t.surface, t.token);
1717
+ countOccurrence(surfaceCounts, spellings, t.surface, t.token);
1744
1718
  itemTokenCount++;
1745
1719
  }
1746
1720
  positionOffset += itemTokenCount;
@@ -1755,7 +1729,7 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
1755
1729
  for (const t of tokenizeIterator(item, language, opts)) {
1756
1730
  tokenCounts.set(t.token, (tokenCounts.get(t.token) ?? 0) + 1);
1757
1731
  fieldTokenList.push(t.token);
1758
- countSurface(surfaceCounts, t.surface, t.token);
1732
+ countOccurrence(surfaceCounts, spellings, t.surface, t.token);
1759
1733
  }
1760
1734
  }
1761
1735
  fieldLengths[fieldPath] = fieldTokenList.length;
@@ -1767,17 +1741,28 @@ function indexStringArrayField(state, internalId, fieldPath, arr, language, opti
1767
1741
  tokensByField[fieldPath] = fieldTokenList;
1768
1742
  }
1769
1743
  function indexDocument(state, docId, document, fields, language, options) {
1770
- const fieldLengths = {};
1771
- const tokensByField = {};
1744
+ const indexed = emptyIndexedText(fields);
1745
+ const { fieldLengths, tokensByField } = indexed;
1772
1746
  const internalId = resolveInternalId(state, docId);
1773
- for (const { path: fieldPath, base, pattern } of fields) {
1747
+ for (const field of fields) {
1748
+ const { path: fieldPath, base, pattern } = field;
1774
1749
  const value = getNestedValue(document, fieldPath);
1775
1750
  if (value === void 0 || value === null) continue;
1776
1751
  recordValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
1777
1752
  ensureFieldIndex(state, fieldPath, base);
1778
1753
  if (pattern) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
1779
1754
  if (base === "string") {
1780
- indexStringField(state, internalId, fieldPath, value, language, options, fieldLengths, tokensByField);
1755
+ indexStringField(
1756
+ state,
1757
+ internalId,
1758
+ fieldPath,
1759
+ value,
1760
+ language,
1761
+ options,
1762
+ fieldLengths,
1763
+ tokensByField,
1764
+ partialSpellingsFor(field, indexed)
1765
+ );
1781
1766
  } else if (base === "number") {
1782
1767
  getOrCreateNumericIndex(state, fieldPath).insert(internalId, value);
1783
1768
  } else if (base === "boolean") {
@@ -1796,7 +1781,8 @@ function indexDocument(state, docId, document, fields, language, options) {
1796
1781
  language,
1797
1782
  options,
1798
1783
  fieldLengths,
1799
- tokensByField
1784
+ tokensByField,
1785
+ partialSpellingsFor(field, indexed)
1800
1786
  );
1801
1787
  } else if (base === "number[]") {
1802
1788
  const numIdx = getOrCreateNumericIndex(state, fieldPath);
@@ -1811,15 +1797,17 @@ function indexDocument(state, docId, document, fields, language, options) {
1811
1797
  }
1812
1798
  }
1813
1799
  }
1814
- return { fieldLengths, tokensByField };
1800
+ return indexed;
1815
1801
  }
1816
1802
  function removeFromIndexes(state, docId, storedDoc, schemaFields, language, options) {
1817
- const fieldLengths = {};
1818
- const tokensByField = {};
1803
+ const removed = emptyIndexedText(schemaFields);
1804
+ const { fieldLengths, tokensByField } = removed;
1819
1805
  const fields = storedDoc.fields;
1820
1806
  const internalId = resolveInternalId(state, docId);
1821
1807
  const opts = tokenizeOptions(options);
1822
- for (const { path: fieldPath, base, pattern } of schemaFields) {
1808
+ const occurrencesByToken = /* @__PURE__ */ new Map();
1809
+ for (const field of schemaFields) {
1810
+ const { path: fieldPath, base, pattern } = field;
1823
1811
  const value = getNestedValue(fields, fieldPath);
1824
1812
  if (value === void 0 || value === null) continue;
1825
1813
  forgetValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
@@ -1827,16 +1815,15 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
1827
1815
  if (base === "string") {
1828
1816
  const result = tokenize(value, language, opts);
1829
1817
  fieldLengths[fieldPath] = result.tokens.length;
1830
- const uniqueTokens = /* @__PURE__ */ new Set();
1831
1818
  const fieldTokenList = [];
1832
1819
  const surfaceCounts = /* @__PURE__ */ new Map();
1820
+ const spellings = partialSpellingsFor(field, removed);
1833
1821
  for (let i = 0; i < result.tokens.length; i++) {
1834
1822
  const t = result.tokens[i];
1835
- uniqueTokens.add(t.token);
1836
1823
  fieldTokenList.push(t.token);
1837
- countSurface(surfaceCounts, result.surfaces?.[i], t.token);
1824
+ countOccurrence(surfaceCounts, spellings, result.surfaces?.[i], t.token);
1838
1825
  }
1839
- for (const token of uniqueTokens) state.invertedIdx.remove(token, internalId);
1826
+ addFieldOccurrences(occurrencesByToken, fieldTokenList);
1840
1827
  applySurfaceCounts(state, surfaceCounts, -1);
1841
1828
  tokensByField[fieldPath] = fieldTokenList;
1842
1829
  } else if (base === "number") {
@@ -1849,20 +1836,19 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
1849
1836
  state.geoIndexes.get(fieldPath)?.remove(internalId);
1850
1837
  } else if (base === "string[]") {
1851
1838
  const arr = value;
1852
- const uniqueTokens = /* @__PURE__ */ new Set();
1853
1839
  const fieldTokenList = [];
1854
1840
  const surfaceCounts = /* @__PURE__ */ new Map();
1841
+ const spellings = partialSpellingsFor(field, removed);
1855
1842
  for (const item of arr) {
1856
1843
  const result = tokenize(item, language, opts);
1857
1844
  for (let i = 0; i < result.tokens.length; i++) {
1858
1845
  const t = result.tokens[i];
1859
- uniqueTokens.add(t.token);
1860
1846
  fieldTokenList.push(t.token);
1861
- countSurface(surfaceCounts, result.surfaces?.[i], t.token);
1847
+ countOccurrence(surfaceCounts, spellings, result.surfaces?.[i], t.token);
1862
1848
  }
1863
1849
  }
1864
1850
  fieldLengths[fieldPath] = fieldTokenList.length;
1865
- for (const token of uniqueTokens) state.invertedIdx.remove(token, internalId);
1851
+ addFieldOccurrences(occurrencesByToken, fieldTokenList);
1866
1852
  applySurfaceCounts(state, surfaceCounts, -1);
1867
1853
  tokensByField[fieldPath] = fieldTokenList;
1868
1854
  } else if (base === "number[]") {
@@ -1882,7 +1868,8 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
1882
1868
  }
1883
1869
  }
1884
1870
  }
1885
- return { fieldLengths, tokensByField };
1871
+ for (const [token, occurrences] of occurrencesByToken) state.invertedIdx.remove(token, internalId, occurrences);
1872
+ return removed;
1886
1873
  }
1887
1874
 
1888
1875
  // src/core/partition/field-updates.ts
@@ -1999,6 +1986,8 @@ function evaluatePatternTests(fieldPath, filter, context, bitsets) {
1999
1986
  if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
2000
1987
  if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
2001
1988
  if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
1989
+ if (typeof filter.wildcard === "string") bitsets.push(matching("wildcard", [filter.wildcard]));
1990
+ if (typeof filter.regex === "string") bitsets.push(matching("regex", [filter.regex]));
2002
1991
  }
2003
1992
 
2004
1993
  // src/filters/field-tests.ts
@@ -2203,6 +2192,8 @@ var FIELD_FILTER_OPERATORS = [
2203
2192
  "startsWith",
2204
2193
  "endsWith",
2205
2194
  "contains",
2195
+ "wildcard",
2196
+ "regex",
2206
2197
  "caseFold",
2207
2198
  "containsAll",
2208
2199
  "matchesAny",
@@ -3017,6 +3008,67 @@ function partitionFilterMatches(state, filters, schema, meter) {
3017
3008
  };
3018
3009
  }
3019
3010
 
3011
+ // src/core/scorer.ts
3012
+ var DEFAULT_K1 = 1.2;
3013
+ var DEFAULT_B = 0.75;
3014
+ function computeIDF(docFrequency, totalDocs) {
3015
+ if (totalDocs === 0) return 0;
3016
+ return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
3017
+ }
3018
+ function bm25PruningSound(params) {
3019
+ const k1 = params?.k1 ?? DEFAULT_K1;
3020
+ const b = params?.b ?? DEFAULT_B;
3021
+ return Number.isFinite(k1) && k1 >= 0 && b >= 0 && b <= 1;
3022
+ }
3023
+ function resolveBM25Params(params) {
3024
+ return { k1: params?.k1 ?? DEFAULT_K1, b: params?.b ?? DEFAULT_B };
3025
+ }
3026
+ function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
3027
+ if (avgFieldLength === 0) return 0;
3028
+ const numerator = termFrequency * (k1 + 1);
3029
+ const denominator = termFrequency + k1 * (1 - b + b * fieldLength / avgFieldLength);
3030
+ if (denominator === 0) return 0;
3031
+ return idf * (numerator / denominator);
3032
+ }
3033
+ function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
3034
+ if (totalDocs === 0) return 0;
3035
+ const k1 = params?.k1 ?? DEFAULT_K1;
3036
+ const b = params?.b ?? DEFAULT_B;
3037
+ return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
3038
+ }
3039
+
3040
+ // src/core/partition/field-scoring.ts
3041
+ function loadFieldScoring(state, fields, boost, averageFieldLengths) {
3042
+ const fieldNames = state.fieldNameTable.names;
3043
+ const scoring = {
3044
+ searchable: new Uint8Array(fieldNames.length),
3045
+ boosts: new Float64Array(fieldNames.length),
3046
+ averageLengths: new Float64Array(fieldNames.length),
3047
+ lengthColumns: new Array(fieldNames.length).fill(null)
3048
+ };
3049
+ for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
3050
+ const fieldName = fieldNames[fieldIndex];
3051
+ scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
3052
+ scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
3053
+ scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
3054
+ scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
3055
+ }
3056
+ return scoring;
3057
+ }
3058
+ function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
3059
+ const column = fieldIndex < columns.length ? columns[fieldIndex] : null;
3060
+ if (column === null || internalId >= column.length) return averageLength;
3061
+ const stored = column[internalId];
3062
+ return stored > 0 ? stored : averageLength;
3063
+ }
3064
+ function bestSearchable(searchable, values) {
3065
+ let best = 0;
3066
+ for (let index = 0; index < searchable.length; index++) {
3067
+ if (searchable[index] === 1 && values[index] > best) best = values[index];
3068
+ }
3069
+ return best;
3070
+ }
3071
+
3020
3072
  // src/core/partition/posting-columns.ts
3021
3073
  function postingColumns(list) {
3022
3074
  const deletedDocs = list.deletedDocs;
@@ -3030,39 +3082,174 @@ function postingColumns(list) {
3030
3082
  };
3031
3083
  }
3032
3084
 
3085
+ // src/core/partition/scoring.ts
3086
+ var EMPTY_COMPONENTS = Object.freeze({});
3087
+ function recordComponents(components, internalId, fieldName, token, termFrequency, fieldLength, idf) {
3088
+ const existing = components.get(internalId);
3089
+ if (existing) {
3090
+ existing.termFrequencies[`${fieldName}:${token}`] = termFrequency;
3091
+ existing.fieldLengths[fieldName] = fieldLength;
3092
+ existing.idf[token] = idf;
3093
+ return;
3094
+ }
3095
+ components.set(internalId, {
3096
+ termFrequencies: { [`${fieldName}:${token}`]: termFrequency },
3097
+ fieldLengths: { [fieldName]: fieldLength },
3098
+ idf: { [token]: idf }
3099
+ });
3100
+ }
3101
+ function mergeGroupComponents(components, internalId, contribution) {
3102
+ const existing = components.get(internalId);
3103
+ if (existing) {
3104
+ Object.assign(existing.termFrequencies, contribution.termFrequencies);
3105
+ Object.assign(existing.fieldLengths, contribution.fieldLengths);
3106
+ existing.idf[contribution.token] = contribution.idf;
3107
+ return;
3108
+ }
3109
+ components.set(internalId, {
3110
+ termFrequencies: contribution.termFrequencies,
3111
+ fieldLengths: contribution.fieldLengths,
3112
+ idf: { [contribution.token]: contribution.idf }
3113
+ });
3114
+ }
3115
+
3116
+ // src/core/partition/group-scoring.ts
3117
+ function partialFieldMarks(fieldNames, partialFields) {
3118
+ const marks = new Uint8Array(fieldNames.length);
3119
+ if (partialFields === void 0 || partialFields.length === 0) return marks;
3120
+ for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
3121
+ if (partialFields.includes(fieldNames[fieldIndex])) marks[fieldIndex] = 1;
3122
+ }
3123
+ return marks;
3124
+ }
3125
+ function termGroupOf(groups, token) {
3126
+ if (groups === void 0) return void 0;
3127
+ for (const group of groups) {
3128
+ if (group.token === token) return group;
3129
+ }
3130
+ return void 0;
3131
+ }
3132
+ function typoMatchesFor(index, token, group, exact, tolerance, prefixLength) {
3133
+ if (group !== void 0) {
3134
+ if (!group.typoMatches || exact || tolerance === 0) return [];
3135
+ return index.fuzzyLookup(token, tolerance, prefixLength).filter((match) => match.token !== token);
3136
+ }
3137
+ if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
3138
+ const postingList = index.lookup(token);
3139
+ return postingList ? [{ token, postingList }] : [];
3140
+ }
3141
+ function resolveGroupMatches(context, group) {
3142
+ let groupDocumentFrequency = 0;
3143
+ const found = [];
3144
+ for (const member of group.members) {
3145
+ const documents = context.globalFrequency(member.token) ?? context.index.documentFrequency(member.token);
3146
+ if (documents > groupDocumentFrequency) groupDocumentFrequency = documents;
3147
+ const postingList = context.index.lookup(member.token);
3148
+ if (postingList === void 0) continue;
3149
+ found.push({ token: member.token, factor: member.factor, partialFactor: member.partialFactor, postingList });
3150
+ }
3151
+ if (found.length === 0) return [];
3152
+ const idf = computeIDF(groupDocumentFrequency, context.totalDocs);
3153
+ return found.map((entry) => ({ ...entry, idf }));
3154
+ }
3155
+ function groupPostingCount(matches) {
3156
+ let total = 0;
3157
+ for (const match of matches) total += match.postingList.length;
3158
+ return total;
3159
+ }
3160
+ function computeGroupContributions(context, matches, collect) {
3161
+ const { fields, fieldNames, filterBitset, partialMarks, k1, b } = context;
3162
+ const scoresAreZero = context.totalDocs === 0;
3163
+ const best = /* @__PURE__ */ new Map();
3164
+ for (const match of matches) {
3165
+ const perTerm = /* @__PURE__ */ new Map();
3166
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
3167
+ match.postingList
3168
+ );
3169
+ for (let pi = 0; pi < count; pi++) {
3170
+ const internalId = docIds[pi];
3171
+ if (hasDeleted && deletedDocs.has(internalId)) continue;
3172
+ if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
3173
+ const fieldIndex = fieldNameIndices[pi];
3174
+ if (fields.searchable[fieldIndex] === 0) continue;
3175
+ const factor = partialMarks[fieldIndex] === 1 ? match.partialFactor : match.factor;
3176
+ if (factor === 0) continue;
3177
+ const fieldName = fieldNames[fieldIndex];
3178
+ const termFrequency = termFrequencies[pi];
3179
+ const fieldBoost = fields.boosts[fieldIndex];
3180
+ const avgLen = fields.averageLengths[fieldIndex];
3181
+ const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
3182
+ const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b)) * fieldBoost * factor;
3183
+ const existing = perTerm.get(internalId);
3184
+ if (existing) {
3185
+ existing.score += termScore;
3186
+ if (collect) {
3187
+ existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
3188
+ existing.fieldLengths[fieldName] = actualFieldLength;
3189
+ }
3190
+ } else if (collect) {
3191
+ perTerm.set(internalId, {
3192
+ score: termScore,
3193
+ token: match.token,
3194
+ idf: match.idf,
3195
+ termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
3196
+ fieldLengths: { [fieldName]: actualFieldLength }
3197
+ });
3198
+ } else {
3199
+ perTerm.set(internalId, {
3200
+ score: termScore,
3201
+ token: match.token,
3202
+ idf: match.idf,
3203
+ termFrequencies: EMPTY_COMPONENTS,
3204
+ fieldLengths: EMPTY_COMPONENTS
3205
+ });
3206
+ }
3207
+ }
3208
+ for (const [internalId, contribution] of perTerm) {
3209
+ const current = best.get(internalId);
3210
+ if (!current || contribution.score > current.score) {
3211
+ best.set(internalId, contribution);
3212
+ }
3213
+ }
3214
+ }
3215
+ return best;
3216
+ }
3217
+
3033
3218
  // src/core/partition/matches.ts
3034
- function markPostingList(matched, list, fieldNames, fields, filterBitset) {
3219
+ function markPostingList(matched, list, markedFields, filterBitset) {
3035
3220
  const { docIds, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(list);
3036
3221
  for (let pi = 0; pi < count; pi++) {
3037
3222
  const internalId = docIds[pi];
3038
3223
  if (hasDeleted && deletedDocs.has(internalId)) continue;
3039
3224
  if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
3040
- if (fields && !fields.includes(fieldNames[fieldNameIndices[pi]])) continue;
3225
+ if (markedFields[fieldNameIndices[pi]] === 0) continue;
3041
3226
  bitsetSet(matched, internalId);
3042
3227
  }
3043
3228
  }
3229
+ function searchedFieldMarks(fieldNames, fields) {
3230
+ const marks = new Uint8Array(fieldNames.length);
3231
+ for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
3232
+ if (fields === void 0 || fields.includes(fieldNames[fieldIndex])) marks[fieldIndex] = 1;
3233
+ }
3234
+ return marks;
3235
+ }
3044
3236
  function searchFulltextMatches(state, params) {
3045
- const { queryTokens, prefixExpansion, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
3237
+ const { queryTokens, termGroups, fields, tolerance = 0, prefixLength = 2, exact = false, filterBitset } = params;
3046
3238
  const fieldNames = state.fieldNameTable.names;
3047
3239
  const matched = createBitSet(state.docStore.internalIdCapacity());
3240
+ const searched = searchedFieldMarks(fieldNames, fields);
3241
+ const partial = partialFieldMarks(fieldNames, params.partialFields);
3242
+ const searchedPartial = searched.map((mark, fieldIndex) => mark & partial[fieldIndex]);
3048
3243
  for (const qt of queryTokens) {
3049
- if (prefixExpansion && qt.token === prefixExpansion.token) {
3050
- const seen = /* @__PURE__ */ new Set();
3051
- for (const term of [qt.token, ...prefixExpansion.terms]) {
3052
- if (seen.has(term)) continue;
3053
- seen.add(term);
3054
- const postingList = state.invertedIdx.lookup(term);
3055
- if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
3056
- }
3057
- continue;
3058
- }
3059
- if (exact || tolerance === 0) {
3060
- const postingList = state.invertedIdx.lookup(qt.token);
3061
- if (postingList) markPostingList(matched, postingList, fieldNames, fields, filterBitset);
3062
- continue;
3244
+ const group = termGroupOf(termGroups, qt.token);
3245
+ for (const match of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
3246
+ markPostingList(matched, match.postingList, searched, filterBitset);
3063
3247
  }
3064
- for (const match of state.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength)) {
3065
- markPostingList(matched, match.postingList, fieldNames, fields, filterBitset);
3248
+ if (group === void 0) continue;
3249
+ for (const member of group.members) {
3250
+ const postingList = state.invertedIdx.lookup(member.token);
3251
+ if (postingList === void 0) continue;
3252
+ markPostingList(matched, postingList, member.factor > 0 ? searched : searchedPartial, filterBitset);
3066
3253
  }
3067
3254
  }
3068
3255
  const docStore = state.docStore;
@@ -3150,6 +3337,7 @@ function estimatePartitionBytes(state) {
3150
3337
  bytes += docCount * state.geoIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
3151
3338
  bytes += docCount * state.sizeIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
3152
3339
  for (const index of state.patternIndexes.values()) bytes += index.bytes();
3340
+ bytes += state.partialWords.bytes();
3153
3341
  return bytes + sortColumnBytes(state);
3154
3342
  }
3155
3343
 
@@ -3281,10 +3469,71 @@ function mergeSegmentState(target, source) {
3281
3469
  mergeFieldIndexes(target, source, idMapping);
3282
3470
  mergeStatistics(target, source);
3283
3471
  mergeSurfaceForms(target, source);
3472
+ target.partialWords.afterMerge(source.partialWords.spellings(), [...idMapping.values()]);
3284
3473
  target.sortColumns = null;
3285
3474
  target.scoreBuffer = null;
3286
3475
  }
3287
3476
 
3477
+ // src/core/partition/partial-words/platform.ts
3478
+ function platformPartialWordSearch() {
3479
+ return partialWordSearchModule;
3480
+ }
3481
+
3482
+ // src/core/partition/partial-words/registry.ts
3483
+ var registered = null;
3484
+ function isPartialWordSearchModule(value) {
3485
+ return value !== null && typeof value === "object" && value.name === "partialWords" && "createGrowingSlot" in value && typeof value.createGrowingSlot === "function" && "createFrozenSlot" in value && typeof value.createFrozenSlot === "function" && "buildTable" in value && typeof value.buildTable === "function" && "expand" in value && typeof value.expand === "function";
3486
+ }
3487
+ function registerPartialWordSearch(module) {
3488
+ if (!isPartialWordSearchModule(module)) {
3489
+ throw new NarsilError(
3490
+ ErrorCodes.CONFIG_INVALID,
3491
+ 'Pass registerPartialWordSearch the partialWordSearch object that "@delali/narsil/partial-words" exports'
3492
+ );
3493
+ }
3494
+ registered = module;
3495
+ }
3496
+ function availablePartialWordSearch() {
3497
+ return registered ?? platformPartialWordSearch();
3498
+ }
3499
+ function requirePartialWordSearchFor(schema) {
3500
+ const fields = partialFieldsOf(schema);
3501
+ if (fields.length === 0 || availablePartialWordSearch() !== null) return;
3502
+ throw new NarsilError(
3503
+ ErrorCodes.CONFIG_INVALID,
3504
+ `The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load the index`,
3505
+ { field: fields[0] }
3506
+ );
3507
+ }
3508
+ var EMPTY_GROWING_SLOT = {
3509
+ readerFor: () => null,
3510
+ spellings: () => null,
3511
+ bytes: () => 0,
3512
+ afterDocument: () => void 0,
3513
+ afterMerge: () => void 0,
3514
+ forgetSpellings: () => void 0,
3515
+ clear: () => void 0
3516
+ };
3517
+ var EMPTY_FROZEN_SLOT = {
3518
+ readerFor: () => null,
3519
+ spellings: () => null,
3520
+ bytes: () => 0,
3521
+ afterRemoval: () => void 0
3522
+ };
3523
+ function createGrowingPartialWords(view) {
3524
+ return availablePartialWordSearch()?.createGrowingSlot(view) ?? EMPTY_GROWING_SLOT;
3525
+ }
3526
+ function createFrozenPartialWords(initial, tokenTable, documents) {
3527
+ return availablePartialWordSearch()?.createFrozenSlot(initial, tokenTable, documents) ?? EMPTY_FROZEN_SLOT;
3528
+ }
3529
+ function buildPartialWordTableFrom(source, tokenTable) {
3530
+ return availablePartialWordSearch()?.buildTable(source, tokenTable) ?? null;
3531
+ }
3532
+ function expandPartialWords(subs, analysis, requests) {
3533
+ const module = availablePartialWordSearch();
3534
+ return module === null ? requests.map(() => []) : module.expand(subs, analysis, requests);
3535
+ }
3536
+
3288
3537
  // src/core/partition/rebuild.ts
3289
3538
  var EMPTY_STATS = {
3290
3539
  totalDocuments: 0,
@@ -3340,70 +3589,10 @@ function rebuildTextIndex(state, schema, language, options) {
3340
3589
  }
3341
3590
  } finally {
3342
3591
  state.invertedIdx.endBatch();
3592
+ state.partialWords.forgetSpellings();
3343
3593
  }
3344
3594
  }
3345
3595
 
3346
- // src/core/scorer.ts
3347
- var DEFAULT_K1 = 1.2;
3348
- var DEFAULT_B = 0.75;
3349
- function computeIDF(docFrequency, totalDocs) {
3350
- if (totalDocs === 0) return 0;
3351
- return Math.log((totalDocs - docFrequency + 0.5) / (docFrequency + 0.5) + 1);
3352
- }
3353
- function bm25PruningSound(params) {
3354
- const k1 = params?.k1 ?? DEFAULT_K1;
3355
- const b = params?.b ?? DEFAULT_B;
3356
- return Number.isFinite(k1) && k1 >= 0 && b >= 0 && b <= 1;
3357
- }
3358
- function resolveBM25Params(params) {
3359
- return { k1: params?.k1 ?? DEFAULT_K1, b: params?.b ?? DEFAULT_B };
3360
- }
3361
- function computeBM25WithIDF(termFrequency, idf, fieldLength, avgFieldLength, k1, b) {
3362
- if (avgFieldLength === 0) return 0;
3363
- const numerator = termFrequency * (k1 + 1);
3364
- const denominator = termFrequency + k1 * (1 - b + b * fieldLength / avgFieldLength);
3365
- if (denominator === 0) return 0;
3366
- return idf * (numerator / denominator);
3367
- }
3368
- function computeBM25(termFrequency, docFrequency, totalDocs, fieldLength, avgFieldLength, params) {
3369
- if (totalDocs === 0) return 0;
3370
- const k1 = params?.k1 ?? DEFAULT_K1;
3371
- const b = params?.b ?? DEFAULT_B;
3372
- return computeBM25WithIDF(termFrequency, computeIDF(docFrequency, totalDocs), fieldLength, avgFieldLength, k1, b);
3373
- }
3374
-
3375
- // src/core/partition/field-scoring.ts
3376
- function loadFieldScoring(state, fields, boost, averageFieldLengths) {
3377
- const fieldNames = state.fieldNameTable.names;
3378
- const scoring = {
3379
- searchable: new Uint8Array(fieldNames.length),
3380
- boosts: new Float64Array(fieldNames.length),
3381
- averageLengths: new Float64Array(fieldNames.length),
3382
- lengthColumns: new Array(fieldNames.length).fill(null)
3383
- };
3384
- for (let fieldIndex = 0; fieldIndex < fieldNames.length; fieldIndex++) {
3385
- const fieldName = fieldNames[fieldIndex];
3386
- scoring.searchable[fieldIndex] = fields === void 0 || fields.includes(fieldName) ? 1 : 0;
3387
- scoring.boosts[fieldIndex] = boost?.[fieldName] ?? 1;
3388
- scoring.averageLengths[fieldIndex] = averageFieldLengths[fieldName] ?? 1;
3389
- scoring.lengthColumns[fieldIndex] = state.docStore.fieldLengthColumn(fieldName);
3390
- }
3391
- return scoring;
3392
- }
3393
- function fieldLengthOf(columns, fieldIndex, internalId, averageLength) {
3394
- const column = fieldIndex < columns.length ? columns[fieldIndex] : null;
3395
- if (column === null || internalId >= column.length) return averageLength;
3396
- const stored = column[internalId];
3397
- return stored > 0 ? stored : averageLength;
3398
- }
3399
- function bestSearchable(searchable, values) {
3400
- let best = 0;
3401
- for (let index = 0; index < searchable.length; index++) {
3402
- if (searchable[index] === 1 && values[index] > best) best = values[index];
3403
- }
3404
- return best;
3405
- }
3406
-
3407
3596
  // src/core/partition/block-bounds.ts
3408
3597
  var ABSENT_FIELD_LENGTH_BOUND = 0;
3409
3598
  var boundsByList = /* @__PURE__ */ new WeakMap();
@@ -3499,37 +3688,6 @@ function blockBoundsFor(list, columns) {
3499
3688
  return built;
3500
3689
  }
3501
3690
 
3502
- // src/core/partition/scoring.ts
3503
- var EMPTY_COMPONENTS = Object.freeze({});
3504
- function recordComponents(components, internalId, fieldName, token, termFrequency, fieldLength, idf) {
3505
- const existing = components.get(internalId);
3506
- if (existing) {
3507
- existing.termFrequencies[`${fieldName}:${token}`] = termFrequency;
3508
- existing.fieldLengths[fieldName] = fieldLength;
3509
- existing.idf[token] = idf;
3510
- return;
3511
- }
3512
- components.set(internalId, {
3513
- termFrequencies: { [`${fieldName}:${token}`]: termFrequency },
3514
- fieldLengths: { [fieldName]: fieldLength },
3515
- idf: { [token]: idf }
3516
- });
3517
- }
3518
- function mergePrefixComponents(components, internalId, contribution) {
3519
- const existing = components.get(internalId);
3520
- if (existing) {
3521
- Object.assign(existing.termFrequencies, contribution.termFrequencies);
3522
- Object.assign(existing.fieldLengths, contribution.fieldLengths);
3523
- existing.idf[contribution.token] = contribution.idf;
3524
- return;
3525
- }
3526
- components.set(internalId, {
3527
- termFrequencies: contribution.termFrequencies,
3528
- fieldLengths: contribution.fieldLengths,
3529
- idf: { [contribution.token]: contribution.idf }
3530
- });
3531
- }
3532
-
3533
3691
  // src/core/partition/top-k-heap.ts
3534
3692
  function candidateWorse(a, b) {
3535
3693
  if (a.score !== b.score) return a.score < b.score;
@@ -3693,7 +3851,7 @@ function topKFromBuffer(buffer, k, resolver, components) {
3693
3851
  // src/core/partition/multi-term-topk.ts
3694
3852
  function prunableMultiTermLists(params, index) {
3695
3853
  if (params.queryTokens.length < 2) return null;
3696
- if (params.prefixExpansion !== void 0) return null;
3854
+ if (params.termGroups !== void 0 && params.termGroups.length > 0) return null;
3697
3855
  if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
3698
3856
  if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
3699
3857
  if (params.collectComponents !== false) return null;
@@ -3865,87 +4023,6 @@ function multiTermTopK(request) {
3865
4023
  return { scored: pageFromBuffer(scans, buffer, selection, resolver), totalMatched };
3866
4024
  }
3867
4025
 
3868
- // src/core/partition/prefix-scoring.ts
3869
- function resolvePrefixMatches(context, token, expansionTerms) {
3870
- const found = [];
3871
- const seen = /* @__PURE__ */ new Set();
3872
- for (const term of [token, ...expansionTerms]) {
3873
- if (seen.has(term)) continue;
3874
- seen.add(term);
3875
- const postingList = context.index.lookup(term);
3876
- if (!postingList) continue;
3877
- found.push({ token: term, postingList, docFreq: context.documentFrequency(term, postingList) });
3878
- }
3879
- if (found.length === 0) return [];
3880
- let blendedDf = 0;
3881
- for (const f of found) {
3882
- if (f.docFreq > blendedDf) blendedDf = f.docFreq;
3883
- }
3884
- const blendedIdf = computeIDF(blendedDf, context.totalDocs);
3885
- return found.map((f) => ({
3886
- token: f.token,
3887
- factor: Math.min(1, token.length / f.token.length),
3888
- postingList: f.postingList,
3889
- docFreq: blendedDf,
3890
- idf: blendedIdf
3891
- }));
3892
- }
3893
- function computePrefixContributions(context, matches, collect) {
3894
- const { fields, fieldNames, filterBitset, k1, b } = context;
3895
- const scoresAreZero = context.totalDocs === 0;
3896
- const best = /* @__PURE__ */ new Map();
3897
- for (const match of matches) {
3898
- const perTerm = /* @__PURE__ */ new Map();
3899
- const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
3900
- match.postingList
3901
- );
3902
- for (let pi = 0; pi < count; pi++) {
3903
- const internalId = docIds[pi];
3904
- if (hasDeleted && deletedDocs.has(internalId)) continue;
3905
- if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
3906
- const fieldIndex = fieldNameIndices[pi];
3907
- if (fields.searchable[fieldIndex] === 0) continue;
3908
- const fieldName = fieldNames[fieldIndex];
3909
- const termFrequency = termFrequencies[pi];
3910
- const fieldBoost = fields.boosts[fieldIndex];
3911
- const avgLen = fields.averageLengths[fieldIndex];
3912
- const actualFieldLength = fieldLengthOf(fields.lengthColumns, fieldIndex, internalId, avgLen);
3913
- const termScore = (scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b)) * fieldBoost * match.factor;
3914
- const existing = perTerm.get(internalId);
3915
- if (existing) {
3916
- existing.score += termScore;
3917
- if (collect) {
3918
- existing.termFrequencies[`${fieldName}:${match.token}`] = termFrequency;
3919
- existing.fieldLengths[fieldName] = actualFieldLength;
3920
- }
3921
- } else if (collect) {
3922
- perTerm.set(internalId, {
3923
- score: termScore,
3924
- token: match.token,
3925
- idf: match.idf,
3926
- termFrequencies: { [`${fieldName}:${match.token}`]: termFrequency },
3927
- fieldLengths: { [fieldName]: actualFieldLength }
3928
- });
3929
- } else {
3930
- perTerm.set(internalId, {
3931
- score: termScore,
3932
- token: match.token,
3933
- idf: match.idf,
3934
- termFrequencies: EMPTY_COMPONENTS,
3935
- fieldLengths: EMPTY_COMPONENTS
3936
- });
3937
- }
3938
- }
3939
- for (const [internalId, contribution] of perTerm) {
3940
- const current = best.get(internalId);
3941
- if (!current || contribution.score > current.score) {
3942
- best.set(internalId, contribution);
3943
- }
3944
- }
3945
- }
3946
- return best;
3947
- }
3948
-
3949
4026
  // src/core/partition/single-term-topk.ts
3950
4027
  function fieldsCoverEvery(fields, fieldNames) {
3951
4028
  if (fields === void 0) return true;
@@ -3956,7 +4033,7 @@ function fieldsCoverEvery(fields, fieldNames) {
3956
4033
  }
3957
4034
  function prunableSingleTermList(params, index, fieldNames) {
3958
4035
  if (params.queryTokens.length !== 1) return null;
3959
- if (params.prefixExpansion !== void 0) return null;
4036
+ if (params.termGroups !== void 0 && params.termGroups.length > 0) return null;
3960
4037
  if (params.exact !== true && (params.tolerance ?? 0) !== 0) return null;
3961
4038
  if (params.termMatch !== void 0 && params.termMatch !== "any") return null;
3962
4039
  if (params.collectComponents !== false) return null;
@@ -4066,15 +4143,11 @@ function singleTermTopK(request) {
4066
4143
  function globalDocFreqFor(docFreqs, term, fallback) {
4067
4144
  return Object.hasOwn(docFreqs, term) ? docFreqs[term] : fallback;
4068
4145
  }
4069
- function matchesFor(index, token, exact, tolerance, prefixLength) {
4070
- if (!exact) return index.fuzzyLookup(token, tolerance, prefixLength);
4071
- const postingList = index.lookup(token);
4072
- return postingList ? [{ token, postingList }] : [];
4073
- }
4074
4146
  function searchFulltext(state, params) {
4075
4147
  const {
4076
4148
  queryTokens,
4077
- prefixExpansion,
4149
+ termGroups,
4150
+ partialFields,
4078
4151
  fields,
4079
4152
  boost,
4080
4153
  tolerance = 0,
@@ -4106,19 +4179,75 @@ function searchFulltext(state, params) {
4106
4179
  function documentFrequency(token, list) {
4107
4180
  return globalStats ? globalDocFreqFor(globalStats.docFrequencies, token, list.docIdSet.size) : list.docIdSet.size;
4108
4181
  }
4109
- const prefixContext = {
4182
+ const groupContext = {
4110
4183
  index: state.invertedIdx,
4111
4184
  fields: fieldScoring,
4185
+ partialMarks: partialFieldMarks(fieldNames, partialFields),
4112
4186
  fieldNames,
4113
4187
  totalDocs,
4114
4188
  k1,
4115
4189
  b,
4116
4190
  filterBitset,
4117
- documentFrequency
4191
+ globalFrequency(token) {
4192
+ if (globalStats === void 0 || !Object.hasOwn(globalStats.docFrequencies, token)) return void 0;
4193
+ return globalStats.docFrequencies[token];
4194
+ }
4118
4195
  };
4119
- function mergePrefixContribution(internalId, contribution) {
4196
+ function mergeGroupContribution(internalId, contribution) {
4120
4197
  addScore(scoreBuffer, internalId, contribution.score);
4121
- if (components !== null) mergePrefixComponents(components, internalId, contribution);
4198
+ if (components !== null) mergeGroupComponents(components, internalId, contribution);
4199
+ }
4200
+ function resolveMatch(token, postingList) {
4201
+ const docFreq = documentFrequency(token, postingList);
4202
+ return { token, docFreq, idf: computeIDF(docFreq, totalDocs), postingList };
4203
+ }
4204
+ function scorePostings(match, onlyScored) {
4205
+ const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
4206
+ match.postingList
4207
+ );
4208
+ if (!onlyScored && !hasDeleted && filterBitset === void 0 && components === null) {
4209
+ for (let pi = 0; pi < count; pi++) {
4210
+ const internalId = docIds[pi];
4211
+ const fieldIndex = fieldNameIndices[pi];
4212
+ if (searchable[fieldIndex] === 0) continue;
4213
+ const avgLen = averageLengths[fieldIndex];
4214
+ const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
4215
+ const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[pi], match.idf, actualFieldLength, avgLen, k1, b);
4216
+ addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
4217
+ }
4218
+ return;
4219
+ }
4220
+ for (let pi = 0; pi < count; pi++) {
4221
+ const internalId = docIds[pi];
4222
+ if (hasDeleted && deletedDocs.has(internalId)) continue;
4223
+ if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
4224
+ if (onlyScored && !hasScore(scoreBuffer, internalId)) continue;
4225
+ const fieldIndex = fieldNameIndices[pi];
4226
+ if (searchable[fieldIndex] === 0) continue;
4227
+ const termFrequency = termFrequencies[pi];
4228
+ const avgLen = averageLengths[fieldIndex];
4229
+ const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
4230
+ const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b);
4231
+ addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
4232
+ if (components !== null) {
4233
+ recordComponents(
4234
+ components,
4235
+ internalId,
4236
+ fieldNames[fieldIndex],
4237
+ match.token,
4238
+ termFrequency,
4239
+ actualFieldLength,
4240
+ match.idf
4241
+ );
4242
+ }
4243
+ }
4244
+ }
4245
+ function scoreGroup(group, onlyScored) {
4246
+ if (group === null) return;
4247
+ for (const [internalId, contribution] of computeGroupContributions(groupContext, group, collectComponents)) {
4248
+ if (onlyScored && !hasScore(scoreBuffer, internalId)) continue;
4249
+ mergeGroupContribution(internalId, contribution);
4250
+ }
4122
4251
  }
4123
4252
  const prunableList = prunableSingleTermList(params, state.invertedIdx, fieldNames);
4124
4253
  if (prunableList !== null && maxResults !== void 0) {
@@ -4148,127 +4277,30 @@ function searchFulltext(state, params) {
4148
4277
  }
4149
4278
  if (useIntersection) {
4150
4279
  const resolved = [];
4151
- let prefixMatches = [];
4152
4280
  for (const qt of queryTokens) {
4153
- if (prefixExpansion && qt.token === prefixExpansion.token) {
4154
- prefixMatches = resolvePrefixMatches(prefixContext, qt.token, prefixExpansion.terms);
4155
- let totalPostings2 = 0;
4156
- for (const m of prefixMatches) {
4157
- totalPostings2 += m.postingList.length;
4158
- }
4159
- resolved.push({ token: qt.token, matches: [], totalPostings: totalPostings2, isPrefix: true });
4160
- continue;
4161
- }
4162
- const rawMatches = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
4163
- let totalPostings = 0;
4281
+ const group = termGroupOf(termGroups, qt.token);
4282
+ const groupMatches = group === void 0 ? null : resolveGroupMatches(groupContext, group);
4283
+ let totalPostings = groupMatches === null ? 0 : groupPostingCount(groupMatches);
4164
4284
  const matches = [];
4165
- for (const m of rawMatches) {
4166
- const docFreq = documentFrequency(m.token, m.postingList);
4167
- const idf = computeIDF(docFreq, totalDocs);
4285
+ for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
4168
4286
  totalPostings += m.postingList.length;
4169
- matches.push({ token: m.token, docFreq, idf, postingList: m.postingList });
4287
+ matches.push(resolveMatch(m.token, m.postingList));
4170
4288
  }
4171
- resolved.push({ token: qt.token, matches, totalPostings });
4289
+ resolved.push({ token: qt.token, matches, totalPostings, group: groupMatches });
4172
4290
  }
4173
4291
  resolved.sort((a, b2) => a.totalPostings - b2.totalPostings);
4174
4292
  for (let tokenIndex = 0; tokenIndex < resolved.length; tokenIndex++) {
4175
- if (resolved[tokenIndex].isPrefix) {
4176
- const contributions = computePrefixContributions(prefixContext, prefixMatches, collectComponents);
4177
- for (const [internalId, contribution] of contributions) {
4178
- if (tokenIndex > 0 && !hasScore(scoreBuffer, internalId)) continue;
4179
- mergePrefixContribution(internalId, contribution);
4180
- }
4181
- continue;
4182
- }
4183
- for (const match of resolved[tokenIndex].matches) {
4184
- const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
4185
- match.postingList
4186
- );
4187
- for (let pi = 0; pi < count; pi++) {
4188
- const internalId = docIds[pi];
4189
- if (hasDeleted && deletedDocs.has(internalId)) continue;
4190
- if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
4191
- if (tokenIndex > 0 && !hasScore(scoreBuffer, internalId)) continue;
4192
- const fieldIndex = fieldNameIndices[pi];
4193
- if (searchable[fieldIndex] === 0) continue;
4194
- const termFrequency = termFrequencies[pi];
4195
- const fieldBoost = boosts[fieldIndex];
4196
- const avgLen = averageLengths[fieldIndex];
4197
- const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
4198
- let termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, match.idf, actualFieldLength, avgLen, k1, b);
4199
- termScore *= fieldBoost;
4200
- addScore(scoreBuffer, internalId, termScore);
4201
- if (components !== null) {
4202
- recordComponents(
4203
- components,
4204
- internalId,
4205
- fieldNames[fieldIndex],
4206
- match.token,
4207
- termFrequency,
4208
- actualFieldLength,
4209
- match.idf
4210
- );
4211
- }
4212
- }
4213
- }
4293
+ const onlyScored = tokenIndex > 0;
4294
+ for (const match of resolved[tokenIndex].matches) scorePostings(match, onlyScored);
4295
+ scoreGroup(resolved[tokenIndex].group, onlyScored);
4214
4296
  }
4215
4297
  } else {
4216
4298
  for (const qt of queryTokens) {
4217
- if (prefixExpansion && qt.token === prefixExpansion.token) {
4218
- const contributions = computePrefixContributions(
4219
- prefixContext,
4220
- resolvePrefixMatches(prefixContext, qt.token, prefixExpansion.terms),
4221
- collectComponents
4222
- );
4223
- for (const [internalId, contribution] of contributions) {
4224
- mergePrefixContribution(internalId, contribution);
4225
- }
4226
- continue;
4227
- }
4228
- const matchingPostings = matchesFor(state.invertedIdx, qt.token, exact, tolerance, prefixLength);
4229
- for (const match of matchingPostings) {
4230
- const idf = computeIDF(documentFrequency(match.token, match.postingList), totalDocs);
4231
- const { docIds, termFrequencies, fieldNameIndices, deletedDocs, hasDeleted, count } = postingColumns(
4232
- match.postingList
4233
- );
4234
- if (!hasDeleted && filterBitset === void 0 && components === null) {
4235
- for (let pi = 0; pi < count; pi++) {
4236
- const internalId = docIds[pi];
4237
- const fieldIndex = fieldNameIndices[pi];
4238
- if (searchable[fieldIndex] === 0) continue;
4239
- const avgLen = averageLengths[fieldIndex];
4240
- const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
4241
- const termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequencies[pi], idf, actualFieldLength, avgLen, k1, b);
4242
- addScore(scoreBuffer, internalId, termScore * boosts[fieldIndex]);
4243
- }
4244
- continue;
4245
- }
4246
- for (let pi = 0; pi < count; pi++) {
4247
- const internalId = docIds[pi];
4248
- if (hasDeleted && deletedDocs.has(internalId)) continue;
4249
- if (filterBitset && !bitsetHas(filterBitset, internalId)) continue;
4250
- const fieldIndex = fieldNameIndices[pi];
4251
- if (searchable[fieldIndex] === 0) continue;
4252
- const termFrequency = termFrequencies[pi];
4253
- const fieldBoost = boosts[fieldIndex];
4254
- const avgLen = averageLengths[fieldIndex];
4255
- const actualFieldLength = fieldLengthOf(lengthColumns, fieldIndex, internalId, avgLen);
4256
- let termScore = scoresAreZero ? 0 : computeBM25WithIDF(termFrequency, idf, actualFieldLength, avgLen, k1, b);
4257
- termScore *= fieldBoost;
4258
- addScore(scoreBuffer, internalId, termScore);
4259
- if (components !== null) {
4260
- recordComponents(
4261
- components,
4262
- internalId,
4263
- fieldNames[fieldIndex],
4264
- match.token,
4265
- termFrequency,
4266
- actualFieldLength,
4267
- idf
4268
- );
4269
- }
4270
- }
4299
+ const group = termGroupOf(termGroups, qt.token);
4300
+ for (const m of typoMatchesFor(state.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
4301
+ scorePostings(resolveMatch(m.token, m.postingList), false);
4271
4302
  }
4303
+ if (group !== void 0) scoreGroup(resolveGroupMatches(groupContext, group), false);
4272
4304
  }
4273
4305
  }
4274
4306
  const totalMatched = scoreBuffer.touchedCount;
@@ -4477,6 +4509,7 @@ function encodeSegmentState(state) {
4477
4509
  totalFieldLengths: fieldLengths.totals,
4478
4510
  docFrequencies,
4479
4511
  surfaceForms: state.surfaceRegistry.size() === 0 ? null : state.surfaceRegistry.serialize(),
4512
+ partialSpellings: state.partialWords.spellings(),
4480
4513
  ...encodeFieldIndexes(state, remap)
4481
4514
  };
4482
4515
  }
@@ -4595,6 +4628,8 @@ function mergeSegmentPayload(target, payload, documents) {
4595
4628
  target.surfaceRegistry.add(surface, value[1], value[0]);
4596
4629
  }
4597
4630
  }
4631
+ const mergedIds = Array.from({ length: payload.docIds.length }, (_, at) => ordinalBase + at);
4632
+ target.partialWords.afterMerge(payload.partialSpellings, mergedIds);
4598
4633
  target.sortColumns = null;
4599
4634
  target.scoreBuffer = null;
4600
4635
  }
@@ -4884,72 +4919,7 @@ function deserializePartition(state, data, clearFn, schema) {
4884
4919
  }
4885
4920
  loadPatternIndexes(state, data.fieldIndexes.pattern, schema);
4886
4921
  rebuildSizeIndexes(state, schema);
4887
- }
4888
-
4889
- // src/core/partition/suggestions.ts
4890
- function verbatimOccurrences(state, token) {
4891
- const postingList = state.invertedIdx.lookup(token);
4892
- if (!postingList) return 0;
4893
- const derived = postingList.totalTermFrequency - state.surfaceRegistry.stemChangedTotalFor(token);
4894
- return derived > 0 ? derived : 0;
4895
- }
4896
- function suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit) {
4897
- if (limit <= 0 || surfacePrefix.length === 0) return [];
4898
- const groups = /* @__PURE__ */ new Map();
4899
- for (const candidate of state.surfaceRegistry.candidatesForPrefix(surfacePrefix)) {
4900
- const postingList = state.invertedIdx.lookup(candidate.token);
4901
- if (!postingList || postingList.docIdSet.size === 0) continue;
4902
- let group = groups.get(candidate.token);
4903
- if (!group) {
4904
- group = { documentFrequency: postingList.docIdSet.size, surfaces: [] };
4905
- groups.set(candidate.token, group);
4906
- }
4907
- group.surfaces.push({ surface: candidate.surface, occurrences: candidate.occurrences });
4908
- }
4909
- const termPrefixes = state.surfaceRegistry.size() === 0 && stemmedPrefix !== surfacePrefix && stemmedPrefix.length > 0 ? [surfacePrefix, stemmedPrefix] : [surfacePrefix];
4910
- for (const prefix of termPrefixes) {
4911
- for (const suggestion of state.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
4912
- const occurrences = verbatimOccurrences(state, suggestion.term);
4913
- if (occurrences === 0) continue;
4914
- let group = groups.get(suggestion.term);
4915
- if (!group) {
4916
- group = { documentFrequency: suggestion.documentFrequency, surfaces: [] };
4917
- groups.set(suggestion.term, group);
4918
- } else if (group.surfaces.some((s) => s.surface === suggestion.term)) {
4919
- continue;
4920
- }
4921
- group.surfaces.push({ surface: suggestion.term, occurrences });
4922
- }
4923
- }
4924
- const results = [];
4925
- for (const [token, group] of groups) {
4926
- results.push({ token, documentFrequency: group.documentFrequency, surfaces: group.surfaces });
4927
- }
4928
- results.sort((a, b) => b.documentFrequency - a.documentFrequency || compareCodePoints(a.token, b.token));
4929
- if (results.length > limit) results.length = limit;
4930
- return results;
4931
- }
4932
- function expandTermPrefix(state, surfacePrefix, stemmedToken, maxExpansions) {
4933
- if (maxExpansions <= 0 || surfacePrefix.length === 0) return [];
4934
- const dfByToken = /* @__PURE__ */ new Map();
4935
- for (const candidate of state.surfaceRegistry.candidatesForPrefix(surfacePrefix)) {
4936
- if (dfByToken.has(candidate.token)) continue;
4937
- const postingList = state.invertedIdx.lookup(candidate.token);
4938
- if (!postingList || postingList.docIdSet.size === 0) continue;
4939
- dfByToken.set(candidate.token, postingList.docIdSet.size);
4940
- }
4941
- const termPrefixes = state.surfaceRegistry.size() === 0 && stemmedToken !== surfacePrefix && stemmedToken.length > 0 ? [surfacePrefix, stemmedToken] : [surfacePrefix];
4942
- for (const prefix of termPrefixes) {
4943
- for (const suggestion of state.invertedIdx.prefixSearch(prefix, Number.MAX_SAFE_INTEGER)) {
4944
- if (dfByToken.has(suggestion.term)) continue;
4945
- if (verbatimOccurrences(state, suggestion.term) === 0) continue;
4946
- dfByToken.set(suggestion.term, suggestion.documentFrequency);
4947
- }
4948
- }
4949
- const entries = Array.from(dfByToken.entries());
4950
- entries.sort((a, b) => b[1] - a[1] || compareCodePoints(a[0], b[0]));
4951
- if (entries.length > maxExpansions) entries.length = maxExpansions;
4952
- return entries.map((e) => e[0]);
4922
+ state.partialWords.forgetSpellings();
4953
4923
  }
4954
4924
 
4955
4925
  // src/core/partition/index.ts
@@ -4966,9 +4936,10 @@ function partitionStateOf(partition) {
4966
4936
  }
4967
4937
  function createPartitionIndex(partitionId, trackPositions = true) {
4968
4938
  const fieldNameTable = { names: [], indexMap: /* @__PURE__ */ new Map() };
4939
+ const docStore = createDocumentStore();
4969
4940
  const state = {
4970
4941
  invertedIdx: createInvertedIndex(fieldNameTable),
4971
- docStore: createDocumentStore(),
4942
+ docStore,
4972
4943
  stats: createPartitionStats(),
4973
4944
  surfaceRegistry: createSurfaceRegistry(),
4974
4945
  numericIndexes: /* @__PURE__ */ new Map(),
@@ -4978,6 +4949,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
4978
4949
  patternIndexes: /* @__PURE__ */ new Map(),
4979
4950
  sizeIndexes: /* @__PURE__ */ new Map(),
4980
4951
  fieldNameTable,
4952
+ partialWords: createGrowingPartialWords({ docStore }),
4981
4953
  flatSchemaCache: null,
4982
4954
  lastSchemaRef: null,
4983
4955
  trackPositions,
@@ -4998,6 +4970,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
4998
4970
  state.geoIndexes.clear();
4999
4971
  state.patternIndexes.clear();
5000
4972
  state.sizeIndexes.clear();
4973
+ state.partialWords.clear();
5001
4974
  state.stats.deserialize({ totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} });
5002
4975
  state.flatSchemaCache = null;
5003
4976
  state.lastSchemaRef = null;
@@ -5026,7 +4999,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
5026
4999
  }
5027
5000
  }
5028
5001
  const internalId = state.docStore.ensureInternalId(docId);
5029
- const { fieldLengths, tokensByField } = indexDocument(
5002
+ const { fieldLengths, tokensByField, partialSpellings } = indexDocument(
5030
5003
  state,
5031
5004
  docId,
5032
5005
  document,
@@ -5040,6 +5013,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
5040
5013
  state.docStore.store(docId, document, fieldLengths);
5041
5014
  }
5042
5015
  state.stats.addDocument(fieldLengths, tokensByField);
5016
+ state.partialWords.afterDocument(partialFieldsOf(schema), internalId, partialSpellings, 1);
5043
5017
  recordSortValues(state, internalId, document);
5044
5018
  },
5045
5019
  remove(docId, schema, language, options) {
@@ -5050,7 +5024,7 @@ function createPartitionIndex(partitionId, trackPositions = true) {
5050
5024
  partitionId
5051
5025
  });
5052
5026
  }
5053
- const { fieldLengths, tokensByField } = removeFromIndexes(
5027
+ const { fieldLengths, tokensByField, partialSpellings } = removeFromIndexes(
5054
5028
  state,
5055
5029
  docId,
5056
5030
  stored,
@@ -5061,6 +5035,9 @@ function createPartitionIndex(partitionId, trackPositions = true) {
5061
5035
  const internalId = state.docStore.getInternalId(docId);
5062
5036
  state.docStore.remove(docId);
5063
5037
  state.stats.removeDocument(fieldLengths, tokensByField);
5038
+ if (internalId !== void 0) {
5039
+ state.partialWords.afterDocument(partialFieldsOf(schema), internalId, partialSpellings, -1);
5040
+ }
5064
5041
  forgetSortValues(state, internalId);
5065
5042
  },
5066
5043
  beginBatch() {
@@ -5096,29 +5073,19 @@ function createPartitionIndex(partitionId, trackPositions = true) {
5096
5073
  const fields = schemaFieldsOf(schema);
5097
5074
  const needsTextReindex = textFieldsChanged(stored.fields, document, fields);
5098
5075
  if (needsTextReindex) {
5099
- const { fieldLengths: oldFieldLengths, tokensByField: oldTokens } = removeFromIndexes(
5100
- state,
5101
- docId,
5102
- stored,
5103
- fields,
5104
- language,
5105
- options
5106
- );
5107
- state.stats.removeDocument(oldFieldLengths, oldTokens);
5076
+ const previous = removeFromIndexes(state, docId, stored, fields, language, options);
5077
+ state.stats.removeDocument(previous.fieldLengths, previous.tokensByField);
5108
5078
  const previousInternalId = state.docStore.getInternalId(docId);
5079
+ if (previousInternalId !== void 0) {
5080
+ state.partialWords.afterDocument(partialFieldsOf(schema), previousInternalId, previous.partialSpellings, -1);
5081
+ }
5109
5082
  state.docStore.remove(docId);
5110
5083
  forgetSortValues(state, previousInternalId);
5111
5084
  const internalId = state.docStore.ensureInternalId(docId);
5112
- const { fieldLengths: newFieldLengths, tokensByField: newTokens } = indexDocument(
5113
- state,
5114
- docId,
5115
- document,
5116
- fields,
5117
- language,
5118
- options
5119
- );
5120
- state.docStore.store(docId, document, newFieldLengths);
5121
- state.stats.addDocument(newFieldLengths, newTokens);
5085
+ const next = indexDocument(state, docId, document, fields, language, options);
5086
+ state.docStore.store(docId, document, next.fieldLengths);
5087
+ state.stats.addDocument(next.fieldLengths, next.tokensByField);
5088
+ state.partialWords.afterDocument(partialFieldsOf(schema), internalId, next.partialSpellings, 1);
5122
5089
  recordSortValues(state, internalId, document);
5123
5090
  } else {
5124
5091
  updateFieldIndexOnly(state, docId, stored.fields, document, fields);
@@ -5189,7 +5156,13 @@ function createPartitionIndex(partitionId, trackPositions = true) {
5189
5156
  return suggestDisplayTerms(state, surfacePrefix, stemmedPrefix, limit);
5190
5157
  },
5191
5158
  expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
5192
- return expandTermPrefix(state, surfacePrefix, stemmedToken, maxExpansions);
5159
+ return expandTermPrefix([state], surfacePrefix, stemmedToken, maxExpansions);
5160
+ },
5161
+ expandPartialWords(analysis, requests) {
5162
+ return expandPartialWords([state], analysis, requests);
5163
+ },
5164
+ preparePartialWords(analysis) {
5165
+ expandPartialWords([state], analysis, []);
5193
5166
  },
5194
5167
  serialize(indexName, totalPartitions, language, schema) {
5195
5168
  return serializePartition(state, partitionId, indexName, totalPartitions, language, schema);
@@ -5223,7 +5196,7 @@ function frozenSegmentBytes(segment) {
5223
5196
  bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
5224
5197
  }
5225
5198
  for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
5226
- return bytes;
5199
+ return bytes + segment.partialWords.bytes();
5227
5200
  }
5228
5201
 
5229
5202
  // src/core/partition/composite/compaction.ts
@@ -5358,20 +5331,19 @@ function subMatchesView(matches, layout, subIndex) {
5358
5331
 
5359
5332
  // src/core/partition/composite/stats.ts
5360
5333
  function collectCandidateTerms(subs, params) {
5361
- const { queryTokens, prefixExpansion, tolerance = 0, prefixLength = 2, exact = false } = params;
5334
+ const { queryTokens, termGroups, tolerance = 0, prefixLength = 2, exact = false } = params;
5362
5335
  const terms = /* @__PURE__ */ new Set();
5363
5336
  for (const qt of queryTokens) {
5364
- if (prefixExpansion && qt.token === prefixExpansion.token) {
5365
- terms.add(qt.token);
5366
- for (const term of prefixExpansion.terms) terms.add(term);
5367
- continue;
5337
+ const group = termGroupOf(termGroups, qt.token);
5338
+ if (group !== void 0) {
5339
+ for (const member of group.members) terms.add(member.token);
5368
5340
  }
5369
- if (exact || tolerance === 0) {
5341
+ if (group === void 0 && (exact || tolerance === 0)) {
5370
5342
  terms.add(qt.token);
5371
5343
  continue;
5372
5344
  }
5373
5345
  for (const sub of subs) {
5374
- for (const match of sub.invertedIdx.fuzzyLookup(qt.token, tolerance, prefixLength)) {
5346
+ for (const match of typoMatchesFor(sub.invertedIdx, qt.token, group, exact, tolerance, prefixLength)) {
5375
5347
  terms.add(match.token);
5376
5348
  }
5377
5349
  }
@@ -5381,11 +5353,7 @@ function collectCandidateTerms(subs, params) {
5381
5353
  function resolveQueryTermFrequencies(subs, params) {
5382
5354
  const frequencies = /* @__PURE__ */ Object.create(null);
5383
5355
  for (const term of collectCandidateTerms(subs, params)) {
5384
- let total = 0;
5385
- for (const sub of subs) {
5386
- const list = sub.invertedIdx.lookup(term);
5387
- if (list !== void 0) total += list.docIdSet.size;
5388
- }
5356
+ const total = partitionDocumentFrequency(subs, term);
5389
5357
  if (total > 0) frequencies[term] = total;
5390
5358
  }
5391
5359
  return frequencies;
@@ -5571,9 +5539,10 @@ function compositeSortValues(subs, docId, fields, fieldTypes, modes) {
5571
5539
 
5572
5540
  // src/core/partition/composite/suggest.ts
5573
5541
  function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
5542
+ if (limit <= 0) return [];
5574
5543
  const merged = /* @__PURE__ */ new Map();
5575
5544
  for (const sub of subs) {
5576
- for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix, limit)) {
5545
+ for (const suggestion of suggestDisplayTerms(sub, surfacePrefix, stemmedPrefix, Number.MAX_SAFE_INTEGER)) {
5577
5546
  let entry = merged.get(suggestion.token);
5578
5547
  if (entry === void 0) {
5579
5548
  entry = { documentFrequency: 0, surfaceOccurrences: /* @__PURE__ */ new Map() };
@@ -5598,19 +5567,6 @@ function compositeSuggestTerms(subs, surfacePrefix, stemmedPrefix, limit) {
5598
5567
  if (suggestions.length > limit) suggestions.length = limit;
5599
5568
  return suggestions;
5600
5569
  }
5601
- function compositeExpandTermPrefix(subs, surfacePrefix, stemmedToken, maxExpansions) {
5602
- const expansions = [];
5603
- const seen = /* @__PURE__ */ new Set();
5604
- for (const sub of subs) {
5605
- for (const term of expandTermPrefix(sub, surfacePrefix, stemmedToken, maxExpansions)) {
5606
- if (seen.has(term)) continue;
5607
- seen.add(term);
5608
- expansions.push(term);
5609
- if (expansions.length === maxExpansions) return expansions;
5610
- }
5611
- }
5612
- return expansions;
5613
- }
5614
5570
 
5615
5571
  // src/core/partition/composite/index.ts
5616
5572
  function isCompositePartition(partition) {
@@ -5814,7 +5770,13 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5814
5770
  return compositeSuggestTerms(subs(), surfacePrefix, stemmedPrefix, limit);
5815
5771
  },
5816
5772
  expandTermPrefix(surfacePrefix, stemmedToken, maxExpansions) {
5817
- return compositeExpandTermPrefix(subs(), surfacePrefix, stemmedToken, maxExpansions);
5773
+ return expandTermPrefix(subs(), surfacePrefix, stemmedToken, maxExpansions);
5774
+ },
5775
+ expandPartialWords(analysis, requests) {
5776
+ return expandPartialWords(subs(), analysis, requests);
5777
+ },
5778
+ preparePartialWords(analysis) {
5779
+ expandPartialWords(subs(), analysis, []);
5818
5780
  },
5819
5781
  get(docId, projection) {
5820
5782
  const owner = frozenOwner(docId);
@@ -5875,6 +5837,11 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
5875
5837
  }
5876
5838
 
5877
5839
  // src/partitioning/insert-options.ts
5840
+ function resolvePartialWordAnalysis(config, language, analysis) {
5841
+ const fields = partialFieldsOf(config.schema);
5842
+ if (fields.length === 0) return null;
5843
+ return { fields, language, options: resolvePartitionInsertOptions(config, analysis) };
5844
+ }
5878
5845
  function resolvePartitionInsertOptions(config, analysis, options) {
5879
5846
  const applyStrict = config.strict === true;
5880
5847
  const applyAnalyzer = analysis.stopWords !== void 0 || analysis.customTokenizer !== void 0;
@@ -5933,8 +5900,10 @@ function createCapacityChecks(source) {
5933
5900
  // src/partitioning/manager/index.ts
5934
5901
  function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
5935
5902
  requirePatternSearchFor(config.schema);
5903
+ requirePartialWordSearchFor(config.schema);
5936
5904
  const count = initialPartitionCount ?? 1;
5937
5905
  const analysis = resolveIndexAnalysis(config);
5906
+ const partialWordAnalysis = resolvePartialWordAnalysis(config, language, analysis);
5938
5907
  const trackPositions = config.trackPositions ?? true;
5939
5908
  const vecIndexes = vectorIndexes ?? /* @__PURE__ */ new Map();
5940
5909
  let partitions = [];
@@ -5954,6 +5923,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
5954
5923
  function resolveInsertOptions(options) {
5955
5924
  return resolvePartitionInsertOptions(config, analysis, options);
5956
5925
  }
5926
+ function preparePartialWords(partitionId) {
5927
+ if (partialWordAnalysis !== null) partitions[partitionId].preparePartialWords(partialWordAnalysis);
5928
+ }
5957
5929
  function locateDocument(docId) {
5958
5930
  return docPartitionMap.get(docId);
5959
5931
  }
@@ -6009,6 +5981,9 @@ function createPartitionManager(indexName, config, language, router, initialPart
6009
5981
  get analysis() {
6010
5982
  return analysis;
6011
5983
  },
5984
+ get partialWordAnalysis() {
5985
+ return partialWordAnalysis;
5986
+ },
6012
5987
  getPartition(partitionId) {
6013
5988
  validatePartitionId(partitionId);
6014
5989
  return partitions[partitionId];
@@ -6085,6 +6060,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
6085
6060
  rebuildTextIndex(partitionId) {
6086
6061
  validatePartitionId(partitionId);
6087
6062
  partitions[partitionId].rebuildTextIndex(config.schema, language, resolveInsertOptions());
6063
+ preparePartialWords(partitionId);
6088
6064
  },
6089
6065
  beginBatchRemove() {
6090
6066
  for (let i = 0; i < partitions.length; i++) {
@@ -6186,6 +6162,7 @@ function createPartitionManager(indexName, config, language, router, initialPart
6186
6162
  deserializePartition(partitionId, data) {
6187
6163
  validatePartitionId(partitionId);
6188
6164
  partitions[partitionId].deserialize(data, config.schema);
6165
+ preparePartialWords(partitionId);
6189
6166
  rebuildDocPartitionMapForPartition(partitionId);
6190
6167
  },
6191
6168
  getAggregateStats() {
@@ -8681,6 +8658,15 @@ function decodeStoredDocument(bytes, discardedFields) {
8681
8658
  }
8682
8659
  return document;
8683
8660
  }
8661
+ function* storedDocumentsOf(data) {
8662
+ const count = data.offsets.length - 1;
8663
+ for (let ordinal = 0; ordinal < count; ordinal++) {
8664
+ yield decodeStoredDocument(
8665
+ data.blob.subarray(data.offsets[ordinal], data.offsets[ordinal + 1]),
8666
+ data.discardedFields
8667
+ );
8668
+ }
8669
+ }
8684
8670
  function decodeEveryStoredDocument(data) {
8685
8671
  const count = data.offsets.length - 1;
8686
8672
  for (let ordinal = 0; ordinal < count; ordinal++) {
@@ -8724,6 +8710,9 @@ function wrapEncodedDocumentTable(data) {
8724
8710
  decodedBytes += documentValueBytes(document);
8725
8711
  }
8726
8712
  return document;
8713
+ },
8714
+ readAt(ordinal) {
8715
+ return decoded[ordinal] ?? decodeStoredDocument(blob.subarray(offsets[ordinal], offsets[ordinal + 1]), data.discardedFields);
8727
8716
  }
8728
8717
  };
8729
8718
  }
@@ -8737,75 +8726,15 @@ function wrapDocumentArray(documents) {
8737
8726
  byteLength: residentBytes,
8738
8727
  docAt(ordinal) {
8739
8728
  return documents[ordinal];
8729
+ },
8730
+ readAt(ordinal) {
8731
+ return documents[ordinal];
8740
8732
  }
8741
8733
  };
8742
8734
  }
8743
8735
 
8744
- // src/core/partition/frozen/string-blob.ts
8745
- var SURROGATE = /[\ud800-\udfff]/;
8746
- var encoder = new TextEncoder();
8747
- function encodeOneByOne(strings) {
8748
- const encoded = new Array(strings.length);
8749
- let blobLength = 0;
8750
- for (let i = 0; i < strings.length; i++) {
8751
- encoded[i] = encoder.encode(strings[i]);
8752
- blobLength += encoded[i].length;
8753
- }
8754
- const blob = new Uint8Array(blobLength);
8755
- const offsets = new Uint32Array(strings.length + 1);
8756
- let cursor = 0;
8757
- for (let i = 0; i < strings.length; i++) {
8758
- blob.set(encoded[i], cursor);
8759
- cursor += encoded[i].length;
8760
- offsets[i + 1] = cursor;
8761
- }
8762
- return { blob, offsets };
8763
- }
8764
- function basicPlaneUtf8Bytes(text) {
8765
- let bytes = text.length;
8766
- for (let i = 0; i < text.length; i++) {
8767
- const unit = text.charCodeAt(i);
8768
- if (unit >= 128) bytes += unit < 2048 ? 1 : 2;
8769
- }
8770
- return bytes;
8771
- }
8772
- function encodeStringBlob(strings) {
8773
- const joined = strings.join("");
8774
- if (SURROGATE.test(joined)) return encodeOneByOne(strings);
8775
- const blob = encoder.encode(joined);
8776
- const offsets = new Uint32Array(strings.length + 1);
8777
- let cursor = 0;
8778
- for (let i = 0; i < strings.length; i++) {
8779
- cursor += basicPlaneUtf8Bytes(strings[i]);
8780
- offsets[i + 1] = cursor;
8781
- }
8782
- return cursor === blob.length ? { blob, offsets } : encodeOneByOne(strings);
8783
- }
8784
- function orderByComparator(strings) {
8785
- const order = new Array(strings.length);
8786
- for (let i = 0; i < strings.length; i++) order[i] = i;
8787
- return order.sort((a, b) => compareCodePoints(strings[a], strings[b]));
8788
- }
8789
- function alreadyInCodePointOrder(strings) {
8790
- for (let i = 1; i < strings.length; i++) {
8791
- if (compareCodePoints(strings[i - 1], strings[i]) > 0) return false;
8792
- }
8793
- return true;
8794
- }
8795
- function codePointOrder(strings) {
8796
- if (alreadyInCodePointOrder(strings)) return strings.map((_, i) => i);
8797
- if (SURROGATE.test(strings.join(""))) return orderByComparator(strings);
8798
- const positions = /* @__PURE__ */ new Map();
8799
- for (let i = 0; i < strings.length; i++) positions.set(strings[i], i);
8800
- if (positions.size !== strings.length) return orderByComparator(strings);
8801
- const order = new Array(strings.length);
8802
- const sorted = [...strings].sort();
8803
- for (let i = 0; i < sorted.length; i++) order[i] = positions.get(sorted[i]) ?? 0;
8804
- return order;
8805
- }
8806
-
8807
8736
  // src/core/partition/frozen/external-ids.ts
8808
- var encoder2 = new TextEncoder();
8737
+ var encoder = new TextEncoder();
8809
8738
  var decoder = new TextDecoder();
8810
8739
  function encodeExternalIdTableData(docIds) {
8811
8740
  const { blob, offsets } = encodeStringBlob(docIds);
@@ -8837,7 +8766,7 @@ function wrapExternalIdTable(data) {
8837
8766
  bytes: blob.byteLength + offsets.byteLength + sortedOrdinals.byteLength,
8838
8767
  idAt,
8839
8768
  ordinalOf(docId) {
8840
- const query = encoder2.encode(docId);
8769
+ const query = encoder.encode(docId);
8841
8770
  let lo = 0;
8842
8771
  let hi = count;
8843
8772
  while (lo < hi) {
@@ -9123,6 +9052,10 @@ function createFrozenInvertedReader(table, views) {
9123
9052
  if (at < 0) return void 0;
9124
9053
  return views.viewReadOnce(table.payloadSlot(at), table.documentFrequencyAt(at));
9125
9054
  },
9055
+ documentFrequency(token) {
9056
+ const at = table.find(token);
9057
+ return at < 0 ? 0 : table.documentFrequencyAt(at);
9058
+ },
9126
9059
  fuzzyLookup(token, tolerance, prefixLength) {
9127
9060
  if (tolerance === 0) {
9128
9061
  const at = table.find(token);
@@ -9277,7 +9210,7 @@ function createLazySurfaceReader(load) {
9277
9210
  }
9278
9211
 
9279
9212
  // src/core/partition/frozen/token-table.ts
9280
- var encoder3 = new TextEncoder();
9213
+ var encoder2 = new TextEncoder();
9281
9214
  var decoder3 = new TextDecoder();
9282
9215
  function compareBytes2(blob, start, end, query) {
9283
9216
  const length = end - start;
@@ -9341,7 +9274,7 @@ function wrapFrozenTokenTable(data) {
9341
9274
  size,
9342
9275
  bytes: blob.byteLength + offsets.byteLength + payloadSlots.byteLength + documentFrequencies.byteLength,
9343
9276
  find(token) {
9344
- const query = encoder3.encode(token);
9277
+ const query = encoder2.encode(token);
9345
9278
  const at = lowerBound3(query);
9346
9279
  if (at >= size) return -1;
9347
9280
  return compareBytes2(blob, offsets[at], offsets[at + 1], query) === 0 ? at : -1;
@@ -9364,7 +9297,7 @@ function wrapFrozenTokenTable(data) {
9364
9297
  return documentFrequencies[sortedIndex];
9365
9298
  },
9366
9299
  prefixRange(prefix) {
9367
- const bytes = encoder3.encode(prefix);
9300
+ const bytes = encoder2.encode(prefix);
9368
9301
  const start = lowerBound3(bytes);
9369
9302
  return { start, end: prefixEnd(start, bytes) };
9370
9303
  },
@@ -9382,8 +9315,8 @@ function wrapFrozenTokenTable(data) {
9382
9315
  fromCodePoint = unit;
9383
9316
  toCodePoint = nextCodePoint(unit);
9384
9317
  }
9385
- const from = encoder3.encode(String.fromCodePoint(fromCodePoint));
9386
- const to = encoder3.encode(String.fromCodePoint(toCodePoint));
9318
+ const from = encoder2.encode(String.fromCodePoint(fromCodePoint));
9319
+ const to = encoder2.encode(String.fromCodePoint(toCodePoint));
9387
9320
  return { start: lowerBound3(from), end: lowerBound3(to) };
9388
9321
  }
9389
9322
  };
@@ -9445,6 +9378,35 @@ function sharedSurfaceTable(payload) {
9445
9378
  if (table.counts.length === 0) return null;
9446
9379
  return { blob: sharedUint8(table.blob), offsets: sharedUint32(table.offsets), counts: sharedUint32(table.counts) };
9447
9380
  }
9381
+ function sharePartialWordTable(table) {
9382
+ if (typeof SharedArrayBuffer !== "function") return table;
9383
+ return {
9384
+ fields: [...table.fields],
9385
+ spellings: sharedUint8(table.spellings),
9386
+ spellingOffsets: sharedUint32(table.spellingOffsets),
9387
+ spellingTokens: sharedUint32(table.spellingTokens),
9388
+ spellingCounts: sharedUint32(table.spellingCounts),
9389
+ runs: sharedUint32(table.runs),
9390
+ runOffsets: sharedUint32(table.runOffsets),
9391
+ runEntries: sharedUint32(table.runEntries)
9392
+ };
9393
+ }
9394
+ function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
9395
+ let source = spellings;
9396
+ if (source === null && analysis !== null && analysis.fields.length > 0) {
9397
+ source = encodePartialSpellings(analysis.fields, spellingCountsOf(storedDocumentsOf(documentData), analysis));
9398
+ }
9399
+ if (source === null || source.fields.length === 0) return null;
9400
+ const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
9401
+ return table === null ? null : sharePartialWordTable(table);
9402
+ }
9403
+ function shareSegmentArrays(arrays, analysis, segmentId) {
9404
+ return {
9405
+ ...arrays,
9406
+ segmentId: segmentId ?? generateId(),
9407
+ partialWords: sharedPartialWordsOf(arrays.tokenTable, null, arrays.documentTable, analysis)
9408
+ };
9409
+ }
9448
9410
  function segmentArraysOf(payload, documentData) {
9449
9411
  const tokenTable = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
9450
9412
  const idTable = encodeExternalIdTableData(payload.docIds);
@@ -9473,11 +9435,11 @@ function segmentArraysOf(payload, documentData) {
9473
9435
  surfaceTable: surfaceTable === null || surfaceTable.counts.length === 0 ? null : surfaceTable
9474
9436
  };
9475
9437
  }
9476
- function freezeSegmentShared(payload, documents, segmentId) {
9438
+ function freezeSegmentShared(payload, documents, segmentId, analysis = null) {
9477
9439
  if (typeof SharedArrayBuffer !== "function") return null;
9478
- return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId);
9440
+ return freezeEncodedSegmentShared(payload, encodeDocumentTableData(documents), segmentId, analysis);
9479
9441
  }
9480
- function freezeEncodedSegmentShared(payload, documentData, segmentId) {
9442
+ function freezeEncodedSegmentShared(payload, documentData, segmentId, analysis = null) {
9481
9443
  if (typeof SharedArrayBuffer !== "function") return null;
9482
9444
  const tokenData = encodeFrozenTokenTableData(payload.tokens, payload.docFrequencies);
9483
9445
  const idData = encodeExternalIdTableData(payload.docIds);
@@ -9544,7 +9506,8 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId) {
9544
9506
  offsets: sharedUint32(entry.offsets),
9545
9507
  postings: sharedUint32(entry.postings)
9546
9508
  })),
9547
- surfaceTable: sharedSurfaceTable(payload)
9509
+ surfaceTable: sharedSurfaceTable(payload),
9510
+ partialWords: sharedPartialWordsOf(tokenData, payload.partialSpellings, documentData, analysis)
9548
9511
  };
9549
9512
  }
9550
9513
 
@@ -9580,7 +9543,12 @@ function buildGeoReaders(entries) {
9580
9543
  }
9581
9544
  return readers;
9582
9545
  }
9583
- function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot) {
9546
+ function* survivingDocuments(documentSource, tombstones) {
9547
+ for (let ordinal = 0; ordinal < documentSource.count; ordinal++) {
9548
+ if (!tombstones.has(ordinal)) yield documentSource.readAt(ordinal);
9549
+ }
9550
+ }
9551
+ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot, partialWordTable) {
9584
9552
  const tombstones = createFrozenTombstones();
9585
9553
  const postingViews = createFrozenPostingViews(source, tombstones);
9586
9554
  const lengthColumns = /* @__PURE__ */ new Map();
@@ -9588,6 +9556,10 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
9588
9556
  lengthColumns.set(source.fieldLengthNames[i], source.fieldLengthColumns[i]);
9589
9557
  }
9590
9558
  const docStore = createFrozenDocTable(source.idTable, lengthColumns, documentSource, tombstones);
9559
+ const partialWords = createFrozenPartialWords(partialWordTable, source.tokenTable, {
9560
+ documentAt: (ordinal) => documentSource.readAt(ordinal),
9561
+ survivors: () => survivingDocuments(documentSource, tombstones)
9562
+ });
9591
9563
  const numericIndexes = /* @__PURE__ */ new Map();
9592
9564
  for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
9593
9565
  const sizeIndexes = /* @__PURE__ */ new Map();
@@ -9620,6 +9592,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
9620
9592
  patternIndexes,
9621
9593
  sizeIndexes,
9622
9594
  fieldNameTable,
9595
+ partialWords,
9623
9596
  trackPositions: source.positionOffsets !== null,
9624
9597
  flatSchemaCache: null,
9625
9598
  lastSchemaRef: null,
@@ -9639,6 +9612,7 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
9639
9612
  if (ordinal === void 0) return false;
9640
9613
  const added = tombstones.add(ordinal);
9641
9614
  if (added) {
9615
+ partialWords.afterRemoval(ordinal);
9642
9616
  docStore.releaseSortedDocIds();
9643
9617
  segment.sortColumns = null;
9644
9618
  segment.scoreBuffer = null;
@@ -9654,18 +9628,20 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
9654
9628
  return segment;
9655
9629
  }
9656
9630
  function createFrozenSegment(payload, documents, segmentId) {
9631
+ const tokenTable = buildFrozenTokenTable(payload.tokens, payload.docFrequencies);
9657
9632
  return assembleFrozenSegment(
9658
9633
  segmentId ?? generateId(),
9659
9634
  {
9660
9635
  ...payload,
9661
- tokenTable: buildFrozenTokenTable(payload.tokens, payload.docFrequencies),
9636
+ tokenTable,
9662
9637
  idTable: buildExternalIdTable(payload.docIds),
9663
9638
  documentTable: null,
9664
9639
  readSurfaceForms: () => payload.surfaceForms,
9665
9640
  docFrequencies: () => payload.docFrequencies
9666
9641
  },
9667
9642
  wrapDocumentArray(documents),
9668
- null
9643
+ null,
9644
+ payload.partialSpellings === null ? null : buildPartialWordTableFrom(payload.partialSpellings, tokenTable)
9669
9645
  );
9670
9646
  }
9671
9647
  function createSharedFrozenSegment(snapshot) {
@@ -9687,7 +9663,8 @@ function createSharedFrozenSegment(snapshot) {
9687
9663
  }
9688
9664
  },
9689
9665
  wrapEncodedDocumentTable(snapshot.documentTable),
9690
- snapshot
9666
+ snapshot,
9667
+ snapshot.partialWords
9691
9668
  );
9692
9669
  }
9693
9670
  var FORBIDDEN_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
@@ -10269,10 +10246,10 @@ function mergeDocuments2(inputs, documentCount) {
10269
10246
  }
10270
10247
  return { blob, offsets };
10271
10248
  }
10272
- function mergeFrozenSegments(segments) {
10249
+ function mergeFrozenSegments(segments, analysis) {
10273
10250
  if (typeof SharedArrayBuffer !== "function") return null;
10274
10251
  const merged = mergeSegmentContents(segments.map((segment) => ({ segment })));
10275
- return freezeEncodedSegmentShared(merged.payload, merged.documents);
10252
+ return freezeEncodedSegmentShared(merged.payload, merged.documents, void 0, analysis);
10276
10253
  }
10277
10254
  function mergeSegmentContents(selections) {
10278
10255
  const docIds = [];
@@ -10301,6 +10278,7 @@ function mergeSegmentContents(selections) {
10301
10278
  totalFieldLengths: lengths.totalFieldLengths,
10302
10279
  docFrequencies: postings.docFrequencies,
10303
10280
  surfaceForms: mergeSurfaceForms2(inputs),
10281
+ partialSpellings: null,
10304
10282
  ...mergeFieldIndexes2(inputs)
10305
10283
  };
10306
10284
  return { payload, documents: mergeDocuments2(inputs, documentCount) };
@@ -10571,14 +10549,18 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
10571
10549
  patternIndexes: /* @__PURE__ */ new Map(),
10572
10550
  sizeIndexes: /* @__PURE__ */ new Map()
10573
10551
  };
10552
+ const partialFields = partialFieldsOf(schema);
10553
+ const spellings = /* @__PURE__ */ new Map();
10554
+ let fieldSpellings = null;
10574
10555
  let positionOffset = 0;
10575
10556
  const addToken = (token, position, surface) => {
10576
10557
  postings.addToken(token, positionOffset + position);
10577
10558
  surfaces.count(surface, token);
10559
+ if (fieldSpellings !== null) countSpelling(fieldSpellings, token, surface ?? token, 1);
10578
10560
  };
10579
10561
  for (let ordinal = 0; ordinal < documents.length; ordinal++) {
10580
10562
  const document = documents[ordinal].document;
10581
- for (const { path: fieldPath, base: fieldType, pattern } of fields) {
10563
+ for (const { path: fieldPath, base: fieldType, pattern, partial } of fields) {
10582
10564
  const value = getNestedValue(document, fieldPath);
10583
10565
  if (value === void 0 || value === null) continue;
10584
10566
  recordValueSize(filters.sizeIndexes, ordinal, fieldPath, fieldType, value);
@@ -10587,6 +10569,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
10587
10569
  indexFilterValue(filters, ordinal, fieldPath, fieldType, value);
10588
10570
  continue;
10589
10571
  }
10572
+ fieldSpellings = partial ? spellings : null;
10590
10573
  postings.beginField();
10591
10574
  positionOffset = 0;
10592
10575
  for (const text of fieldType === "string" ? [value] : value) {
@@ -10607,6 +10590,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
10607
10590
  fieldLengthColumns: lengths.columns,
10608
10591
  totalFieldLengths: lengths.totalsByName(),
10609
10592
  surfaceForms: surfaces.encode(),
10593
+ partialSpellings: partialFields.length === 0 ? null : encodePartialSpellings(partialFields, spellings),
10610
10594
  ...encodeFieldIndexes(filters, identity)
10611
10595
  };
10612
10596
  }
@@ -10673,7 +10657,7 @@ function runSegmentAction(entry, action) {
10673
10657
  }
10674
10658
  case "compactSegments": {
10675
10659
  const partition = requireComposite(entry, action.indexName, action.partitionId, "compact");
10676
- return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds));
10660
+ return mergeFrozenSegments(partition.frozenSegmentsById(action.segmentIds), entry.manager.partialWordAnalysis);
10677
10661
  }
10678
10662
  case "swapSegments": {
10679
10663
  requireComposite(entry, action.indexName, action.partitionId, "swap");
@@ -11150,4 +11134,4 @@ function createDirectExecutor(options) {
11150
11134
  };
11151
11135
  }
11152
11136
 
11153
- export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerPatternSearch, registerStopWords, registerTokenizer, requirePatternSearchFor, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };
11137
+ export { CHECKSUM_YIELD_CHUNK_BYTES, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, MAX_FILE_BYTES, NATIVE_CRC32_MIN_BYTES, NO_LIVE_VECTOR_AT_POSITION, SHUTDOWN_TIMEOUT_MS, VALID_HNSW_METRICS, VECTOR_FILE_MAX_VECTORS, VECTOR_FILE_PAYLOAD_VERSION, assembleGraphParts, buildSegmentPayload, bytesToVectors, countDead, createDirectExecutor, createFrozenSegment, createPartitionIndex, createPartitionManager, createPartitionRouter, createSharedFrozenSegment, deadBitsWhere, deadBytesFor, decodeEveryStoredDocument, decodeMapWithoutFields, decodeStoredDocument, decodeVectorFilePayload, decodeVectorGraphPartPayload, decodeVectorGraphPayload, decodeVectorIndexParts, encodeSegmentState, freezeSegmentShared, generateId, getStopWords, getTokenizer, graphPartsOfEarlierPayload, hasStopWords, hasTokenizer, isCompositePartition, isDead, mergeFrozenSegments, mergeSegmentContents, missingWorkerEntry, nodeThreadId, partitionStateOf, readHeapStatistics, registerPartialWordSearch, registerStopWords, registerTokenizer, requirePartialWordSearchFor, resolveIndexAnalysis, resolvePartitionInsertOptions, resolveRequestThreadCount, resolveWorkerCount, resolveWorkerEntry, segmentArraysOf, selectionKeeps, shareSegmentArrays, spawnNodeWorker, splitWorkerBudget, survivorDocumentsOf, vectorsPerFile, vectorsToBytes, workerResourceLimits };