@delali/narsil 0.3.1-next.79 → 0.3.1-next.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +3 -3
  2. package/dist/{capabilities-1lEYL3WK.d.ts → capabilities-CcTDWh_P.d.ts} +2 -2
  3. package/dist/{chunk-AMGDG4ZJ.mjs → chunk-23HJ5QXQ.mjs} +1 -1
  4. package/dist/{chunk-R6WBLBTC.mjs → chunk-4QDQEJDV.mjs} +9 -45
  5. package/dist/chunk-DQABW5UQ.mjs +140 -0
  6. package/dist/chunk-GG2FBGVZ.mjs +30 -0
  7. package/dist/{chunk-XSZTF4ZQ.mjs → chunk-HJASKX4F.mjs} +1 -1
  8. package/dist/{chunk-DFW3U5UW.mjs → chunk-K4FZ55Q2.mjs} +21 -192
  9. package/dist/{chunk-PJWG547Z.mjs → chunk-MEOOYKWL.mjs} +3 -3
  10. package/dist/{chunk-55WEJKWT.mjs → chunk-NSXQCE4K.mjs} +17 -63
  11. package/dist/{chunk-PXDQFYD5.mjs → chunk-OSF3SGJV.mjs} +23 -139
  12. package/dist/{chunk-RTR44V2A.mjs → chunk-P2PYEZZE.mjs} +586 -4
  13. package/dist/{chunk-Z3BXCZXS.mjs → chunk-QPYV22XH.mjs} +2 -1
  14. package/dist/{chunk-4344I4MR.mjs → chunk-R2XCJIO7.mjs} +335 -12
  15. package/dist/{chunk-IBR6634K.mjs → chunk-R6A773WK.mjs} +2 -69
  16. package/dist/chunk-WQBCBKY3.mjs +8 -0
  17. package/dist/{chunk-P2BEMMOD.mjs → chunk-YGLSR7BC.mjs} +150 -203
  18. package/dist/chunk-YTJCGKM4.mjs +56 -0
  19. package/dist/{client-ZXcbSViH.d.ts → client-CsybN673.d.ts} +2 -2
  20. package/dist/client.d.ts +6 -6
  21. package/dist/{config-DTe5EbWW.d.ts → config-BCNQtszf.d.ts} +1 -1
  22. package/dist/{config-DHL2KsfD.d.ts → config-CFonR_gY.d.ts} +1 -1
  23. package/dist/direct-executor-VNMQ6NIH.mjs +19 -0
  24. package/dist/distribution/coordinator/etcd.mjs +2 -1
  25. package/dist/distribution/coordinator/in-memory.mjs +2 -1
  26. package/dist/distribution.d.ts +5 -5
  27. package/dist/distribution.mjs +15 -13
  28. package/dist/{engine-BjPBzWkA.d.ts → engine-B0oIF1_r.d.ts} +1 -1
  29. package/dist/index.browser.mjs +113 -516
  30. package/dist/index.d.ts +5 -5
  31. package/dist/index.mjs +14 -12
  32. package/dist/invalidation/filesystem.mjs +2 -1
  33. package/dist/{memory-DW3OJ0im.d.ts → memory-B6SBOKlS.d.ts} +1287 -1287
  34. package/dist/partial-words.mjs +5 -4
  35. package/dist/pattern.mjs +4 -1
  36. package/dist/persistence/durability/checkpoint-worker.d.ts +7 -7
  37. package/dist/persistence/durability/checkpoint-worker.mjs +14 -12
  38. package/dist/react.d.ts +6 -6
  39. package/dist/server.d.ts +8 -8
  40. package/dist/server.mjs +10 -7
  41. package/dist/{tasks-BTbwBqar.d.ts → tasks-CYAQe6qd.d.ts} +1 -1
  42. package/dist/{thread-XBDVXKUE.mjs → thread-P6QJ2Q5O.mjs} +11 -8
  43. package/dist/vector/search-worker.mjs +3 -2
  44. package/dist/workers/entry.mjs +3 -3
  45. package/package.json +9 -7
  46. package/dist/chunk-IOZPBNSG.mjs +0 -55
  47. package/dist/chunk-WFVOBZS4.mjs +0 -485
  48. package/dist/direct-executor-WB2HKU4N.mjs +0 -17
@@ -371,20 +371,6 @@ function appendFoldedCodePoint(codePoint, out) {
371
371
  }
372
372
  for (const foldedCodePoint of folded) out.push(foldedCodePoint);
373
373
  }
374
- function appendFoldedCodePoints(text, out) {
375
- const single = singleFoldTable();
376
- const multi = multiFoldTable();
377
- const cursor = { index: 0, pending: null, pendingIndex: 0 };
378
- for (let codePoint = nextFoldedCodePoint(text, cursor, single, multi); codePoint !== EXHAUSTED; ) {
379
- out.push(codePoint);
380
- codePoint = nextFoldedCodePoint(text, cursor, single, multi);
381
- }
382
- }
383
- function foldedCodePointCount(text) {
384
- const folded = [];
385
- appendFoldedCodePoints(text, folded);
386
- return folded.length;
387
- }
388
374
 
389
375
  // src/core/ordering/constants.ts
390
376
  var SORT_VALUE_MAX_CODE_POINTS = 512;
@@ -797,8 +783,21 @@ function extractVectorFieldsFromSchema(schema) {
797
783
 
798
784
  // src/core/partition/partial-words/registry.ts
799
785
  var registered = null;
786
+ var MODULE_FUNCTIONS = [
787
+ "createGrowingSlot",
788
+ "createFrozenSlot",
789
+ "buildTable",
790
+ "expand",
791
+ "countSpelling",
792
+ "encodeSpellings",
793
+ "spellingCountsOf",
794
+ "highlightMatcher",
795
+ "addTermMembers",
796
+ "highlightWords"
797
+ ];
800
798
  function isPartialWordSearchModule(value) {
801
- return value !== null && typeof value === "object" && value.name === "partialWords" && "createGrowingSlot" in value && typeof value.createGrowingSlot === "function" && "createFrozenSlot" in value && typeof value.createFrozenSlot === "function" && "buildTable" in value && typeof value.buildTable === "function" && "expand" in value && typeof value.expand === "function";
799
+ if (value === null || typeof value !== "object" || value.name !== "partialWords") return false;
800
+ return MODULE_FUNCTIONS.every((name) => typeof Reflect.get(value, name) === "function");
802
801
  }
803
802
  function registerPartialWordSearch(module) {
804
803
  if (!isPartialWordSearchModule(module)) {
@@ -812,12 +811,21 @@ function registerPartialWordSearch(module) {
812
811
  function availablePartialWordSearch() {
813
812
  return registered ?? platformPartialWordSearch();
814
813
  }
814
+ var PARTIAL_WORD_IMPORT_ADVICE = `Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load an index with a partial field`;
815
+ function partialWordModule() {
816
+ const module = availablePartialWordSearch();
817
+ if (module !== null) return module;
818
+ throw new NarsilError(
819
+ ErrorCodes.CONFIG_INVALID,
820
+ `This bundle contains no partial-word search code. ${PARTIAL_WORD_IMPORT_ADVICE}`
821
+ );
822
+ }
815
823
  function requirePartialWordSearchFor(schema) {
816
824
  const fields = partialFieldsOf(schema);
817
825
  if (fields.length === 0 || availablePartialWordSearch() !== null) return;
818
826
  throw new NarsilError(
819
827
  ErrorCodes.CONFIG_INVALID,
820
- `The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load the index`,
828
+ `The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. ${PARTIAL_WORD_IMPORT_ADVICE}`,
821
829
  { field: fields[0] }
822
830
  );
823
831
  }
@@ -851,6 +859,25 @@ function expandPartialWords(subs, analysis, requests) {
851
859
  const module = availablePartialWordSearch();
852
860
  return module === null ? requests.map(() => []) : module.expand(subs, analysis, requests);
853
861
  }
862
+ function countPartialSpelling(counts, token, spelling, change) {
863
+ partialWordModule().countSpelling(counts, token, spelling, change);
864
+ }
865
+ function encodePartialSpellingsOf(fields, counts) {
866
+ return partialWordModule().encodeSpellings(fields, counts);
867
+ }
868
+ function partialSpellingsOfDocuments(documents, analysis) {
869
+ const module = partialWordModule();
870
+ return module.encodeSpellings(analysis.fields, module.spellingCountsOf(documents, analysis));
871
+ }
872
+ function partialWordHighlighter(text, language, partial) {
873
+ return partialWordModule().highlightMatcher(text, language, partial);
874
+ }
875
+ function addPartialWordMembers(query, expand, groupOf) {
876
+ partialWordModule().addTermMembers(query, expand, groupOf);
877
+ }
878
+ function partialWordsToHighlight(queryTokens, written) {
879
+ return partialWordModule().highlightWords(queryTokens, written);
880
+ }
854
881
 
855
882
  // src/core/pattern-index/platform.browser.ts
856
883
  function platformPatternSearch() {
@@ -859,8 +886,22 @@ function platformPatternSearch() {
859
886
 
860
887
  // src/core/pattern-index/registry.ts
861
888
  var registered2 = null;
889
+ var MODULE_FUNCTIONS2 = [
890
+ "createIndex",
891
+ "readArrays",
892
+ "mergeArrays",
893
+ "matchInto",
894
+ "checkExpression",
895
+ "expressionFootprint",
896
+ "largestFootprint",
897
+ "checkOperands",
898
+ "evaluateTests",
899
+ "serializeIndex",
900
+ "decodeIndex"
901
+ ];
862
902
  function isPatternSearchModule(value) {
863
- return value !== null && typeof value === "object" && value.name === "pattern" && "createIndex" in value && typeof value.createIndex === "function" && "readArrays" in value && typeof value.readArrays === "function" && "mergeArrays" in value && typeof value.mergeArrays === "function" && "matchInto" in value && typeof value.matchInto === "function" && "checkExpression" in value && typeof value.checkExpression === "function" && "expressionFootprint" in value && typeof value.expressionFootprint === "function";
903
+ if (value === null || typeof value !== "object" || value.name !== "pattern") return false;
904
+ return MODULE_FUNCTIONS2.every((name) => typeof Reflect.get(value, name) === "function");
864
905
  }
865
906
  function registerPatternSearch(module) {
866
907
  if (!isPatternSearchModule(module)) {
@@ -2584,17 +2625,11 @@ function generateId() {
2584
2625
 
2585
2626
  // src/core/pattern-index/constants.ts
2586
2627
  var DEFAULT_PATTERN_WORK_CAP = 25e6;
2587
- var MAX_PATTERN_TEXT_CODE_POINTS = 1024;
2588
- var MAX_PATTERN_POSITIONS = 2048;
2589
2628
  var MATCHER_STATE_BUDGET_BYTES = 1e6;
2590
- var MATCHER_CODE_POINT_BYTES = 8;
2591
- var MATCHER_SIMULATION_BYTES_PER_POSITION = 16;
2592
- var MATCHER_STATE_OVERHEAD_BYTES = 96;
2593
2629
  var PATTERN_WORK_FLUSH_UNITS = 65536;
2594
- var PATTERN_RUN_CODE_POINTS = 3;
2595
- var MAX_PATTERN_TOLERANCE = 2;
2596
2630
  var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "contains"];
2597
2631
  var PATTERN_EXPRESSION_OPERATORS = ["wildcard", "regex"];
2632
+ var PATTERN_ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
2598
2633
 
2599
2634
  // src/geo/constants.ts
2600
2635
  var DEG_TO_RAD = Math.PI / 180;
@@ -4124,14 +4159,6 @@ function bitsetHas(bs, index) {
4124
4159
  if (word >= bs.length) return false;
4125
4160
  return (bs[word] & 1 << (index & 31)) !== 0;
4126
4161
  }
4127
- function bitsetAnd(a, b) {
4128
- const len = Math.min(a.length, b.length);
4129
- const result = new Uint32Array(len);
4130
- for (let i = 0; i < len; i++) {
4131
- result[i] = a[i] & b[i];
4132
- }
4133
- return result;
4134
- }
4135
4162
  function bitsetAndInto(target, other) {
4136
4163
  const shared2 = Math.min(target.length, other.length);
4137
4164
  for (let i = 0; i < shared2; i++) target[i] &= other[i];
@@ -4145,18 +4172,6 @@ function bitsetAndNotInto(target, removed) {
4145
4172
  const shared2 = Math.min(target.length, removed.length);
4146
4173
  for (let i = 0; i < shared2; i++) target[i] &= ~removed[i];
4147
4174
  }
4148
- function bitsetNot(bs, capacity) {
4149
- const result = new Uint32Array(bs.length);
4150
- const fullWords = capacity >>> 5;
4151
- for (let i = 0; i < fullWords; i++) {
4152
- result[i] = ~bs[i];
4153
- }
4154
- const remainder = capacity & 31;
4155
- if (remainder > 0 && fullWords < bs.length) {
4156
- result[fullWords] = ~bs[fullWords] & (1 << remainder) - 1;
4157
- }
4158
- return result;
4159
- }
4160
4175
  function bitsetIsEmpty(bs) {
4161
4176
  for (let i = 0; i < bs.length; i++) {
4162
4177
  if (bs[i] !== 0) return false;
@@ -4378,24 +4393,7 @@ function computeFacets(state, matched, config, schema) {
4378
4393
  }
4379
4394
 
4380
4395
  // src/core/pattern-index/host.ts
4381
- var NO_SOURCES = [];
4382
- var foldSources = null;
4383
- function sourcesByFold() {
4384
- if (foldSources !== null) return foldSources;
4385
- const sources = /* @__PURE__ */ new Map();
4386
- const add = (source, folded) => {
4387
- const known = sources.get(folded);
4388
- if (known === void 0) sources.set(folded, [source]);
4389
- else known.push(source);
4390
- };
4391
- for (const [source, folded] of singleFoldTable()) add(source, folded);
4392
- foldSources = sources;
4393
- return sources;
4394
- }
4395
- function singleFoldSources(folded) {
4396
- return sourcesByFold().get(folded) ?? NO_SOURCES;
4397
- }
4398
- var PATTERN_HOST = { appendFoldedCodePoint, singleFoldSources };
4396
+ var PATTERN_HOST = { appendFoldedCodePoint, singleFolds: singleFoldTable };
4399
4397
  function patternIndexOf(indexes, fieldPath) {
4400
4398
  let index = indexes.get(fieldPath);
4401
4399
  if (index === void 0) {
@@ -5238,78 +5236,6 @@ function createNumericIndex() {
5238
5236
  };
5239
5237
  }
5240
5238
 
5241
- // src/core/partition/partial-words/constants.ts
5242
- var PARTIAL_WORD_RUN_CODE_POINTS = 3;
5243
-
5244
- // src/core/partition/partial-words/spellings.ts
5245
- var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
5246
- function codePointCountOf(text) {
5247
- let count = 0;
5248
- for (let index = 0; index < text.length; count++) {
5249
- index += (text.codePointAt(index) ?? 0) > LAST_BASIC_PLANE_CODE_POINT2 ? 2 : 1;
5250
- }
5251
- return count;
5252
- }
5253
- function holdsRun(spelling) {
5254
- if (spelling.length > 2 * PARTIAL_WORD_RUN_CODE_POINTS) return true;
5255
- if (spelling.length <= PARTIAL_WORD_RUN_CODE_POINTS) return false;
5256
- return codePointCountOf(spelling) > PARTIAL_WORD_RUN_CODE_POINTS;
5257
- }
5258
- function countSpelling(counts, token, spelling, change) {
5259
- if (!holdsRun(spelling)) return;
5260
- let byToken = counts.get(token);
5261
- if (byToken === void 0) {
5262
- byToken = /* @__PURE__ */ new Map();
5263
- counts.set(token, byToken);
5264
- }
5265
- const next = (byToken.get(spelling) ?? 0) + change;
5266
- if (next > 0) {
5267
- byToken.set(spelling, next);
5268
- return;
5269
- }
5270
- byToken.delete(spelling);
5271
- if (byToken.size === 0) counts.delete(token);
5272
- }
5273
- function textsOf(value) {
5274
- if (typeof value === "string") return [value];
5275
- if (!Array.isArray(value)) return [];
5276
- return value.filter((element) => typeof element === "string");
5277
- }
5278
- function countDocumentSpellings(counts, document2, analysis, direction) {
5279
- const tokenizing = { ...tokenizeOptions(analysis.options), collectSurfaces: true };
5280
- const countOccurrence2 = (token, _position, surface) => {
5281
- countSpelling(counts, token, surface ?? token, direction);
5282
- };
5283
- for (const field of analysis.fields) {
5284
- for (const text of textsOf(getNestedValue(document2, field))) {
5285
- visitTokens(text, analysis.language, tokenizing, countOccurrence2);
5286
- }
5287
- }
5288
- }
5289
- function spellingCountsOf(documents, analysis) {
5290
- const counts = /* @__PURE__ */ new Map();
5291
- for (const document2 of documents) countDocumentSpellings(counts, document2, analysis, 1);
5292
- return counts;
5293
- }
5294
- function encodePartialSpellings(fields, counts) {
5295
- const encoded = { fields: [...fields], tokens: [], spellings: [], counts: [] };
5296
- for (const [token, byToken] of counts) {
5297
- for (const [spelling, count] of byToken) {
5298
- encoded.tokens.push(token);
5299
- encoded.spellings.push(spelling);
5300
- encoded.counts.push(count);
5301
- }
5302
- }
5303
- return encoded;
5304
- }
5305
- function sameFields(left, right) {
5306
- if (left.length !== right.length) return false;
5307
- for (let at = 0; at < left.length; at++) {
5308
- if (left[at] !== right[at]) return false;
5309
- }
5310
- return true;
5311
- }
5312
-
5313
5239
  // src/core/partition/value-sizes.ts
5314
5240
  function keepsValueSize(base) {
5315
5241
  return base === "string" || base === "verbatim" || base.endsWith("[]");
@@ -5401,7 +5327,7 @@ function countSurface(counts, surface, token) {
5401
5327
  }
5402
5328
  function countOccurrence(counts, spellings, surface, token) {
5403
5329
  countSurface(counts, surface, token);
5404
- if (spellings !== null) countSpelling(spellings, token, surface ?? token, 1);
5330
+ if (spellings !== null) countPartialSpelling(spellings, token, surface ?? token, 1);
5405
5331
  }
5406
5332
  function addFieldOccurrences(occurrencesByToken, fieldTokens) {
5407
5333
  const fieldCounts = /* @__PURE__ */ new Map();
@@ -5730,73 +5656,6 @@ var FILTER_OPERAND_COPY_DEPTH = 3;
5730
5656
  var MAX_FILTER_ARRAY_SIZE = 65536;
5731
5657
  var GEO_UNIT_METRES = { km: 1e3, mi: 1609.344, m: 1 };
5732
5658
 
5733
- // src/core/pattern-index/typo-runs.ts
5734
- var BASIC_PLANE_LAST = 65535;
5735
- function isPatternTolerance(value) {
5736
- if (value === "auto") return true;
5737
- return typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= MAX_PATTERN_TOLERANCE;
5738
- }
5739
- function foldTypoText(text, host) {
5740
- const folded = [];
5741
- let longestFold = 0;
5742
- for (let at = 0; at < text.length; at++) {
5743
- const codePoint = text.codePointAt(at) ?? 0;
5744
- if (codePoint > BASIC_PLANE_LAST) at++;
5745
- const before = folded.length;
5746
- host.appendFoldedCodePoint(codePoint, folded);
5747
- longestFold = Math.max(longestFold, folded.length - before);
5748
- }
5749
- return { folded, longestFold };
5750
- }
5751
- function distinctTypoRuns(text, limit) {
5752
- const { folded } = text;
5753
- const seen = /* @__PURE__ */ new Set();
5754
- const runs = [];
5755
- for (let at = 0; at + PATTERN_RUN_CODE_POINTS <= folded.length && seen.size < limit; at++) {
5756
- const key = String.fromCodePoint(folded[at], folded[at + 1], folded[at + 2]);
5757
- if (seen.has(key)) continue;
5758
- seen.add(key);
5759
- runs.push(folded[at], folded[at + 1], folded[at + 2]);
5760
- }
5761
- return runs;
5762
- }
5763
- function typoRunsNeeded(tolerance, longestFold) {
5764
- return tolerance * (2 + longestFold) + 1;
5765
- }
5766
- function distinctTypoRunCount(text, limit) {
5767
- return distinctTypoRuns(text, limit).length / PATTERN_RUN_CODE_POINTS;
5768
- }
5769
-
5770
- // src/filters/pattern-tests.ts
5771
- function excluding(context, fieldPath, excluded) {
5772
- const universe = bitsetAnd(context.presentBitset(fieldPath), context.allDocIdsBitset);
5773
- return bitsetAnd(universe, bitsetNot(excluded, context.capacity));
5774
- }
5775
- function textsOf2(operand) {
5776
- const texts = /* @__PURE__ */ new Set();
5777
- if (!Array.isArray(operand)) return texts;
5778
- for (const text of operand) if (typeof text === "string") texts.add(text);
5779
- return texts;
5780
- }
5781
- function evaluatePatternTests(fieldPath, filter, context, bitsets) {
5782
- const caseFold = filter.caseFold === true;
5783
- const matching = (operator, texts, tolerance = 0) => {
5784
- const into = createBitSet(context.capacity);
5785
- for (const text of texts) context.matchPattern(fieldPath, operator, text, caseFold, tolerance, into);
5786
- return into;
5787
- };
5788
- const eqTolerance = isPatternTolerance(filter.tolerance) ? filter.tolerance : 0;
5789
- if (typeof filter.eq === "string") bitsets.push(matching("eq", [filter.eq], eqTolerance));
5790
- if (typeof filter.ne === "string") bitsets.push(excluding(context, fieldPath, matching("eq", [filter.ne])));
5791
- if (filter.in !== void 0) bitsets.push(matching("eq", textsOf2(filter.in)));
5792
- if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq", textsOf2(filter.nin))));
5793
- if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
5794
- if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
5795
- if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
5796
- if (typeof filter.wildcard === "string") bitsets.push(matching("wildcard", [filter.wildcard]));
5797
- if (typeof filter.regex === "string") bitsets.push(matching("regex", [filter.regex]));
5798
- }
5799
-
5800
5659
  // src/filters/field-tests.ts
5801
5660
  var COMPARISON_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
5802
5661
  var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
@@ -5976,7 +5835,7 @@ function narrowByFieldTests(fieldPath, f, parsed, context, narrow) {
5976
5835
  narrowByTextRange(fieldPath, f, context, narrow);
5977
5836
  if (parsed.pattern) {
5978
5837
  const patternBits = [];
5979
- evaluatePatternTests(fieldPath, f, context, patternBits);
5838
+ requirePatternSearch(fieldPath).evaluateTests(fieldPath, f, context, patternBits);
5980
5839
  for (const matched of patternBits) narrow(matched);
5981
5840
  } else {
5982
5841
  narrowByValue(fieldPath, f, parsed, context, bits, narrow);
@@ -6227,50 +6086,14 @@ function requirePatternField(fieldPath, operator, type) {
6227
6086
  operator
6228
6087
  );
6229
6088
  }
6230
- function requireTextLength(fieldPath, operator, text) {
6231
- if (!exceedsCodePoints(text, MAX_PATTERN_TEXT_CODE_POINTS)) return;
6232
- invalid(
6233
- `The text of "${operator}" on pattern field "${fieldPath}" holds more than ${MAX_PATTERN_TEXT_CODE_POINTS} code points`,
6234
- fieldPath,
6235
- operator
6236
- );
6237
- }
6238
- function requirePatternText(fieldPath, operator, text, caseFold) {
6239
- requireTextLength(fieldPath, operator, text);
6240
- if (caseFold && foldedCodePointCount(text) > MAX_PATTERN_POSITIONS) {
6241
- invalid(
6242
- `The text of "${operator}" on pattern field "${fieldPath}" has a case fold of more than ${MAX_PATTERN_POSITIONS} code points, while a pattern holds at most ${MAX_PATTERN_POSITIONS} positions`,
6243
- fieldPath,
6244
- operator
6245
- );
6246
- }
6247
- }
6248
- var ELEMENT_OPERATORS2 = ["matchesAny", "containsAll"];
6249
6089
  var PATTERN_OPERAND_KEYS = [
6250
6090
  "contains",
6251
6091
  "caseFold",
6252
6092
  "tolerance",
6253
6093
  ...PATTERN_EXPRESSION_OPERATORS,
6254
6094
  ...PATTERN_TEST_OPERATORS,
6255
- ...ELEMENT_OPERATORS2
6095
+ ...PATTERN_ELEMENT_OPERATORS
6256
6096
  ];
6257
- function requireExpressions(fieldPath, filter, type) {
6258
- for (const operator of PATTERN_EXPRESSION_OPERATORS) {
6259
- const pattern = filter[operator];
6260
- if (pattern === void 0) continue;
6261
- if (typeof pattern !== "string")
6262
- invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
6263
- requirePatternField(fieldPath, operator, type);
6264
- requireTextLength(fieldPath, operator, pattern);
6265
- requirePatternSearch(fieldPath).checkExpression(
6266
- operator,
6267
- pattern,
6268
- filter.caseFold === true,
6269
- fieldPath,
6270
- PATTERN_HOST
6271
- );
6272
- }
6273
- }
6274
6097
  function requirePatternOperands(fieldPath, filter, type) {
6275
6098
  if (filter.contains !== void 0) {
6276
6099
  if (typeof filter.contains !== "string") {
@@ -6284,50 +6107,16 @@ function requirePatternOperands(fieldPath, filter, type) {
6284
6107
  }
6285
6108
  requirePatternField(fieldPath, "caseFold", type);
6286
6109
  }
6287
- requireExpressions(fieldPath, filter, type);
6110
+ for (const operator of PATTERN_EXPRESSION_OPERATORS) {
6111
+ const pattern = filter[operator];
6112
+ if (pattern === void 0) continue;
6113
+ if (typeof pattern !== "string")
6114
+ invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
6115
+ requirePatternField(fieldPath, operator, type);
6116
+ }
6288
6117
  if (filter.tolerance !== void 0) requirePatternField(fieldPath, "tolerance", type);
6289
6118
  if (parsedTypeOf(type)?.pattern !== true) return;
6290
- const caseFold = filter.caseFold === true;
6291
- for (const operator of PATTERN_TEST_OPERATORS) {
6292
- const operand = filter[operator];
6293
- if (typeof operand === "string") requirePatternText(fieldPath, operator, operand, caseFold);
6294
- if (!Array.isArray(operand)) continue;
6295
- for (const text of operand) {
6296
- if (typeof text === "string") requirePatternText(fieldPath, operator, text, caseFold);
6297
- }
6298
- }
6299
- for (const operator of ELEMENT_OPERATORS2) {
6300
- const operand = filter[operator];
6301
- if (!Array.isArray(operand)) continue;
6302
- for (const text of operand) {
6303
- if (typeof text === "string") requirePatternText(fieldPath, operator, text, false);
6304
- }
6305
- }
6306
- requireTolerance(fieldPath, filter);
6307
- }
6308
- function requireTolerance(fieldPath, filter) {
6309
- const { tolerance, eq } = filter;
6310
- if (tolerance === void 0) return;
6311
- if (!isPatternTolerance(tolerance)) {
6312
- invalid(`"tolerance" on field "${fieldPath}" must be 0, 1, 2, or "auto"`, fieldPath, "tolerance");
6313
- }
6314
- if (typeof eq !== "string") {
6315
- invalid(
6316
- `"tolerance" applies only to an "eq" test, while the filter on field "${fieldPath}" sets no "eq" with text`,
6317
- fieldPath,
6318
- "tolerance"
6319
- );
6320
- }
6321
- if (tolerance === 0 || tolerance === "auto") return;
6322
- const text = foldTypoText(eq, PATTERN_HOST);
6323
- const requiredRuns = typoRunsNeeded(tolerance, text.longestFold);
6324
- const distinctRuns = distinctTypoRunCount(text, requiredRuns);
6325
- if (distinctRuns >= requiredRuns) return;
6326
- throw new NarsilError(
6327
- ErrorCodes.SEARCH_INVALID_FILTER,
6328
- `A tolerance of ${tolerance} on "eq" of pattern field "${fieldPath}" needs ${requiredRuns} distinct runs of three code points in the case fold of the text, while the text holds ${distinctRuns}. Lengthen the text, lower the tolerance, or set it to "auto"`,
6329
- { fieldPath, operator: "tolerance", requiredRuns, distinctRuns }
6330
- );
6119
+ requirePatternSearch(fieldPath).checkOperands(fieldPath, filter, PATTERN_HOST);
6331
6120
  }
6332
6121
  function requireWellFormedOperand(fieldPath, operator, text) {
6333
6122
  if (typeof text !== "string" || !holdsUnpairedSurrogate(text)) return;
@@ -8923,99 +8712,18 @@ function mergeSegmentPayload(target, payload, documents) {
8923
8712
  }
8924
8713
 
8925
8714
  // src/core/partition/pattern-snapshot.ts
8926
- function loadFailed(fieldPath, problem) {
8927
- return new NarsilError(
8928
- ErrorCodes.PERSISTENCE_LOAD_FAILED,
8929
- `The pattern index of field "${fieldPath}" in this partition is corrupt: ${problem}`,
8930
- { field: fieldPath }
8931
- );
8932
- }
8933
- function serializePatternIndex(state, fieldPath, index) {
8934
- const resolver = state.docStore.resolver();
8935
- const arrays = index.toArrays(fieldPath, null);
8936
- const positionOf = /* @__PURE__ */ new Map();
8937
- const docIds = [];
8938
- for (const ordinal of arrays.docIds) {
8939
- const externalId = resolver.toExternal(ordinal);
8940
- if (externalId === void 0) continue;
8941
- positionOf.set(ordinal, docIds.length);
8942
- docIds.push(externalId);
8943
- }
8944
- const runs = /* @__PURE__ */ Object.create(null);
8945
- for (let run = 0; run + 1 < arrays.offsets.length; run++) {
8946
- const positions = [];
8947
- for (let at = arrays.offsets[run]; at < arrays.offsets[run + 1]; at++) {
8948
- const position = positionOf.get(arrays.postings[at]);
8949
- if (position !== void 0) positions.push(position);
8950
- }
8951
- if (positions.length === 0) continue;
8952
- const base = run * PATTERN_RUN_CODE_POINTS;
8953
- runs[String.fromCodePoint(arrays.runs[base], arrays.runs[base + 1], arrays.runs[base + 2])] = positions;
8954
- }
8955
- return { docIds, runs };
8956
- }
8957
8715
  function serializePatternIndexes(state, schema) {
8958
8716
  const patternFields = schemaFieldsOf(schema).filter((field) => field.pattern);
8959
8717
  if (patternFields.length === 0) return void 0;
8718
+ const resolver = state.docStore.resolver();
8719
+ const toExternal = (ordinal) => resolver.toExternal(ordinal);
8960
8720
  const serialized = /* @__PURE__ */ Object.create(null);
8961
8721
  for (const { path: fieldPath } of patternFields) {
8962
8722
  const index = state.patternIndexes.get(fieldPath);
8963
- serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } : serializePatternIndex(state, fieldPath, index);
8723
+ serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } : requirePatternSearch(fieldPath).serializeIndex(fieldPath, index, toExternal);
8964
8724
  }
8965
8725
  return serialized;
8966
8726
  }
8967
- function appendRunCodePoints(fieldPath, key, out) {
8968
- const start = out.length;
8969
- for (const character of key) out.push(character.codePointAt(0) ?? 0);
8970
- const count = out.length - start;
8971
- if (count !== PATTERN_RUN_CODE_POINTS) {
8972
- throw loadFailed(
8973
- fieldPath,
8974
- `the key "${key}" holds ${count} code points, while each key must hold ${PATTERN_RUN_CODE_POINTS}`
8975
- );
8976
- }
8977
- }
8978
- function requirePositions(fieldPath, key, list, documentCount) {
8979
- if (!Array.isArray(list)) throw loadFailed(fieldPath, `the run "${key}" must map to a list of positions`);
8980
- let previous = -1;
8981
- for (const position of list) {
8982
- if (!Number.isInteger(position) || position < 0 || position >= documentCount) {
8983
- throw loadFailed(fieldPath, `the run "${key}" names position ${String(position)} outside its document list`);
8984
- }
8985
- if (position <= previous) {
8986
- throw loadFailed(fieldPath, `the positions of the run "${key}" are out of ascending order`);
8987
- }
8988
- previous = position;
8989
- }
8990
- return list;
8991
- }
8992
- function requireDistinctDocuments(fieldPath, docIds) {
8993
- const seen = /* @__PURE__ */ new Set();
8994
- for (const docId of docIds) {
8995
- if (seen.has(docId)) throw loadFailed(fieldPath, `the document list names "${docId}" twice`);
8996
- seen.add(docId);
8997
- }
8998
- }
8999
- function decodePatternEntry(fieldPath, entry) {
9000
- requireDistinctDocuments(fieldPath, entry.docIds);
9001
- const runs = [];
9002
- const offsets = [0];
9003
- const postings = [];
9004
- for (const [key, list] of Object.entries(entry.runs)) {
9005
- appendRunCodePoints(fieldPath, key, runs);
9006
- for (const position of requirePositions(fieldPath, key, list, entry.docIds.length)) postings.push(position);
9007
- offsets.push(postings.length);
9008
- }
9009
- const positions = new Uint32Array(entry.docIds.length);
9010
- for (let position = 0; position < positions.length; position++) positions[position] = position;
9011
- return {
9012
- fieldPath,
9013
- docIds: positions,
9014
- runs: Uint32Array.from(runs),
9015
- offsets: Uint32Array.from(offsets),
9016
- postings: Uint32Array.from(postings)
9017
- };
9018
- }
9019
8727
  function rebuildPatternIndex(state, fieldPath) {
9020
8728
  for (const [docId, stored] of state.docStore.all()) {
9021
8729
  const value = getNestedValue(stored.fields, fieldPath);
@@ -9033,7 +8741,7 @@ function loadPatternIndexes(state, serialized, schema) {
9033
8741
  rebuildPatternIndex(state, fieldPath);
9034
8742
  continue;
9035
8743
  }
9036
- const decoded = decodePatternEntry(fieldPath, entry);
8744
+ const decoded = requirePatternSearch(fieldPath).decodeIndex(fieldPath, entry);
9037
8745
  const ordinals = entry.docIds.map((docId) => resolver.toInternal(docId) ?? -1);
9038
8746
  patternIndexOf(state.patternIndexes, fieldPath).addArrays(decoded, (position) => ordinals[position]);
9039
8747
  }
@@ -10273,8 +9981,6 @@ var MAX_SORT_FIELDS = 8;
10273
9981
  var MAX_SORT_FIELD_NAME_LENGTH = 255;
10274
9982
  var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
10275
9983
  var PREFIX_MAX_EXPANSIONS = 50;
10276
- var PARTIAL_WORDS_ADDED_PER_QUERY = 50;
10277
- var PARTIAL_WORD_MIN_CODE_POINTS = 3;
10278
9984
  var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
10279
9985
  var MAX_TOLERANCE = 10;
10280
9986
  var MAX_PREFIX_LENGTH = 1024;
@@ -10624,18 +10330,6 @@ function addCompletions(groups, partition, lastToken, typedPrefix) {
10624
10330
  group.members.push({ token: completion, factor, partialFactor: factor });
10625
10331
  }
10626
10332
  }
10627
- function addPartialWords(group, added) {
10628
- const membersByToken = /* @__PURE__ */ new Map();
10629
- for (const member of group.members) membersByToken.set(member.token, member);
10630
- for (const word of added) {
10631
- const member = membersByToken.get(word.token);
10632
- if (member === void 0) {
10633
- group.members.push({ token: word.token, factor: 0, partialFactor: word.factor });
10634
- } else if (word.factor > member.partialFactor) {
10635
- member.partialFactor = word.factor;
10636
- }
10637
- }
10638
- }
10639
10333
  function partialWordAnalysisOf(fields, language, analysis) {
10640
10334
  return {
10641
10335
  fields,
@@ -10654,29 +10348,6 @@ function searchesAPartialField(params, partialFields) {
10654
10348
  }
10655
10349
  return false;
10656
10350
  }
10657
- function partialWordRequests(params, queryTokens, written, prefixToken) {
10658
- const eligible = [];
10659
- for (const queryToken of queryTokens) {
10660
- const word = written.get(queryToken.position);
10661
- if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
10662
- eligible.push({ token: queryToken.token, written: word });
10663
- if (eligible.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
10664
- }
10665
- const count = eligible.length;
10666
- if (count === 0) return [];
10667
- const evenShare = Math.floor(PARTIAL_WORDS_ADDED_PER_QUERY / count);
10668
- const remainder = PARTIAL_WORDS_ADDED_PER_QUERY % count;
10669
- return eligible.map((entry, order) => ({
10670
- token: entry.token,
10671
- request: {
10672
- written: entry.written,
10673
- ownToken: entry.token,
10674
- share: evenShare + (order < remainder ? 1 : 0),
10675
- tolerance: entry.token === prefixToken ? 0 : params.tolerance ?? 0,
10676
- prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
10677
- }
10678
- }));
10679
- }
10680
10351
  function resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, analysis) {
10681
10352
  const term = params.term;
10682
10353
  if (term === void 0 || params.exact === true) return {};
@@ -10689,16 +10360,18 @@ function resolveTermGroups(partition, params, language, schema, queryTokens, las
10689
10360
  if (usesPrefix) addCompletions(groups, partition, lastToken, written.get(lastToken.position));
10690
10361
  const prefixToken = groups.has(lastToken.token) ? lastToken.token : void 0;
10691
10362
  if (usesPartialWords) {
10692
- const requests = partialWordRequests(params, queryTokens, written, prefixToken);
10693
- const added = partition.expandPartialWords(
10694
- partialWordAnalysisOf(partialFields, language, analysis),
10695
- requests.map((entry) => entry.request)
10363
+ addPartialWordMembers(
10364
+ {
10365
+ queryTokens,
10366
+ written,
10367
+ prefixToken,
10368
+ tolerance: params.tolerance ?? 0,
10369
+ prefixLength: params.prefixLength,
10370
+ analysis: partialWordAnalysisOf(partialFields, language, analysis)
10371
+ },
10372
+ (partialAnalysis, requests) => partition.expandPartialWords(partialAnalysis, requests),
10373
+ (token, typoMatches) => groupFor(groups, token, typoMatches)
10696
10374
  );
10697
- for (let at = 0; at < requests.length; at++) {
10698
- if (added[at].length === 0) continue;
10699
- const token = requests[at].token;
10700
- addPartialWords(groupFor(groups, token, token !== prefixToken), added[at]);
10701
- }
10702
10375
  }
10703
10376
  const termGroups = [...groups.values()];
10704
10377
  if (termGroups.length === 0) return {};
@@ -19248,6 +18921,15 @@ function createFrozenTombstones() {
19248
18921
  };
19249
18922
  }
19250
18923
 
18924
+ // src/core/partition/partial-words/spellings.ts
18925
+ function sameFields(left, right) {
18926
+ if (left.length !== right.length) return false;
18927
+ for (let at = 0; at < left.length; at++) {
18928
+ if (left[at] !== right[at]) return false;
18929
+ }
18930
+ return true;
18931
+ }
18932
+
19251
18933
  // src/core/partition/frozen/shared-snapshot.ts
19252
18934
  function sharedUint32(source) {
19253
18935
  const copy = new Uint32Array(new SharedArrayBuffer(source.length * 4));
@@ -19291,7 +18973,7 @@ function sharePartialWordTable(table) {
19291
18973
  function sharedPartialWordsOf(tokenData, spellings, documentData, analysis) {
19292
18974
  let source = spellings;
19293
18975
  if (source === null && analysis !== null && analysis.fields.length > 0) {
19294
- source = encodePartialSpellings(analysis.fields, spellingCountsOf(storedDocumentsOf(documentData), analysis));
18976
+ source = partialSpellingsOfDocuments(storedDocumentsOf(documentData), analysis);
19295
18977
  }
19296
18978
  if (source === null || source.fields.length === 0) return null;
19297
18979
  const table = buildPartialWordTableFrom(source, wrapFrozenTokenTable(tokenData));
@@ -20158,7 +19840,7 @@ function mergePartialWords(inputs, mergedTokenCount, analysis) {
20158
19840
  }
20159
19841
  const counts = /* @__PURE__ */ new Map();
20160
19842
  for (const input of inputs) input.segment.partialWords.countKeptSpellings(counts, analysis, input.remap);
20161
- return { spellings: encodePartialSpellings(analysis.fields, counts), reused: null };
19843
+ return { spellings: encodePartialSpellingsOf(analysis.fields, counts), reused: null };
20162
19844
  }
20163
19845
  function mergeFrozenSegments(segments, analysis) {
20164
19846
  if (typeof SharedArrayBuffer !== "function") return null;
@@ -20504,7 +20186,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
20504
20186
  const addToken = (token, position, surface) => {
20505
20187
  postings.addToken(token, positionOffset + position);
20506
20188
  surfaces.count(surface, token);
20507
- if (fieldSpellings !== null) countSpelling(fieldSpellings, token, surface ?? token, 1);
20189
+ if (fieldSpellings !== null) countPartialSpelling(fieldSpellings, token, surface ?? token, 1);
20508
20190
  };
20509
20191
  for (let ordinal = 0; ordinal < documents.length; ordinal++) {
20510
20192
  const document2 = documents[ordinal].document;
@@ -20538,7 +20220,7 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
20538
20220
  fieldLengthColumns: lengths.columns,
20539
20221
  totalFieldLengths: lengths.totalsByName(),
20540
20222
  surfaceForms: surfaces.encode(),
20541
- partialSpellings: partialFields.length === 0 ? null : encodePartialSpellings(partialFields, spellings),
20223
+ partialSpellings: partialFields.length === 0 ? null : encodePartialSpellingsOf(partialFields, spellings),
20542
20224
  ...encodeFieldIndexes(filters, identity)
20543
20225
  };
20544
20226
  }
@@ -22112,7 +21794,7 @@ async function packChecksummedEnvelope(payloadChunks, envelopeFormatVersion, wha
22112
21794
  checksum});
22113
21795
  return { chunks: [header2, ...payloadChunks], byteLength: HEADER_SIZE + payloadLength, checksum };
22114
21796
  }
22115
- function loadFailed2(message, details) {
21797
+ function loadFailed(message, details) {
22116
21798
  throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, message, details);
22117
21799
  }
22118
21800
  async function gunzipWithinLimit(compressed, what) {
@@ -22125,7 +21807,7 @@ async function gunzipWithinLimit(compressed, what) {
22125
21807
  try {
22126
21808
  step = await reader.read();
22127
21809
  } catch (error) {
22128
- loadFailed2(`The ${what} holds a gzip payload that fails to inflate`, {
21810
+ loadFailed(`The ${what} holds a gzip payload that fails to inflate`, {
22129
21811
  cause: error instanceof Error ? error.message : String(error)
22130
21812
  });
22131
21813
  }
@@ -22133,7 +21815,7 @@ async function gunzipWithinLimit(compressed, what) {
22133
21815
  total += step.value.byteLength;
22134
21816
  if (total > MAX_FILE_BYTES) {
22135
21817
  await reader.cancel();
22136
- loadFailed2(`The ${what} inflates past the file size limit of ${MAX_FILE_BYTES} bytes`);
21818
+ loadFailed(`The ${what} inflates past the file size limit of ${MAX_FILE_BYTES} bytes`);
22137
21819
  }
22138
21820
  chunks.push(step.value);
22139
21821
  }
@@ -22162,7 +21844,7 @@ async function openChecksummedEnvelope(data, expected) {
22162
21844
  );
22163
21845
  }
22164
21846
  if (header2.flags.compressionEnabled && !expected.allowsCompression) {
22165
- loadFailed2(`The ${what} is compressed, while its format stores it raw`);
21847
+ loadFailed(`The ${what} is compressed, while its format stores it raw`);
22166
21848
  }
22167
21849
  if (header2.flags.compressionEnabled && header2.flags.compressionAlgorithm !== "gzip") {
22168
21850
  throw new NarsilError(
@@ -22170,9 +21852,9 @@ async function openChecksummedEnvelope(data, expected) {
22170
21852
  `The ${what} uses ${header2.flags.compressionAlgorithm} compression, which this build cannot read`
22171
21853
  );
22172
21854
  }
22173
- if (!header2.flags.checksumPresent) loadFailed2(`The ${what} carries no checksum, while its format requires one`);
21855
+ if (!header2.flags.checksumPresent) loadFailed(`The ${what} carries no checksum, while its format requires one`);
22174
21856
  if (data.byteLength !== HEADER_SIZE + header2.payloadLength) {
22175
- loadFailed2(
21857
+ loadFailed(
22176
21858
  `The ${what} holds ${data.byteLength} bytes where its header gives ${HEADER_SIZE + header2.payloadLength}`,
22177
21859
  { bytes: data.byteLength, expected: HEADER_SIZE + header2.payloadLength }
22178
21860
  );
@@ -30974,65 +30656,18 @@ function stopIdleSweep(state) {
30974
30656
  }
30975
30657
 
30976
30658
  // src/core/pattern-index/matcher-budget.ts
30977
- var POSITIONS_PER_WORD = 32;
30978
- function positionWords(positions) {
30979
- return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
30980
- }
30981
- function smallestMatcherBytes(positions) {
30982
- return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
30983
- }
30984
30659
  function widestPatternSplit(footprintBytes) {
30985
30660
  return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / Math.max(1, footprintBytes)));
30986
30661
  }
30987
30662
 
30988
30663
  // src/core/pattern-index/query-positions.ts
30989
- function isRecord6(value) {
30990
- return value !== null && typeof value === "object" && !Array.isArray(value);
30991
- }
30992
- function literalFootprint(text, caseFold) {
30993
- if (typeof text !== "string") return null;
30994
- return smallestMatcherBytes(caseFold ? foldedCodePointCount(text) : codePointCount(text));
30995
- }
30996
- function larger(left, right) {
30997
- if (left === null) return right;
30998
- if (right === null) return left;
30999
- return Math.max(left, right);
31000
- }
31001
- function largestTestOf(filter, module) {
31002
- const caseFold = filter.caseFold === true;
31003
- let largest = null;
31004
- for (const operator of PATTERN_TEST_OPERATORS) {
31005
- const operand = filter[operator];
31006
- if (operand === void 0) continue;
31007
- const texts = Array.isArray(operand) ? operand : [operand];
31008
- for (const text of texts) largest = larger(largest, literalFootprint(text, caseFold));
31009
- }
31010
- for (const operator of PATTERN_EXPRESSION_OPERATORS) {
31011
- const pattern = filter[operator];
31012
- if (typeof pattern !== "string") continue;
31013
- largest = larger(largest, module.expressionFootprint(operator, pattern, caseFold, PATTERN_HOST));
31014
- }
31015
- return largest;
31016
- }
31017
- function largestIn(expression, patternFields, module) {
31018
- let largest = null;
31019
- if (isRecord6(expression.fields)) {
31020
- for (const [fieldPath, filter] of Object.entries(expression.fields)) {
31021
- if (patternFields.has(fieldPath) && isRecord6(filter)) largest = larger(largest, largestTestOf(filter, module));
31022
- }
31023
- }
31024
- const nested = [...expression.and ?? [], ...expression.or ?? []];
31025
- if (expression.not !== void 0) nested.push(expression.not);
31026
- for (const child of nested) largest = larger(largest, largestIn(child, patternFields, module));
31027
- return largest;
31028
- }
31029
30664
  function largestPatternFootprint(expression, schema) {
31030
30665
  if (expression === void 0) return null;
31031
30666
  const module = availablePatternSearch();
31032
30667
  if (module === null) return null;
31033
30668
  const patternFields = /* @__PURE__ */ new Set();
31034
30669
  for (const { path, pattern } of schemaFieldsOf(schema)) if (pattern) patternFields.add(path);
31035
- return patternFields.size === 0 ? null : largestIn(expression, patternFields, module);
30670
+ return patternFields.size === 0 ? null : module.largestFootprint(expression, patternFields, PATTERN_HOST);
31036
30671
  }
31037
30672
 
31038
30673
  // src/engine/orchestration/search.ts
@@ -34270,7 +33905,7 @@ function refuse(path, reason) {
34270
33905
  field: path
34271
33906
  });
34272
33907
  }
34273
- function isRecord7(value) {
33908
+ function isRecord6(value) {
34274
33909
  return typeof value === "object" && value !== null && !Array.isArray(value);
34275
33910
  }
34276
33911
  function startsWithTagPrefix(key) {
@@ -34337,7 +33972,7 @@ function untagTag(tag, value, path, depth) {
34337
33972
  case NUMBER_TAG:
34338
33973
  return numberOf(value, path);
34339
33974
  case OBJECT_TAG:
34340
- if (!isRecord7(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
33975
+ if (!isRecord6(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
34341
33976
  return untagMap(value, path, depth);
34342
33977
  default:
34343
33978
  return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
@@ -34560,7 +34195,7 @@ function documentExportStream(header2, checkpoint) {
34560
34195
  function invalidHeader(message) {
34561
34196
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
34562
34197
  }
34563
- function isRecord8(value) {
34198
+ function isRecord7(value) {
34564
34199
  return typeof value === "object" && value !== null && !Array.isArray(value);
34565
34200
  }
34566
34201
  function configOfHeader(line) {
@@ -34576,7 +34211,7 @@ function configOfHeader(line) {
34576
34211
  } catch {
34577
34212
  invalidHeader("is not JSON");
34578
34213
  }
34579
- if (!isRecord8(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
34214
+ if (!isRecord7(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
34580
34215
  invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
34581
34216
  }
34582
34217
  const configuration = requireConfigurationShape(raw.index, "document export header");
@@ -34669,7 +34304,7 @@ async function takeLine(state, entry) {
34669
34304
  refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
34670
34305
  return;
34671
34306
  }
34672
- if (!isRecord8(document2) || typeof document2.id !== "string" || document2.id.length === 0) {
34307
+ if (!isRecord7(document2) || typeof document2.id !== "string" || document2.id.length === 0) {
34673
34308
  refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
34674
34309
  return;
34675
34310
  }
@@ -36331,31 +35966,6 @@ function applySnippetTruncation(taggedText, originalText, positions, maxLen, pre
36331
35966
  if (needsTrailingEllipsis) snippet = `${snippet}...`;
36332
35967
  return { snippet, visiblePositions };
36333
35968
  }
36334
- function containsPartialWord(spelling, words) {
36335
- for (const word of words) {
36336
- if (spelling.length > word.length && spelling.includes(word)) return true;
36337
- }
36338
- return false;
36339
- }
36340
- function partialWordMatcher(text, language, partial) {
36341
- const indexed = tokenize(text, language, {
36342
- stem: true,
36343
- removeStopWords: true,
36344
- collectSurfaces: true,
36345
- customTokenizer: partial.customTokenizer,
36346
- stopWordOverride: partial.stopWordOverride
36347
- });
36348
- const positions = /* @__PURE__ */ new Set();
36349
- const spellings = /* @__PURE__ */ new Set();
36350
- for (let at = 0; at < indexed.tokens.length; at++) {
36351
- const spelling = indexed.surfaces?.[at] ?? indexed.tokens[at].token;
36352
- if (!containsPartialWord(spelling, partial.words)) continue;
36353
- positions.add(indexed.tokens[at].position);
36354
- spellings.add(spelling);
36355
- }
36356
- if (partial.customTokenizer !== void 0) return (_position, fieldToken) => spellings.has(fieldToken);
36357
- return (position) => positions.has(position);
36358
- }
36359
35969
  function highlightField(text, queryTokens, language, options) {
36360
35970
  const preTag = options?.preTag ?? "<mark>";
36361
35971
  const postTag = options?.postTag ?? "</mark>";
@@ -36392,7 +36002,7 @@ function highlightField(text, queryTokens, language, options) {
36392
36002
  matchByFieldToken.set(fieldToken, matches);
36393
36003
  return matches;
36394
36004
  };
36395
- const matchesInside = options?.partialWords === void 0 ? null : partialWordMatcher(text, language, options.partialWords);
36005
+ const matchesInside = options?.partialWords === void 0 ? null : partialWordHighlighter(text, language, options.partialWords);
36396
36006
  for (let i = 0; i < fieldResult.tokens.length; i++) {
36397
36007
  if (i >= charOffsets.length) break;
36398
36008
  const fieldToken = fieldResult.tokens[i].token;
@@ -36442,20 +36052,7 @@ function partialWordTagging(params, schema, queryTokens, written, analysis) {
36442
36052
  if (params.partialWords !== true || params.exact === true) return null;
36443
36053
  const fields = partialFieldsOf(schema).filter((field) => params.fields === void 0 || params.fields.includes(field));
36444
36054
  if (fields.length === 0) return null;
36445
- const writtenAt = /* @__PURE__ */ new Map();
36446
- for (const word of written) {
36447
- if (!writtenAt.has(word.position)) writtenAt.set(word.position, word.token);
36448
- }
36449
- const seenTokens = /* @__PURE__ */ new Set();
36450
- const words = [];
36451
- for (const queryToken of queryTokens) {
36452
- if (seenTokens.has(queryToken.token)) continue;
36453
- seenTokens.add(queryToken.token);
36454
- const word = writtenAt.get(queryToken.position);
36455
- if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
36456
- words.push(word);
36457
- if (words.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
36458
- }
36055
+ const words = partialWordsToHighlight(queryTokens, written);
36459
36056
  if (words.length === 0) return null;
36460
36057
  return {
36461
36058
  fields: new Set(fields),