@delali/narsil 0.3.1-next.78 → 0.3.1-next.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +3 -3
  2. package/dist/{capabilities-1lEYL3WK.d.ts → capabilities-CcTDWh_P.d.ts} +2 -2
  3. package/dist/{chunk-AMGDG4ZJ.mjs → chunk-23HJ5QXQ.mjs} +1 -1
  4. package/dist/{chunk-R6WBLBTC.mjs → chunk-4QDQEJDV.mjs} +9 -45
  5. package/dist/chunk-DQABW5UQ.mjs +140 -0
  6. package/dist/chunk-GG2FBGVZ.mjs +30 -0
  7. package/dist/{chunk-XSZTF4ZQ.mjs → chunk-HJASKX4F.mjs} +1 -1
  8. package/dist/{chunk-DFW3U5UW.mjs → chunk-K4FZ55Q2.mjs} +21 -192
  9. package/dist/{chunk-PJWG547Z.mjs → chunk-MEOOYKWL.mjs} +3 -3
  10. package/dist/{chunk-55WEJKWT.mjs → chunk-NSXQCE4K.mjs} +17 -63
  11. package/dist/{chunk-PXDQFYD5.mjs → chunk-OSF3SGJV.mjs} +23 -139
  12. package/dist/{chunk-RTR44V2A.mjs → chunk-P2PYEZZE.mjs} +586 -4
  13. package/dist/{chunk-Z3BXCZXS.mjs → chunk-QPYV22XH.mjs} +2 -1
  14. package/dist/{chunk-4344I4MR.mjs → chunk-R2XCJIO7.mjs} +335 -12
  15. package/dist/{chunk-IBR6634K.mjs → chunk-R6A773WK.mjs} +2 -69
  16. package/dist/chunk-WQBCBKY3.mjs +8 -0
  17. package/dist/{chunk-P2BEMMOD.mjs → chunk-YGLSR7BC.mjs} +150 -203
  18. package/dist/chunk-YTJCGKM4.mjs +56 -0
  19. package/dist/{client-ZXcbSViH.d.ts → client-CsybN673.d.ts} +2 -2
  20. package/dist/client.d.ts +6 -6
  21. package/dist/{config-DTe5EbWW.d.ts → config-BCNQtszf.d.ts} +1 -1
  22. package/dist/{config-DHL2KsfD.d.ts → config-CFonR_gY.d.ts} +1 -1
  23. package/dist/direct-executor-VNMQ6NIH.mjs +19 -0
  24. package/dist/distribution/coordinator/etcd.mjs +2 -1
  25. package/dist/distribution/coordinator/in-memory.mjs +2 -1
  26. package/dist/distribution.d.ts +5 -5
  27. package/dist/distribution.mjs +15 -13
  28. package/dist/{engine-BjPBzWkA.d.ts → engine-B0oIF1_r.d.ts} +1 -1
  29. package/dist/index.browser.mjs +113 -516
  30. package/dist/index.d.ts +5 -5
  31. package/dist/index.mjs +14 -12
  32. package/dist/invalidation/filesystem.mjs +2 -1
  33. package/dist/{memory-DW3OJ0im.d.ts → memory-B6SBOKlS.d.ts} +1287 -1287
  34. package/dist/partial-words.mjs +5 -4
  35. package/dist/pattern.mjs +4 -1
  36. package/dist/persistence/durability/checkpoint-worker.d.ts +7 -7
  37. package/dist/persistence/durability/checkpoint-worker.mjs +14 -12
  38. package/dist/react.d.ts +6 -6
  39. package/dist/server.d.ts +8 -8
  40. package/dist/server.mjs +10 -7
  41. package/dist/{tasks-BTbwBqar.d.ts → tasks-CYAQe6qd.d.ts} +1 -1
  42. package/dist/{thread-XBDVXKUE.mjs → thread-P6QJ2Q5O.mjs} +11 -8
  43. package/dist/vector/search-worker.mjs +3 -2
  44. package/dist/workers/entry.mjs +3 -3
  45. package/package.json +9 -7
  46. package/dist/chunk-IOZPBNSG.mjs +0 -55
  47. package/dist/chunk-WFVOBZS4.mjs +0 -485
  48. package/dist/direct-executor-WB2HKU4N.mjs +0 -17
@@ -1,6 +1,198 @@
1
- import { tokenizeOptions, getNestedValue, visitTokens } from './chunk-WFVOBZS4.mjs';
1
+ import { flattenSchema, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-OSF3SGJV.mjs';
2
2
  import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
3
- import { compareCodePoints } from './chunk-IBR6634K.mjs';
3
+ import { isWordIndexedBase } from './chunk-DQABW5UQ.mjs';
4
+ import { compareCodePoints, codePointCount } from './chunk-YTJCGKM4.mjs';
5
+ import { createNarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
6
+
7
+ // src/core/tokenizer/constants.ts
8
+ var TOKEN_CACHE_SIZE_FLOOR = 5e4;
9
+ var TOKEN_CACHE_SIZE_CEILING = 2e6;
10
+ var TOKEN_CACHE_BYTES_PER_ENTRY = 200;
11
+ var NODE_TOKEN_CACHE_ENTRIES = 1e6;
12
+ var UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES = 2e5;
13
+ var CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION = 0.05;
14
+ var SMALL_DEVICE_MEMORY_GB = 1;
15
+ var MEDIUM_DEVICE_MEMORY_GB = 4;
16
+ var SMALL_DEVICE_TOKEN_CACHE_ENTRIES = 1e5;
17
+ var MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES = 25e4;
18
+ var LARGE_DEVICE_TOKEN_CACHE_ENTRIES = 5e5;
19
+ var DEFAULT_MIN_TOKEN_LENGTH = 1;
20
+
21
+ // src/core/tokenizer/normalize.ts
22
+ var DOTTED_CAPITAL_I = "\u0130";
23
+ var COMBINING_DOT_ABOVE = "\u0307";
24
+ var ARMENIAN_MARKS = /[\u055B-\u055F]/;
25
+ var FOLDED_FORMS = /\u0307|[\u0130\u055B-\u055F\u2018\u2019\u02BC\u02BB]/g;
26
+ var WIDTH_FORMS = /[\uFF01-\uFF5E\uFF61-\uFF9F]/;
27
+ var WIDTH_FORMS_GLOBAL = /[\uFF01-\uFF5E\uFF61-\uFF9F]/g;
28
+ var FULLWIDTH_ASCII_END = 65374;
29
+ var FULLWIDTH_ASCII_OFFSET = 65248;
30
+ var HALFWIDTH_KANA_START = 65377;
31
+ var HALFWIDTH_VOICED_MARK = 65438;
32
+ var HALFWIDTH_SEMI_VOICED_MARK = 65439;
33
+ var COMBINING_VOICED_MARK = "\u3099";
34
+ var COMBINING_SEMI_VOICED_MARK = "\u309A";
35
+ var HALFWIDTH_KANA_TARGETS = "\u3002\u300C\u300D\u3001\u30FB\u30F2\u30A1\u30A3\u30A5\u30A7\u30A9\u30E3\u30E5\u30E7\u30C3\u30FC\u30A2\u30A4\u30A6\u30A8\u30AA\u30AB\u30AD\u30AF\u30B1\u30B3\u30B5\u30B7\u30B9\u30BB\u30BD\u30BF\u30C1\u30C4\u30C6\u30C8\u30CA\u30CB\u30CC\u30CD\u30CE\u30CF\u30D2\u30D5\u30D8\u30DB\u30DE\u30DF\u30E0\u30E1\u30E2\u30E4\u30E6\u30E8\u30E9\u30EA\u30EB\u30EC\u30ED\u30EF\u30F3";
36
+ function foldForm(match) {
37
+ if (match === DOTTED_CAPITAL_I) return "i";
38
+ if (match === COMBINING_DOT_ABOVE || ARMENIAN_MARKS.test(match)) return "";
39
+ return "'";
40
+ }
41
+ function foldWidth(match) {
42
+ const codePoint = match.charCodeAt(0);
43
+ if (codePoint <= FULLWIDTH_ASCII_END) return String.fromCharCode(codePoint - FULLWIDTH_ASCII_OFFSET);
44
+ if (codePoint === HALFWIDTH_VOICED_MARK) return COMBINING_VOICED_MARK;
45
+ if (codePoint === HALFWIDTH_SEMI_VOICED_MARK) return COMBINING_SEMI_VOICED_MARK;
46
+ return HALFWIDTH_KANA_TARGETS[codePoint - HALFWIDTH_KANA_START];
47
+ }
48
+ function isAscii(text) {
49
+ for (let i = 0; i < text.length; i++) {
50
+ if (text.charCodeAt(i) > 127) return false;
51
+ }
52
+ return true;
53
+ }
54
+ function foldWidthForms(text) {
55
+ if (!WIDTH_FORMS.test(text)) return text;
56
+ return text.replace(WIDTH_FORMS_GLOBAL, foldWidth);
57
+ }
58
+ function normalizeForSplitting(text) {
59
+ if (isAscii(text)) return text.toLowerCase();
60
+ return foldWidthForms(text).normalize("NFC").replace(FOLDED_FORMS, foldForm).toLowerCase();
61
+ }
62
+ function stripDiacritics(text) {
63
+ return text.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
64
+ }
65
+
66
+ // src/core/tokenizer/cache.ts
67
+ function clampCacheSize(entries) {
68
+ return Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(entries, TOKEN_CACHE_SIZE_CEILING));
69
+ }
70
+ function computeDefaultCacheSize() {
71
+ try {
72
+ if (typeof process !== "undefined" && typeof process.versions?.node === "string" && typeof window === "undefined") {
73
+ const constrainedMemory = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
74
+ if (constrainedMemory > 0 && constrainedMemory < Number.MAX_SAFE_INTEGER) {
75
+ const budget = constrainedMemory * CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION;
76
+ return clampCacheSize(Math.floor(budget / TOKEN_CACHE_BYTES_PER_ENTRY));
77
+ }
78
+ return clampCacheSize(NODE_TOKEN_CACHE_ENTRIES);
79
+ }
80
+ if (typeof navigator !== "undefined") {
81
+ const mem = navigator.deviceMemory;
82
+ if (typeof mem === "number" && mem > 0) {
83
+ let entries;
84
+ if (mem <= SMALL_DEVICE_MEMORY_GB) entries = SMALL_DEVICE_TOKEN_CACHE_ENTRIES;
85
+ else if (mem <= MEDIUM_DEVICE_MEMORY_GB) entries = MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES;
86
+ else entries = LARGE_DEVICE_TOKEN_CACHE_ENTRIES;
87
+ return clampCacheSize(entries);
88
+ }
89
+ }
90
+ } catch {
91
+ return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
92
+ }
93
+ return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
94
+ }
95
+ var normalizationCache = /* @__PURE__ */ new Map();
96
+ var normalizationCacheSize = 0;
97
+ var maxCacheSize = computeDefaultCacheSize();
98
+ var cachedLangName = "";
99
+ var cachedFlags = "";
100
+ var cachedBucket = null;
101
+ function configureNormalizationCache(maxSize) {
102
+ if (!Number.isFinite(maxSize)) {
103
+ throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must be a finite number", {
104
+ received: maxSize
105
+ });
106
+ }
107
+ if (maxSize < 0) {
108
+ throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must not be negative", {
109
+ received: maxSize
110
+ });
111
+ }
112
+ if (maxSize === 0) {
113
+ throw createNarsilError(
114
+ ErrorCodes.CONFIG_INVALID,
115
+ "tokenizerCacheSize must be greater than zero; the normalization cache cannot be disabled",
116
+ {
117
+ received: maxSize
118
+ }
119
+ );
120
+ }
121
+ maxCacheSize = Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(Math.floor(maxSize), TOKEN_CACHE_SIZE_CEILING));
122
+ if (normalizationCacheSize > maxCacheSize) {
123
+ evictOldestEntries(normalizationCacheSize - maxCacheSize);
124
+ }
125
+ }
126
+ function getCacheBucket(language, flags) {
127
+ const langKey = `${language.name}@${language.revision}`;
128
+ if (cachedBucket && langKey === cachedLangName && flags === cachedFlags) return cachedBucket;
129
+ const key = `${langKey}:${flags}`;
130
+ let bucket = normalizationCache.get(key);
131
+ if (!bucket) {
132
+ bucket = /* @__PURE__ */ new Map();
133
+ normalizationCache.set(key, bucket);
134
+ }
135
+ cachedLangName = langKey;
136
+ cachedFlags = flags;
137
+ cachedBucket = bucket;
138
+ return bucket;
139
+ }
140
+ function evictOldestEntries(count) {
141
+ let remaining = count;
142
+ for (const [key, bucket] of normalizationCache) {
143
+ if (remaining <= 0) break;
144
+ if (bucket.size <= remaining) {
145
+ remaining -= bucket.size;
146
+ normalizationCacheSize -= bucket.size;
147
+ normalizationCache.delete(key);
148
+ if (bucket === cachedBucket) cachedBucket = null;
149
+ } else {
150
+ let deleted = 0;
151
+ for (const raw of bucket.keys()) {
152
+ if (deleted >= remaining) break;
153
+ bucket.delete(raw);
154
+ deleted++;
155
+ }
156
+ normalizationCacheSize -= deleted;
157
+ remaining = 0;
158
+ }
159
+ }
160
+ }
161
+ function transformToken(raw, language, stem, removeDiacritics) {
162
+ const flags = (stem ? "s" : "") + (removeDiacritics ? "d" : "");
163
+ let bucket = getCacheBucket(language, flags);
164
+ const cached = bucket.get(raw);
165
+ if (cached !== void 0) return cached;
166
+ let normalized = raw;
167
+ if (language.normalizer) {
168
+ normalized = language.normalizer(normalized);
169
+ }
170
+ if (removeDiacritics) {
171
+ normalized = stripDiacritics(normalized);
172
+ }
173
+ if (stem && language.stemmer) {
174
+ normalized = language.stemmer(normalized);
175
+ }
176
+ if (normalizationCacheSize >= maxCacheSize) {
177
+ evictOldestEntries(Math.max(1, maxCacheSize >>> 2));
178
+ bucket = getCacheBucket(language, flags);
179
+ }
180
+ bucket.set(raw, normalized);
181
+ normalizationCacheSize++;
182
+ return normalized;
183
+ }
184
+ function clearNormalizationCache() {
185
+ normalizationCache.clear();
186
+ normalizationCacheSize = 0;
187
+ cachedBucket = null;
188
+ }
189
+ function resetNormalizationCache() {
190
+ clearNormalizationCache();
191
+ maxCacheSize = computeDefaultCacheSize();
192
+ }
193
+ function getNormalizationCacheSize() {
194
+ return normalizationCacheSize;
195
+ }
4
196
 
5
197
  // src/core/partition/suggestions.ts
6
198
  function verbatimOccurrences(state, token) {
@@ -246,6 +438,297 @@ function codePointOrder(strings) {
246
438
  return order;
247
439
  }
248
440
 
441
+ // src/core/tokenizer/ngrams.ts
442
+ var OTHER_SCRIPT = 0;
443
+ var HAN_SCRIPT = 1;
444
+ var HIRAGANA_SCRIPT = 2;
445
+ var KATAKANA_SCRIPT = 3;
446
+ var HANGUL_SCRIPT = 4;
447
+ var THAI_SCRIPT = 5;
448
+ var LAO_SCRIPT = 6;
449
+ var KHMER_SCRIPT = 7;
450
+ var MYANMAR_SCRIPT = 8;
451
+ var PROLONGED_SOUND_MARK = 12540;
452
+ var IDEOGRAPHIC_ITERATION_MARK = 12293;
453
+ var LOWEST_NGRAM_CODE_POINT = 3584;
454
+ var LOWEST_MARK_CODE_POINT = 768;
455
+ var THAI_BLOCK_END = 3711;
456
+ var LAO_BLOCK_END = 3839;
457
+ var MYANMAR_BLOCK_START = 4096;
458
+ var MYANMAR_BLOCK_END = 4255;
459
+ var KHMER_BLOCK_START = 6016;
460
+ var KHMER_BLOCK_END = 6655;
461
+ var SOUTHEAST_ASIAN_BLOCKS_END = 6656;
462
+ var HIRAGANA_BLOCK_START = 12352;
463
+ var HIRAGANA_BLOCK_END = 12447;
464
+ var COMBINING_VOICED_MARK2 = 12441;
465
+ var COMBINING_SEMI_VOICED_MARK2 = 12442;
466
+ var KATAKANA_BLOCK_START = 12448;
467
+ var KATAKANA_BLOCK_END = 12543;
468
+ var HAN_EXTENSION_A_START = 13312;
469
+ var HAN_EXTENSION_A_END = 19903;
470
+ var HAN_UNIFIED_START = 19968;
471
+ var HAN_UNIFIED_END = 40959;
472
+ var HANGUL_SYLLABLE_START = 44032;
473
+ var HANGUL_SYLLABLE_END = 55203;
474
+ var HAN_COMPATIBILITY_START = 63744;
475
+ var HAN_COMPATIBILITY_END = 64255;
476
+ var SUPPLEMENTARY_PLANE_START = 65536;
477
+ var MARK = /\p{M}/uy;
478
+ var HAN = /\p{Script=Han}/uy;
479
+ var HIRAGANA = /\p{Script=Hiragana}/uy;
480
+ var KATAKANA = /\p{Script=Katakana}/uy;
481
+ var HANGUL = /\p{Script=Hangul}/uy;
482
+ var THAI = /\p{Script=Thai}/uy;
483
+ var LAO = /\p{Script=Lao}/uy;
484
+ var KHMER = /\p{Script=Khmer}/uy;
485
+ var MYANMAR = /\p{Script=Myanmar}/uy;
486
+ function matchesAt(pattern, part, index) {
487
+ pattern.lastIndex = index;
488
+ return pattern.test(part);
489
+ }
490
+ function isMark(part, index, codePoint) {
491
+ if (codePoint < LOWEST_MARK_CODE_POINT) return false;
492
+ if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return false;
493
+ if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) {
494
+ return codePoint === COMBINING_VOICED_MARK2 || codePoint === COMBINING_SEMI_VOICED_MARK2;
495
+ }
496
+ if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return false;
497
+ if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return false;
498
+ return matchesAt(MARK, part, index);
499
+ }
500
+ function classifyOutsideCommonBlocks(part, index) {
501
+ if (matchesAt(HAN, part, index)) return HAN_SCRIPT;
502
+ if (matchesAt(HIRAGANA, part, index)) return HIRAGANA_SCRIPT;
503
+ if (matchesAt(KATAKANA, part, index)) return KATAKANA_SCRIPT;
504
+ if (matchesAt(HANGUL, part, index)) return HANGUL_SCRIPT;
505
+ if (matchesAt(THAI, part, index)) return THAI_SCRIPT;
506
+ if (matchesAt(LAO, part, index)) return LAO_SCRIPT;
507
+ if (matchesAt(KHMER, part, index)) return KHMER_SCRIPT;
508
+ if (matchesAt(MYANMAR, part, index)) return MYANMAR_SCRIPT;
509
+ return OTHER_SCRIPT;
510
+ }
511
+ function classifySoutheastAsian(part, index, codePoint) {
512
+ if (codePoint <= THAI_BLOCK_END) return matchesAt(THAI, part, index) ? THAI_SCRIPT : OTHER_SCRIPT;
513
+ if (codePoint <= LAO_BLOCK_END) return matchesAt(LAO, part, index) ? LAO_SCRIPT : OTHER_SCRIPT;
514
+ if (codePoint >= MYANMAR_BLOCK_START && codePoint <= MYANMAR_BLOCK_END) {
515
+ return matchesAt(MYANMAR, part, index) ? MYANMAR_SCRIPT : OTHER_SCRIPT;
516
+ }
517
+ if (codePoint >= KHMER_BLOCK_START && codePoint <= KHMER_BLOCK_END) {
518
+ return matchesAt(KHMER, part, index) ? KHMER_SCRIPT : OTHER_SCRIPT;
519
+ }
520
+ return classifyOutsideCommonBlocks(part, index);
521
+ }
522
+ function classify(part, index, codePoint, current) {
523
+ if (codePoint < LOWEST_NGRAM_CODE_POINT) return OTHER_SCRIPT;
524
+ if (codePoint < SOUTHEAST_ASIAN_BLOCKS_END) return classifySoutheastAsian(part, index, codePoint);
525
+ if (codePoint === PROLONGED_SOUND_MARK) {
526
+ return current === HIRAGANA_SCRIPT ? HIRAGANA_SCRIPT : KATAKANA_SCRIPT;
527
+ }
528
+ if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= HIRAGANA_BLOCK_END) return HIRAGANA_SCRIPT;
529
+ if (codePoint >= KATAKANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) return KATAKANA_SCRIPT;
530
+ if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return HAN_SCRIPT;
531
+ if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return HAN_SCRIPT;
532
+ if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return HANGUL_SCRIPT;
533
+ if (codePoint >= HAN_COMPATIBILITY_START && codePoint <= HAN_COMPATIBILITY_END) return HAN_SCRIPT;
534
+ if (codePoint === IDEOGRAPHIC_ITERATION_MARK) return HAN_SCRIPT;
535
+ return classifyOutsideCommonBlocks(part, index);
536
+ }
537
+ function emitRun(part, offsets, kind, size, out) {
538
+ const count = offsets.length - 1;
539
+ if (count < 1) return;
540
+ if (kind === OTHER_SCRIPT || count <= size) {
541
+ out.push(part.slice(offsets[0], offsets[count]));
542
+ return;
543
+ }
544
+ for (let start = 0; start + size <= count; start++) {
545
+ out.push(part.slice(offsets[start], offsets[start + size]));
546
+ }
547
+ }
548
+ function expandNgrams(part, size, out) {
549
+ const length = part.length;
550
+ if (length === 0) return;
551
+ const offsets = [];
552
+ let kind = OTHER_SCRIPT;
553
+ let index = 0;
554
+ while (index < length) {
555
+ const codePoint = part.codePointAt(index);
556
+ if (codePoint === void 0) break;
557
+ const width = codePoint >= SUPPLEMENTARY_PLANE_START ? 2 : 1;
558
+ if (offsets.length > 0 && isMark(part, index, codePoint)) {
559
+ index += width;
560
+ continue;
561
+ }
562
+ const next = classify(part, index, codePoint, kind);
563
+ if (offsets.length > 0 && next !== kind) {
564
+ offsets.push(index);
565
+ emitRun(part, offsets, kind, size, out);
566
+ offsets.length = 0;
567
+ }
568
+ kind = next;
569
+ offsets.push(index);
570
+ index += width;
571
+ }
572
+ offsets.push(length);
573
+ emitRun(part, offsets, kind, size, out);
574
+ }
575
+
576
+ // src/core/tokenizer/tokenize.ts
577
+ function producesSurfaceForms(language, options) {
578
+ if (!options?.collectSurfaces || options.customTokenizer) return false;
579
+ return (options.stem ?? true) && language.stemmer !== null;
580
+ }
581
+ var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
582
+ function resolveStopWords(language, override) {
583
+ if (!override) return language.stopWords;
584
+ if (override instanceof Set) return override;
585
+ return override(language.stopWords);
586
+ }
587
+ function splitText(text, language) {
588
+ const pattern = language.tokenizer?.splitPattern ?? DEFAULT_SPLIT_PATTERN;
589
+ const parts = text.split(pattern);
590
+ const ngramSize = language.tokenizer?.ngramSize;
591
+ if (ngramSize === void 0 || ngramSize < 1) return parts;
592
+ const size = Math.floor(ngramSize);
593
+ const expanded = [];
594
+ for (const part of parts) {
595
+ expandNgrams(part, size, expanded);
596
+ }
597
+ return expanded;
598
+ }
599
+ var CHAR_APOSTROPHE = 39;
600
+ var CHAR_S = 115;
601
+ function stripPossessive(token) {
602
+ const len = token.length;
603
+ if (len >= 2 && token.charCodeAt(len - 1) === CHAR_S && token.charCodeAt(len - 2) === CHAR_APOSTROPHE) {
604
+ return token.slice(0, -2);
605
+ }
606
+ if (len >= 1 && token.charCodeAt(len - 1) === CHAR_APOSTROPHE) {
607
+ return token.slice(0, -1);
608
+ }
609
+ return token;
610
+ }
611
+ function visitTokens(text, language, options, visit) {
612
+ const {
613
+ stem = true,
614
+ removeStopWords = true,
615
+ removeDiacritics = false,
616
+ collectSurfaces = false,
617
+ customTokenizer,
618
+ stopWordOverride
619
+ } = options ?? {};
620
+ if (customTokenizer) {
621
+ for (const entry of customTokenizer.tokenize(text)) visit(entry.token, entry.position, void 0, entry.token);
622
+ return;
623
+ }
624
+ const rawParts = splitText(normalizeForSplitting(text), language);
625
+ const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
626
+ const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
627
+ const wantSurfaces = collectSurfaces && stem && language.stemmer !== null;
628
+ const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : null;
629
+ const stripPossessives = language.tokenizer?.stripPossessive ?? false;
630
+ for (let position = 0; position < rawParts.length; position++) {
631
+ const part = rawParts[position];
632
+ const candidate = stripPossessives ? stripPossessive(part) : part;
633
+ if (candidate.length < minLength || stopWords?.has(candidate)) continue;
634
+ const processed = transformToken(candidate, language, stem, effectiveDiacritics);
635
+ if (processed.length === 0) continue;
636
+ if (!wantSurfaces) {
637
+ visit(processed, position, void 0, part);
638
+ continue;
639
+ }
640
+ const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
641
+ visit(processed, position, surface === processed ? void 0 : surface, part);
642
+ }
643
+ }
644
+ function tokenize(text, language, options) {
645
+ if (options?.customTokenizer) {
646
+ const customResult = options.customTokenizer.tokenize(text);
647
+ return { tokens: customResult, originalTokens: customResult.map((entry) => entry.token) };
648
+ }
649
+ const tokens = [];
650
+ const originalTokens = [];
651
+ const surfaces = producesSurfaceForms(language, options) ? [] : void 0;
652
+ visitTokens(text, language, options, (token, position, surface, original) => {
653
+ tokens.push({ token, position });
654
+ originalTokens.push(original);
655
+ surfaces?.push(surface);
656
+ });
657
+ return { tokens, originalTokens, surfaces };
658
+ }
659
+
660
+ // src/core/partition/utils.ts
661
+ function getOrCreateFieldNameIndex(table, fieldName) {
662
+ const existing = table.indexMap.get(fieldName);
663
+ if (existing !== void 0) return existing;
664
+ const idx = table.names.length;
665
+ table.names.push(fieldName);
666
+ table.indexMap.set(fieldName, idx);
667
+ return idx;
668
+ }
669
+ var VECTOR_PATTERN = /^vector\[(\d+)]$/;
670
+ function getNestedValue(obj, path) {
671
+ const parts = path.split(".");
672
+ let current = obj;
673
+ for (let i = 0; i < parts.length; i++) {
674
+ if (current === null || current === void 0 || typeof current !== "object") return void 0;
675
+ if (!Object.hasOwn(current, parts[i])) return void 0;
676
+ current = current[parts[i]];
677
+ }
678
+ return current;
679
+ }
680
+ function stringArraysEqual(a, b) {
681
+ if (a === b) return true;
682
+ if (!a || !b) return false;
683
+ if (a.length !== b.length) return false;
684
+ for (let i = 0; i < a.length; i++) {
685
+ if (a[i] !== b[i]) return false;
686
+ }
687
+ return true;
688
+ }
689
+ function textFieldsChanged(oldDoc, newDoc, fields) {
690
+ for (const { path, base } of fields) {
691
+ if (!isWordIndexedBase(base)) continue;
692
+ const oldVal = getNestedValue(oldDoc, path);
693
+ const newVal = getNestedValue(newDoc, path);
694
+ if (base === "string") {
695
+ if (oldVal !== newVal) return true;
696
+ } else {
697
+ if (!stringArraysEqual(oldVal, newVal)) return true;
698
+ }
699
+ }
700
+ return false;
701
+ }
702
+ function parseVectorDimension(fieldType) {
703
+ const match = VECTOR_PATTERN.exec(fieldType);
704
+ return match ? Number.parseInt(match[1], 10) : null;
705
+ }
706
+ function tokenizeOptions(options) {
707
+ return {
708
+ stem: true,
709
+ removeStopWords: true,
710
+ collectSurfaces: options?.collectSurfaces === true,
711
+ stopWordOverride: options?.stopWordOverride,
712
+ customTokenizer: options?.customTokenizer
713
+ };
714
+ }
715
+ function getFlatSchema(state, schema) {
716
+ if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
717
+ state.flatSchemaCache = flattenSchema(schema);
718
+ state.lastSchemaRef = schema;
719
+ return state.flatSchemaCache;
720
+ }
721
+ function getFieldValueForDoc(docStore, docId, fieldPath) {
722
+ const stored = docStore.get(docId);
723
+ if (!stored) return void 0;
724
+ return getNestedValue(stored.fields, fieldPath);
725
+ }
726
+ function getFieldValueByInternalId(docStore, internalId, fieldPath) {
727
+ const externalId = docStore.getExternalId(internalId);
728
+ if (externalId === void 0) return void 0;
729
+ return getFieldValueForDoc(docStore, externalId, fieldPath);
730
+ }
731
+
249
732
  // src/core/partition/partial-words/spellings.ts
250
733
  var LAST_BASIC_PLANE_CODE_POINT2 = 65535;
251
734
  function codePointCountOf(text) {
@@ -806,13 +1289,112 @@ function createGrowingPartialWordSlot(view) {
806
1289
  return slot;
807
1290
  }
808
1291
 
1292
+ // src/core/partition/partial-words/highlight.ts
1293
+ function containsPartialWord(spelling, words) {
1294
+ for (const word of words) {
1295
+ if (spelling.length > word.length && spelling.includes(word)) return true;
1296
+ }
1297
+ return false;
1298
+ }
1299
+ function partialWordMatcher(text, language, partial) {
1300
+ const indexed = tokenize(text, language, {
1301
+ stem: true,
1302
+ removeStopWords: true,
1303
+ collectSurfaces: true,
1304
+ customTokenizer: partial.customTokenizer,
1305
+ stopWordOverride: partial.stopWordOverride
1306
+ });
1307
+ const positions = /* @__PURE__ */ new Set();
1308
+ const spellings = /* @__PURE__ */ new Set();
1309
+ for (let at = 0; at < indexed.tokens.length; at++) {
1310
+ const spelling = indexed.surfaces?.[at] ?? indexed.tokens[at].token;
1311
+ if (!containsPartialWord(spelling, partial.words)) continue;
1312
+ positions.add(indexed.tokens[at].position);
1313
+ spellings.add(spelling);
1314
+ }
1315
+ if (partial.customTokenizer !== void 0) return (_position, fieldToken) => spellings.has(fieldToken);
1316
+ return (position) => positions.has(position);
1317
+ }
1318
+
1319
+ // src/core/partition/partial-words/query.ts
1320
+ function addPartialWords(group, added) {
1321
+ const membersByToken = /* @__PURE__ */ new Map();
1322
+ for (const member of group.members) membersByToken.set(member.token, member);
1323
+ for (const word of added) {
1324
+ const member = membersByToken.get(word.token);
1325
+ if (member === void 0) {
1326
+ group.members.push({ token: word.token, factor: 0, partialFactor: word.factor });
1327
+ } else if (word.factor > member.partialFactor) {
1328
+ member.partialFactor = word.factor;
1329
+ }
1330
+ }
1331
+ }
1332
+ function partialWordRequests(query) {
1333
+ const eligible = [];
1334
+ for (const queryToken of query.queryTokens) {
1335
+ const word = query.written.get(queryToken.position);
1336
+ if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
1337
+ eligible.push({ token: queryToken.token, written: word });
1338
+ if (eligible.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
1339
+ }
1340
+ const count = eligible.length;
1341
+ if (count === 0) return [];
1342
+ const evenShare = Math.floor(PARTIAL_WORDS_ADDED_PER_QUERY / count);
1343
+ const remainder = PARTIAL_WORDS_ADDED_PER_QUERY % count;
1344
+ return eligible.map((entry, order) => ({
1345
+ token: entry.token,
1346
+ request: {
1347
+ written: entry.written,
1348
+ ownToken: entry.token,
1349
+ share: evenShare + (order < remainder ? 1 : 0),
1350
+ tolerance: entry.token === query.prefixToken ? 0 : query.tolerance,
1351
+ prefixLength: query.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH
1352
+ }
1353
+ }));
1354
+ }
1355
+ function addPartialWordMembers(query, expand, groupOf) {
1356
+ const requests = partialWordRequests(query);
1357
+ const added = expand(
1358
+ query.analysis,
1359
+ requests.map((entry) => entry.request)
1360
+ );
1361
+ for (let at = 0; at < requests.length; at++) {
1362
+ if (added[at].length === 0) continue;
1363
+ const token = requests[at].token;
1364
+ addPartialWords(groupOf(token, token !== query.prefixToken), added[at]);
1365
+ }
1366
+ }
1367
+ function partialWordsToHighlight(queryTokens, writtenWords) {
1368
+ const writtenAt = /* @__PURE__ */ new Map();
1369
+ for (const word of writtenWords) {
1370
+ if (!writtenAt.has(word.position)) writtenAt.set(word.position, word.token);
1371
+ }
1372
+ const seenTokens = /* @__PURE__ */ new Set();
1373
+ const words = [];
1374
+ for (const queryToken of queryTokens) {
1375
+ if (seenTokens.has(queryToken.token)) continue;
1376
+ seenTokens.add(queryToken.token);
1377
+ const word = writtenAt.get(queryToken.position);
1378
+ if (word === void 0 || codePointCount(word) < PARTIAL_WORD_MIN_CODE_POINTS) continue;
1379
+ words.push(word);
1380
+ if (words.length === PARTIAL_WORDS_ADDED_PER_QUERY) break;
1381
+ }
1382
+ return words;
1383
+ }
1384
+
809
1385
  // src/core/partition/partial-words/module.ts
810
1386
  var partialWordSearchModule = {
811
1387
  name: "partialWords",
812
1388
  createGrowingSlot: createGrowingPartialWordSlot,
813
1389
  createFrozenSlot: createFrozenPartialWordSlot,
814
1390
  buildTable: buildPartialWordTable,
815
- expand: expandPartialWords
1391
+ expand: expandPartialWords,
1392
+ countSpelling,
1393
+ encodeSpellings: encodePartialSpellings,
1394
+ spellingCountsOf,
1395
+ highlightMatcher: partialWordMatcher,
1396
+ addTermMembers: addPartialWordMembers,
1397
+ highlightWords: partialWordsToHighlight
816
1398
  };
817
1399
 
818
- export { PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES, codePointOrder, countSpelling, encodeLookupKey, encodePartialSpellings, encodeStringBlob, expandTermPrefix, partialWordSearchModule, partitionDocumentFrequency, sameFields, spellingCountsOf, suggestDisplayTerms };
1400
+ export { PRUNING_REJECTION_SLACK, SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY, SEGMENT_FIELD_SCRATCH_START, SEGMENT_POSITION_POOL_START, SEGMENT_POSTING_LOG_START, SEGMENT_TOKEN_TABLE_START, SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD, SORT_COLUMN_REBUILD_FRACTION_SHIFT, TARGET_BLOCK_ENTRIES, clearNormalizationCache, codePointOrder, configureNormalizationCache, encodeLookupKey, encodeStringBlob, expandTermPrefix, getFieldValueByInternalId, getFieldValueForDoc, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, parseVectorDimension, partialWordSearchModule, partitionDocumentFrequency, resetNormalizationCache, sameFields, suggestDisplayTerms, textFieldsChanged, tokenize, tokenizeOptions, visitTokens };
@@ -1,5 +1,6 @@
1
- import { MAX_FILTER_DEPTH, MAX_SORT_FIELDS } from './chunk-IOZPBNSG.mjs';
2
1
  import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-2E35UM3G.mjs';
2
+ import { MAX_FILTER_DEPTH } from './chunk-WQBCBKY3.mjs';
3
+ import { MAX_SORT_FIELDS } from './chunk-OSF3SGJV.mjs';
3
4
  import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-I64LXKO3.mjs';
4
5
  import { mkdtemp, open, rm } from 'fs/promises';
5
6
  import { tmpdir } from 'os';