@delali/narsil 0.2.4-next.48 → 0.2.4-next.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +2 -2
  2. package/dist/adapters/indexeddb.mjs +15 -10
  3. package/dist/{options-BC68VPoE.d.ts → capabilities-JrVQOwi4.d.ts} +55 -47
  4. package/dist/{chunk-BFIH7OV5.mjs → chunk-2GWVQWGA.mjs} +3 -1
  5. package/dist/chunk-AV4IJKD2.mjs +35 -0
  6. package/dist/{chunk-EV45L35L.mjs → chunk-BEIK3EOZ.mjs} +845 -2407
  7. package/dist/chunk-CBLS46IE.mjs +10 -0
  8. package/dist/chunk-CMVG2UEF.mjs +2038 -0
  9. package/dist/chunk-CV5H7XL7.mjs +876 -0
  10. package/dist/{chunk-JQUL2T6I.mjs → chunk-GCM3JFVA.mjs} +1 -2
  11. package/dist/{chunk-XWICBGYT.mjs → chunk-MHE7YWAU.mjs} +3 -5
  12. package/dist/{chunk-UZTT2BKQ.mjs → chunk-ND34I3SV.mjs} +1202 -1881
  13. package/dist/{chunk-PM52SMBT.mjs → chunk-OOSIDHA3.mjs} +2 -1
  14. package/dist/chunk-ORBLD3EL.mjs +6 -0
  15. package/dist/chunk-PIWKZPP7.mjs +64 -0
  16. package/dist/{chunk-4COAUBLJ.mjs → chunk-PMBHH44R.mjs} +124 -8
  17. package/dist/chunk-RP23PWMV.mjs +11 -0
  18. package/dist/chunk-UNEEFTEU.mjs +7 -0
  19. package/dist/{chunk-WTONLDU3.mjs → chunk-WYJXSLER.mjs} +9 -19
  20. package/dist/{chunk-DFZDV5FV.mjs → chunk-X6YZO4MQ.mjs} +19 -15
  21. package/dist/chunk-XE6XR2OE.mjs +23 -0
  22. package/dist/chunk-YEO6G2NG.mjs +6 -0
  23. package/dist/chunk-ZHUQKZBE.mjs +1292 -0
  24. package/dist/{client-C5BkUqB8.d.ts → client-CBOVHdjp.d.ts} +31 -7
  25. package/dist/client.d.ts +7 -7
  26. package/dist/client.mjs +250 -232
  27. package/dist/{config-B-ZI-ft5.d.ts → config-Czh2YNDi.d.ts} +2 -2
  28. package/dist/{config-BZRExglL.d.ts → config-OON7IJ_B.d.ts} +50 -13
  29. package/dist/direct-executor-OREXU72C.mjs +12 -0
  30. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  31. package/dist/distribution/coordinator/etcd.mjs +2 -3
  32. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  33. package/dist/distribution/coordinator/in-memory.mjs +2 -2
  34. package/dist/distribution/transport/grpc.mjs +7 -9
  35. package/dist/distribution/transport/in-memory.mjs +2 -1
  36. package/dist/distribution/transport/tcp.mjs +3 -2
  37. package/dist/distribution.d.ts +8 -8
  38. package/dist/distribution.mjs +142 -101
  39. package/dist/embeddings/openai.mjs +5 -3
  40. package/dist/{engine-Cjr1uBLa.d.ts → engine-9PtFY50q.d.ts} +125 -69
  41. package/dist/index.browser.mjs +4689 -3330
  42. package/dist/index.d.ts +7 -7
  43. package/dist/index.mjs +14 -8
  44. package/dist/invalidation/broadcast-channel.mjs +2 -1
  45. package/dist/invalidation/filesystem.mjs +3 -4
  46. package/dist/{search-DDHjfXhJ.d.ts → memory-Bzd5xcEW.d.ts} +89 -63
  47. package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
  48. package/dist/persistence/durability/checkpoint-worker.mjs +14 -8
  49. package/dist/react.d.ts +4 -4
  50. package/dist/react.mjs +7 -5
  51. package/dist/{schema-B7juWg8Q.d.ts → schema-BqRx9t4E.d.ts} +22 -3
  52. package/dist/server.d.ts +47 -42
  53. package/dist/server.mjs +338 -821
  54. package/dist/thread-PZLEO6JL.mjs +310 -0
  55. package/dist/{types-DsSSjY4d.d.ts → types-DXPxaXPO.d.ts} +1 -1
  56. package/dist/vector/hnsw-build-worker.mjs +3 -4
  57. package/dist/vector/search-worker.mjs +4 -123
  58. package/dist/workers/entry.mjs +47 -3
  59. package/package.json +5 -1
  60. package/dist/direct-executor-EOTFGXTO.mjs +0 -8
@@ -0,0 +1,2038 @@
1
+ import { MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, RESULT_WINDOW, DEFAULT_PAGE_SIZE, PREFIX_MAX_EXPANSIONS, MAX_CURSOR_ANCHOR_CODE_POINTS } from './chunk-CBLS46IE.mjs';
2
+ import { MAX_M } from './chunk-RP23PWMV.mjs';
3
+ import { compareCodePoints, compareComparableKeys, toComparableSortValue, readSortField, SORT_VALUE_MAX_CODE_POINTS, compareSortValues } from './chunk-2GWVQWGA.mjs';
4
+ import { createNarsilError, ErrorCodes, NarsilError } from './chunk-P4ELXVJQ.mjs';
5
+
6
+ // src/core/tokenizer/constants.ts
7
+ var TOKEN_CACHE_SIZE_FLOOR = 5e4;
8
+ var TOKEN_CACHE_SIZE_CEILING = 2e6;
9
+ var TOKEN_CACHE_BYTES_PER_ENTRY = 200;
10
+ var NODE_TOKEN_CACHE_ENTRIES = 1e6;
11
+ var UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES = 2e5;
12
+ var CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION = 0.05;
13
+ var SMALL_DEVICE_MEMORY_GB = 1;
14
+ var MEDIUM_DEVICE_MEMORY_GB = 4;
15
+ var SMALL_DEVICE_TOKEN_CACHE_ENTRIES = 1e5;
16
+ var MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES = 25e4;
17
+ var LARGE_DEVICE_TOKEN_CACHE_ENTRIES = 5e5;
18
+ var DEFAULT_MIN_TOKEN_LENGTH = 1;
19
+
20
+ // src/core/tokenizer/normalize.ts
21
+ var DOTTED_CAPITAL_I = "\u0130";
22
+ var COMBINING_DOT_ABOVE = "\u0307";
23
+ var ARMENIAN_MARKS = /[\u055B-\u055F]/;
24
+ var FOLDED_FORMS = /\u0307|[\u0130\u055B-\u055F\u2018\u2019\u02BC\u02BB]/g;
25
+ var WIDTH_FORMS = /[\uFF01-\uFF5E\uFF61-\uFF9F]/;
26
+ var WIDTH_FORMS_GLOBAL = /[\uFF01-\uFF5E\uFF61-\uFF9F]/g;
27
+ var FULLWIDTH_ASCII_END = 65374;
28
+ var FULLWIDTH_ASCII_OFFSET = 65248;
29
+ var HALFWIDTH_KANA_START = 65377;
30
+ var HALFWIDTH_VOICED_MARK = 65438;
31
+ var HALFWIDTH_SEMI_VOICED_MARK = 65439;
32
+ var COMBINING_VOICED_MARK = "\u3099";
33
+ var COMBINING_SEMI_VOICED_MARK = "\u309A";
34
+ var HALFWIDTH_KANA_TARGETS = "\u3002\u300C\u300D\u3001\u30FB\u30F2\u30A1\u30A3\u30A5\u30A7\u30A9\u30E3\u30E5\u30E7\u30C3\u30FC\u30A2\u30A4\u30A6\u30A8\u30AA\u30AB\u30AD\u30AF\u30B1\u30B3\u30B5\u30B7\u30B9\u30BB\u30BD\u30BF\u30C1\u30C4\u30C6\u30C8\u30CA\u30CB\u30CC\u30CD\u30CE\u30CF\u30D2\u30D5\u30D8\u30DB\u30DE\u30DF\u30E0\u30E1\u30E2\u30E4\u30E6\u30E8\u30E9\u30EA\u30EB\u30EC\u30ED\u30EF\u30F3";
35
+ function foldForm(match) {
36
+ if (match === DOTTED_CAPITAL_I) return "i";
37
+ if (match === COMBINING_DOT_ABOVE || ARMENIAN_MARKS.test(match)) return "";
38
+ return "'";
39
+ }
40
+ function foldWidth(match) {
41
+ const codePoint = match.charCodeAt(0);
42
+ if (codePoint <= FULLWIDTH_ASCII_END) return String.fromCharCode(codePoint - FULLWIDTH_ASCII_OFFSET);
43
+ if (codePoint === HALFWIDTH_VOICED_MARK) return COMBINING_VOICED_MARK;
44
+ if (codePoint === HALFWIDTH_SEMI_VOICED_MARK) return COMBINING_SEMI_VOICED_MARK;
45
+ return HALFWIDTH_KANA_TARGETS[codePoint - HALFWIDTH_KANA_START];
46
+ }
47
+ function isAscii(text) {
48
+ for (let i = 0; i < text.length; i++) {
49
+ if (text.charCodeAt(i) > 127) return false;
50
+ }
51
+ return true;
52
+ }
53
+ function foldWidthForms(text) {
54
+ if (!WIDTH_FORMS.test(text)) return text;
55
+ return text.replace(WIDTH_FORMS_GLOBAL, foldWidth);
56
+ }
57
+ function normalizeForSplitting(text) {
58
+ if (isAscii(text)) return text.toLowerCase();
59
+ return foldWidthForms(text).normalize("NFC").replace(FOLDED_FORMS, foldForm).toLowerCase();
60
+ }
61
+ function stripDiacritics(text) {
62
+ return text.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
63
+ }
64
+
65
+ // src/core/tokenizer/cache.ts
66
+ function clampCacheSize(entries) {
67
+ return Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(entries, TOKEN_CACHE_SIZE_CEILING));
68
+ }
69
+ function computeDefaultCacheSize() {
70
+ try {
71
+ if (typeof process !== "undefined" && typeof process.versions?.node === "string" && typeof window === "undefined") {
72
+ const constrainedMemory = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
73
+ if (constrainedMemory > 0) {
74
+ const budget = constrainedMemory * CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION;
75
+ return clampCacheSize(Math.floor(budget / TOKEN_CACHE_BYTES_PER_ENTRY));
76
+ }
77
+ return clampCacheSize(NODE_TOKEN_CACHE_ENTRIES);
78
+ }
79
+ if (typeof navigator !== "undefined") {
80
+ const mem = navigator.deviceMemory;
81
+ if (typeof mem === "number" && mem > 0) {
82
+ let entries;
83
+ if (mem <= SMALL_DEVICE_MEMORY_GB) entries = SMALL_DEVICE_TOKEN_CACHE_ENTRIES;
84
+ else if (mem <= MEDIUM_DEVICE_MEMORY_GB) entries = MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES;
85
+ else entries = LARGE_DEVICE_TOKEN_CACHE_ENTRIES;
86
+ return clampCacheSize(entries);
87
+ }
88
+ }
89
+ } catch {
90
+ return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
91
+ }
92
+ return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
93
+ }
94
+ var normalizationCache = /* @__PURE__ */ new Map();
95
+ var normalizationCacheSize = 0;
96
+ var maxCacheSize = computeDefaultCacheSize();
97
+ var cachedLangName = "";
98
+ var cachedFlags = "";
99
+ var cachedBucket = null;
100
+ function configureNormalizationCache(maxSize) {
101
+ if (!Number.isFinite(maxSize)) {
102
+ throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must be a finite number", {
103
+ received: maxSize
104
+ });
105
+ }
106
+ if (maxSize < 0) {
107
+ throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must not be negative", {
108
+ received: maxSize
109
+ });
110
+ }
111
+ if (maxSize === 0) {
112
+ throw createNarsilError(
113
+ ErrorCodes.CONFIG_INVALID,
114
+ "tokenizerCacheSize must be greater than zero; the normalization cache cannot be disabled",
115
+ {
116
+ received: maxSize
117
+ }
118
+ );
119
+ }
120
+ maxCacheSize = Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(Math.floor(maxSize), TOKEN_CACHE_SIZE_CEILING));
121
+ if (normalizationCacheSize > maxCacheSize) {
122
+ evictOldestEntries(normalizationCacheSize - maxCacheSize);
123
+ }
124
+ }
125
+ function getCacheBucket(language, flags) {
126
+ const langKey = `${language.name}@${language.revision}`;
127
+ if (cachedBucket && langKey === cachedLangName && flags === cachedFlags) return cachedBucket;
128
+ const key = `${langKey}:${flags}`;
129
+ let bucket = normalizationCache.get(key);
130
+ if (!bucket) {
131
+ bucket = /* @__PURE__ */ new Map();
132
+ normalizationCache.set(key, bucket);
133
+ }
134
+ cachedLangName = langKey;
135
+ cachedFlags = flags;
136
+ cachedBucket = bucket;
137
+ return bucket;
138
+ }
139
+ function evictOldestEntries(count) {
140
+ let remaining = count;
141
+ for (const [key, bucket] of normalizationCache) {
142
+ if (remaining <= 0) break;
143
+ if (bucket.size <= remaining) {
144
+ remaining -= bucket.size;
145
+ normalizationCacheSize -= bucket.size;
146
+ normalizationCache.delete(key);
147
+ if (bucket === cachedBucket) cachedBucket = null;
148
+ } else {
149
+ let deleted = 0;
150
+ for (const raw of bucket.keys()) {
151
+ if (deleted >= remaining) break;
152
+ bucket.delete(raw);
153
+ deleted++;
154
+ }
155
+ normalizationCacheSize -= deleted;
156
+ remaining = 0;
157
+ }
158
+ }
159
+ }
160
+ function transformToken(raw, language, stem, removeDiacritics) {
161
+ const flags = (stem ? "s" : "") + (removeDiacritics ? "d" : "");
162
+ let bucket = getCacheBucket(language, flags);
163
+ const cached = bucket.get(raw);
164
+ if (cached !== void 0) return cached;
165
+ let normalized = raw;
166
+ if (language.normalizer) {
167
+ normalized = language.normalizer(normalized);
168
+ }
169
+ if (removeDiacritics) {
170
+ normalized = stripDiacritics(normalized);
171
+ }
172
+ if (stem && language.stemmer) {
173
+ normalized = language.stemmer(normalized);
174
+ }
175
+ if (normalizationCacheSize >= maxCacheSize) {
176
+ evictOldestEntries(Math.max(1, maxCacheSize >>> 2));
177
+ bucket = getCacheBucket(language, flags);
178
+ }
179
+ bucket.set(raw, normalized);
180
+ normalizationCacheSize++;
181
+ return normalized;
182
+ }
183
+ function clearNormalizationCache() {
184
+ normalizationCache.clear();
185
+ normalizationCacheSize = 0;
186
+ cachedBucket = null;
187
+ }
188
+ function resetNormalizationCache() {
189
+ clearNormalizationCache();
190
+ maxCacheSize = computeDefaultCacheSize();
191
+ }
192
+ function getNormalizationCacheSize() {
193
+ return normalizationCacheSize;
194
+ }
195
+
196
+ // src/core/tokenizer/ngrams.ts
197
+ var OTHER_SCRIPT = 0;
198
+ var HAN_SCRIPT = 1;
199
+ var HIRAGANA_SCRIPT = 2;
200
+ var KATAKANA_SCRIPT = 3;
201
+ var HANGUL_SCRIPT = 4;
202
+ var THAI_SCRIPT = 5;
203
+ var LAO_SCRIPT = 6;
204
+ var KHMER_SCRIPT = 7;
205
+ var MYANMAR_SCRIPT = 8;
206
+ var PROLONGED_SOUND_MARK = 12540;
207
+ var IDEOGRAPHIC_ITERATION_MARK = 12293;
208
+ var LOWEST_NGRAM_CODE_POINT = 3584;
209
+ var LOWEST_MARK_CODE_POINT = 768;
210
+ var THAI_BLOCK_END = 3711;
211
+ var LAO_BLOCK_END = 3839;
212
+ var MYANMAR_BLOCK_START = 4096;
213
+ var MYANMAR_BLOCK_END = 4255;
214
+ var KHMER_BLOCK_START = 6016;
215
+ var KHMER_BLOCK_END = 6655;
216
+ var SOUTHEAST_ASIAN_BLOCKS_END = 6656;
217
+ var HIRAGANA_BLOCK_START = 12352;
218
+ var HIRAGANA_BLOCK_END = 12447;
219
+ var COMBINING_VOICED_MARK2 = 12441;
220
+ var COMBINING_SEMI_VOICED_MARK2 = 12442;
221
+ var KATAKANA_BLOCK_START = 12448;
222
+ var KATAKANA_BLOCK_END = 12543;
223
+ var HAN_EXTENSION_A_START = 13312;
224
+ var HAN_EXTENSION_A_END = 19903;
225
+ var HAN_UNIFIED_START = 19968;
226
+ var HAN_UNIFIED_END = 40959;
227
+ var HANGUL_SYLLABLE_START = 44032;
228
+ var HANGUL_SYLLABLE_END = 55203;
229
+ var HAN_COMPATIBILITY_START = 63744;
230
+ var HAN_COMPATIBILITY_END = 64255;
231
+ var SUPPLEMENTARY_PLANE_START = 65536;
232
+ var MARK = /\p{M}/uy;
233
+ var HAN = /\p{Script=Han}/uy;
234
+ var HIRAGANA = /\p{Script=Hiragana}/uy;
235
+ var KATAKANA = /\p{Script=Katakana}/uy;
236
+ var HANGUL = /\p{Script=Hangul}/uy;
237
+ var THAI = /\p{Script=Thai}/uy;
238
+ var LAO = /\p{Script=Lao}/uy;
239
+ var KHMER = /\p{Script=Khmer}/uy;
240
+ var MYANMAR = /\p{Script=Myanmar}/uy;
241
+ function matchesAt(pattern, part, index) {
242
+ pattern.lastIndex = index;
243
+ return pattern.test(part);
244
+ }
245
+ function isMark(part, index, codePoint) {
246
+ if (codePoint < LOWEST_MARK_CODE_POINT) return false;
247
+ if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return false;
248
+ if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) {
249
+ return codePoint === COMBINING_VOICED_MARK2 || codePoint === COMBINING_SEMI_VOICED_MARK2;
250
+ }
251
+ if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return false;
252
+ if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return false;
253
+ return matchesAt(MARK, part, index);
254
+ }
255
+ function classifyOutsideCommonBlocks(part, index) {
256
+ if (matchesAt(HAN, part, index)) return HAN_SCRIPT;
257
+ if (matchesAt(HIRAGANA, part, index)) return HIRAGANA_SCRIPT;
258
+ if (matchesAt(KATAKANA, part, index)) return KATAKANA_SCRIPT;
259
+ if (matchesAt(HANGUL, part, index)) return HANGUL_SCRIPT;
260
+ if (matchesAt(THAI, part, index)) return THAI_SCRIPT;
261
+ if (matchesAt(LAO, part, index)) return LAO_SCRIPT;
262
+ if (matchesAt(KHMER, part, index)) return KHMER_SCRIPT;
263
+ if (matchesAt(MYANMAR, part, index)) return MYANMAR_SCRIPT;
264
+ return OTHER_SCRIPT;
265
+ }
266
+ function classifySoutheastAsian(part, index, codePoint) {
267
+ if (codePoint <= THAI_BLOCK_END) return matchesAt(THAI, part, index) ? THAI_SCRIPT : OTHER_SCRIPT;
268
+ if (codePoint <= LAO_BLOCK_END) return matchesAt(LAO, part, index) ? LAO_SCRIPT : OTHER_SCRIPT;
269
+ if (codePoint >= MYANMAR_BLOCK_START && codePoint <= MYANMAR_BLOCK_END) {
270
+ return matchesAt(MYANMAR, part, index) ? MYANMAR_SCRIPT : OTHER_SCRIPT;
271
+ }
272
+ if (codePoint >= KHMER_BLOCK_START && codePoint <= KHMER_BLOCK_END) {
273
+ return matchesAt(KHMER, part, index) ? KHMER_SCRIPT : OTHER_SCRIPT;
274
+ }
275
+ return classifyOutsideCommonBlocks(part, index);
276
+ }
277
+ function classify(part, index, codePoint, current) {
278
+ if (codePoint < LOWEST_NGRAM_CODE_POINT) return OTHER_SCRIPT;
279
+ if (codePoint < SOUTHEAST_ASIAN_BLOCKS_END) return classifySoutheastAsian(part, index, codePoint);
280
+ if (codePoint === PROLONGED_SOUND_MARK) {
281
+ return current === HIRAGANA_SCRIPT ? HIRAGANA_SCRIPT : KATAKANA_SCRIPT;
282
+ }
283
+ if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= HIRAGANA_BLOCK_END) return HIRAGANA_SCRIPT;
284
+ if (codePoint >= KATAKANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) return KATAKANA_SCRIPT;
285
+ if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return HAN_SCRIPT;
286
+ if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return HAN_SCRIPT;
287
+ if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return HANGUL_SCRIPT;
288
+ if (codePoint >= HAN_COMPATIBILITY_START && codePoint <= HAN_COMPATIBILITY_END) return HAN_SCRIPT;
289
+ if (codePoint === IDEOGRAPHIC_ITERATION_MARK) return HAN_SCRIPT;
290
+ return classifyOutsideCommonBlocks(part, index);
291
+ }
292
+ function emitRun(part, offsets, kind, size, out) {
293
+ const count = offsets.length - 1;
294
+ if (count < 1) return;
295
+ if (kind === OTHER_SCRIPT || count <= size) {
296
+ out.push(part.slice(offsets[0], offsets[count]));
297
+ return;
298
+ }
299
+ for (let start = 0; start + size <= count; start++) {
300
+ out.push(part.slice(offsets[start], offsets[start + size]));
301
+ }
302
+ }
303
+ function expandNgrams(part, size, out) {
304
+ const length = part.length;
305
+ if (length === 0) return;
306
+ const offsets = [];
307
+ let kind = OTHER_SCRIPT;
308
+ let index = 0;
309
+ while (index < length) {
310
+ const codePoint = part.codePointAt(index);
311
+ if (codePoint === void 0) break;
312
+ const width = codePoint >= SUPPLEMENTARY_PLANE_START ? 2 : 1;
313
+ if (offsets.length > 0 && isMark(part, index, codePoint)) {
314
+ index += width;
315
+ continue;
316
+ }
317
+ const next = classify(part, index, codePoint, kind);
318
+ if (offsets.length > 0 && next !== kind) {
319
+ offsets.push(index);
320
+ emitRun(part, offsets, kind, size, out);
321
+ offsets.length = 0;
322
+ }
323
+ kind = next;
324
+ offsets.push(index);
325
+ index += width;
326
+ }
327
+ offsets.push(length);
328
+ emitRun(part, offsets, kind, size, out);
329
+ }
330
+
331
+ // src/core/tokenizer/tokenize.ts
332
+ var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
333
+ function resolveStopWords(language, override) {
334
+ if (!override) return language.stopWords;
335
+ if (override instanceof Set) return override;
336
+ return override(language.stopWords);
337
+ }
338
+ function splitText(text, language) {
339
+ const pattern = language.tokenizer?.splitPattern ?? DEFAULT_SPLIT_PATTERN;
340
+ const parts = text.split(pattern);
341
+ const ngramSize = language.tokenizer?.ngramSize;
342
+ if (ngramSize === void 0 || ngramSize < 1) return parts;
343
+ const size = Math.floor(ngramSize);
344
+ const expanded = [];
345
+ for (const part of parts) {
346
+ expandNgrams(part, size, expanded);
347
+ }
348
+ return expanded;
349
+ }
350
+ var CHAR_APOSTROPHE = 39;
351
+ var CHAR_S = 115;
352
+ function stripPossessive(token) {
353
+ const len = token.length;
354
+ if (len >= 2 && token.charCodeAt(len - 1) === CHAR_S && token.charCodeAt(len - 2) === CHAR_APOSTROPHE) {
355
+ return token.slice(0, -2);
356
+ }
357
+ if (len >= 1 && token.charCodeAt(len - 1) === CHAR_APOSTROPHE) {
358
+ return token.slice(0, -1);
359
+ }
360
+ return token;
361
+ }
362
+ function tokenize(text, language, options) {
363
+ const {
364
+ stem = true,
365
+ removeStopWords = true,
366
+ removeDiacritics = false,
367
+ collectSurfaces = false,
368
+ customTokenizer,
369
+ stopWordOverride
370
+ } = options ?? {};
371
+ if (customTokenizer) {
372
+ const customResult = customTokenizer.tokenize(text);
373
+ const originals = customResult.map((t) => t.token);
374
+ return {
375
+ tokens: customResult,
376
+ originalTokens: originals
377
+ };
378
+ }
379
+ const normalized = normalizeForSplitting(text);
380
+ const rawParts = splitText(normalized, language);
381
+ const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
382
+ const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
383
+ const stemsTokens = stem && language.stemmer !== null;
384
+ const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
385
+ const stripPossessives = language.tokenizer?.stripPossessive ?? false;
386
+ const tokens = [];
387
+ const originalTokens = [];
388
+ const surfaces = collectSurfaces && stemsTokens ? [] : void 0;
389
+ let position = 0;
390
+ for (const part of rawParts) {
391
+ const candidate = stripPossessives ? stripPossessive(part) : part;
392
+ if (candidate.length < minLength) {
393
+ position++;
394
+ continue;
395
+ }
396
+ if (stopWords.has(candidate)) {
397
+ position++;
398
+ continue;
399
+ }
400
+ const processed = transformToken(candidate, language, stem, effectiveDiacritics);
401
+ if (processed.length > 0) {
402
+ tokens.push({ token: processed, position });
403
+ originalTokens.push(part);
404
+ if (surfaces) {
405
+ const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
406
+ surfaces.push(surface === processed ? void 0 : surface);
407
+ }
408
+ }
409
+ position++;
410
+ }
411
+ return { tokens, originalTokens, surfaces };
412
+ }
413
+ function* tokenizeIterator(text, language, options) {
414
+ const {
415
+ stem = true,
416
+ removeStopWords = true,
417
+ removeDiacritics = false,
418
+ collectSurfaces = false,
419
+ customTokenizer,
420
+ stopWordOverride
421
+ } = options ?? {};
422
+ if (customTokenizer) {
423
+ const customResult = customTokenizer.tokenize(text);
424
+ for (const entry of customResult) {
425
+ yield entry;
426
+ }
427
+ return;
428
+ }
429
+ const normalized = normalizeForSplitting(text);
430
+ const rawParts = splitText(normalized, language);
431
+ const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
432
+ const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
433
+ const stemsTokens = stem && language.stemmer !== null;
434
+ const wantSurfaces = collectSurfaces && stemsTokens;
435
+ const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
436
+ const stripPossessives = language.tokenizer?.stripPossessive ?? false;
437
+ let position = 0;
438
+ for (const part of rawParts) {
439
+ const candidate = stripPossessives ? stripPossessive(part) : part;
440
+ if (candidate.length < minLength) {
441
+ position++;
442
+ continue;
443
+ }
444
+ if (stopWords.has(candidate)) {
445
+ position++;
446
+ continue;
447
+ }
448
+ const processed = transformToken(candidate, language, stem, effectiveDiacritics);
449
+ if (processed.length > 0) {
450
+ if (wantSurfaces) {
451
+ const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
452
+ yield { token: processed, position, surface: surface === processed ? void 0 : surface };
453
+ } else {
454
+ yield { token: processed, position };
455
+ }
456
+ }
457
+ position++;
458
+ }
459
+ }
460
+
461
+ // src/schema/constants.ts
462
+ var MAX_NESTING_DEPTH = 4;
463
+ var MAX_DOCUMENT_NESTING_DEPTH = 32;
464
+
465
+ // src/schema/validator/shared.ts
466
+ var SORTABLE_TEXT_FIELD_TYPE = "string:sortable";
467
+ var SCALAR_FIELD_TYPES = /* @__PURE__ */ new Set([
468
+ "string",
469
+ SORTABLE_TEXT_FIELD_TYPE,
470
+ "number",
471
+ "boolean",
472
+ "enum",
473
+ "geopoint",
474
+ "string[]",
475
+ "number[]",
476
+ "boolean[]",
477
+ "enum[]"
478
+ ]);
479
+ function isTextFieldType(fieldType) {
480
+ return fieldType === "string" || fieldType === SORTABLE_TEXT_FIELD_TYPE;
481
+ }
482
+ var VECTOR_PATTERN = /^vector\[(\d+)]$/;
483
+ var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
484
+ var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
485
+ var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
486
+ function isPlainObject(value) {
487
+ return typeof value === "object" && value !== null && !Array.isArray(value);
488
+ }
489
+
490
+ // src/schema/validator/schema.ts
491
+ function validateSchemaFields(schema, depth, prefix) {
492
+ if (depth > MAX_NESTING_DEPTH) {
493
+ throw new NarsilError(
494
+ ErrorCodes.SCHEMA_DEPTH_EXCEEDED,
495
+ `Schema nesting exceeds the maximum depth of ${MAX_NESTING_DEPTH} at "${prefix}"`,
496
+ { path: prefix, maxDepth: MAX_NESTING_DEPTH }
497
+ );
498
+ }
499
+ for (const [field, type] of Object.entries(schema)) {
500
+ const path = prefix ? `${prefix}.${field}` : field;
501
+ if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
502
+ throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field name "${field}" is not allowed in a schema`, {
503
+ field,
504
+ path
505
+ });
506
+ }
507
+ if (!FIELD_NAME_PATTERN.test(field)) {
508
+ throw new NarsilError(
509
+ ErrorCodes.SCHEMA_INVALID_TYPE,
510
+ `Field name "${field}" contains characters that are not allowed; use letters, digits, and underscores only`,
511
+ { field, path }
512
+ );
513
+ }
514
+ if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
515
+ throw new NarsilError(
516
+ ErrorCodes.SCHEMA_INVALID_TYPE,
517
+ `Field "${field}" is reserved and cannot be defined in a schema`,
518
+ { field, path }
519
+ );
520
+ }
521
+ if (isPlainObject(type)) {
522
+ validateSchemaFields(type, depth + 1, path);
523
+ continue;
524
+ }
525
+ if (typeof type !== "string") {
526
+ throw new NarsilError(
527
+ ErrorCodes.SCHEMA_INVALID_TYPE,
528
+ `Field "${path}" has an invalid type definition: ${String(type)}`,
529
+ {
530
+ field: path,
531
+ type: String(type)
532
+ }
533
+ );
534
+ }
535
+ if (SCALAR_FIELD_TYPES.has(type)) continue;
536
+ const vectorMatch = VECTOR_PATTERN.exec(type);
537
+ if (vectorMatch) {
538
+ const dimension = Number.parseInt(vectorMatch[1], 10);
539
+ if (dimension <= 0) {
540
+ throw new NarsilError(
541
+ ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
542
+ `Vector field "${path}" must have a positive dimension, got ${dimension}`,
543
+ { field: path, dimension }
544
+ );
545
+ }
546
+ continue;
547
+ }
548
+ throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field "${path}" has an unsupported type: "${type}"`, {
549
+ field: path,
550
+ type
551
+ });
552
+ }
553
+ }
554
+ function validateSchema(schema) {
555
+ if (!isPlainObject(schema)) {
556
+ throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must be a plain object", {
557
+ received: typeof schema
558
+ });
559
+ }
560
+ if (Object.keys(schema).length === 0) {
561
+ throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
562
+ }
563
+ validateSchemaFields(schema, 1, "");
564
+ }
565
+ function flattenRecursive(schema, prefix, result) {
566
+ for (const [field, type] of Object.entries(schema)) {
567
+ const path = prefix ? `${prefix}.${field}` : field;
568
+ if (isPlainObject(type)) {
569
+ flattenRecursive(type, path, result);
570
+ } else {
571
+ result[path] = type;
572
+ }
573
+ }
574
+ }
575
+ function flattenSchema(schema) {
576
+ const result = {};
577
+ flattenRecursive(schema, "", result);
578
+ return result;
579
+ }
580
+ function extractVectorFieldsFromSchema(schema) {
581
+ const flat = flattenSchema(schema);
582
+ const result = /* @__PURE__ */ new Map();
583
+ for (const [fieldPath, fieldType] of Object.entries(flat)) {
584
+ const match = VECTOR_PATTERN.exec(fieldType);
585
+ if (match) {
586
+ result.set(fieldPath, Number.parseInt(match[1], 10));
587
+ }
588
+ }
589
+ return result;
590
+ }
591
+
592
+ // src/schema/validator/field-values.ts
593
+ function validateGeopoint(path, value) {
594
+ if (!isPlainObject(value)) {
595
+ throw new NarsilError(
596
+ ErrorCodes.SCHEMA_INVALID_GEOPOINT,
597
+ `Field "${path}" expected a geopoint object with lat and lon`,
598
+ {
599
+ field: path,
600
+ received: Array.isArray(value) ? "array" : typeof value
601
+ }
602
+ );
603
+ }
604
+ if (typeof value.lat !== "number" || typeof value.lon !== "number" || Number.isNaN(value.lat) || Number.isNaN(value.lon)) {
605
+ throw new NarsilError(ErrorCodes.SCHEMA_INVALID_GEOPOINT, `Field "${path}" geopoint requires numeric lat and lon`, {
606
+ field: path,
607
+ lat: Number.isNaN(value.lat) ? "NaN" : typeof value.lat,
608
+ lon: Number.isNaN(value.lon) ? "NaN" : typeof value.lon
609
+ });
610
+ }
611
+ if (value.lat < -90 || value.lat > 90) {
612
+ throw new NarsilError(
613
+ ErrorCodes.SCHEMA_INVALID_GEOPOINT,
614
+ `Field "${path}" geopoint lat must be between -90 and 90`,
615
+ {
616
+ field: path,
617
+ lat: value.lat
618
+ }
619
+ );
620
+ }
621
+ if (value.lon < -180 || value.lon > 180) {
622
+ throw new NarsilError(
623
+ ErrorCodes.SCHEMA_INVALID_GEOPOINT,
624
+ `Field "${path}" geopoint lon must be between -180 and 180`,
625
+ {
626
+ field: path,
627
+ lon: value.lon
628
+ }
629
+ );
630
+ }
631
+ }
632
+ function validateVector(path, value, expectedDimension) {
633
+ const isArray = Array.isArray(value);
634
+ const isFloat32 = value instanceof Float32Array;
635
+ if (!isArray && !isFloat32) {
636
+ throw new NarsilError(
637
+ ErrorCodes.DOC_VALIDATION_FAILED,
638
+ `Field "${path}" expected a vector (Array or Float32Array)`,
639
+ { field: path, received: typeof value }
640
+ );
641
+ }
642
+ const arr = value;
643
+ if (arr.length !== expectedDimension) {
644
+ throw new NarsilError(
645
+ ErrorCodes.DOC_VALIDATION_FAILED,
646
+ `Field "${path}" expected vector of dimension ${expectedDimension}, got ${arr.length}`,
647
+ { field: path, expected: expectedDimension, received: arr.length }
648
+ );
649
+ }
650
+ for (let i = 0; i < arr.length; i++) {
651
+ if (typeof arr[i] !== "number" || Number.isNaN(arr[i])) {
652
+ throw new NarsilError(
653
+ ErrorCodes.DOC_VALIDATION_FAILED,
654
+ `Field "${path}" vector element at index ${i} is not a valid number`,
655
+ { field: path, index: i, value: arr[i] }
656
+ );
657
+ }
658
+ }
659
+ }
660
+ function validateTypedArray(path, value, elementType) {
661
+ if (!Array.isArray(value)) {
662
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an array`, {
663
+ field: path,
664
+ received: typeof value
665
+ });
666
+ }
667
+ for (let i = 0; i < value.length; i++) {
668
+ const el = value[i];
669
+ if (typeof el !== elementType) {
670
+ throw new NarsilError(
671
+ ErrorCodes.DOC_VALIDATION_FAILED,
672
+ `Field "${path}" array element at index ${i} expected ${elementType}, got ${typeof el}`,
673
+ { field: path, index: i, expected: elementType, received: typeof el }
674
+ );
675
+ }
676
+ if (elementType === "number" && Number.isNaN(el)) {
677
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" array element at index ${i} is NaN`, {
678
+ field: path,
679
+ index: i
680
+ });
681
+ }
682
+ }
683
+ }
684
+ function validateFieldValue(path, value, type) {
685
+ switch (type) {
686
+ case "string":
687
+ case SORTABLE_TEXT_FIELD_TYPE:
688
+ if (typeof value !== "string") {
689
+ throw new NarsilError(
690
+ ErrorCodes.DOC_VALIDATION_FAILED,
691
+ `Field "${path}" expected string, got ${typeof value}`,
692
+ {
693
+ field: path,
694
+ expected: "string",
695
+ received: typeof value
696
+ }
697
+ );
698
+ }
699
+ return;
700
+ case "number":
701
+ if (typeof value !== "number" || Number.isNaN(value)) {
702
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a valid number`, {
703
+ field: path,
704
+ expected: "number",
705
+ received: Number.isNaN(value) ? "NaN" : typeof value
706
+ });
707
+ }
708
+ return;
709
+ case "boolean":
710
+ if (typeof value !== "boolean") {
711
+ throw new NarsilError(
712
+ ErrorCodes.DOC_VALIDATION_FAILED,
713
+ `Field "${path}" expected boolean, got ${typeof value}`,
714
+ {
715
+ field: path,
716
+ expected: "boolean",
717
+ received: typeof value
718
+ }
719
+ );
720
+ }
721
+ return;
722
+ case "enum":
723
+ if (typeof value !== "string") {
724
+ throw new NarsilError(
725
+ ErrorCodes.DOC_VALIDATION_FAILED,
726
+ `Field "${path}" expected enum (string), got ${typeof value}`,
727
+ {
728
+ field: path,
729
+ expected: "enum",
730
+ received: typeof value
731
+ }
732
+ );
733
+ }
734
+ return;
735
+ case "geopoint":
736
+ validateGeopoint(path, value);
737
+ return;
738
+ case "string[]":
739
+ validateTypedArray(path, value, "string");
740
+ return;
741
+ case "number[]":
742
+ validateTypedArray(path, value, "number");
743
+ return;
744
+ case "boolean[]":
745
+ validateTypedArray(path, value, "boolean");
746
+ return;
747
+ case "enum[]":
748
+ validateTypedArray(path, value, "string");
749
+ return;
750
+ default: {
751
+ const vectorMatch = VECTOR_PATTERN.exec(type);
752
+ if (vectorMatch) {
753
+ validateVector(path, value, Number.parseInt(vectorMatch[1], 10));
754
+ }
755
+ }
756
+ }
757
+ }
758
+
759
+ // src/schema/validator/storable.ts
760
+ var RESERVED_OBJECT_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
761
+ function reject(path, reason) {
762
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
763
+ field: path
764
+ });
765
+ }
766
+ function isPlainObject2(value) {
767
+ const proto = Object.getPrototypeOf(value);
768
+ return proto === Object.prototype || proto === null;
769
+ }
770
+ function walkStorable(value, path, depth, ancestors, skip) {
771
+ if (value === null || value === void 0) return;
772
+ const kind = typeof value;
773
+ if (kind === "string" || kind === "boolean") return;
774
+ if (kind === "number") return;
775
+ if (kind === "bigint" || kind === "symbol" || kind === "function") {
776
+ reject(path, `holds a ${kind}, which cannot be stored or recovered`);
777
+ }
778
+ const container = value;
779
+ if (container instanceof Date || container instanceof Uint8Array) return;
780
+ const isArray = Array.isArray(container);
781
+ if (!isArray && !isPlainObject2(container)) {
782
+ reject(
783
+ path,
784
+ "holds a value that does not survive storage; use plain objects, arrays, strings, numbers, booleans, dates, or Uint8Array bytes"
785
+ );
786
+ }
787
+ if (depth >= MAX_DOCUMENT_NESTING_DEPTH) {
788
+ reject(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
789
+ }
790
+ if (ancestors.has(container)) {
791
+ reject(path, "refers back to one of its own ancestors");
792
+ }
793
+ ancestors.add(container);
794
+ if (isArray) {
795
+ const items = container;
796
+ for (let i = 0; i < items.length; i++) {
797
+ walkStorable(items[i], `${path}[${i}]`, depth + 1, ancestors, skip);
798
+ }
799
+ } else {
800
+ const record = container;
801
+ for (const key of Object.keys(record)) {
802
+ const keyPath = path === "" ? key : `${path}.${key}`;
803
+ if (RESERVED_OBJECT_KEYS.has(key)) {
804
+ reject(keyPath, "uses a reserved object key");
805
+ }
806
+ if (skip.has(keyPath)) continue;
807
+ walkStorable(record[key], keyPath, depth + 1, ancestors, skip);
808
+ }
809
+ }
810
+ ancestors.delete(container);
811
+ }
812
+ function assertStorableDocument(document, skipFieldPaths) {
813
+ walkStorable(document, "", 0, /* @__PURE__ */ new Set(), skipFieldPaths ?? /* @__PURE__ */ new Set());
814
+ }
815
+
816
+ // src/schema/validator/document.ts
817
+ var vectorPathCache = /* @__PURE__ */ new WeakMap();
818
+ function vectorFieldPaths(schema) {
819
+ let paths = vectorPathCache.get(schema);
820
+ if (paths !== void 0) return paths;
821
+ paths = /* @__PURE__ */ new Set();
822
+ const collect = (level, prefix) => {
823
+ for (const [field, type] of Object.entries(level)) {
824
+ const path = prefix === "" ? field : `${prefix}.${field}`;
825
+ if (typeof type === "string" && VECTOR_PATTERN.test(type)) {
826
+ paths?.add(path);
827
+ } else if (isPlainObject(type)) {
828
+ collect(type, path);
829
+ }
830
+ }
831
+ };
832
+ collect(schema, "");
833
+ vectorPathCache.set(schema, paths);
834
+ return paths;
835
+ }
836
+ function validateDocumentFields(doc, schema, prefix) {
837
+ for (const [field, type] of Object.entries(schema)) {
838
+ const path = prefix ? `${prefix}.${field}` : field;
839
+ const value = doc[field];
840
+ if (value === void 0 || value === null) continue;
841
+ if (isPlainObject(type)) {
842
+ if (!isPlainObject(value)) {
843
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an object`, {
844
+ field: path,
845
+ received: Array.isArray(value) ? "array" : typeof value
846
+ });
847
+ }
848
+ validateDocumentFields(value, type, path);
849
+ continue;
850
+ }
851
+ validateFieldValue(path, value, type);
852
+ }
853
+ }
854
+ function validateDocument(document, schema) {
855
+ if (!isPlainObject(document)) {
856
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document must be a plain object", {
857
+ received: typeof document
858
+ });
859
+ }
860
+ assertStorableDocument(document, vectorFieldPaths(schema));
861
+ validateDocumentFields(document, schema, "");
862
+ }
863
+ function collectExtraFields(doc, schema, prefix, extras) {
864
+ for (const key of Object.keys(doc)) {
865
+ if (key === "id") continue;
866
+ const path = prefix ? `${prefix}.${key}` : key;
867
+ if (!(key in schema)) {
868
+ extras.push(path);
869
+ continue;
870
+ }
871
+ const schemaType = schema[key];
872
+ if (isPlainObject(schemaType) && isPlainObject(doc[key])) {
873
+ collectExtraFields(doc[key], schemaType, path, extras);
874
+ }
875
+ }
876
+ }
877
+ function validateDocumentStrict(document, schema) {
878
+ const extras = [];
879
+ collectExtraFields(document, schema, "", extras);
880
+ if (extras.length > 0) {
881
+ throw new NarsilError(
882
+ ErrorCodes.DOC_VALIDATION_FAILED,
883
+ `Document contains fields not defined in schema: ${extras.join(", ")}`,
884
+ { extraFields: extras }
885
+ );
886
+ }
887
+ }
888
+ function resolveNestedValue(document, path) {
889
+ if (!path.includes(".")) return document[path];
890
+ const segments = path.split(".");
891
+ let current = document;
892
+ for (const segment of segments) {
893
+ if (current === null || current === void 0 || typeof current !== "object") return void 0;
894
+ current = current[segment];
895
+ }
896
+ return current;
897
+ }
898
+ function validateRequiredFields(document, required) {
899
+ if (required.length === 0) return;
900
+ for (const field of required) {
901
+ const value = resolveNestedValue(document, field);
902
+ if (value === void 0 || value === null) {
903
+ throw new NarsilError(ErrorCodes.DOC_MISSING_REQUIRED_FIELD, `Document is missing required field "${field}"`, {
904
+ field
905
+ });
906
+ }
907
+ }
908
+ }
909
+
910
+ // src/schema/validator/vector-promotion.ts
911
+ function fail(message, details) {
912
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, message, details);
913
+ }
914
+ function isPositiveInteger(value) {
915
+ return Number.isInteger(value) && value >= 1;
916
+ }
917
+ function validateVectorPromotion(config) {
918
+ if (config === void 0) return;
919
+ if (config.threshold !== void 0 && !isPositiveInteger(config.threshold)) {
920
+ fail("vectorPromotion.threshold must be a positive integer", { threshold: config.threshold });
921
+ }
922
+ const quantization = config.quantization;
923
+ if (quantization !== void 0 && quantization !== "sq8" && quantization !== "none") {
924
+ fail("vectorPromotion.quantization must be 'sq8' or 'none'", { quantization });
925
+ }
926
+ const hnsw = config.hnswConfig;
927
+ if (hnsw === void 0) return;
928
+ if (hnsw.m !== void 0 && (!isPositiveInteger(hnsw.m) || hnsw.m > MAX_M)) {
929
+ fail(`vectorPromotion.hnswConfig.m must be a positive integer no greater than ${MAX_M}`, { m: hnsw.m });
930
+ }
931
+ if (hnsw.efConstruction !== void 0 && !isPositiveInteger(hnsw.efConstruction)) {
932
+ fail("vectorPromotion.hnswConfig.efConstruction must be a positive integer", {
933
+ efConstruction: hnsw.efConstruction
934
+ });
935
+ }
936
+ const metric = hnsw.metric;
937
+ if (metric !== void 0 && metric !== "cosine" && metric !== "dotProduct" && metric !== "euclidean") {
938
+ fail("vectorPromotion.hnswConfig.metric must be 'cosine', 'dotProduct', or 'euclidean'", { metric });
939
+ }
940
+ }
941
+
942
+ // src/partitioning/distributed-scoring.ts
943
+ function analysedQueryTokens(term, language, analysis) {
944
+ const { tokens } = tokenize(term, language, {
945
+ stem: true,
946
+ removeStopWords: true,
947
+ customTokenizer: analysis.customTokenizer,
948
+ stopWordOverride: analysis.stopWords
949
+ });
950
+ const unique = /* @__PURE__ */ new Set();
951
+ for (const { token } of tokens) {
952
+ unique.add(token);
953
+ }
954
+ return unique;
955
+ }
956
+ function copyNumericRecord(source) {
957
+ const copy = /* @__PURE__ */ Object.create(null);
958
+ for (const [key, value] of Object.entries(source)) {
959
+ if (typeof value === "number" && Number.isFinite(value)) {
960
+ copy[key] = value;
961
+ }
962
+ }
963
+ return copy;
964
+ }
965
+ function sanitizeGlobalStats(stats) {
966
+ return {
967
+ totalDocuments: typeof stats.totalDocuments === "number" && Number.isFinite(stats.totalDocuments) ? stats.totalDocuments : 0,
968
+ docFrequencies: copyNumericRecord(stats.docFrequencies),
969
+ totalFieldLengths: copyNumericRecord(stats.totalFieldLengths),
970
+ averageFieldLengths: copyNumericRecord(stats.averageFieldLengths)
971
+ };
972
+ }
973
+ function pruneStatsToQueryTerms(stats, term, language, analysis) {
974
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
975
+ for (const token of analysedQueryTokens(term, language, analysis)) {
976
+ if (Object.hasOwn(stats.docFrequencies, token)) {
977
+ docFrequencies[token] = stats.docFrequencies[token];
978
+ }
979
+ }
980
+ return {
981
+ totalDocuments: stats.totalDocuments,
982
+ docFrequencies,
983
+ totalFieldLengths: stats.totalFieldLengths,
984
+ averageFieldLengths: stats.averageFieldLengths
985
+ };
986
+ }
987
+ function collectQueryTermStats(manager, term, language, analysis, partitionIds) {
988
+ const queryTokens = analysedQueryTokens(term, language, analysis);
989
+ let totalDocuments = 0;
990
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
991
+ const totalFieldLengths = /* @__PURE__ */ Object.create(null);
992
+ const statsPartitions = partitionIds === void 0 ? manager.getAllPartitions() : partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
993
+ for (const partition of statsPartitions) {
994
+ const stats = partition.stats;
995
+ totalDocuments += stats.totalDocuments;
996
+ for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
997
+ totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
998
+ }
999
+ for (const token of queryTokens) {
1000
+ if (Object.hasOwn(stats.docFrequencies, token)) {
1001
+ docFrequencies[token] = (docFrequencies[token] ?? 0) + stats.docFrequencies[token];
1002
+ }
1003
+ }
1004
+ }
1005
+ const averageFieldLengths = /* @__PURE__ */ Object.create(null);
1006
+ if (totalDocuments > 0) {
1007
+ for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
1008
+ averageFieldLengths[field] = totalLength / totalDocuments;
1009
+ }
1010
+ }
1011
+ return { totalDocuments, docFrequencies, totalFieldLengths, averageFieldLengths };
1012
+ }
1013
+ function mergePartitionStats(statsArray) {
1014
+ let totalDocuments = 0;
1015
+ const docFrequencies = /* @__PURE__ */ Object.create(null);
1016
+ const totalFieldLengths = /* @__PURE__ */ Object.create(null);
1017
+ for (const stats of statsArray) {
1018
+ totalDocuments += stats.totalDocuments;
1019
+ for (const [term, freq] of Object.entries(stats.docFrequencies)) {
1020
+ docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
1021
+ }
1022
+ for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
1023
+ totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
1024
+ }
1025
+ }
1026
+ const averageFieldLengths = /* @__PURE__ */ Object.create(null);
1027
+ if (totalDocuments > 0) {
1028
+ for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
1029
+ averageFieldLengths[field] = totalLength / totalDocuments;
1030
+ }
1031
+ }
1032
+ return {
1033
+ totalDocuments,
1034
+ docFrequencies,
1035
+ totalFieldLengths,
1036
+ averageFieldLengths
1037
+ };
1038
+ }
1039
+
1040
+ // src/core/partition/scored-merge.ts
1041
+ function kWayMerge(arrays) {
1042
+ const nonEmpty = arrays.filter((a) => a.length > 0);
1043
+ if (nonEmpty.length === 0) return [];
1044
+ if (nonEmpty.length === 1) return nonEmpty[0];
1045
+ if (nonEmpty.length <= 4) {
1046
+ return sequentialMerge(nonEmpty);
1047
+ }
1048
+ return heapMerge(nonEmpty);
1049
+ }
1050
+ function sequentialMerge(arrays) {
1051
+ let merged = arrays[0];
1052
+ for (let i = 1; i < arrays.length; i++) {
1053
+ merged = mergeTwoSorted(merged, arrays[i]);
1054
+ }
1055
+ return merged;
1056
+ }
1057
+ function mergeTwoSorted(a, b) {
1058
+ const result = new Array(a.length + b.length);
1059
+ let ai = 0;
1060
+ let bi = 0;
1061
+ let ri = 0;
1062
+ while (ai < a.length && bi < b.length) {
1063
+ if (a[ai].score > b[bi].score || a[ai].score === b[bi].score && compareCodePoints(a[ai].docId, b[bi].docId) <= 0) {
1064
+ result[ri++] = a[ai++];
1065
+ } else {
1066
+ result[ri++] = b[bi++];
1067
+ }
1068
+ }
1069
+ while (ai < a.length) {
1070
+ result[ri++] = a[ai++];
1071
+ }
1072
+ while (bi < b.length) {
1073
+ result[ri++] = b[bi++];
1074
+ }
1075
+ return result;
1076
+ }
1077
+ function heapMerge(arrays) {
1078
+ const heap = [];
1079
+ let totalSize = 0;
1080
+ for (let i = 0; i < arrays.length; i++) {
1081
+ if (arrays[i].length > 0) {
1082
+ totalSize += arrays[i].length;
1083
+ heapPush(heap, {
1084
+ score: arrays[i][0].score,
1085
+ docId: arrays[i][0].docId,
1086
+ partitionIdx: i,
1087
+ resultIdx: 0
1088
+ });
1089
+ }
1090
+ }
1091
+ const result = new Array(totalSize);
1092
+ let writeIdx = 0;
1093
+ while (heap.length > 0) {
1094
+ const top = heapPop(heap);
1095
+ result[writeIdx++] = arrays[top.partitionIdx][top.resultIdx];
1096
+ const nextIdx = top.resultIdx + 1;
1097
+ if (nextIdx < arrays[top.partitionIdx].length) {
1098
+ const nextDoc = arrays[top.partitionIdx][nextIdx];
1099
+ heapPush(heap, {
1100
+ score: nextDoc.score,
1101
+ docId: nextDoc.docId,
1102
+ partitionIdx: top.partitionIdx,
1103
+ resultIdx: nextIdx
1104
+ });
1105
+ }
1106
+ }
1107
+ return result;
1108
+ }
1109
+ function heapNodeGreater(a, b) {
1110
+ if (a.score !== b.score) return a.score > b.score;
1111
+ return compareCodePoints(a.docId, b.docId) < 0;
1112
+ }
1113
+ function heapPush(heap, node) {
1114
+ heap.push(node);
1115
+ let idx = heap.length - 1;
1116
+ while (idx > 0) {
1117
+ const parentIdx = idx - 1 >> 1;
1118
+ if (heapNodeGreater(heap[idx], heap[parentIdx])) {
1119
+ const tmp = heap[idx];
1120
+ heap[idx] = heap[parentIdx];
1121
+ heap[parentIdx] = tmp;
1122
+ idx = parentIdx;
1123
+ } else {
1124
+ break;
1125
+ }
1126
+ }
1127
+ }
1128
+ function heapPop(heap) {
1129
+ const top = heap[0];
1130
+ const last = heap.pop();
1131
+ if (heap.length > 0) {
1132
+ heap[0] = last;
1133
+ let idx = 0;
1134
+ while (true) {
1135
+ const left = 2 * idx + 1;
1136
+ const right = 2 * idx + 2;
1137
+ let largest = idx;
1138
+ if (left < heap.length && heapNodeGreater(heap[left], heap[largest])) {
1139
+ largest = left;
1140
+ }
1141
+ if (right < heap.length && heapNodeGreater(heap[right], heap[largest])) {
1142
+ largest = right;
1143
+ }
1144
+ if (largest !== idx) {
1145
+ const tmp = heap[idx];
1146
+ heap[idx] = heap[largest];
1147
+ heap[largest] = tmp;
1148
+ idx = largest;
1149
+ } else {
1150
+ break;
1151
+ }
1152
+ }
1153
+ }
1154
+ return top;
1155
+ }
1156
+
1157
+ // src/search/facets.ts
1158
+ function mergeFacets(partitionFacets) {
1159
+ const merged = /* @__PURE__ */ new Map();
1160
+ const bounds = /* @__PURE__ */ new Map();
1161
+ for (const partition of partitionFacets) {
1162
+ for (const [field, facetResult] of Object.entries(partition)) {
1163
+ let fieldMap = merged.get(field);
1164
+ if (!fieldMap) {
1165
+ fieldMap = /* @__PURE__ */ new Map();
1166
+ merged.set(field, fieldMap);
1167
+ }
1168
+ bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
1169
+ for (const [value, count] of Object.entries(facetResult.values)) {
1170
+ fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
1171
+ }
1172
+ }
1173
+ }
1174
+ const result = {};
1175
+ for (const [field, valueMap] of merged) {
1176
+ const values = {};
1177
+ for (const [value, count] of valueMap) {
1178
+ values[value] = count;
1179
+ }
1180
+ result[field] = { values, count: valueMap.size, errorBound: bounds.get(field) ?? 0 };
1181
+ }
1182
+ return result;
1183
+ }
1184
+
1185
+ // src/core/bitset.ts
1186
+ function createBitSet(capacity) {
1187
+ return new Uint32Array(capacity + 31 >>> 5 || 1);
1188
+ }
1189
+ function bitsetSet(bs, index) {
1190
+ const word = index >>> 5;
1191
+ if (word < bs.length) {
1192
+ bs[word] |= 1 << (index & 31);
1193
+ }
1194
+ }
1195
+ function bitsetHas(bs, index) {
1196
+ const word = index >>> 5;
1197
+ if (word >= bs.length) return false;
1198
+ return (bs[word] & 1 << (index & 31)) !== 0;
1199
+ }
1200
+ function bitsetAnd(a, b) {
1201
+ const len = Math.min(a.length, b.length);
1202
+ const result = new Uint32Array(len);
1203
+ for (let i = 0; i < len; i++) {
1204
+ result[i] = a[i] & b[i];
1205
+ }
1206
+ return result;
1207
+ }
1208
+ function bitsetOr(a, b) {
1209
+ const longer = a.length >= b.length ? a : b;
1210
+ const shorter = a.length >= b.length ? b : a;
1211
+ const result = new Uint32Array(longer.length);
1212
+ for (let i = 0; i < shorter.length; i++) {
1213
+ result[i] = longer[i] | shorter[i];
1214
+ }
1215
+ for (let i = shorter.length; i < longer.length; i++) {
1216
+ result[i] = longer[i];
1217
+ }
1218
+ return result;
1219
+ }
1220
+ function bitsetNot(bs, capacity) {
1221
+ const result = new Uint32Array(bs.length);
1222
+ const fullWords = capacity >>> 5;
1223
+ for (let i = 0; i < fullWords; i++) {
1224
+ result[i] = ~bs[i];
1225
+ }
1226
+ const remainder = capacity & 31;
1227
+ if (remainder > 0 && fullWords < bs.length) {
1228
+ result[fullWords] = ~bs[fullWords] & (1 << remainder) - 1;
1229
+ }
1230
+ return result;
1231
+ }
1232
+ function bitsetIsEmpty(bs) {
1233
+ for (let i = 0; i < bs.length; i++) {
1234
+ if (bs[i] !== 0) return false;
1235
+ }
1236
+ return true;
1237
+ }
1238
+ function bitsetFromSet(set, capacity) {
1239
+ const bs = createBitSet(capacity);
1240
+ for (const id of set) {
1241
+ bs[id >>> 5] |= 1 << (id & 31);
1242
+ }
1243
+ return bs;
1244
+ }
1245
+
1246
+ // src/core/fuzzy.ts
1247
+ function boundedLevenshtein(a, b, tolerance) {
1248
+ if (tolerance < 0) return { distance: -1, withinTolerance: false };
1249
+ if (a === b) return { distance: 0, withinTolerance: true };
1250
+ const m = a.length;
1251
+ const n = b.length;
1252
+ if (m === 0) {
1253
+ return n <= tolerance ? { distance: n, withinTolerance: true } : { distance: n, withinTolerance: false };
1254
+ }
1255
+ if (n === 0) {
1256
+ return m <= tolerance ? { distance: m, withinTolerance: true } : { distance: m, withinTolerance: false };
1257
+ }
1258
+ if (a.startsWith(b)) {
1259
+ const diff2 = m - n;
1260
+ return diff2 <= tolerance ? { distance: diff2, withinTolerance: true } : { distance: diff2, withinTolerance: false };
1261
+ }
1262
+ if (b.startsWith(a)) {
1263
+ return { distance: 0, withinTolerance: true };
1264
+ }
1265
+ const diff = Math.abs(m - n);
1266
+ if (diff > tolerance) return { distance: -1, withinTolerance: false };
1267
+ const matrix = [];
1268
+ for (let i = 0; i <= m; i++) {
1269
+ matrix[i] = [i];
1270
+ for (let j = 1; j <= n; j++) {
1271
+ matrix[i][j] = i === 0 ? j : 0;
1272
+ }
1273
+ }
1274
+ for (let i = 1; i <= m; i++) {
1275
+ let rowMin = Infinity;
1276
+ for (let j = 1; j <= n; j++) {
1277
+ if (a[i - 1] === b[j - 1]) {
1278
+ matrix[i][j] = matrix[i - 1][j - 1];
1279
+ } else {
1280
+ matrix[i][j] = Math.min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + 1);
1281
+ }
1282
+ rowMin = Math.min(rowMin, matrix[i][j]);
1283
+ }
1284
+ if (rowMin > tolerance) {
1285
+ return { distance: -1, withinTolerance: false };
1286
+ }
1287
+ }
1288
+ const finalDistance = matrix[m][n];
1289
+ return {
1290
+ distance: finalDistance,
1291
+ withinTolerance: finalDistance <= tolerance
1292
+ };
1293
+ }
1294
+
1295
+ // src/search/cursor-codec.ts
1296
+ function encodeCursorText(text) {
1297
+ if (typeof Buffer !== "undefined") return Buffer.from(text, "utf-8").toString("base64");
1298
+ const bytes = new TextEncoder().encode(text);
1299
+ let binary = "";
1300
+ for (const byte of bytes) binary += String.fromCharCode(byte);
1301
+ return btoa(binary);
1302
+ }
1303
+ function decodeCursorText(encoded) {
1304
+ if (typeof Buffer !== "undefined") return Buffer.from(encoded, "base64").toString("utf-8");
1305
+ const binary = atob(encoded);
1306
+ const bytes = new Uint8Array(binary.length);
1307
+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
1308
+ return new TextDecoder().decode(bytes);
1309
+ }
1310
+
1311
+ // src/search/sorting.ts
1312
+ function isFieldList(sort) {
1313
+ return Array.isArray(sort);
1314
+ }
1315
+ function normalizeSort(sort) {
1316
+ if (sort === void 0) return [];
1317
+ if (isFieldList(sort)) {
1318
+ return sort.map((entry) => ({ field: entry.field, direction: entry.direction }));
1319
+ }
1320
+ return Object.entries(sort).map(([field, direction]) => ({ field, direction }));
1321
+ }
1322
+ function requireSortableFields(sort, schema) {
1323
+ const fields = normalizeSort(sort);
1324
+ if (fields.length === 0) return;
1325
+ const flatSchema = flattenSchema(schema);
1326
+ for (const entry of fields) {
1327
+ if (flatSchema[entry.field] !== "string") continue;
1328
+ throw new NarsilError(
1329
+ ErrorCodes.SEARCH_INVALID_FIELD,
1330
+ `A sort names text field "${entry.field}" only where the schema declares it "${SORTABLE_TEXT_FIELD_TYPE}", because ordering text costs far more memory per document than ordering a number`,
1331
+ { field: entry.field, fieldType: "string" }
1332
+ );
1333
+ }
1334
+ }
1335
+ function readFieldValue(obj, path) {
1336
+ return readSortField(obj, path);
1337
+ }
1338
+ function readSortValues(document, fields) {
1339
+ if (!document) return fields.map(() => void 0);
1340
+ return fields.map((field) => readFieldValue(document, field));
1341
+ }
1342
+ function applySorting(hits, sort, sortKeyOf) {
1343
+ const normalized = normalizeSort(sort);
1344
+ if (normalized.length === 0) return hits;
1345
+ const directions = normalized.map((entry) => entry.direction);
1346
+ const keyCache = /* @__PURE__ */ new Map();
1347
+ for (const hit of hits) {
1348
+ keyCache.set(hit.id, sortKeyOf(hit.id));
1349
+ }
1350
+ const sorted = hits.slice();
1351
+ sorted.sort(
1352
+ (a, b) => compareComparableKeys(keyCache.get(a.id) ?? [], keyCache.get(b.id) ?? [], directions) || compareCodePoints(a.id, b.id)
1353
+ );
1354
+ return sorted;
1355
+ }
1356
+
1357
+ // src/search/cursor.ts
1358
+ var CURSOR_VERSION = 3;
1359
+ function invalidCursor(cursor, reason) {
1360
+ const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
1361
+ return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
1362
+ }
1363
+ function exceedsCodePoints(value, maximum) {
1364
+ if (value.length <= maximum) return false;
1365
+ let counted = 0;
1366
+ let i = 0;
1367
+ while (i < value.length) {
1368
+ const unit = value.charCodeAt(i);
1369
+ if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
1370
+ const low = value.charCodeAt(i + 1);
1371
+ if (low >= 56320 && low < 57344) i++;
1372
+ }
1373
+ i++;
1374
+ counted++;
1375
+ if (counted > maximum) return true;
1376
+ }
1377
+ return false;
1378
+ }
1379
+ function sortSignatureOf(sort) {
1380
+ const fields = normalizeSort(sort);
1381
+ if (fields.length === 0) return null;
1382
+ if (fields.length > MAX_SORT_FIELDS) {
1383
+ throw new NarsilError(
1384
+ ErrorCodes.SEARCH_INVALID_FIELD,
1385
+ `A sort names at most ${MAX_SORT_FIELDS} fields, because the paging cursor carries one value per field`,
1386
+ { count: fields.length, limit: MAX_SORT_FIELDS }
1387
+ );
1388
+ }
1389
+ for (const entry of fields) {
1390
+ if (entry.field.length > MAX_SORT_FIELD_NAME_LENGTH) {
1391
+ throw new NarsilError(
1392
+ ErrorCodes.SEARCH_INVALID_FIELD,
1393
+ `A sort field name holds at most ${MAX_SORT_FIELD_NAME_LENGTH} characters, because the paging cursor carries the sort with every page`,
1394
+ { field: entry.field.slice(0, 64), length: entry.field.length, limit: MAX_SORT_FIELD_NAME_LENGTH }
1395
+ );
1396
+ }
1397
+ }
1398
+ return JSON.stringify(fields.map((entry) => [entry.field, entry.direction]));
1399
+ }
1400
+ function encodePageCursor(cursor) {
1401
+ const payload = { v: CURSOR_VERSION, a: cursor.anchor };
1402
+ if (cursor.sortKey !== null && cursor.sortSignature !== null) {
1403
+ payload.k = cursor.sortKey.map(toComparableSortValue);
1404
+ payload.o = cursor.sortSignature;
1405
+ } else if (cursor.score !== null) {
1406
+ payload.s = cursor.score;
1407
+ }
1408
+ payload.q = cursor.binding;
1409
+ return encodeCursorText(JSON.stringify(payload));
1410
+ }
1411
+ function decodeSortKey(value, cursor) {
1412
+ if (!Array.isArray(value)) throw invalidCursor(cursor, '"k" must be an array');
1413
+ if (value.length > MAX_SORT_FIELDS) throw invalidCursor(cursor, '"k" carries too many sort values');
1414
+ const key = [];
1415
+ for (const entry of value) {
1416
+ if (entry === null || typeof entry === "boolean") {
1417
+ key.push(entry);
1418
+ continue;
1419
+ }
1420
+ if (typeof entry === "number") {
1421
+ if (!Number.isFinite(entry)) throw invalidCursor(cursor, '"k" holds a number that is not finite');
1422
+ key.push(entry);
1423
+ continue;
1424
+ }
1425
+ if (typeof entry === "string") {
1426
+ if (exceedsCodePoints(entry, SORT_VALUE_MAX_CODE_POINTS)) {
1427
+ throw invalidCursor(cursor, '"k" holds an oversized sort value');
1428
+ }
1429
+ key.push(entry);
1430
+ continue;
1431
+ }
1432
+ throw invalidCursor(cursor, '"k" accepts a string, a finite number, a boolean, or null');
1433
+ }
1434
+ return key;
1435
+ }
1436
+ function decodePageCursor(cursor) {
1437
+ if (cursor.length > MAX_CURSOR_LENGTH) throw invalidCursor(cursor, "the cursor is too long");
1438
+ let json;
1439
+ try {
1440
+ json = decodeCursorText(cursor);
1441
+ } catch {
1442
+ throw invalidCursor(cursor, "invalid base64 encoding");
1443
+ }
1444
+ let parsed;
1445
+ try {
1446
+ parsed = JSON.parse(json);
1447
+ } catch {
1448
+ throw invalidCursor(cursor, "invalid JSON");
1449
+ }
1450
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
1451
+ throw invalidCursor(cursor, "expected an object");
1452
+ }
1453
+ const { v, a, s, k, o, q } = parsed;
1454
+ if (v !== CURSOR_VERSION) throw invalidCursor(cursor, `unsupported cursor version ${String(v)}`);
1455
+ if (typeof q !== "string" || !/^[0-9a-f]{8}$/.test(q)) {
1456
+ throw invalidCursor(cursor, '"q" must be 8 lowercase hex digits');
1457
+ }
1458
+ if (typeof a !== "string" || a.length === 0) throw invalidCursor(cursor, '"a" must be a non-empty string');
1459
+ if (exceedsCodePoints(a, MAX_CURSOR_ANCHOR_CODE_POINTS)) {
1460
+ throw invalidCursor(cursor, '"a" is too long to be a document id');
1461
+ }
1462
+ if (s !== void 0 && (typeof s !== "number" || !Number.isFinite(s))) {
1463
+ throw invalidCursor(cursor, '"s" must be a finite number');
1464
+ }
1465
+ if (s !== void 0 && k !== void 0) throw invalidCursor(cursor, 'a cursor carries "s" or "k", never both');
1466
+ const sortKey = k === void 0 ? null : decodeSortKey(k, cursor);
1467
+ if (o !== void 0 && typeof o !== "string") throw invalidCursor(cursor, '"o" must be a string');
1468
+ if (sortKey === null !== (o === void 0)) {
1469
+ throw invalidCursor(cursor, '"k" and "o" have to arrive together');
1470
+ }
1471
+ return {
1472
+ anchor: a,
1473
+ score: typeof s === "number" ? s : null,
1474
+ sortKey,
1475
+ sortSignature: typeof o === "string" ? o : null,
1476
+ binding: q
1477
+ };
1478
+ }
1479
+ function requireMatchingCursor(cursor, encoded, signature, scoreAnchored, binding) {
1480
+ if (cursor.binding !== binding) {
1481
+ throw invalidCursor(encoded, "the cursor belongs to a different query");
1482
+ }
1483
+ if (signature !== null) {
1484
+ if (cursor.sortSignature !== signature) {
1485
+ throw invalidCursor(encoded, "the cursor belongs to a different sort order");
1486
+ }
1487
+ return;
1488
+ }
1489
+ if (cursor.sortSignature !== null) {
1490
+ throw invalidCursor(encoded, "the cursor belongs to a sorted request");
1491
+ }
1492
+ if (scoreAnchored && cursor.score === null) {
1493
+ throw invalidCursor(encoded, "the cursor carries no score to seek from");
1494
+ }
1495
+ if (!scoreAnchored && cursor.score !== null) {
1496
+ throw invalidCursor(encoded, "the cursor belongs to a search rather than a listing");
1497
+ }
1498
+ }
1499
+
1500
+ // src/search/pagination.ts
1501
+ function clampRowCount(value, fallback) {
1502
+ if (value === void 0 || !Number.isFinite(value)) return fallback;
1503
+ return Math.max(0, Math.floor(value));
1504
+ }
1505
+ function requireWithinResultWindow(limit, offset) {
1506
+ const depth = offset + limit;
1507
+ if (depth <= RESULT_WINDOW) return;
1508
+ throw new NarsilError(
1509
+ ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
1510
+ `A request reaches the first ${RESULT_WINDOW} results, and offset + limit is ${depth}. Page past that with the cursor each result carries`,
1511
+ { limit, offset, window: RESULT_WINDOW }
1512
+ );
1513
+ }
1514
+ function lastOrganicResult(sliced, pinnedIds) {
1515
+ for (let i = sliced.length - 1; i >= 0; i--) {
1516
+ if (pinnedIds === void 0 || !pinnedIds.has(sliced[i].id)) return sliced[i];
1517
+ }
1518
+ return void 0;
1519
+ }
1520
+ function ordersAfterAnchor(result, anchor, sort) {
1521
+ if (sort !== void 0 && anchor.sortKey !== null) {
1522
+ const comparison = compareSortValues(sort.sortKeyOf(result.id), anchor.sortKey, sort.directions) || compareCodePoints(result.id, anchor.anchor);
1523
+ return comparison > 0;
1524
+ }
1525
+ if (anchor.score === null) return true;
1526
+ if (result.score === void 0) return true;
1527
+ if (result.score < anchor.score) return true;
1528
+ return result.score === anchor.score && compareCodePoints(result.id, anchor.anchor) > 0;
1529
+ }
1530
+ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedIds) {
1531
+ const decoded = cursor ? decodePageCursor(cursor) : null;
1532
+ if (decoded !== null && cursor !== void 0) {
1533
+ requireMatchingCursor(decoded, cursor, sort?.signature ?? null, true, binding);
1534
+ }
1535
+ if (limit <= 0) {
1536
+ return { paginated: [] };
1537
+ }
1538
+ let startIndex = 0;
1539
+ if (decoded !== null) {
1540
+ startIndex = results.length;
1541
+ for (let i = 0; i < results.length; i++) {
1542
+ if (ordersAfterAnchor(results[i], decoded, sort)) {
1543
+ startIndex = i;
1544
+ break;
1545
+ }
1546
+ }
1547
+ }
1548
+ const afterOffset = startIndex + offset;
1549
+ const sliced = results.slice(afterOffset, afterOffset + limit);
1550
+ let nextCursor;
1551
+ const hasMore = afterOffset + limit < results.length;
1552
+ const lastResult = lastOrganicResult(sliced, pinnedIds);
1553
+ if (hasMore && lastResult !== void 0) {
1554
+ nextCursor = sort !== void 0 ? encodePageCursor({
1555
+ anchor: lastResult.id,
1556
+ score: null,
1557
+ sortKey: sort.sortKeyOf(lastResult.id).map(toComparableSortValue),
1558
+ sortSignature: sort.signature,
1559
+ binding
1560
+ }) : encodePageCursor({
1561
+ anchor: lastResult.id,
1562
+ score: lastResult.score ?? null,
1563
+ sortKey: null,
1564
+ sortSignature: null,
1565
+ binding
1566
+ });
1567
+ }
1568
+ return { paginated: sliced, nextCursor };
1569
+ }
1570
+
1571
+ // src/search/fulltext.ts
1572
+ function resolvePrefixExpansion(partition, params, language, options, queryTokens, lastToken) {
1573
+ const term = params.term;
1574
+ if (term === void 0) return void 0;
1575
+ const unstemmed = tokenize(term, language, {
1576
+ stem: false,
1577
+ removeStopWords: true,
1578
+ customTokenizer: options?.customTokenizer,
1579
+ stopWordOverride: options?.stopWords
1580
+ });
1581
+ let surfacePrefix;
1582
+ for (const t of unstemmed.tokens) {
1583
+ if (t.position === lastToken.position) {
1584
+ surfacePrefix = t.token;
1585
+ break;
1586
+ }
1587
+ }
1588
+ if (surfacePrefix === void 0 || surfacePrefix.length === 0) return void 0;
1589
+ const expansions = partition.expandTermPrefix(surfacePrefix, lastToken.token, PREFIX_MAX_EXPANSIONS);
1590
+ const terms = [];
1591
+ for (const expanded of expansions) {
1592
+ if (expanded !== lastToken.token) terms.push(expanded);
1593
+ }
1594
+ for (const qt of queryTokens) {
1595
+ if (qt.token === lastToken.token) {
1596
+ return { token: qt.token, terms };
1597
+ }
1598
+ }
1599
+ return void 0;
1600
+ }
1601
+ function prepareFulltextQuery(partition, params, language, schema, options) {
1602
+ if (!params.term || params.term.trim().length === 0) return null;
1603
+ if (params.fields && params.fields.length === 0) return null;
1604
+ const flatSchema = flattenSchema(schema);
1605
+ if (params.fields) {
1606
+ validateSearchFields(params.fields, flatSchema);
1607
+ }
1608
+ const queryTokenResult = tokenize(params.term, language, {
1609
+ stem: true,
1610
+ removeStopWords: true,
1611
+ customTokenizer: options?.customTokenizer,
1612
+ stopWordOverride: options?.stopWords
1613
+ });
1614
+ if (queryTokenResult.tokens.length === 0) return null;
1615
+ const queryTokens = deduplicateTokens(queryTokenResult.tokens);
1616
+ let prefixExpansion;
1617
+ if (params.prefix === true && params.exact !== true) {
1618
+ const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
1619
+ prefixExpansion = resolvePrefixExpansion(partition, params, language, options, queryTokens, lastToken);
1620
+ }
1621
+ let filterBitset;
1622
+ if (params.filters) {
1623
+ filterBitset = partition.applyFiltersBitset(params.filters, schema);
1624
+ if (bitsetIsEmpty(filterBitset)) return null;
1625
+ }
1626
+ return { queryTokens, prefixExpansion, filterBitset };
1627
+ }
1628
+ function fulltextMatches(partition, params, language, schema, options) {
1629
+ const prepared = prepareFulltextQuery(partition, params, language, schema, options);
1630
+ if (prepared === null) return null;
1631
+ return partition.searchFulltextMatches({
1632
+ queryTokens: prepared.queryTokens,
1633
+ prefixExpansion: prepared.prefixExpansion,
1634
+ fields: params.fields,
1635
+ tolerance: params.tolerance ?? 0,
1636
+ prefixLength: params.prefixLength ?? 2,
1637
+ exact: params.exact ?? false,
1638
+ filterBitset: prepared.filterBitset
1639
+ });
1640
+ }
1641
+ function fulltextSearch(partition, params, language, schema, options) {
1642
+ const prepared = prepareFulltextQuery(partition, params, language, schema, options);
1643
+ if (prepared === null) {
1644
+ return { scored: [], totalMatched: 0 };
1645
+ }
1646
+ const { queryTokens, prefixExpansion, filterBitset } = prepared;
1647
+ const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
1648
+ const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
1649
+ const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
1650
+ const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
1651
+ const rawResult = partition.searchFulltext({
1652
+ queryTokens,
1653
+ prefixExpansion,
1654
+ fields: params.fields,
1655
+ boost: params.boost,
1656
+ tolerance: params.tolerance ?? 0,
1657
+ prefixLength: params.prefixLength ?? 2,
1658
+ exact: params.exact ?? false,
1659
+ bm25Params: options?.bm25Params,
1660
+ globalStats: options?.globalStats,
1661
+ maxResults,
1662
+ termMatch: params.termMatch,
1663
+ filterBitset,
1664
+ collectComponents,
1665
+ collectMatchedSet
1666
+ });
1667
+ let scored = rawResult.scored;
1668
+ const termMatch = params.termMatch ?? "any";
1669
+ if (termMatch !== "any") {
1670
+ scored = filterByTermCoverage(
1671
+ scored,
1672
+ queryTokens,
1673
+ termMatch,
1674
+ params.tolerance ?? 0,
1675
+ params.exact ?? false,
1676
+ prefixExpansion
1677
+ );
1678
+ }
1679
+ if (params.minScore !== void 0 && params.minScore > 0) {
1680
+ const threshold = params.minScore;
1681
+ scored = scored.filter((doc) => doc.score >= threshold);
1682
+ }
1683
+ const totalMatched = needsAllResults ? scored.length : rawResult.totalMatched;
1684
+ if (params.facets === void 0) {
1685
+ return { scored, totalMatched };
1686
+ }
1687
+ if (needsAllResults) {
1688
+ return { scored, totalMatched, matchedIds: scored.map((doc) => doc.docId) };
1689
+ }
1690
+ return { scored, totalMatched, matchedOrdinalBitset: rawResult.matchedOrdinalBitset };
1691
+ }
1692
+ function validateSearchFields(fields, flatSchema) {
1693
+ for (const field of fields) {
1694
+ const fieldType = flatSchema[field];
1695
+ if (!fieldType) {
1696
+ throw new NarsilError(ErrorCodes.SEARCH_INVALID_FIELD, `Field "${field}" does not exist in the schema`, { field });
1697
+ }
1698
+ if (!isTextFieldType(fieldType) && fieldType !== "string[]") {
1699
+ throw new NarsilError(
1700
+ ErrorCodes.SEARCH_INVALID_FIELD,
1701
+ `Field "${field}" has type "${fieldType}" which cannot be used for full-text search`,
1702
+ { field, fieldType }
1703
+ );
1704
+ }
1705
+ }
1706
+ }
1707
+ function deduplicateTokens(tokens) {
1708
+ const seen = /* @__PURE__ */ new Set();
1709
+ const result = [];
1710
+ for (const t of tokens) {
1711
+ if (!seen.has(t.token)) {
1712
+ seen.add(t.token);
1713
+ result.push(t);
1714
+ }
1715
+ }
1716
+ return result;
1717
+ }
1718
+ function filterByTermCoverage(scored, queryTokens, policy, tolerance, exact, prefixExpansion) {
1719
+ const requiredCount = policy === "all" ? queryTokens.length : policy;
1720
+ if (requiredCount <= 0) return scored;
1721
+ if (requiredCount > queryTokens.length) return [];
1722
+ const expansionTerms = prefixExpansion ? new Set(prefixExpansion.terms) : void 0;
1723
+ return scored.filter((doc) => {
1724
+ const matched = countDocTermMatches(doc, queryTokens, tolerance, exact, prefixExpansion?.token, expansionTerms);
1725
+ return matched >= requiredCount;
1726
+ });
1727
+ }
1728
+ function countDocTermMatches(doc, queryTokens, tolerance, exact, prefixToken, expansionTerms) {
1729
+ const indexTokens = Object.keys(doc.idf);
1730
+ if (indexTokens.length === 0) return 0;
1731
+ let count = 0;
1732
+ for (const qt of queryTokens) {
1733
+ if (prefixToken !== void 0 && qt.token === prefixToken) {
1734
+ if (prefixTokenSatisfied(qt.token, indexTokens, expansionTerms)) {
1735
+ count++;
1736
+ }
1737
+ continue;
1738
+ }
1739
+ if (queryTermSatisfied(qt.token, indexTokens, tolerance, exact)) {
1740
+ count++;
1741
+ }
1742
+ }
1743
+ return count;
1744
+ }
1745
+ function prefixTokenSatisfied(prefixToken, indexTokens, expansionTerms) {
1746
+ for (const indexToken of indexTokens) {
1747
+ if (indexToken === prefixToken) return true;
1748
+ if (expansionTerms?.has(indexToken)) return true;
1749
+ }
1750
+ return false;
1751
+ }
1752
+ function queryTermSatisfied(queryToken, indexTokens, tolerance, exact) {
1753
+ if (exact || tolerance === 0) {
1754
+ return indexTokens.includes(queryToken);
1755
+ }
1756
+ for (const indexToken of indexTokens) {
1757
+ if (boundedLevenshtein(queryToken, indexToken, tolerance).withinTolerance) {
1758
+ return true;
1759
+ }
1760
+ }
1761
+ return false;
1762
+ }
1763
+
1764
+ // src/partitioning/partition-selection.ts
1765
+ function partitionsIn(manager, partitionIds) {
1766
+ if (partitionIds === void 0) {
1767
+ return manager.getAllPartitions();
1768
+ }
1769
+ const partitions = [];
1770
+ for (const partitionId of partitionIds) {
1771
+ const partition = manager.partitionAt(partitionId);
1772
+ if (partition !== void 0) {
1773
+ partitions.push(partition);
1774
+ }
1775
+ }
1776
+ return partitions;
1777
+ }
1778
+
1779
+ // src/partitioning/fan-out.ts
1780
+ async function fanOutQuery(manager, params, language, schema, config, searchOptions) {
1781
+ const partitions = partitionsIn(manager, config.partitionIds);
1782
+ if (partitions.length === 0) {
1783
+ return { scored: [], totalMatched: 0 };
1784
+ }
1785
+ let globalStats;
1786
+ const effectiveMode = resolveEffectiveMode(config);
1787
+ if (effectiveMode === "dfs") {
1788
+ if (params.term !== void 0 && params.term.trim().length > 0) {
1789
+ globalStats = collectQueryTermStats(
1790
+ manager,
1791
+ params.term,
1792
+ language,
1793
+ {
1794
+ stopWords: searchOptions?.stopWords,
1795
+ customTokenizer: searchOptions?.customTokenizer
1796
+ },
1797
+ config.partitionIds
1798
+ );
1799
+ }
1800
+ } else if (effectiveMode === "broadcast") {
1801
+ globalStats = config.globalStats;
1802
+ }
1803
+ const options = buildSearchOptions(searchOptions, globalStats);
1804
+ if (partitions.length === 1 && !config.dispatcher) {
1805
+ const result = dispatchSinglePartition(partitions[0], params, language, schema, options);
1806
+ let facets2;
1807
+ if (params.facets) {
1808
+ facets2 = partitions[0].computeFacets(facetMatchSetOf(result), params.facets, schema);
1809
+ }
1810
+ return { scored: result.scored, totalMatched: result.totalMatched, facets: facets2 };
1811
+ }
1812
+ let outcomes;
1813
+ let facets;
1814
+ if (config.dispatcher) {
1815
+ outcomes = await dispatchWithDispatcher(partitions, params, language, schema, options, config.dispatcher);
1816
+ if (params.facets) {
1817
+ facets = collectAndMergeFacets(outcomes, params, schema);
1818
+ }
1819
+ } else {
1820
+ outcomes = [];
1821
+ const partitionFacets = [];
1822
+ for (const partition of partitions) {
1823
+ const result = dispatchSinglePartition(partition, params, language, schema, options);
1824
+ if (params.facets) {
1825
+ partitionFacets.push(partition.computeFacets(facetMatchSetOf(result), params.facets, schema));
1826
+ }
1827
+ outcomes.push({ result, partition });
1828
+ }
1829
+ if (params.facets) {
1830
+ facets = mergeFacets(partitionFacets);
1831
+ }
1832
+ }
1833
+ const allScoredArrays = outcomes.map((o) => o.result.scored);
1834
+ const merged = kWayMerge(allScoredArrays);
1835
+ let totalMatched = 0;
1836
+ for (const outcome of outcomes) {
1837
+ totalMatched += outcome.result.totalMatched;
1838
+ }
1839
+ return { scored: merged, totalMatched, facets };
1840
+ }
1841
+ function facetMatchSetOf(result) {
1842
+ if (result.matchedOrdinalBitset !== void 0) {
1843
+ return { ordinalBitset: result.matchedOrdinalBitset };
1844
+ }
1845
+ return new Set(result.matchedIds ?? []);
1846
+ }
1847
+ function resolveEffectiveMode(config) {
1848
+ if (config.scoringMode === "broadcast" && !config.globalStats) {
1849
+ return "dfs";
1850
+ }
1851
+ return config.scoringMode;
1852
+ }
1853
+ function buildSearchOptions(base, globalStats) {
1854
+ return {
1855
+ ...base,
1856
+ globalStats: globalStats ?? base?.globalStats
1857
+ };
1858
+ }
1859
+ function dispatchWithDispatcher(partitions, params, language, schema, options, dispatcher) {
1860
+ const promises = partitions.map(
1861
+ (partition) => dispatcher.dispatch(partition, params, language, schema, options).then((result) => ({ result, partition }))
1862
+ );
1863
+ return Promise.all(promises);
1864
+ }
1865
+ function dispatchSinglePartition(partition, params, language, schema, options) {
1866
+ return fulltextSearch(partition, params, language, schema, options);
1867
+ }
1868
+ function collectAndMergeFacets(outcomes, params, schema) {
1869
+ const partitionFacets = [];
1870
+ for (const outcome of outcomes) {
1871
+ if (!params.facets) continue;
1872
+ const facetResult = outcome.partition.computeFacets(facetMatchSetOf(outcome.result), params.facets, schema);
1873
+ partitionFacets.push(facetResult);
1874
+ }
1875
+ return mergeFacets(partitionFacets);
1876
+ }
1877
+
1878
+ // src/partitioning/match-count.ts
1879
+ function countsWithoutScores(params) {
1880
+ return params.minScore === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
1881
+ }
1882
+ function fanOutMatchCount(manager, params, language, schema, options) {
1883
+ let count = 0;
1884
+ for (const partition of partitionsIn(manager, options?.partitionIds)) {
1885
+ const matches = fulltextMatches(partition, params, language, schema, options?.searchOptions);
1886
+ if (matches !== null) count += matches.count;
1887
+ }
1888
+ return count;
1889
+ }
1890
+
1891
+ // src/core/projection.ts
1892
+ function splitPaths(paths) {
1893
+ if (!paths) return [];
1894
+ const split = [];
1895
+ for (const path of paths) {
1896
+ const segments = path.split(".").filter((segment) => segment.length > 0);
1897
+ if (segments.length === 0) continue;
1898
+ if (segments.some((segment) => PROTOTYPE_POLLUTION_KEYS.has(segment))) continue;
1899
+ split.push(segments);
1900
+ }
1901
+ return split;
1902
+ }
1903
+ function overlaps(a, b) {
1904
+ const shared = Math.min(a.length, b.length);
1905
+ for (let i = 0; i < shared; i++) {
1906
+ if (a[i] !== b[i]) return false;
1907
+ }
1908
+ return true;
1909
+ }
1910
+ function covers(outer, inner) {
1911
+ if (outer.length > inner.length) return false;
1912
+ for (let i = 0; i < outer.length; i++) {
1913
+ if (outer[i] !== inner[i]) return false;
1914
+ }
1915
+ return true;
1916
+ }
1917
+ function readPath(source, path) {
1918
+ let current = source;
1919
+ for (const segment of path) {
1920
+ if (typeof current !== "object" || current === null || Array.isArray(current))
1921
+ return { found: false, value: void 0 };
1922
+ const record = current;
1923
+ if (!Object.hasOwn(record, segment)) return { found: false, value: void 0 };
1924
+ current = record[segment];
1925
+ }
1926
+ return { found: true, value: current };
1927
+ }
1928
+ function writePath(target, path, value) {
1929
+ let current = target;
1930
+ for (let i = 0; i < path.length - 1; i++) {
1931
+ const segment = path[i];
1932
+ const existing = current[segment];
1933
+ if (typeof existing === "object" && existing !== null && !Array.isArray(existing)) {
1934
+ current = existing;
1935
+ continue;
1936
+ }
1937
+ const created = {};
1938
+ current[segment] = created;
1939
+ current = created;
1940
+ }
1941
+ current[path[path.length - 1]] = value;
1942
+ }
1943
+ function omitPath(source, path) {
1944
+ const segment = path[0];
1945
+ if (!Object.hasOwn(source, segment)) return source;
1946
+ const copy = { ...source };
1947
+ if (path.length === 1) {
1948
+ delete copy[segment];
1949
+ return copy;
1950
+ }
1951
+ const child = copy[segment];
1952
+ if (typeof child !== "object" || child === null || Array.isArray(child)) return copy;
1953
+ copy[segment] = omitPath(child, path.slice(1));
1954
+ return copy;
1955
+ }
1956
+ function cloneExcept(value, path, exclude) {
1957
+ const reachesDeeper = exclude.some((excluded) => excluded.length > path.length && covers(path, excluded));
1958
+ if (!reachesDeeper) return structuredClone(value);
1959
+ if (typeof value !== "object" || value === null || Array.isArray(value)) return structuredClone(value);
1960
+ const source = value;
1961
+ const kept = {};
1962
+ for (const key of Object.keys(source)) {
1963
+ const childPath = [...path, key];
1964
+ if (exclude.some((excluded) => covers(excluded, childPath))) continue;
1965
+ kept[key] = cloneExcept(source[key], childPath, exclude);
1966
+ }
1967
+ return kept;
1968
+ }
1969
+ function resolveProjection(projection) {
1970
+ if (projection === void 0 || projection === true) return { kind: "full" };
1971
+ if (projection === false) return { kind: "none" };
1972
+ if (typeof projection !== "object" || projection === null) return { kind: "full" };
1973
+ const askedInclude = Array.isArray(projection.include) && projection.include.length > 0;
1974
+ const askedExclude = Array.isArray(projection.exclude) && projection.exclude.length > 0;
1975
+ if (!askedInclude && !askedExclude) return { kind: "full" };
1976
+ return {
1977
+ kind: "fields",
1978
+ include: askedInclude ? splitPaths(projection.include) : null,
1979
+ exclude: splitPaths(projection.exclude)
1980
+ };
1981
+ }
1982
+ function projectionKeepsField(projection, fieldPath) {
1983
+ if (projection.kind === "full") return true;
1984
+ if (projection.kind === "none") return false;
1985
+ const path = fieldPath.split(".").filter((segment) => segment.length > 0);
1986
+ if (path.length === 0) return false;
1987
+ if (projection.include && !projection.include.some((included) => overlaps(included, path))) return false;
1988
+ return !projection.exclude.some((excluded) => covers(excluded, path));
1989
+ }
1990
+ function cloneProjected(document, projection) {
1991
+ if (projection === void 0 || projection.kind === "full") return structuredClone(document);
1992
+ if (projection.kind === "none") return {};
1993
+ if (projection.include === null) return cloneExcept(document, [], projection.exclude);
1994
+ const projected = {};
1995
+ for (const path of projection.include) {
1996
+ if (projection.exclude.some((excluded) => covers(excluded, path))) continue;
1997
+ const { found, value } = readPath(document, path);
1998
+ if (!found) continue;
1999
+ writePath(projected, path, cloneExcept(value, path, projection.exclude));
2000
+ }
2001
+ return projected;
2002
+ }
2003
+ function applyProjection(document, projection) {
2004
+ if (projection.kind === "full") return document;
2005
+ if (projection.kind === "none") return {};
2006
+ let projected;
2007
+ if (projection.include) {
2008
+ projected = {};
2009
+ for (const path of projection.include) {
2010
+ const { found, value } = readPath(document, path);
2011
+ if (found) writePath(projected, path, value);
2012
+ }
2013
+ } else {
2014
+ projected = document;
2015
+ }
2016
+ for (const path of projection.exclude) {
2017
+ projected = omitPath(projected, path);
2018
+ }
2019
+ return projected;
2020
+ }
2021
+
2022
+ // src/core/hash.ts
2023
+ var FNV_OFFSET_BASIS = 2166136261;
2024
+ var FNV_PRIME = 16777619;
2025
+ var encoder = new TextEncoder();
2026
+ function fnv1aBytes(bytes) {
2027
+ let hash = FNV_OFFSET_BASIS;
2028
+ for (let i = 0; i < bytes.length; i++) {
2029
+ hash ^= bytes[i];
2030
+ hash = Math.imul(hash, FNV_PRIME);
2031
+ }
2032
+ return hash >>> 0;
2033
+ }
2034
+ function fnv1a(input) {
2035
+ return fnv1aBytes(encoder.encode(input));
2036
+ }
2037
+
2038
+ export { MAX_NESTING_DEPTH, VECTOR_PATTERN, applyPagination, applyProjection, applySorting, bitsetAnd, bitsetFromSet, bitsetHas, bitsetNot, bitsetOr, bitsetSet, boundedLevenshtein, clampRowCount, clearNormalizationCache, cloneProjected, collectQueryTermStats, configureNormalizationCache, countsWithoutScores, createBitSet, decodePageCursor, encodePageCursor, extractVectorFieldsFromSchema, fanOutMatchCount, fanOutQuery, flattenSchema, fnv1a, fnv1aBytes, fulltextMatches, getNormalizationCacheSize, isTextFieldType, kWayMerge, mergeFacets, mergePartitionStats, normalizeSort, partitionsIn, projectionKeepsField, pruneStatsToQueryTerms, readSortValues, requireMatchingCursor, requireSortableFields, requireWithinResultWindow, resetNormalizationCache, resolveProjection, sanitizeGlobalStats, sortSignatureOf, tokenize, tokenizeIterator, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateVectorPromotion };