@delali/narsil 0.2.4-next.59 → 0.3.1-next.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +9 -8
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
  8. package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
  9. package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
  10. package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
  11. package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
  12. package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
  13. package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
  14. package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
  15. package/dist/browser/chunk-RHG4O77A.mjs +206 -0
  16. package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
  17. package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
  18. package/dist/browser/module-CBB2ETYP.mjs +3111 -0
  19. package/dist/browser/release-76MXRPGV.mjs +33 -0
  20. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  21. package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
  22. package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
  23. package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
  24. package/dist/browser/write-7D6VIK6O.mjs +208 -0
  25. package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
  26. package/dist/chunk-2RIRGOUL.mjs +113 -0
  27. package/dist/chunk-37CD3OKS.mjs +87 -0
  28. package/dist/chunk-44W3CBNG.mjs +7 -0
  29. package/dist/chunk-46D6JQIG.mjs +426 -0
  30. package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
  31. package/dist/chunk-747OHOUA.mjs +11 -0
  32. package/dist/chunk-75XIEF6H.mjs +126 -0
  33. package/dist/chunk-7X5JST5L.mjs +114 -0
  34. package/dist/chunk-BOFBLJFZ.mjs +209 -0
  35. package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
  36. package/dist/chunk-C7SLA6CM.mjs +19 -0
  37. package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
  38. package/dist/chunk-CVXLLEQ3.mjs +1011 -0
  39. package/dist/chunk-DJHHF2L5.mjs +17 -0
  40. package/dist/chunk-E74YLAIG.mjs +24 -0
  41. package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
  42. package/dist/chunk-EWZLIYHV.mjs +57 -0
  43. package/dist/chunk-GBEEG5AN.mjs +35 -0
  44. package/dist/chunk-GES7U2PM.mjs +68 -0
  45. package/dist/chunk-HJSN7UYG.mjs +10 -0
  46. package/dist/chunk-HLPRHOSO.mjs +17 -0
  47. package/dist/chunk-HMFNFVP7.mjs +14 -0
  48. package/dist/chunk-J5QP52HA.mjs +1751 -0
  49. package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
  50. package/dist/chunk-KOKM4PQN.mjs +52 -0
  51. package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
  52. package/dist/chunk-KZ3NX2P7.mjs +21 -0
  53. package/dist/chunk-L5ORED7V.mjs +25 -0
  54. package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
  55. package/dist/chunk-LKOB5COA.mjs +117 -0
  56. package/dist/chunk-LPQOEU5M.mjs +96 -0
  57. package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
  58. package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
  59. package/dist/chunk-OM362BRO.mjs +948 -0
  60. package/dist/chunk-OZDFA4WC.mjs +67 -0
  61. package/dist/chunk-P36MUFEJ.mjs +10 -0
  62. package/dist/chunk-PC2G76VW.mjs +2540 -0
  63. package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
  64. package/dist/chunk-PNFKUCG3.mjs +1130 -0
  65. package/dist/chunk-Q6XWVUZX.mjs +44 -0
  66. package/dist/chunk-R27EMH5M.mjs +617 -0
  67. package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
  68. package/dist/chunk-RF7PLG43.mjs +106 -0
  69. package/dist/chunk-RFBLQLLZ.mjs +16 -0
  70. package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
  71. package/dist/chunk-S7FL6UAH.mjs +521 -0
  72. package/dist/chunk-S7GNDRPS.mjs +651 -0
  73. package/dist/chunk-SP3NM3ZG.mjs +3537 -0
  74. package/dist/chunk-U476IXKQ.mjs +10 -0
  75. package/dist/chunk-UA3AUGNV.mjs +462 -0
  76. package/dist/chunk-UTG5YNXV.mjs +507 -0
  77. package/dist/chunk-VI4R6W25.mjs +8484 -0
  78. package/dist/chunk-WQBCBKY3.mjs +8 -0
  79. package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
  80. package/dist/chunk-X5CV2E2B.mjs +1 -0
  81. package/dist/chunk-XADZPPZI.mjs +614 -0
  82. package/dist/chunk-YEEQHT4V.mjs +212 -0
  83. package/dist/chunk-YTJCGKM4.mjs +56 -0
  84. package/dist/chunk-YYQIODMF.mjs +275 -0
  85. package/dist/chunk-ZI34TVOA.mjs +144 -0
  86. package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
  87. package/dist/chunk-ZM7KU4YW.mjs +35 -0
  88. package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
  89. package/dist/client.d.ts +11 -10
  90. package/dist/client.mjs +209 -45
  91. package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
  92. package/dist/config-DffQlmRq.d.ts +421 -0
  93. package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
  94. package/dist/distribution/coordinator/etcd.d.ts +79 -18
  95. package/dist/distribution/coordinator/etcd.mjs +553 -109
  96. package/dist/distribution/coordinator/in-memory.d.ts +4 -3
  97. package/dist/distribution/coordinator/in-memory.mjs +22 -3
  98. package/dist/distribution/transport/grpc.d.ts +2 -2
  99. package/dist/distribution/transport/grpc.mjs +79 -14
  100. package/dist/distribution/transport/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/in-memory.mjs +12 -7
  102. package/dist/distribution/transport/tcp.d.ts +2 -2
  103. package/dist/distribution/transport/tcp.mjs +50 -28
  104. package/dist/distribution.d.ts +144 -23
  105. package/dist/distribution.mjs +14479 -8173
  106. package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
  107. package/dist/embeddings/openai.d.ts +2 -2
  108. package/dist/embeddings/openai.mjs +22 -133
  109. package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
  110. package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
  111. package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
  112. package/dist/index.browser.mjs +4 -32180
  113. package/dist/index.d.ts +91 -20
  114. package/dist/index.mjs +33 -16
  115. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  116. package/dist/invalidation/filesystem.d.ts +1 -1
  117. package/dist/invalidation/filesystem.mjs +2 -1
  118. package/dist/invalidation/noop.d.ts +1 -1
  119. package/dist/module-3S65MVCM.mjs +2151 -0
  120. package/dist/partial-words-DTIHO8G4.d.ts +19 -0
  121. package/dist/partial-words.d.ts +22 -0
  122. package/dist/partial-words.mjs +17 -0
  123. package/dist/pattern-Cledccbl.d.ts +18 -0
  124. package/dist/pattern.d.ts +22 -0
  125. package/dist/pattern.mjs +12 -0
  126. package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
  127. package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
  128. package/dist/react.d.ts +279 -183
  129. package/dist/react.mjs +201 -48
  130. package/dist/release-BH3DKW3T.mjs +33 -0
  131. package/dist/replace-CEO5KJJ3.mjs +57 -0
  132. package/dist/rerank-K7C4WZFG.mjs +60 -0
  133. package/dist/rerankers/http.d.ts +50 -0
  134. package/dist/rerankers/http.mjs +104 -0
  135. package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
  136. package/dist/runtime-6Y4ZYXOG.mjs +177 -0
  137. package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
  138. package/dist/server.d.ts +13 -12
  139. package/dist/server.mjs +611 -80
  140. package/dist/snapshot-QGZPODM5.mjs +799 -0
  141. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  142. package/dist/synonyms.d.ts +22 -0
  143. package/dist/synonyms.mjs +12 -0
  144. package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
  145. package/dist/thread-N3IUGUS3.mjs +697 -0
  146. package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
  147. package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
  148. package/dist/vector/search-worker.d.ts +46 -4
  149. package/dist/vector/search-worker.mjs +21 -11
  150. package/dist/workers/entry.mjs +9 -6
  151. package/dist/write-U5RI7MAX.mjs +241 -0
  152. package/package.json +55 -9
  153. package/dist/chunk-3L4U6EJQ.mjs +0 -49
  154. package/dist/chunk-IFBKLBGB.mjs +0 -7
  155. package/dist/chunk-ORBLD3EL.mjs +0 -6
  156. package/dist/chunk-U7KYW4BA.mjs +0 -135
  157. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  158. package/dist/chunk-ZNZBXMP5.mjs +0 -2117
  159. package/dist/chunk-ZZT3EAUC.mjs +0 -11168
  160. package/dist/config-BhldZgaK.d.ts +0 -194
  161. package/dist/direct-executor-ALAVE7FC.mjs +0 -12
  162. package/dist/serialization/crc32-worker.d.ts +0 -20
  163. package/dist/serialization/crc32-worker.mjs +0 -62
  164. package/dist/simd-DW5AtJ6B.d.ts +0 -41
  165. package/dist/thread-DYHEQRVY.mjs +0 -402
@@ -1,2117 +0,0 @@
1
- import { isTextFieldType, flattenSchema, SORTABLE_TEXT_FIELD_TYPE, isGeopointOrVectorType, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PROTOTYPE_POLLUTION_KEYS, PREFIX_MAX_EXPANSIONS, MAX_CURSOR_ANCHOR_CODE_POINTS } from './chunk-L6TNPLAR.mjs';
2
- import { compareCodePoints, sortModeOf, defaultSortMode, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, isSortMode, SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, compareSortValues } from './chunk-YMZCMH5M.mjs';
3
- import { createNarsilError, ErrorCodes, NarsilError } from './chunk-USKFU7J5.mjs';
4
-
5
- // src/core/tokenizer/constants.ts
6
- var TOKEN_CACHE_SIZE_FLOOR = 5e4;
7
- var TOKEN_CACHE_SIZE_CEILING = 2e6;
8
- var TOKEN_CACHE_BYTES_PER_ENTRY = 200;
9
- var NODE_TOKEN_CACHE_ENTRIES = 1e6;
10
- var UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES = 2e5;
11
- var CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION = 0.05;
12
- var SMALL_DEVICE_MEMORY_GB = 1;
13
- var MEDIUM_DEVICE_MEMORY_GB = 4;
14
- var SMALL_DEVICE_TOKEN_CACHE_ENTRIES = 1e5;
15
- var MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES = 25e4;
16
- var LARGE_DEVICE_TOKEN_CACHE_ENTRIES = 5e5;
17
- var DEFAULT_MIN_TOKEN_LENGTH = 1;
18
-
19
- // src/core/tokenizer/normalize.ts
20
- var DOTTED_CAPITAL_I = "\u0130";
21
- var COMBINING_DOT_ABOVE = "\u0307";
22
- var ARMENIAN_MARKS = /[\u055B-\u055F]/;
23
- var FOLDED_FORMS = /\u0307|[\u0130\u055B-\u055F\u2018\u2019\u02BC\u02BB]/g;
24
- var WIDTH_FORMS = /[\uFF01-\uFF5E\uFF61-\uFF9F]/;
25
- var WIDTH_FORMS_GLOBAL = /[\uFF01-\uFF5E\uFF61-\uFF9F]/g;
26
- var FULLWIDTH_ASCII_END = 65374;
27
- var FULLWIDTH_ASCII_OFFSET = 65248;
28
- var HALFWIDTH_KANA_START = 65377;
29
- var HALFWIDTH_VOICED_MARK = 65438;
30
- var HALFWIDTH_SEMI_VOICED_MARK = 65439;
31
- var COMBINING_VOICED_MARK = "\u3099";
32
- var COMBINING_SEMI_VOICED_MARK = "\u309A";
33
- var HALFWIDTH_KANA_TARGETS = "\u3002\u300C\u300D\u3001\u30FB\u30F2\u30A1\u30A3\u30A5\u30A7\u30A9\u30E3\u30E5\u30E7\u30C3\u30FC\u30A2\u30A4\u30A6\u30A8\u30AA\u30AB\u30AD\u30AF\u30B1\u30B3\u30B5\u30B7\u30B9\u30BB\u30BD\u30BF\u30C1\u30C4\u30C6\u30C8\u30CA\u30CB\u30CC\u30CD\u30CE\u30CF\u30D2\u30D5\u30D8\u30DB\u30DE\u30DF\u30E0\u30E1\u30E2\u30E4\u30E6\u30E8\u30E9\u30EA\u30EB\u30EC\u30ED\u30EF\u30F3";
34
- function foldForm(match) {
35
- if (match === DOTTED_CAPITAL_I) return "i";
36
- if (match === COMBINING_DOT_ABOVE || ARMENIAN_MARKS.test(match)) return "";
37
- return "'";
38
- }
39
- function foldWidth(match) {
40
- const codePoint = match.charCodeAt(0);
41
- if (codePoint <= FULLWIDTH_ASCII_END) return String.fromCharCode(codePoint - FULLWIDTH_ASCII_OFFSET);
42
- if (codePoint === HALFWIDTH_VOICED_MARK) return COMBINING_VOICED_MARK;
43
- if (codePoint === HALFWIDTH_SEMI_VOICED_MARK) return COMBINING_SEMI_VOICED_MARK;
44
- return HALFWIDTH_KANA_TARGETS[codePoint - HALFWIDTH_KANA_START];
45
- }
46
- function isAscii(text) {
47
- for (let i = 0; i < text.length; i++) {
48
- if (text.charCodeAt(i) > 127) return false;
49
- }
50
- return true;
51
- }
52
- function foldWidthForms(text) {
53
- if (!WIDTH_FORMS.test(text)) return text;
54
- return text.replace(WIDTH_FORMS_GLOBAL, foldWidth);
55
- }
56
- function normalizeForSplitting(text) {
57
- if (isAscii(text)) return text.toLowerCase();
58
- return foldWidthForms(text).normalize("NFC").replace(FOLDED_FORMS, foldForm).toLowerCase();
59
- }
60
- function stripDiacritics(text) {
61
- return text.normalize("NFD").replace(/[\u0300-\u036f]/g, "");
62
- }
63
-
64
- // src/core/tokenizer/cache.ts
65
- function clampCacheSize(entries) {
66
- return Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(entries, TOKEN_CACHE_SIZE_CEILING));
67
- }
68
- function computeDefaultCacheSize() {
69
- try {
70
- if (typeof process !== "undefined" && typeof process.versions?.node === "string" && typeof window === "undefined") {
71
- const constrainedMemory = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
72
- if (constrainedMemory > 0 && constrainedMemory < Number.MAX_SAFE_INTEGER) {
73
- const budget = constrainedMemory * CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION;
74
- return clampCacheSize(Math.floor(budget / TOKEN_CACHE_BYTES_PER_ENTRY));
75
- }
76
- return clampCacheSize(NODE_TOKEN_CACHE_ENTRIES);
77
- }
78
- if (typeof navigator !== "undefined") {
79
- const mem = navigator.deviceMemory;
80
- if (typeof mem === "number" && mem > 0) {
81
- let entries;
82
- if (mem <= SMALL_DEVICE_MEMORY_GB) entries = SMALL_DEVICE_TOKEN_CACHE_ENTRIES;
83
- else if (mem <= MEDIUM_DEVICE_MEMORY_GB) entries = MEDIUM_DEVICE_TOKEN_CACHE_ENTRIES;
84
- else entries = LARGE_DEVICE_TOKEN_CACHE_ENTRIES;
85
- return clampCacheSize(entries);
86
- }
87
- }
88
- } catch {
89
- return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
90
- }
91
- return clampCacheSize(UNKNOWN_MEMORY_TOKEN_CACHE_ENTRIES);
92
- }
93
- var normalizationCache = /* @__PURE__ */ new Map();
94
- var normalizationCacheSize = 0;
95
- var maxCacheSize = computeDefaultCacheSize();
96
- var cachedLangName = "";
97
- var cachedFlags = "";
98
- var cachedBucket = null;
99
- function configureNormalizationCache(maxSize) {
100
- if (!Number.isFinite(maxSize)) {
101
- throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must be a finite number", {
102
- received: maxSize
103
- });
104
- }
105
- if (maxSize < 0) {
106
- throw createNarsilError(ErrorCodes.CONFIG_INVALID, "tokenizerCacheSize must not be negative", {
107
- received: maxSize
108
- });
109
- }
110
- if (maxSize === 0) {
111
- throw createNarsilError(
112
- ErrorCodes.CONFIG_INVALID,
113
- "tokenizerCacheSize must be greater than zero; the normalization cache cannot be disabled",
114
- {
115
- received: maxSize
116
- }
117
- );
118
- }
119
- maxCacheSize = Math.max(TOKEN_CACHE_SIZE_FLOOR, Math.min(Math.floor(maxSize), TOKEN_CACHE_SIZE_CEILING));
120
- if (normalizationCacheSize > maxCacheSize) {
121
- evictOldestEntries(normalizationCacheSize - maxCacheSize);
122
- }
123
- }
124
- function getCacheBucket(language, flags) {
125
- const langKey = `${language.name}@${language.revision}`;
126
- if (cachedBucket && langKey === cachedLangName && flags === cachedFlags) return cachedBucket;
127
- const key = `${langKey}:${flags}`;
128
- let bucket = normalizationCache.get(key);
129
- if (!bucket) {
130
- bucket = /* @__PURE__ */ new Map();
131
- normalizationCache.set(key, bucket);
132
- }
133
- cachedLangName = langKey;
134
- cachedFlags = flags;
135
- cachedBucket = bucket;
136
- return bucket;
137
- }
138
- function evictOldestEntries(count) {
139
- let remaining = count;
140
- for (const [key, bucket] of normalizationCache) {
141
- if (remaining <= 0) break;
142
- if (bucket.size <= remaining) {
143
- remaining -= bucket.size;
144
- normalizationCacheSize -= bucket.size;
145
- normalizationCache.delete(key);
146
- if (bucket === cachedBucket) cachedBucket = null;
147
- } else {
148
- let deleted = 0;
149
- for (const raw of bucket.keys()) {
150
- if (deleted >= remaining) break;
151
- bucket.delete(raw);
152
- deleted++;
153
- }
154
- normalizationCacheSize -= deleted;
155
- remaining = 0;
156
- }
157
- }
158
- }
159
- function transformToken(raw, language, stem, removeDiacritics) {
160
- const flags = (stem ? "s" : "") + (removeDiacritics ? "d" : "");
161
- let bucket = getCacheBucket(language, flags);
162
- const cached = bucket.get(raw);
163
- if (cached !== void 0) return cached;
164
- let normalized = raw;
165
- if (language.normalizer) {
166
- normalized = language.normalizer(normalized);
167
- }
168
- if (removeDiacritics) {
169
- normalized = stripDiacritics(normalized);
170
- }
171
- if (stem && language.stemmer) {
172
- normalized = language.stemmer(normalized);
173
- }
174
- if (normalizationCacheSize >= maxCacheSize) {
175
- evictOldestEntries(Math.max(1, maxCacheSize >>> 2));
176
- bucket = getCacheBucket(language, flags);
177
- }
178
- bucket.set(raw, normalized);
179
- normalizationCacheSize++;
180
- return normalized;
181
- }
182
- function clearNormalizationCache() {
183
- normalizationCache.clear();
184
- normalizationCacheSize = 0;
185
- cachedBucket = null;
186
- }
187
- function resetNormalizationCache() {
188
- clearNormalizationCache();
189
- maxCacheSize = computeDefaultCacheSize();
190
- }
191
- function getNormalizationCacheSize() {
192
- return normalizationCacheSize;
193
- }
194
-
195
- // src/core/tokenizer/ngrams.ts
196
- var OTHER_SCRIPT = 0;
197
- var HAN_SCRIPT = 1;
198
- var HIRAGANA_SCRIPT = 2;
199
- var KATAKANA_SCRIPT = 3;
200
- var HANGUL_SCRIPT = 4;
201
- var THAI_SCRIPT = 5;
202
- var LAO_SCRIPT = 6;
203
- var KHMER_SCRIPT = 7;
204
- var MYANMAR_SCRIPT = 8;
205
- var PROLONGED_SOUND_MARK = 12540;
206
- var IDEOGRAPHIC_ITERATION_MARK = 12293;
207
- var LOWEST_NGRAM_CODE_POINT = 3584;
208
- var LOWEST_MARK_CODE_POINT = 768;
209
- var THAI_BLOCK_END = 3711;
210
- var LAO_BLOCK_END = 3839;
211
- var MYANMAR_BLOCK_START = 4096;
212
- var MYANMAR_BLOCK_END = 4255;
213
- var KHMER_BLOCK_START = 6016;
214
- var KHMER_BLOCK_END = 6655;
215
- var SOUTHEAST_ASIAN_BLOCKS_END = 6656;
216
- var HIRAGANA_BLOCK_START = 12352;
217
- var HIRAGANA_BLOCK_END = 12447;
218
- var COMBINING_VOICED_MARK2 = 12441;
219
- var COMBINING_SEMI_VOICED_MARK2 = 12442;
220
- var KATAKANA_BLOCK_START = 12448;
221
- var KATAKANA_BLOCK_END = 12543;
222
- var HAN_EXTENSION_A_START = 13312;
223
- var HAN_EXTENSION_A_END = 19903;
224
- var HAN_UNIFIED_START = 19968;
225
- var HAN_UNIFIED_END = 40959;
226
- var HANGUL_SYLLABLE_START = 44032;
227
- var HANGUL_SYLLABLE_END = 55203;
228
- var HAN_COMPATIBILITY_START = 63744;
229
- var HAN_COMPATIBILITY_END = 64255;
230
- var SUPPLEMENTARY_PLANE_START = 65536;
231
- var MARK = /\p{M}/uy;
232
- var HAN = /\p{Script=Han}/uy;
233
- var HIRAGANA = /\p{Script=Hiragana}/uy;
234
- var KATAKANA = /\p{Script=Katakana}/uy;
235
- var HANGUL = /\p{Script=Hangul}/uy;
236
- var THAI = /\p{Script=Thai}/uy;
237
- var LAO = /\p{Script=Lao}/uy;
238
- var KHMER = /\p{Script=Khmer}/uy;
239
- var MYANMAR = /\p{Script=Myanmar}/uy;
240
- function matchesAt(pattern, part, index) {
241
- pattern.lastIndex = index;
242
- return pattern.test(part);
243
- }
244
- function isMark(part, index, codePoint) {
245
- if (codePoint < LOWEST_MARK_CODE_POINT) return false;
246
- if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return false;
247
- if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) {
248
- return codePoint === COMBINING_VOICED_MARK2 || codePoint === COMBINING_SEMI_VOICED_MARK2;
249
- }
250
- if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return false;
251
- if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return false;
252
- return matchesAt(MARK, part, index);
253
- }
254
- function classifyOutsideCommonBlocks(part, index) {
255
- if (matchesAt(HAN, part, index)) return HAN_SCRIPT;
256
- if (matchesAt(HIRAGANA, part, index)) return HIRAGANA_SCRIPT;
257
- if (matchesAt(KATAKANA, part, index)) return KATAKANA_SCRIPT;
258
- if (matchesAt(HANGUL, part, index)) return HANGUL_SCRIPT;
259
- if (matchesAt(THAI, part, index)) return THAI_SCRIPT;
260
- if (matchesAt(LAO, part, index)) return LAO_SCRIPT;
261
- if (matchesAt(KHMER, part, index)) return KHMER_SCRIPT;
262
- if (matchesAt(MYANMAR, part, index)) return MYANMAR_SCRIPT;
263
- return OTHER_SCRIPT;
264
- }
265
- function classifySoutheastAsian(part, index, codePoint) {
266
- if (codePoint <= THAI_BLOCK_END) return matchesAt(THAI, part, index) ? THAI_SCRIPT : OTHER_SCRIPT;
267
- if (codePoint <= LAO_BLOCK_END) return matchesAt(LAO, part, index) ? LAO_SCRIPT : OTHER_SCRIPT;
268
- if (codePoint >= MYANMAR_BLOCK_START && codePoint <= MYANMAR_BLOCK_END) {
269
- return matchesAt(MYANMAR, part, index) ? MYANMAR_SCRIPT : OTHER_SCRIPT;
270
- }
271
- if (codePoint >= KHMER_BLOCK_START && codePoint <= KHMER_BLOCK_END) {
272
- return matchesAt(KHMER, part, index) ? KHMER_SCRIPT : OTHER_SCRIPT;
273
- }
274
- return classifyOutsideCommonBlocks(part, index);
275
- }
276
- function classify(part, index, codePoint, current) {
277
- if (codePoint < LOWEST_NGRAM_CODE_POINT) return OTHER_SCRIPT;
278
- if (codePoint < SOUTHEAST_ASIAN_BLOCKS_END) return classifySoutheastAsian(part, index, codePoint);
279
- if (codePoint === PROLONGED_SOUND_MARK) {
280
- return current === HIRAGANA_SCRIPT ? HIRAGANA_SCRIPT : KATAKANA_SCRIPT;
281
- }
282
- if (codePoint >= HIRAGANA_BLOCK_START && codePoint <= HIRAGANA_BLOCK_END) return HIRAGANA_SCRIPT;
283
- if (codePoint >= KATAKANA_BLOCK_START && codePoint <= KATAKANA_BLOCK_END) return KATAKANA_SCRIPT;
284
- if (codePoint >= HAN_UNIFIED_START && codePoint <= HAN_UNIFIED_END) return HAN_SCRIPT;
285
- if (codePoint >= HAN_EXTENSION_A_START && codePoint <= HAN_EXTENSION_A_END) return HAN_SCRIPT;
286
- if (codePoint >= HANGUL_SYLLABLE_START && codePoint <= HANGUL_SYLLABLE_END) return HANGUL_SCRIPT;
287
- if (codePoint >= HAN_COMPATIBILITY_START && codePoint <= HAN_COMPATIBILITY_END) return HAN_SCRIPT;
288
- if (codePoint === IDEOGRAPHIC_ITERATION_MARK) return HAN_SCRIPT;
289
- return classifyOutsideCommonBlocks(part, index);
290
- }
291
- function emitRun(part, offsets, kind, size, out) {
292
- const count = offsets.length - 1;
293
- if (count < 1) return;
294
- if (kind === OTHER_SCRIPT || count <= size) {
295
- out.push(part.slice(offsets[0], offsets[count]));
296
- return;
297
- }
298
- for (let start = 0; start + size <= count; start++) {
299
- out.push(part.slice(offsets[start], offsets[start + size]));
300
- }
301
- }
302
- function expandNgrams(part, size, out) {
303
- const length = part.length;
304
- if (length === 0) return;
305
- const offsets = [];
306
- let kind = OTHER_SCRIPT;
307
- let index = 0;
308
- while (index < length) {
309
- const codePoint = part.codePointAt(index);
310
- if (codePoint === void 0) break;
311
- const width = codePoint >= SUPPLEMENTARY_PLANE_START ? 2 : 1;
312
- if (offsets.length > 0 && isMark(part, index, codePoint)) {
313
- index += width;
314
- continue;
315
- }
316
- const next = classify(part, index, codePoint, kind);
317
- if (offsets.length > 0 && next !== kind) {
318
- offsets.push(index);
319
- emitRun(part, offsets, kind, size, out);
320
- offsets.length = 0;
321
- }
322
- kind = next;
323
- offsets.push(index);
324
- index += width;
325
- }
326
- offsets.push(length);
327
- emitRun(part, offsets, kind, size, out);
328
- }
329
-
330
- // src/core/tokenizer/tokenize.ts
331
- function producesSurfaceForms(language, options) {
332
- if (!options?.collectSurfaces || options.customTokenizer) return false;
333
- return (options.stem ?? true) && language.stemmer !== null;
334
- }
335
- var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
336
- function resolveStopWords(language, override) {
337
- if (!override) return language.stopWords;
338
- if (override instanceof Set) return override;
339
- return override(language.stopWords);
340
- }
341
- function splitText(text, language) {
342
- const pattern = language.tokenizer?.splitPattern ?? DEFAULT_SPLIT_PATTERN;
343
- const parts = text.split(pattern);
344
- const ngramSize = language.tokenizer?.ngramSize;
345
- if (ngramSize === void 0 || ngramSize < 1) return parts;
346
- const size = Math.floor(ngramSize);
347
- const expanded = [];
348
- for (const part of parts) {
349
- expandNgrams(part, size, expanded);
350
- }
351
- return expanded;
352
- }
353
- var CHAR_APOSTROPHE = 39;
354
- var CHAR_S = 115;
355
- function stripPossessive(token) {
356
- const len = token.length;
357
- if (len >= 2 && token.charCodeAt(len - 1) === CHAR_S && token.charCodeAt(len - 2) === CHAR_APOSTROPHE) {
358
- return token.slice(0, -2);
359
- }
360
- if (len >= 1 && token.charCodeAt(len - 1) === CHAR_APOSTROPHE) {
361
- return token.slice(0, -1);
362
- }
363
- return token;
364
- }
365
- function visitTokens(text, language, options, visit) {
366
- const {
367
- stem = true,
368
- removeStopWords = true,
369
- removeDiacritics = false,
370
- collectSurfaces = false,
371
- customTokenizer,
372
- stopWordOverride
373
- } = options ?? {};
374
- if (customTokenizer) {
375
- for (const entry of customTokenizer.tokenize(text)) visit(entry.token, entry.position, void 0, entry.token);
376
- return;
377
- }
378
- const rawParts = splitText(normalizeForSplitting(text), language);
379
- const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
380
- const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
381
- const wantSurfaces = collectSurfaces && stem && language.stemmer !== null;
382
- const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : null;
383
- const stripPossessives = language.tokenizer?.stripPossessive ?? false;
384
- for (let position = 0; position < rawParts.length; position++) {
385
- const part = rawParts[position];
386
- const candidate = stripPossessives ? stripPossessive(part) : part;
387
- if (candidate.length < minLength || stopWords?.has(candidate)) continue;
388
- const processed = transformToken(candidate, language, stem, effectiveDiacritics);
389
- if (processed.length === 0) continue;
390
- if (!wantSurfaces) {
391
- visit(processed, position, void 0, part);
392
- continue;
393
- }
394
- const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
395
- visit(processed, position, surface === processed ? void 0 : surface, part);
396
- }
397
- }
398
- function tokenize(text, language, options) {
399
- if (options?.customTokenizer) {
400
- const customResult = options.customTokenizer.tokenize(text);
401
- return { tokens: customResult, originalTokens: customResult.map((entry) => entry.token) };
402
- }
403
- const tokens = [];
404
- const originalTokens = [];
405
- const surfaces = producesSurfaceForms(language, options) ? [] : void 0;
406
- visitTokens(text, language, options, (token, position, surface, original) => {
407
- tokens.push({ token, position });
408
- originalTokens.push(original);
409
- surfaces?.push(surface);
410
- });
411
- return { tokens, originalTokens, surfaces };
412
- }
413
- function* tokenizeIterator(text, language, options) {
414
- const {
415
- stem = true,
416
- removeStopWords = true,
417
- removeDiacritics = false,
418
- collectSurfaces = false,
419
- customTokenizer,
420
- stopWordOverride
421
- } = options ?? {};
422
- if (customTokenizer) {
423
- const customResult = customTokenizer.tokenize(text);
424
- for (const entry of customResult) {
425
- yield entry;
426
- }
427
- return;
428
- }
429
- const normalized = normalizeForSplitting(text);
430
- const rawParts = splitText(normalized, language);
431
- const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
432
- const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
433
- const stemsTokens = stem && language.stemmer !== null;
434
- const wantSurfaces = collectSurfaces && stemsTokens;
435
- const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : /* @__PURE__ */ new Set();
436
- const stripPossessives = language.tokenizer?.stripPossessive ?? false;
437
- let position = 0;
438
- for (const part of rawParts) {
439
- const candidate = stripPossessives ? stripPossessive(part) : part;
440
- if (candidate.length < minLength) {
441
- position++;
442
- continue;
443
- }
444
- if (stopWords.has(candidate)) {
445
- position++;
446
- continue;
447
- }
448
- const processed = transformToken(candidate, language, stem, effectiveDiacritics);
449
- if (processed.length > 0) {
450
- if (wantSurfaces) {
451
- const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
452
- yield { token: processed, position, surface: surface === processed ? void 0 : surface };
453
- } else {
454
- yield { token: processed, position };
455
- }
456
- }
457
- position++;
458
- }
459
- }
460
-
461
- // src/partitioning/distributed-scoring.ts
462
- function analysedQueryTokens(term, language, analysis) {
463
- const { tokens } = tokenize(term, language, {
464
- stem: true,
465
- removeStopWords: true,
466
- customTokenizer: analysis.customTokenizer,
467
- stopWordOverride: analysis.stopWords
468
- });
469
- const unique = /* @__PURE__ */ new Set();
470
- for (const { token } of tokens) {
471
- unique.add(token);
472
- }
473
- return unique;
474
- }
475
- function copyNumericRecord(source) {
476
- const copy = /* @__PURE__ */ Object.create(null);
477
- for (const [key, value] of Object.entries(source)) {
478
- if (typeof value === "number" && Number.isFinite(value)) {
479
- copy[key] = value;
480
- }
481
- }
482
- return copy;
483
- }
484
- function sanitizeGlobalStats(stats) {
485
- return {
486
- totalDocuments: typeof stats.totalDocuments === "number" && Number.isFinite(stats.totalDocuments) ? stats.totalDocuments : 0,
487
- docFrequencies: copyNumericRecord(stats.docFrequencies),
488
- totalFieldLengths: copyNumericRecord(stats.totalFieldLengths),
489
- averageFieldLengths: copyNumericRecord(stats.averageFieldLengths)
490
- };
491
- }
492
- function pruneStatsToQueryTerms(stats, term, language, analysis) {
493
- const docFrequencies = /* @__PURE__ */ Object.create(null);
494
- for (const token of analysedQueryTokens(term, language, analysis)) {
495
- if (Object.hasOwn(stats.docFrequencies, token)) {
496
- docFrequencies[token] = stats.docFrequencies[token];
497
- }
498
- }
499
- return {
500
- totalDocuments: stats.totalDocuments,
501
- docFrequencies,
502
- totalFieldLengths: stats.totalFieldLengths,
503
- averageFieldLengths: stats.averageFieldLengths
504
- };
505
- }
506
- function collectQueryTermStats(manager, term, language, analysis, partitionIds) {
507
- const queryTokens = analysedQueryTokens(term, language, analysis);
508
- let totalDocuments = 0;
509
- const docFrequencies = /* @__PURE__ */ Object.create(null);
510
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
511
- const statsPartitions = partitionIds === void 0 ? manager.getAllPartitions() : partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
512
- for (const partition of statsPartitions) {
513
- const stats = partition.stats;
514
- totalDocuments += stats.totalDocuments;
515
- for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
516
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
517
- }
518
- for (const token of queryTokens) {
519
- if (Object.hasOwn(stats.docFrequencies, token)) {
520
- docFrequencies[token] = (docFrequencies[token] ?? 0) + stats.docFrequencies[token];
521
- }
522
- }
523
- }
524
- const averageFieldLengths = /* @__PURE__ */ Object.create(null);
525
- if (totalDocuments > 0) {
526
- for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
527
- averageFieldLengths[field] = totalLength / totalDocuments;
528
- }
529
- }
530
- return { totalDocuments, docFrequencies, totalFieldLengths, averageFieldLengths };
531
- }
532
- function mergePartitionStats(statsArray) {
533
- let totalDocuments = 0;
534
- const docFrequencies = /* @__PURE__ */ Object.create(null);
535
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
536
- for (const stats of statsArray) {
537
- totalDocuments += stats.totalDocuments;
538
- for (const [term, freq] of Object.entries(stats.docFrequencies)) {
539
- docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
540
- }
541
- for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
542
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
543
- }
544
- }
545
- const averageFieldLengths = /* @__PURE__ */ Object.create(null);
546
- if (totalDocuments > 0) {
547
- for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
548
- averageFieldLengths[field] = totalLength / totalDocuments;
549
- }
550
- }
551
- return {
552
- totalDocuments,
553
- docFrequencies,
554
- totalFieldLengths,
555
- averageFieldLengths
556
- };
557
- }
558
-
559
- // src/core/partition/scored-merge.ts
560
- function kWayMerge(arrays) {
561
- const nonEmpty = arrays.filter((a) => a.length > 0);
562
- if (nonEmpty.length === 0) return [];
563
- if (nonEmpty.length === 1) return nonEmpty[0];
564
- if (nonEmpty.length <= 4) {
565
- return sequentialMerge(nonEmpty);
566
- }
567
- return heapMerge(nonEmpty);
568
- }
569
- function sequentialMerge(arrays) {
570
- let merged = arrays[0];
571
- for (let i = 1; i < arrays.length; i++) {
572
- merged = mergeTwoSorted(merged, arrays[i]);
573
- }
574
- return merged;
575
- }
576
- function mergeTwoSorted(a, b) {
577
- const result = new Array(a.length + b.length);
578
- let ai = 0;
579
- let bi = 0;
580
- let ri = 0;
581
- while (ai < a.length && bi < b.length) {
582
- if (a[ai].score > b[bi].score || a[ai].score === b[bi].score && compareCodePoints(a[ai].docId, b[bi].docId) <= 0) {
583
- result[ri++] = a[ai++];
584
- } else {
585
- result[ri++] = b[bi++];
586
- }
587
- }
588
- while (ai < a.length) {
589
- result[ri++] = a[ai++];
590
- }
591
- while (bi < b.length) {
592
- result[ri++] = b[bi++];
593
- }
594
- return result;
595
- }
596
- function heapMerge(arrays) {
597
- const heap = [];
598
- let totalSize = 0;
599
- for (let i = 0; i < arrays.length; i++) {
600
- if (arrays[i].length > 0) {
601
- totalSize += arrays[i].length;
602
- heapPush(heap, {
603
- score: arrays[i][0].score,
604
- docId: arrays[i][0].docId,
605
- partitionIdx: i,
606
- resultIdx: 0
607
- });
608
- }
609
- }
610
- const result = new Array(totalSize);
611
- let writeIdx = 0;
612
- while (heap.length > 0) {
613
- const top = heapPop(heap);
614
- result[writeIdx++] = arrays[top.partitionIdx][top.resultIdx];
615
- const nextIdx = top.resultIdx + 1;
616
- if (nextIdx < arrays[top.partitionIdx].length) {
617
- const nextDoc = arrays[top.partitionIdx][nextIdx];
618
- heapPush(heap, {
619
- score: nextDoc.score,
620
- docId: nextDoc.docId,
621
- partitionIdx: top.partitionIdx,
622
- resultIdx: nextIdx
623
- });
624
- }
625
- }
626
- return result;
627
- }
628
- function heapNodeGreater(a, b) {
629
- if (a.score !== b.score) return a.score > b.score;
630
- return compareCodePoints(a.docId, b.docId) < 0;
631
- }
632
- function heapPush(heap, node) {
633
- heap.push(node);
634
- let idx = heap.length - 1;
635
- while (idx > 0) {
636
- const parentIdx = idx - 1 >> 1;
637
- if (heapNodeGreater(heap[idx], heap[parentIdx])) {
638
- const tmp = heap[idx];
639
- heap[idx] = heap[parentIdx];
640
- heap[parentIdx] = tmp;
641
- idx = parentIdx;
642
- } else {
643
- break;
644
- }
645
- }
646
- }
647
- function heapPop(heap) {
648
- const top = heap[0];
649
- const last = heap.pop();
650
- if (heap.length > 0) {
651
- heap[0] = last;
652
- let idx = 0;
653
- while (true) {
654
- const left = 2 * idx + 1;
655
- const right = 2 * idx + 2;
656
- let largest = idx;
657
- if (left < heap.length && heapNodeGreater(heap[left], heap[largest])) {
658
- largest = left;
659
- }
660
- if (right < heap.length && heapNodeGreater(heap[right], heap[largest])) {
661
- largest = right;
662
- }
663
- if (largest !== idx) {
664
- const tmp = heap[idx];
665
- heap[idx] = heap[largest];
666
- heap[largest] = tmp;
667
- idx = largest;
668
- } else {
669
- break;
670
- }
671
- }
672
- }
673
- return top;
674
- }
675
-
676
- // src/core/bitset.ts
677
- function createBitSet(capacity) {
678
- return new Uint32Array(capacity + 31 >>> 5 || 1);
679
- }
680
- function bitsetSet(bs, index) {
681
- const word = index >>> 5;
682
- if (word < bs.length) {
683
- bs[word] |= 1 << (index & 31);
684
- }
685
- }
686
- function bitsetHas(bs, index) {
687
- const word = index >>> 5;
688
- if (word >= bs.length) return false;
689
- return (bs[word] & 1 << (index & 31)) !== 0;
690
- }
691
- function bitsetAnd(a, b) {
692
- const len = Math.min(a.length, b.length);
693
- const result = new Uint32Array(len);
694
- for (let i = 0; i < len; i++) {
695
- result[i] = a[i] & b[i];
696
- }
697
- return result;
698
- }
699
- function bitsetOr(a, b) {
700
- const longer = a.length >= b.length ? a : b;
701
- const shorter = a.length >= b.length ? b : a;
702
- const result = new Uint32Array(longer.length);
703
- for (let i = 0; i < shorter.length; i++) {
704
- result[i] = longer[i] | shorter[i];
705
- }
706
- for (let i = shorter.length; i < longer.length; i++) {
707
- result[i] = longer[i];
708
- }
709
- return result;
710
- }
711
- function bitsetNot(bs, capacity) {
712
- const result = new Uint32Array(bs.length);
713
- const fullWords = capacity >>> 5;
714
- for (let i = 0; i < fullWords; i++) {
715
- result[i] = ~bs[i];
716
- }
717
- const remainder = capacity & 31;
718
- if (remainder > 0 && fullWords < bs.length) {
719
- result[fullWords] = ~bs[fullWords] & (1 << remainder) - 1;
720
- }
721
- return result;
722
- }
723
- function bitsetIsEmpty(bs) {
724
- for (let i = 0; i < bs.length; i++) {
725
- if (bs[i] !== 0) return false;
726
- }
727
- return true;
728
- }
729
- function bitsetFromSet(set, capacity) {
730
- const bs = createBitSet(capacity);
731
- for (const id of set) {
732
- bs[id >>> 5] |= 1 << (id & 31);
733
- }
734
- return bs;
735
- }
736
-
737
- // src/core/partition/utils.ts
738
- function getOrCreateFieldNameIndex(table, fieldName) {
739
- const existing = table.indexMap.get(fieldName);
740
- if (existing !== void 0) return existing;
741
- const idx = table.names.length;
742
- table.names.push(fieldName);
743
- table.indexMap.set(fieldName, idx);
744
- return idx;
745
- }
746
- function getNestedValue(obj, path) {
747
- const parts = path.split(".");
748
- let current = obj;
749
- for (let i = 0; i < parts.length; i++) {
750
- if (current === null || current === void 0 || typeof current !== "object") return void 0;
751
- if (!Object.hasOwn(current, parts[i])) return void 0;
752
- current = current[parts[i]];
753
- }
754
- return current;
755
- }
756
- function stringArraysEqual(a, b) {
757
- if (a === b) return true;
758
- if (!a || !b) return false;
759
- if (a.length !== b.length) return false;
760
- for (let i = 0; i < a.length; i++) {
761
- if (a[i] !== b[i]) return false;
762
- }
763
- return true;
764
- }
765
- function textFieldsChanged(oldDoc, newDoc, flatSchema) {
766
- for (const [path, fieldType] of Object.entries(flatSchema)) {
767
- if (!isTextFieldType(fieldType) && fieldType !== "string[]") continue;
768
- const oldVal = getNestedValue(oldDoc, path);
769
- const newVal = getNestedValue(newDoc, path);
770
- if (fieldType !== "string[]") {
771
- if (oldVal !== newVal) return true;
772
- } else {
773
- if (!stringArraysEqual(oldVal, newVal)) return true;
774
- }
775
- }
776
- return false;
777
- }
778
- function tokenizeOptions(options) {
779
- return {
780
- stem: true,
781
- removeStopWords: true,
782
- collectSurfaces: options?.collectSurfaces === true,
783
- stopWordOverride: options?.stopWordOverride,
784
- customTokenizer: options?.customTokenizer
785
- };
786
- }
787
- function getFlatSchema(state, schema) {
788
- if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
789
- state.flatSchemaCache = flattenSchema(schema);
790
- state.lastSchemaRef = schema;
791
- return state.flatSchemaCache;
792
- }
793
- function getFieldValueForDoc(docStore, docId, fieldPath) {
794
- const stored = docStore.get(docId);
795
- if (!stored) return void 0;
796
- return getNestedValue(stored.fields, fieldPath);
797
- }
798
- function getFieldValueByInternalId(docStore, internalId, fieldPath) {
799
- const externalId = docStore.getExternalId(internalId);
800
- if (externalId === void 0) return void 0;
801
- return getFieldValueForDoc(docStore, externalId, fieldPath);
802
- }
803
- function getAllInternalDocIds(docStore) {
804
- const ids = /* @__PURE__ */ new Set();
805
- for (const internalId of docStore.allInternalIds()) {
806
- ids.add(internalId);
807
- }
808
- return ids;
809
- }
810
-
811
- // src/core/partition/facets.ts
812
- var UNDECLARED_FIELD_TYPE = "";
813
- function keptFacetValues(limit) {
814
- return limit !== void 0 && limit > 0 ? Math.floor(limit) : void 0;
815
- }
816
- function ordinalBitsetOf(state, matched) {
817
- if ("ordinalBitset" in matched) return matched.ordinalBitset;
818
- const bitset = createBitSet(state.docStore.internalIdCapacity());
819
- for (const docId of matched) {
820
- const ordinal = state.docStore.getInternalId(docId);
821
- if (ordinal !== void 0) bitsetSet(bitset, ordinal);
822
- }
823
- return bitset;
824
- }
825
- function* matchedFieldValues(state, fieldPath, matched) {
826
- if ("ordinalBitset" in matched) {
827
- const bits = matched.ordinalBitset;
828
- for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
829
- let word = bits[wordIndex];
830
- if (word === 0) continue;
831
- const base = wordIndex << 5;
832
- while (word !== 0) {
833
- const trailingZeros = Math.clz32(word & -word) ^ 31;
834
- yield getFieldValueByInternalId(state.docStore, base + trailingZeros, fieldPath);
835
- word &= word - 1;
836
- }
837
- }
838
- return;
839
- }
840
- for (const docId of matched) {
841
- yield getFieldValueForDoc(state.docStore, docId, fieldPath);
842
- }
843
- }
844
- function countValuesFromDocuments(state, fieldPath, matched) {
845
- const counts = /* @__PURE__ */ new Map();
846
- for (const value of matchedFieldValues(state, fieldPath, matched)) {
847
- if (value === void 0 || value === null) continue;
848
- if (Array.isArray(value)) {
849
- for (const item of value) {
850
- const key = String(item);
851
- counts.set(key, (counts.get(key) ?? 0) + 1);
852
- }
853
- } else {
854
- const key = String(value);
855
- counts.set(key, (counts.get(key) ?? 0) + 1);
856
- }
857
- }
858
- return counts;
859
- }
860
- function countRangeFromDocuments(state, fieldPath, fieldType, range, matched) {
861
- let count = 0;
862
- for (const value of matchedFieldValues(state, fieldPath, matched)) {
863
- if ((fieldType === "number[]" || fieldType === UNDECLARED_FIELD_TYPE) && Array.isArray(value)) {
864
- for (const v of value) {
865
- if (v >= range.from && v < range.to) {
866
- count++;
867
- break;
868
- }
869
- }
870
- } else if (typeof value === "number" && value >= range.from && value < range.to) {
871
- count++;
872
- }
873
- }
874
- return count;
875
- }
876
- function countNumericRanges(state, fieldPath, fieldType, ranges, bitset, matched) {
877
- const counts = /* @__PURE__ */ new Map();
878
- const index = state.numericIndexes.get(fieldPath);
879
- for (const range of ranges) {
880
- const label = `${range.from}-${range.to}`;
881
- if (index === void 0) {
882
- counts.set(label, countRangeFromDocuments(state, fieldPath, fieldType, range, matched));
883
- } else if (Number.isNaN(range.from) || Number.isNaN(range.to)) {
884
- counts.set(label, 0);
885
- } else {
886
- counts.set(label, index.facetRangeCount(range.from, range.to, bitset()));
887
- }
888
- }
889
- return counts;
890
- }
891
- function countFieldValues(state, fieldPath, fieldType, bitset, matched) {
892
- if (fieldType === "enum" || fieldType === "enum[]") {
893
- const index = state.enumIndexes.get(fieldPath);
894
- if (index !== void 0) return index.facetCounts(bitset());
895
- } else if (fieldType === "boolean" || fieldType === "boolean[]") {
896
- const index = state.booleanIndexes.get(fieldPath);
897
- if (index !== void 0) {
898
- const { trueCount, falseCount } = index.facetCounts(bitset());
899
- const counts = /* @__PURE__ */ new Map();
900
- if (trueCount > 0) counts.set("true", trueCount);
901
- if (falseCount > 0) counts.set("false", falseCount);
902
- return counts;
903
- }
904
- } else if (fieldType === "number" || fieldType === "number[]") {
905
- const index = state.numericIndexes.get(fieldPath);
906
- if (index !== void 0) {
907
- const counts = /* @__PURE__ */ new Map();
908
- for (const [value, count] of index.facetValueCounts(bitset())) {
909
- counts.set(String(value), count);
910
- }
911
- return counts;
912
- }
913
- }
914
- return countValuesFromDocuments(state, fieldPath, matched);
915
- }
916
- function computeFacets(state, matched, config, schema) {
917
- const result = {};
918
- const flatSchema = getFlatSchema(state, schema);
919
- let matchedBitset = null;
920
- const bitset = () => {
921
- if (matchedBitset === null) matchedBitset = ordinalBitsetOf(state, matched);
922
- return matchedBitset;
923
- };
924
- for (const [fieldPath, facetOpts] of Object.entries(config)) {
925
- const fieldType = flatSchema[fieldPath] ?? UNDECLARED_FIELD_TYPE;
926
- const valueCounts = facetOpts.ranges && (fieldType === "number" || fieldType === "number[]" || fieldType === UNDECLARED_FIELD_TYPE) ? countNumericRanges(state, fieldPath, fieldType, facetOpts.ranges, bitset, matched) : countFieldValues(state, fieldPath, fieldType, bitset, matched);
927
- const ordered = Array.from(valueCounts.entries());
928
- const sortDir = facetOpts.sort ?? "desc";
929
- ordered.sort((a, b) => (sortDir === "asc" ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
930
- const keep = Math.min(keptFacetValues(facetOpts.limit) ?? ordered.length, ordered.length);
931
- const entries = ordered.slice(0, keep);
932
- let errorBound = 0;
933
- for (let index = keep; index < ordered.length; index++) {
934
- if (ordered[index][1] > errorBound) errorBound = ordered[index][1];
935
- }
936
- const values = {};
937
- for (const [key, count] of entries) {
938
- values[key] = count;
939
- }
940
- result[fieldPath] = { values, count: entries.length, errorBound };
941
- }
942
- return result;
943
- }
944
-
945
- // src/search/oversample.ts
946
- function oversampledShardSize(size) {
947
- return Math.ceil(size * 1.5) + 10;
948
- }
949
-
950
- // src/search/facets.ts
951
- function keptValueCount(fieldConfig) {
952
- return keptFacetValues(fieldConfig?.limit);
953
- }
954
- function everyValueFacetConfig(config) {
955
- const widened = {};
956
- for (const [field, fieldConfig] of Object.entries(config)) {
957
- widened[field] = { ...fieldConfig, limit: void 0 };
958
- }
959
- return widened;
960
- }
961
- function oversampledFacetConfig(config) {
962
- const widened = {};
963
- for (const [field, fieldConfig] of Object.entries(config)) {
964
- if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
965
- widened[field] = { ...fieldConfig, limit: void 0 };
966
- continue;
967
- }
968
- const kept = keptValueCount(fieldConfig);
969
- widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
970
- }
971
- return widened;
972
- }
973
- function mergeFacets(partitionFacets, config) {
974
- const merged = /* @__PURE__ */ new Map();
975
- const bounds = /* @__PURE__ */ new Map();
976
- for (const partition of partitionFacets) {
977
- for (const [field, facetResult] of Object.entries(partition)) {
978
- let fieldMap = merged.get(field);
979
- if (!fieldMap) {
980
- fieldMap = /* @__PURE__ */ new Map();
981
- merged.set(field, fieldMap);
982
- }
983
- bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
984
- for (const [value, count] of Object.entries(facetResult.values)) {
985
- fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
986
- }
987
- }
988
- }
989
- const result = {};
990
- for (const [field, valueMap] of merged) {
991
- const fieldConfig = config[field];
992
- const ascending = fieldConfig?.sort === "asc";
993
- const ordered = Array.from(valueMap.entries());
994
- ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
995
- const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
996
- let largestLeftOut = 0;
997
- for (let index = kept; index < ordered.length; index++) {
998
- if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
999
- }
1000
- const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
1001
- const values = {};
1002
- for (let index = 0; index < kept; index++) {
1003
- values[ordered[index][0]] = ordered[index][1];
1004
- }
1005
- result[field] = { values, count: kept, errorBound };
1006
- }
1007
- return result;
1008
- }
1009
-
1010
- // src/core/fuzzy.ts
1011
- function fuzzyTermMatches(queryTerm, indexTerm, tolerance, prefixLength) {
1012
- if (queryTerm === indexTerm) return true;
1013
- if (tolerance <= 0) return false;
1014
- if (prefixLength > 0 && queryTerm.length >= prefixLength && !indexTerm.startsWith(queryTerm.slice(0, prefixLength))) {
1015
- return false;
1016
- }
1017
- return boundedLevenshtein(queryTerm, indexTerm, tolerance).withinTolerance;
1018
- }
1019
- function boundedLevenshtein(a, b, tolerance) {
1020
- if (tolerance < 0) return { distance: -1, withinTolerance: false };
1021
- if (a === b) return { distance: 0, withinTolerance: true };
1022
- const m = a.length;
1023
- const n = b.length;
1024
- if (m === 0) {
1025
- return n <= tolerance ? { distance: n, withinTolerance: true } : { distance: n, withinTolerance: false };
1026
- }
1027
- if (n === 0) {
1028
- return m <= tolerance ? { distance: m, withinTolerance: true } : { distance: m, withinTolerance: false };
1029
- }
1030
- if (a.startsWith(b)) {
1031
- const diff2 = m - n;
1032
- return diff2 <= tolerance ? { distance: diff2, withinTolerance: true } : { distance: diff2, withinTolerance: false };
1033
- }
1034
- if (b.startsWith(a)) {
1035
- return { distance: 0, withinTolerance: true };
1036
- }
1037
- const diff = Math.abs(m - n);
1038
- if (diff > tolerance) return { distance: -1, withinTolerance: false };
1039
- const matrix = [];
1040
- for (let i = 0; i <= m; i++) {
1041
- matrix[i] = [i];
1042
- for (let j = 1; j <= n; j++) {
1043
- matrix[i][j] = i === 0 ? j : 0;
1044
- }
1045
- }
1046
- for (let i = 1; i <= m; i++) {
1047
- let rowMin = Infinity;
1048
- for (let j = 1; j <= n; j++) {
1049
- if (a[i - 1] === b[j - 1]) {
1050
- matrix[i][j] = matrix[i - 1][j - 1];
1051
- } else {
1052
- matrix[i][j] = Math.min(matrix[i - 1][j] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j - 1] + 1);
1053
- }
1054
- rowMin = Math.min(rowMin, matrix[i][j]);
1055
- }
1056
- if (rowMin > tolerance) {
1057
- return { distance: -1, withinTolerance: false };
1058
- }
1059
- }
1060
- const finalDistance = matrix[m][n];
1061
- return {
1062
- distance: finalDistance,
1063
- withinTolerance: finalDistance <= tolerance
1064
- };
1065
- }
1066
-
1067
- // src/search/cursor-codec.ts
1068
- function encodeCursorText(text) {
1069
- if (typeof Buffer !== "undefined") return Buffer.from(text, "utf-8").toString("base64");
1070
- const bytes = new TextEncoder().encode(text);
1071
- let binary = "";
1072
- for (const byte of bytes) binary += String.fromCharCode(byte);
1073
- return btoa(binary);
1074
- }
1075
- function decodeCursorText(encoded) {
1076
- if (typeof Buffer !== "undefined") return Buffer.from(encoded, "base64").toString("utf-8");
1077
- const binary = atob(encoded);
1078
- const bytes = new Uint8Array(binary.length);
1079
- for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
1080
- return new TextDecoder().decode(bytes);
1081
- }
1082
-
1083
- // src/geo/constants.ts
1084
- var DEG_TO_RAD = Math.PI / 180;
1085
- var MIN_POLYGON_POINTS = 3;
1086
- var ANTIMERIDIAN_SPAN_DEGREES = 180;
1087
- var FULL_TURN_DEGREES = 360;
1088
- var VINCENTY_MAX_ITERATIONS = 200;
1089
- var VINCENTY_CONVERGENCE_THRESHOLD = 1e-12;
1090
-
1091
- // src/geo/polygon.ts
1092
- function requirePolygonRing(points) {
1093
- if (!Array.isArray(points) || points.length < MIN_POLYGON_POINTS) {
1094
- throw new NarsilError(
1095
- ErrorCodes.SEARCH_INVALID_FILTER,
1096
- `A polygon encloses an area only with at least ${MIN_POLYGON_POINTS} points`,
1097
- { points: Array.isArray(points) ? points.length : typeof points, minimum: MIN_POLYGON_POINTS }
1098
- );
1099
- }
1100
- points.forEach((point, position) => {
1101
- const { lat, lon } = point ?? {};
1102
- if (typeof lat === "number" && Number.isFinite(lat) && typeof lon === "number" && Number.isFinite(lon)) return;
1103
- throw new NarsilError(
1104
- ErrorCodes.SEARCH_INVALID_FILTER,
1105
- `A polygon point carries a finite "lat" and "lon", and the point at position ${position} does not`,
1106
- { position }
1107
- );
1108
- });
1109
- }
1110
- function signedArea(polygon) {
1111
- let area = 0;
1112
- for (let i = 0, j = polygon.length - 1; i < polygon.length; j = i++) {
1113
- area += polygon[j].lon * polygon[i].lat - polygon[i].lon * polygon[j].lat;
1114
- }
1115
- return area;
1116
- }
1117
- function crossesAntimeridian(polygon) {
1118
- let west = Number.POSITIVE_INFINITY;
1119
- let east = Number.NEGATIVE_INFINITY;
1120
- for (const point of polygon) {
1121
- if (point.lon < west) west = point.lon;
1122
- if (point.lon > east) east = point.lon;
1123
- }
1124
- return east - west >= ANTIMERIDIAN_SPAN_DEGREES && signedArea(polygon) < 0;
1125
- }
1126
- function eastward(lon) {
1127
- return lon < 0 ? lon + FULL_TURN_DEGREES : lon;
1128
- }
1129
- function polygonContainment(polygon) {
1130
- const n = polygon.length;
1131
- if (n < MIN_POLYGON_POINTS) return () => false;
1132
- const unwrap = crossesAntimeridian(polygon);
1133
- const lats = new Float64Array(n);
1134
- const lons = new Float64Array(n);
1135
- for (let i = 0; i < n; i++) {
1136
- lats[i] = polygon[i].lat;
1137
- lons[i] = unwrap ? eastward(polygon[i].lon) : polygon[i].lon;
1138
- }
1139
- return (lat, rawLon) => {
1140
- const lon = unwrap ? eastward(rawLon) : rawLon;
1141
- let inside = false;
1142
- for (let i = 0, j = n - 1; i < n; j = i++) {
1143
- const lonI = lons[i];
1144
- const lonJ = lons[j];
1145
- if (lonI > lon !== lonJ > lon) {
1146
- const intersectLat = lats[i] + (lats[j] - lats[i]) / (lonJ - lonI) * (lon - lonI);
1147
- if (lat < intersectLat) inside = !inside;
1148
- }
1149
- }
1150
- return inside;
1151
- };
1152
- }
1153
-
1154
- // src/filters/constants.ts
1155
- var MAX_FILTER_DEPTH = 30;
1156
-
1157
- // src/filters/operands.ts
1158
- var RANGE_OPERATORS = ["gt", "lt", "gte", "lte"];
1159
- var TEXT_OPERATORS = ["startsWith", "endsWith"];
1160
- var EQUALITY_OPERATORS = ["eq", "ne"];
1161
- var LIST_OPERATORS = ["in", "nin"];
1162
- function operandKindOf(type) {
1163
- if (type === void 0) return void 0;
1164
- if (type === "number" || type === "number[]") return "number";
1165
- if (type === "boolean" || type === "boolean[]") return "boolean";
1166
- if (type === "geopoint") return "geopoint";
1167
- if (type.startsWith("vector[")) return "vector";
1168
- return "text";
1169
- }
1170
- function isFiniteNumber(value) {
1171
- return typeof value === "number" && Number.isFinite(value);
1172
- }
1173
- function kindOfValue(value) {
1174
- if (isFiniteNumber(value)) return "number";
1175
- if (typeof value === "boolean") return "boolean";
1176
- if (typeof value === "string") return "text";
1177
- return void 0;
1178
- }
1179
- function invalid(message, fieldPath, operator) {
1180
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message, { fieldPath, operator });
1181
- }
1182
- function requireFieldKind(fieldPath, operator, type, accepted) {
1183
- const kind = operandKindOf(type);
1184
- if (kind === void 0 || accepted.includes(kind)) return;
1185
- invalid(
1186
- `The engine applies "${operator}" to ${accepted.join(" or ")} fields alone, and the schema declares field "${fieldPath}" as "${type}"`,
1187
- fieldPath,
1188
- operator
1189
- );
1190
- }
1191
- function requireRadius(fieldPath, radius) {
1192
- const circle = radius;
1193
- if (circle === null || typeof circle !== "object") {
1194
- invalid(`"radius" on field "${fieldPath}" must be an object`, fieldPath, "radius");
1195
- }
1196
- for (const key of ["lat", "lon", "distance"]) {
1197
- if (!isFiniteNumber(circle[key])) {
1198
- invalid(`"radius.${key}" on field "${fieldPath}" must be a finite number`, fieldPath, "radius");
1199
- }
1200
- }
1201
- if (circle.distance < 0) {
1202
- invalid(
1203
- `"radius.distance" on field "${fieldPath}" must be zero or more, and this filter sets ${String(circle.distance)}`,
1204
- fieldPath,
1205
- "radius"
1206
- );
1207
- }
1208
- }
1209
- function invalidShape(message) {
1210
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message);
1211
- }
1212
- function isObjectRecord(value) {
1213
- return value !== null && typeof value === "object" && !Array.isArray(value);
1214
- }
1215
- function undeclaredFieldOnStrictIndex(field, option) {
1216
- return new NarsilError(
1217
- ErrorCodes.SEARCH_INVALID_FIELD,
1218
- `The index is strict and its schema declares no field "${field}", so no stored document has a value in that field for this ${option}`,
1219
- { field, option }
1220
- );
1221
- }
1222
- function requireValidExpression(expression, rules, depth) {
1223
- if (depth > MAX_FILTER_DEPTH) invalidShape(`A filter expression nests at most ${MAX_FILTER_DEPTH} levels deep`);
1224
- if (!isObjectRecord(expression)) invalidShape("Each filter expression must be an object");
1225
- const { fields, and, or, not } = expression;
1226
- if (fields !== void 0) {
1227
- if (!isObjectRecord(fields)) invalidShape('"fields" in a filter expression must be an object keyed by field path');
1228
- for (const [fieldPath, filter] of Object.entries(fields)) {
1229
- if (rules.declaredOnly && !Object.hasOwn(rules.fieldTypes, fieldPath)) {
1230
- throw undeclaredFieldOnStrictIndex(fieldPath, "filter");
1231
- }
1232
- if (isObjectRecord(filter)) requireValidOperands(fieldPath, filter, rules.fieldTypes[fieldPath]);
1233
- }
1234
- }
1235
- for (const [clause, list] of [
1236
- ["and", and],
1237
- ["or", or]
1238
- ]) {
1239
- if (list === void 0) continue;
1240
- if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
1241
- for (const nested of list) requireValidExpression(nested, rules, depth + 1);
1242
- }
1243
- if (not !== void 0) requireValidExpression(not, rules, depth + 1);
1244
- }
1245
- function requireValidFilter(expression, fieldTypes, declaredOnly = false) {
1246
- if (expression === void 0 || expression === null) return;
1247
- requireValidExpression(expression, { fieldTypes, declaredOnly }, 0);
1248
- }
1249
- function requireValidOperands(fieldPath, filter, type) {
1250
- for (const operator of RANGE_OPERATORS) {
1251
- if (filter[operator] === void 0) continue;
1252
- if (!isFiniteNumber(filter[operator])) {
1253
- invalid(`"${operator}" on field "${fieldPath}" must be a finite number`, fieldPath, operator);
1254
- }
1255
- requireFieldKind(fieldPath, operator, type, ["number"]);
1256
- }
1257
- if (filter.between !== void 0) {
1258
- const bounds = filter.between;
1259
- if (!Array.isArray(bounds) || bounds.length !== 2 || !isFiniteNumber(bounds[0]) || !isFiniteNumber(bounds[1])) {
1260
- invalid(
1261
- `"between" on field "${fieldPath}" must be two finite numbers, the lower bound first`,
1262
- fieldPath,
1263
- "between"
1264
- );
1265
- }
1266
- requireFieldKind(fieldPath, "between", type, ["number"]);
1267
- }
1268
- for (const operator of TEXT_OPERATORS) {
1269
- if (filter[operator] === void 0) continue;
1270
- if (typeof filter[operator] !== "string") {
1271
- invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
1272
- }
1273
- requireFieldKind(fieldPath, operator, type, ["text"]);
1274
- }
1275
- for (const operator of EQUALITY_OPERATORS) {
1276
- if (filter[operator] === void 0) continue;
1277
- const kind = kindOfValue(filter[operator]);
1278
- if (kind === void 0) {
1279
- invalid(
1280
- `"${operator}" on field "${fieldPath}" must be a string, a finite number, or a boolean`,
1281
- fieldPath,
1282
- operator
1283
- );
1284
- }
1285
- requireFieldKind(fieldPath, operator, type, [kind]);
1286
- }
1287
- for (const operator of LIST_OPERATORS) {
1288
- if (filter[operator] === void 0) continue;
1289
- const values = filter[operator];
1290
- if (!Array.isArray(values) || values.some((value) => typeof value !== "string")) {
1291
- invalid(`"${operator}" on field "${fieldPath}" must be a list of strings`, fieldPath, operator);
1292
- }
1293
- requireFieldKind(fieldPath, operator, type, ["text"]);
1294
- }
1295
- if (filter.size !== void 0) {
1296
- const size = filter.size;
1297
- if (size === null || typeof size !== "object") {
1298
- invalid(`"size" on field "${fieldPath}" must be a comparison object`, fieldPath, "size");
1299
- }
1300
- requireValidOperands(`${fieldPath}.size`, size, "number");
1301
- }
1302
- if (filter.radius !== void 0) requireRadius(fieldPath, filter.radius);
1303
- if (filter.polygon !== void 0) {
1304
- if (!isObjectRecord(filter.polygon)) {
1305
- invalid(`"polygon" on field "${fieldPath}" must be an object`, fieldPath, "polygon");
1306
- }
1307
- requirePolygonRing(filter.polygon.points);
1308
- }
1309
- }
1310
-
1311
- // src/search/sorting.ts
1312
- function isFieldList(sort) {
1313
- return Array.isArray(sort);
1314
- }
1315
- function normalizeSort(sort) {
1316
- if (sort === void 0) return [];
1317
- if (isFieldList(sort)) {
1318
- return sort.map(
1319
- (entry) => entry.mode === void 0 ? { field: entry.field, direction: entry.direction } : { field: entry.field, direction: entry.direction, mode: entry.mode }
1320
- );
1321
- }
1322
- return Object.entries(sort).map(([field, direction]) => ({ field, direction }));
1323
- }
1324
- function sortModesOf(fields) {
1325
- return fields.map(sortModeOf);
1326
- }
1327
- function sortSignatureEntry(entry) {
1328
- const mode = sortModeOf(entry);
1329
- return mode === defaultSortMode(entry.direction) ? [entry.field, entry.direction] : [entry.field, entry.direction, mode];
1330
- }
1331
- function requireSortMode(entry, fieldType) {
1332
- if (entry.mode === void 0) return;
1333
- if (!isSortMode(entry.mode)) {
1334
- throw new NarsilError(
1335
- ErrorCodes.SEARCH_INVALID_MODE,
1336
- `A sort mode is one of ${SORT_MODES.map((mode) => `"${mode}"`).join(", ")}, and the mode on "${entry.field}" is "${String(entry.mode)}"`,
1337
- { field: entry.field, mode: String(entry.mode) }
1338
- );
1339
- }
1340
- const averages = entry.mode === "avg" || entry.mode === "median";
1341
- if (averages && fieldType !== void 0 && fieldType !== "number" && fieldType !== "number[]") {
1342
- throw new NarsilError(
1343
- ErrorCodes.SEARCH_INVALID_FIELD,
1344
- `The engine applies the "${entry.mode}" mode to a number field alone, and "${entry.field}" is a ${fieldType} field`,
1345
- { field: entry.field, fieldType, mode: entry.mode }
1346
- );
1347
- }
1348
- }
1349
- function requireSortableFields(sort, schema, strict = false) {
1350
- const fields = normalizeSort(sort);
1351
- if (fields.length === 0) return;
1352
- const flatSchema = flattenSchema(schema);
1353
- for (const entry of fields) {
1354
- const fieldType = flatSchema[entry.field];
1355
- if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(entry.field, "sort");
1356
- if (entry.direction !== "asc" && entry.direction !== "desc") {
1357
- throw new NarsilError(
1358
- ErrorCodes.SEARCH_INVALID_MODE,
1359
- `A sort direction is "asc" or "desc", and the direction on "${entry.field}" is "${String(entry.direction)}"`,
1360
- { field: entry.field, direction: String(entry.direction) }
1361
- );
1362
- }
1363
- requireSortMode(entry, fieldType);
1364
- if (fieldType === "string") {
1365
- throw new NarsilError(
1366
- ErrorCodes.SEARCH_INVALID_FIELD,
1367
- `The engine sorts by text field "${entry.field}" only where the schema declares it "${SORTABLE_TEXT_FIELD_TYPE}", because ordering text takes far more memory per document than ordering a number`,
1368
- { field: entry.field, fieldType }
1369
- );
1370
- }
1371
- if (isGeopointOrVectorType(fieldType)) {
1372
- throw new NarsilError(
1373
- ErrorCodes.SEARCH_INVALID_FIELD,
1374
- `The engine sorts by a number, boolean, enum, or sortable text field, and "${entry.field}" is a ${fieldType} field`,
1375
- { field: entry.field, fieldType }
1376
- );
1377
- }
1378
- }
1379
- }
1380
- function readSortValues(document, fields, modes) {
1381
- if (!document) return fields.map(() => null);
1382
- return fields.map((field, index) => toReducedSortValue(readSortField(document, field), modes[index] ?? "min"));
1383
- }
1384
- function applySorting(hits, sort, sortKeyOf) {
1385
- const normalized = normalizeSort(sort);
1386
- if (normalized.length === 0) return hits;
1387
- const directions = normalized.map((entry) => entry.direction);
1388
- const keyCache = /* @__PURE__ */ new Map();
1389
- for (const hit of hits) {
1390
- keyCache.set(hit.id, sortKeyOf(hit.id));
1391
- }
1392
- const sorted = hits.slice();
1393
- sorted.sort(
1394
- (a, b) => compareComparableKeys(keyCache.get(a.id) ?? [], keyCache.get(b.id) ?? [], directions) || compareCodePoints(a.id, b.id)
1395
- );
1396
- return sorted;
1397
- }
1398
-
1399
- // src/search/cursor.ts
1400
- var CURSOR_VERSION = 4;
1401
- function invalidCursor(cursor, reason) {
1402
- const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
1403
- return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
1404
- }
1405
- function exceedsCodePoints(value, maximum) {
1406
- if (value.length <= maximum) return false;
1407
- let counted = 0;
1408
- let i = 0;
1409
- while (i < value.length) {
1410
- const unit = value.charCodeAt(i);
1411
- if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
1412
- const low = value.charCodeAt(i + 1);
1413
- if (low >= 56320 && low < 57344) i++;
1414
- }
1415
- i++;
1416
- counted++;
1417
- if (counted > maximum) return true;
1418
- }
1419
- return false;
1420
- }
1421
- function sortSignatureOf(sort) {
1422
- const fields = normalizeSort(sort);
1423
- if (fields.length === 0) return null;
1424
- if (fields.length > MAX_SORT_FIELDS) {
1425
- throw new NarsilError(
1426
- ErrorCodes.SEARCH_INVALID_FIELD,
1427
- `A sort names at most ${MAX_SORT_FIELDS} fields, because the paging cursor carries one value per field`,
1428
- { count: fields.length, limit: MAX_SORT_FIELDS }
1429
- );
1430
- }
1431
- for (const entry of fields) {
1432
- if (entry.field.length > MAX_SORT_FIELD_NAME_LENGTH) {
1433
- throw new NarsilError(
1434
- ErrorCodes.SEARCH_INVALID_FIELD,
1435
- `A sort field name holds at most ${MAX_SORT_FIELD_NAME_LENGTH} characters, because the paging cursor carries the sort with every page`,
1436
- { field: entry.field.slice(0, 64), length: entry.field.length, limit: MAX_SORT_FIELD_NAME_LENGTH }
1437
- );
1438
- }
1439
- }
1440
- return JSON.stringify(fields.map(sortSignatureEntry));
1441
- }
1442
- function encodePageCursor(cursor) {
1443
- const payload = { v: CURSOR_VERSION, a: cursor.anchor };
1444
- if (cursor.sortKey !== null && cursor.sortSignature !== null) {
1445
- payload.k = cursor.sortKey.map(toComparableSortValue);
1446
- payload.o = cursor.sortSignature;
1447
- } else if (cursor.score !== null) {
1448
- payload.s = cursor.score;
1449
- }
1450
- if (cursor.depth !== void 0 && cursor.depth > 0) payload.d = cursor.depth;
1451
- payload.q = cursor.binding;
1452
- return encodeCursorText(JSON.stringify(payload));
1453
- }
1454
- function decodeSortKey(value, cursor) {
1455
- if (!Array.isArray(value)) throw invalidCursor(cursor, '"k" must be an array');
1456
- if (value.length > MAX_SORT_FIELDS) throw invalidCursor(cursor, '"k" carries too many sort values');
1457
- const key = [];
1458
- for (const entry of value) {
1459
- if (entry === null || typeof entry === "boolean") {
1460
- key.push(entry);
1461
- continue;
1462
- }
1463
- if (typeof entry === "number") {
1464
- if (!Number.isFinite(entry)) throw invalidCursor(cursor, '"k" holds a number that is not finite');
1465
- key.push(entry);
1466
- continue;
1467
- }
1468
- if (typeof entry === "string") {
1469
- if (exceedsCodePoints(entry, SORT_VALUE_MAX_CODE_POINTS)) {
1470
- throw invalidCursor(cursor, '"k" holds an oversized sort value');
1471
- }
1472
- key.push(entry);
1473
- continue;
1474
- }
1475
- throw invalidCursor(cursor, '"k" accepts a string, a finite number, a boolean, or null');
1476
- }
1477
- return key;
1478
- }
1479
- function decodePageCursor(cursor) {
1480
- if (cursor.length > MAX_CURSOR_LENGTH) throw invalidCursor(cursor, "the cursor is too long");
1481
- let json;
1482
- try {
1483
- json = decodeCursorText(cursor);
1484
- } catch {
1485
- throw invalidCursor(cursor, "invalid base64 encoding");
1486
- }
1487
- let parsed;
1488
- try {
1489
- parsed = JSON.parse(json);
1490
- } catch {
1491
- throw invalidCursor(cursor, "invalid JSON");
1492
- }
1493
- if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
1494
- throw invalidCursor(cursor, "expected an object");
1495
- }
1496
- const { v, a, s, k, o, d, q } = parsed;
1497
- if (v !== CURSOR_VERSION) throw invalidCursor(cursor, `unsupported cursor version ${String(v)}`);
1498
- if (typeof q !== "string" || !/^[0-9a-f]{8}$/.test(q)) {
1499
- throw invalidCursor(cursor, '"q" must be 8 lowercase hex digits');
1500
- }
1501
- if (typeof a !== "string" || a.length === 0) throw invalidCursor(cursor, '"a" must be a non-empty string');
1502
- if (exceedsCodePoints(a, MAX_CURSOR_ANCHOR_CODE_POINTS)) {
1503
- throw invalidCursor(cursor, '"a" is too long to be a document id');
1504
- }
1505
- if (s !== void 0 && (typeof s !== "number" || !Number.isFinite(s))) {
1506
- throw invalidCursor(cursor, '"s" must be a finite number');
1507
- }
1508
- if (s !== void 0 && k !== void 0) throw invalidCursor(cursor, 'a cursor carries "s" or "k", never both');
1509
- if (d !== void 0 && (typeof d !== "number" || !Number.isSafeInteger(d) || d < 1)) {
1510
- throw invalidCursor(cursor, '"d" must be a positive integer');
1511
- }
1512
- const sortKey = k === void 0 ? null : decodeSortKey(k, cursor);
1513
- if (o !== void 0 && typeof o !== "string") throw invalidCursor(cursor, '"o" must be a string');
1514
- if (sortKey === null !== (o === void 0)) {
1515
- throw invalidCursor(cursor, '"k" and "o" have to arrive together');
1516
- }
1517
- return {
1518
- anchor: a,
1519
- score: typeof s === "number" ? s : null,
1520
- sortKey,
1521
- sortSignature: typeof o === "string" ? o : null,
1522
- depth: typeof d === "number" ? d : 0,
1523
- binding: q
1524
- };
1525
- }
1526
- function requireMatchingCursor(cursor, encoded, signature, scoreAnchored, binding) {
1527
- if (cursor.binding !== binding) {
1528
- throw invalidCursor(encoded, "the cursor belongs to a different query");
1529
- }
1530
- if (signature !== null) {
1531
- if (cursor.sortSignature !== signature) {
1532
- throw invalidCursor(encoded, "the cursor belongs to a different sort order");
1533
- }
1534
- return;
1535
- }
1536
- if (cursor.sortSignature !== null) {
1537
- throw invalidCursor(encoded, "the cursor belongs to a sorted request");
1538
- }
1539
- if (scoreAnchored && cursor.score === null) {
1540
- throw invalidCursor(encoded, "the cursor carries no score to seek from");
1541
- }
1542
- if (!scoreAnchored && cursor.score !== null) {
1543
- throw invalidCursor(encoded, "the cursor belongs to a search rather than a listing");
1544
- }
1545
- }
1546
-
1547
- // src/search/pagination.ts
1548
- function clampRowCount(value, fallback) {
1549
- if (value === void 0 || !Number.isFinite(value)) return fallback;
1550
- return Math.max(0, Math.floor(value));
1551
- }
1552
- function requireWithinResultWindow(limit, offset) {
1553
- const depth = offset + limit;
1554
- if (depth <= RESULT_WINDOW) return;
1555
- throw new NarsilError(
1556
- ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
1557
- `A request reaches the first ${RESULT_WINDOW} results, and offset + limit is ${depth}. Page past that with the cursor each result carries`,
1558
- { limit, offset, window: RESULT_WINDOW }
1559
- );
1560
- }
1561
- function lastOrganicResult(sliced, pinnedIds) {
1562
- for (let i = sliced.length - 1; i >= 0; i--) {
1563
- if (pinnedIds === void 0 || !pinnedIds.has(sliced[i].id)) return sliced[i];
1564
- }
1565
- return void 0;
1566
- }
1567
- function ordersAfterAnchor(result, anchor, sort) {
1568
- if (sort !== void 0 && anchor.sortKey !== null) {
1569
- const comparison = compareSortValues(sort.sortKeyOf(result.id), anchor.sortKey, sort.directions) || compareCodePoints(result.id, anchor.anchor);
1570
- return comparison > 0;
1571
- }
1572
- if (anchor.score === null) return true;
1573
- if (result.score === void 0) return true;
1574
- if (result.score < anchor.score) return true;
1575
- return result.score === anchor.score && compareCodePoints(result.id, anchor.anchor) > 0;
1576
- }
1577
- function applyPagination(results, limit, offset, binding, cursor, sort, pinnedIds) {
1578
- const decoded = cursor ? decodePageCursor(cursor) : null;
1579
- if (decoded !== null && cursor !== void 0) {
1580
- requireMatchingCursor(decoded, cursor, sort?.signature ?? null, true, binding);
1581
- }
1582
- if (limit <= 0) {
1583
- return { paginated: [] };
1584
- }
1585
- let startIndex = 0;
1586
- if (decoded !== null) {
1587
- startIndex = results.length;
1588
- for (let i = 0; i < results.length; i++) {
1589
- if (ordersAfterAnchor(results[i], decoded, sort)) {
1590
- startIndex = i;
1591
- break;
1592
- }
1593
- }
1594
- }
1595
- const afterOffset = startIndex + offset;
1596
- const sliced = results.slice(afterOffset, afterOffset + limit);
1597
- let nextCursor;
1598
- const hasMore = afterOffset + limit < results.length;
1599
- const lastResult = lastOrganicResult(sliced, pinnedIds);
1600
- const depth = afterOffset + sliced.length;
1601
- if (hasMore && lastResult !== void 0) {
1602
- nextCursor = sort !== void 0 ? encodePageCursor({
1603
- anchor: lastResult.id,
1604
- score: null,
1605
- sortKey: sort.sortKeyOf(lastResult.id).map(toComparableSortValue),
1606
- sortSignature: sort.signature,
1607
- depth,
1608
- binding
1609
- }) : encodePageCursor({
1610
- anchor: lastResult.id,
1611
- score: lastResult.score ?? null,
1612
- sortKey: null,
1613
- sortSignature: null,
1614
- depth,
1615
- binding
1616
- });
1617
- }
1618
- return { paginated: sliced, nextCursor };
1619
- }
1620
-
1621
- // src/search/fulltext.ts
1622
- function resolvePrefixExpansion(partition, params, language, options, queryTokens, lastToken) {
1623
- const term = params.term;
1624
- if (term === void 0) return void 0;
1625
- const unstemmed = tokenize(term, language, {
1626
- stem: false,
1627
- removeStopWords: true,
1628
- customTokenizer: options?.customTokenizer,
1629
- stopWordOverride: options?.stopWords
1630
- });
1631
- let surfacePrefix;
1632
- for (const t of unstemmed.tokens) {
1633
- if (t.position === lastToken.position) {
1634
- surfacePrefix = t.token;
1635
- break;
1636
- }
1637
- }
1638
- if (surfacePrefix === void 0 || surfacePrefix.length === 0) return void 0;
1639
- const expansions = partition.expandTermPrefix(surfacePrefix, lastToken.token, PREFIX_MAX_EXPANSIONS);
1640
- const terms = [];
1641
- for (const expanded of expansions) {
1642
- if (expanded !== lastToken.token) terms.push(expanded);
1643
- }
1644
- for (const qt of queryTokens) {
1645
- if (qt.token === lastToken.token) {
1646
- return { token: qt.token, terms };
1647
- }
1648
- }
1649
- return void 0;
1650
- }
1651
- function prepareFulltextQuery(partition, params, language, schema, options) {
1652
- if (!params.term || params.term.trim().length === 0) return null;
1653
- if (params.fields && params.fields.length === 0) return null;
1654
- const flatSchema = flattenSchema(schema);
1655
- if (params.fields) {
1656
- validateSearchFields(params.fields, flatSchema);
1657
- }
1658
- const queryTokenResult = tokenize(params.term, language, {
1659
- stem: true,
1660
- removeStopWords: true,
1661
- customTokenizer: options?.customTokenizer,
1662
- stopWordOverride: options?.stopWords
1663
- });
1664
- if (queryTokenResult.tokens.length === 0) return null;
1665
- const queryTokens = deduplicateTokens(queryTokenResult.tokens);
1666
- let prefixExpansion;
1667
- if (params.prefix === true && params.exact !== true) {
1668
- const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
1669
- prefixExpansion = resolvePrefixExpansion(partition, params, language, options, queryTokens, lastToken);
1670
- }
1671
- let filterBitset;
1672
- if (params.filters) {
1673
- filterBitset = partition.applyFiltersBitset(params.filters, schema);
1674
- if (bitsetIsEmpty(filterBitset)) return null;
1675
- }
1676
- return { queryTokens, prefixExpansion, filterBitset };
1677
- }
1678
- function fulltextMatches(partition, params, language, schema, options) {
1679
- const prepared = prepareFulltextQuery(partition, params, language, schema, options);
1680
- if (prepared === null) return null;
1681
- return partition.searchFulltextMatches({
1682
- queryTokens: prepared.queryTokens,
1683
- prefixExpansion: prepared.prefixExpansion,
1684
- fields: params.fields,
1685
- tolerance: params.tolerance ?? 0,
1686
- prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
1687
- exact: params.exact ?? false,
1688
- filterBitset: prepared.filterBitset
1689
- });
1690
- }
1691
- function fulltextSearch(partition, params, language, schema, options) {
1692
- const prepared = prepareFulltextQuery(partition, params, language, schema, options);
1693
- if (prepared === null) {
1694
- return { scored: [], totalMatched: 0 };
1695
- }
1696
- const { queryTokens, prefixExpansion, filterBitset } = prepared;
1697
- const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
1698
- const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
1699
- const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
1700
- const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
1701
- const rawResult = partition.searchFulltext({
1702
- queryTokens,
1703
- prefixExpansion,
1704
- fields: params.fields,
1705
- boost: params.boost,
1706
- tolerance: params.tolerance ?? 0,
1707
- prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
1708
- exact: params.exact ?? false,
1709
- bm25Params: options?.bm25Params,
1710
- globalStats: options?.globalStats,
1711
- maxResults,
1712
- termMatch: params.termMatch,
1713
- filterBitset,
1714
- collectComponents,
1715
- collectMatchedSet
1716
- });
1717
- let scored = rawResult.scored;
1718
- const termMatch = params.termMatch ?? "any";
1719
- if (termMatch !== "any") {
1720
- scored = filterByTermCoverage(
1721
- scored,
1722
- queryTokens,
1723
- termMatch,
1724
- params.tolerance ?? 0,
1725
- params.exact ?? false,
1726
- prefixExpansion
1727
- );
1728
- }
1729
- if (params.minScore !== void 0 && params.minScore > 0) {
1730
- const threshold = params.minScore;
1731
- scored = scored.filter((doc) => doc.score >= threshold);
1732
- }
1733
- const totalMatched = needsAllResults ? scored.length : rawResult.totalMatched;
1734
- if (params.facets === void 0) {
1735
- return { scored, totalMatched };
1736
- }
1737
- if (needsAllResults) {
1738
- return { scored, totalMatched, matchedIds: scored.map((doc) => doc.docId) };
1739
- }
1740
- return { scored, totalMatched, matchedOrdinalBitset: rawResult.matchedOrdinalBitset };
1741
- }
1742
- function validateSearchFields(fields, flatSchema) {
1743
- for (const field of fields) {
1744
- const fieldType = flatSchema[field];
1745
- if (!fieldType) {
1746
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_FIELD, `Field "${field}" does not exist in the schema`, { field });
1747
- }
1748
- if (!isTextFieldType(fieldType) && fieldType !== "string[]") {
1749
- throw new NarsilError(
1750
- ErrorCodes.SEARCH_INVALID_FIELD,
1751
- `Field "${field}" has type "${fieldType}" which cannot be used for full-text search`,
1752
- { field, fieldType }
1753
- );
1754
- }
1755
- }
1756
- }
1757
- function deduplicateTokens(tokens) {
1758
- const seen = /* @__PURE__ */ new Set();
1759
- const result = [];
1760
- for (const t of tokens) {
1761
- if (!seen.has(t.token)) {
1762
- seen.add(t.token);
1763
- result.push(t);
1764
- }
1765
- }
1766
- return result;
1767
- }
1768
- function filterByTermCoverage(scored, queryTokens, policy, tolerance, exact, prefixExpansion) {
1769
- const requiredCount = policy === "all" ? queryTokens.length : policy;
1770
- if (requiredCount <= 0) return scored;
1771
- if (requiredCount > queryTokens.length) return [];
1772
- const expansionTerms = prefixExpansion ? new Set(prefixExpansion.terms) : void 0;
1773
- return scored.filter((doc) => {
1774
- const matched = countDocTermMatches(doc, queryTokens, tolerance, exact, prefixExpansion?.token, expansionTerms);
1775
- return matched >= requiredCount;
1776
- });
1777
- }
1778
- function countDocTermMatches(doc, queryTokens, tolerance, exact, prefixToken, expansionTerms) {
1779
- const indexTokens = Object.keys(doc.idf);
1780
- if (indexTokens.length === 0) return 0;
1781
- let count = 0;
1782
- for (const qt of queryTokens) {
1783
- if (prefixToken !== void 0 && qt.token === prefixToken) {
1784
- if (prefixTokenSatisfied(qt.token, indexTokens, expansionTerms)) {
1785
- count++;
1786
- }
1787
- continue;
1788
- }
1789
- if (queryTermSatisfied(qt.token, indexTokens, tolerance, exact)) {
1790
- count++;
1791
- }
1792
- }
1793
- return count;
1794
- }
1795
- function prefixTokenSatisfied(prefixToken, indexTokens, expansionTerms) {
1796
- for (const indexToken of indexTokens) {
1797
- if (indexToken === prefixToken) return true;
1798
- if (expansionTerms?.has(indexToken)) return true;
1799
- }
1800
- return false;
1801
- }
1802
- function queryTermSatisfied(queryToken, indexTokens, tolerance, exact) {
1803
- if (exact || tolerance === 0) {
1804
- return indexTokens.includes(queryToken);
1805
- }
1806
- for (const indexToken of indexTokens) {
1807
- if (boundedLevenshtein(queryToken, indexToken, tolerance).withinTolerance) {
1808
- return true;
1809
- }
1810
- }
1811
- return false;
1812
- }
1813
-
1814
- // src/partitioning/partition-selection.ts
1815
- function partitionsIn(manager, partitionIds) {
1816
- if (partitionIds === void 0) {
1817
- return manager.getAllPartitions();
1818
- }
1819
- const partitions = [];
1820
- for (const partitionId of partitionIds) {
1821
- const partition = manager.partitionAt(partitionId);
1822
- if (partition !== void 0) {
1823
- partitions.push(partition);
1824
- }
1825
- }
1826
- return partitions;
1827
- }
1828
-
1829
- // src/partitioning/fan-out.ts
1830
- async function fanOutQuery(manager, params, language, schema, config, searchOptions) {
1831
- const partitions = partitionsIn(manager, config.partitionIds);
1832
- if (partitions.length === 0) {
1833
- return { scored: [], totalMatched: 0 };
1834
- }
1835
- let globalStats;
1836
- const effectiveMode = resolveEffectiveMode(config);
1837
- if (effectiveMode === "dfs") {
1838
- if (params.term !== void 0 && params.term.trim().length > 0) {
1839
- globalStats = collectQueryTermStats(
1840
- manager,
1841
- params.term,
1842
- language,
1843
- {
1844
- stopWords: searchOptions?.stopWords,
1845
- customTokenizer: searchOptions?.customTokenizer
1846
- },
1847
- config.partitionIds
1848
- );
1849
- }
1850
- } else if (effectiveMode === "broadcast") {
1851
- globalStats = config.globalStats;
1852
- }
1853
- const options = buildSearchOptions(searchOptions, globalStats);
1854
- if (partitions.length === 1 && !config.dispatcher) {
1855
- const result = dispatchSinglePartition(partitions[0], params, language, schema, options);
1856
- let facets2;
1857
- if (params.facets) {
1858
- facets2 = partitions[0].computeFacets(facetMatchSetOf(result), params.facets, schema);
1859
- }
1860
- return { scored: result.scored, totalMatched: result.totalMatched, facets: facets2 };
1861
- }
1862
- let outcomes;
1863
- let facets;
1864
- if (config.dispatcher) {
1865
- outcomes = await dispatchWithDispatcher(partitions, params, language, schema, options, config.dispatcher);
1866
- if (params.facets) {
1867
- facets = collectAndMergeFacets(outcomes, params.facets, schema);
1868
- }
1869
- } else {
1870
- outcomes = [];
1871
- const partitionFacets = [];
1872
- const everyValue = params.facets ? everyValueFacetConfig(params.facets) : void 0;
1873
- for (const partition of partitions) {
1874
- const result = dispatchSinglePartition(partition, params, language, schema, options);
1875
- if (everyValue) {
1876
- partitionFacets.push(partition.computeFacets(facetMatchSetOf(result), everyValue, schema));
1877
- }
1878
- outcomes.push({ result, partition });
1879
- }
1880
- if (params.facets) {
1881
- facets = mergeFacets(partitionFacets, params.facets);
1882
- }
1883
- }
1884
- const allScoredArrays = outcomes.map((o) => o.result.scored);
1885
- const merged = kWayMerge(allScoredArrays);
1886
- let totalMatched = 0;
1887
- for (const outcome of outcomes) {
1888
- totalMatched += outcome.result.totalMatched;
1889
- }
1890
- return { scored: merged, totalMatched, facets };
1891
- }
1892
- function facetMatchSetOf(result) {
1893
- if (result.matchedOrdinalBitset !== void 0) {
1894
- return { ordinalBitset: result.matchedOrdinalBitset };
1895
- }
1896
- return new Set(result.matchedIds ?? []);
1897
- }
1898
- function resolveEffectiveMode(config) {
1899
- if (config.scoringMode === "broadcast" && !config.globalStats) {
1900
- return "dfs";
1901
- }
1902
- return config.scoringMode;
1903
- }
1904
- function buildSearchOptions(base, globalStats) {
1905
- return {
1906
- ...base,
1907
- globalStats: globalStats ?? base?.globalStats
1908
- };
1909
- }
1910
- function dispatchWithDispatcher(partitions, params, language, schema, options, dispatcher) {
1911
- const promises = partitions.map(
1912
- (partition) => dispatcher.dispatch(partition, params, language, schema, options).then((result) => ({ result, partition }))
1913
- );
1914
- return Promise.all(promises);
1915
- }
1916
- function dispatchSinglePartition(partition, params, language, schema, options) {
1917
- return fulltextSearch(partition, params, language, schema, options);
1918
- }
1919
- function collectAndMergeFacets(outcomes, facetConfig, schema) {
1920
- const everyValue = everyValueFacetConfig(facetConfig);
1921
- const partitionFacets = [];
1922
- for (const outcome of outcomes) {
1923
- partitionFacets.push(outcome.partition.computeFacets(facetMatchSetOf(outcome.result), everyValue, schema));
1924
- }
1925
- return mergeFacets(partitionFacets, facetConfig);
1926
- }
1927
-
1928
- // src/partitioning/match-count.ts
1929
- function countsWithoutScores(params) {
1930
- return params.minScore === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
1931
- }
1932
- function fanOutMatchCount(manager, params, language, schema, options) {
1933
- let count = 0;
1934
- for (const partition of partitionsIn(manager, options?.partitionIds)) {
1935
- const matches = fulltextMatches(partition, params, language, schema, options?.searchOptions);
1936
- if (matches !== null) count += matches.count;
1937
- }
1938
- return count;
1939
- }
1940
-
1941
- // src/core/projection.ts
1942
- function splitPaths(paths) {
1943
- if (!paths) return [];
1944
- const split = [];
1945
- for (const path of paths) {
1946
- const segments = path.split(".").filter((segment) => segment.length > 0);
1947
- if (segments.length === 0) continue;
1948
- if (segments.some((segment) => PROTOTYPE_POLLUTION_KEYS.has(segment))) continue;
1949
- split.push(segments);
1950
- }
1951
- return split;
1952
- }
1953
- function overlaps(a, b) {
1954
- const shared = Math.min(a.length, b.length);
1955
- for (let i = 0; i < shared; i++) {
1956
- if (a[i] !== b[i]) return false;
1957
- }
1958
- return true;
1959
- }
1960
- function covers(outer, inner) {
1961
- if (outer.length > inner.length) return false;
1962
- for (let i = 0; i < outer.length; i++) {
1963
- if (outer[i] !== inner[i]) return false;
1964
- }
1965
- return true;
1966
- }
1967
- function readPath(source, path) {
1968
- let current = source;
1969
- for (const segment of path) {
1970
- if (typeof current !== "object" || current === null || Array.isArray(current))
1971
- return { found: false, value: void 0 };
1972
- const record = current;
1973
- if (!Object.hasOwn(record, segment)) return { found: false, value: void 0 };
1974
- current = record[segment];
1975
- }
1976
- return { found: true, value: current };
1977
- }
1978
- function writePath(target, path, value) {
1979
- let current = target;
1980
- for (let i = 0; i < path.length - 1; i++) {
1981
- const segment = path[i];
1982
- const existing = current[segment];
1983
- if (typeof existing === "object" && existing !== null && !Array.isArray(existing)) {
1984
- current = existing;
1985
- continue;
1986
- }
1987
- const created = {};
1988
- current[segment] = created;
1989
- current = created;
1990
- }
1991
- current[path[path.length - 1]] = value;
1992
- }
1993
- function omitPath(source, path) {
1994
- const segment = path[0];
1995
- if (!Object.hasOwn(source, segment)) return source;
1996
- const copy = { ...source };
1997
- if (path.length === 1) {
1998
- delete copy[segment];
1999
- return copy;
2000
- }
2001
- const child = copy[segment];
2002
- if (typeof child !== "object" || child === null || Array.isArray(child)) return copy;
2003
- copy[segment] = omitPath(child, path.slice(1));
2004
- return copy;
2005
- }
2006
- function cloneExcept(value, path, exclude) {
2007
- const reachesDeeper = exclude.some((excluded) => excluded.length > path.length && covers(path, excluded));
2008
- if (!reachesDeeper) return structuredClone(value);
2009
- if (typeof value !== "object" || value === null || Array.isArray(value)) return structuredClone(value);
2010
- const source = value;
2011
- const kept = {};
2012
- for (const key of Object.keys(source)) {
2013
- const childPath = [...path, key];
2014
- if (exclude.some((excluded) => covers(excluded, childPath))) continue;
2015
- kept[key] = cloneExcept(source[key], childPath, exclude);
2016
- }
2017
- return kept;
2018
- }
2019
- function resolveProjection(projection) {
2020
- if (projection === void 0 || projection === true) return { kind: "full" };
2021
- if (projection === false) return { kind: "none" };
2022
- if (typeof projection !== "object" || projection === null) return { kind: "full" };
2023
- const askedInclude = Array.isArray(projection.include) && projection.include.length > 0;
2024
- const askedExclude = Array.isArray(projection.exclude) && projection.exclude.length > 0;
2025
- if (!askedInclude && !askedExclude) return { kind: "full" };
2026
- return {
2027
- kind: "fields",
2028
- include: askedInclude ? splitPaths(projection.include) : null,
2029
- exclude: splitPaths(projection.exclude)
2030
- };
2031
- }
2032
- function projectionKeepsField(projection, fieldPath) {
2033
- if (projection.kind === "full") return true;
2034
- if (projection.kind === "none") return false;
2035
- const path = fieldPath.split(".").filter((segment) => segment.length > 0);
2036
- if (path.length === 0) return false;
2037
- if (projection.include && !projection.include.some((included) => overlaps(included, path))) return false;
2038
- return !projection.exclude.some((excluded) => covers(excluded, path));
2039
- }
2040
- function cloneProjected(document, projection) {
2041
- if (projection === void 0 || projection.kind === "full") return structuredClone(document);
2042
- if (projection.kind === "none") return {};
2043
- if (projection.include === null) return cloneExcept(document, [], projection.exclude);
2044
- const projected = {};
2045
- for (const path of projection.include) {
2046
- if (projection.exclude.some((excluded) => covers(excluded, path))) continue;
2047
- const { found, value } = readPath(document, path);
2048
- if (!found) continue;
2049
- writePath(projected, path, cloneExcept(value, path, projection.exclude));
2050
- }
2051
- return projected;
2052
- }
2053
- function applyProjection(document, projection) {
2054
- if (projection.kind === "full") return document;
2055
- if (projection.kind === "none") return {};
2056
- let projected;
2057
- if (projection.include) {
2058
- projected = {};
2059
- for (const path of projection.include) {
2060
- const { found, value } = readPath(document, path);
2061
- if (found) writePath(projected, path, value);
2062
- }
2063
- } else {
2064
- projected = document;
2065
- }
2066
- for (const path of projection.exclude) {
2067
- projected = omitPath(projected, path);
2068
- }
2069
- return projected;
2070
- }
2071
-
2072
- // src/core/hash.ts
2073
- var FNV_OFFSET_BASIS = 2166136261;
2074
- var FNV_PRIME = 16777619;
2075
- var encoder = new TextEncoder();
2076
- function fnv1aBytes(bytes) {
2077
- let hash = FNV_OFFSET_BASIS;
2078
- for (let i = 0; i < bytes.length; i++) {
2079
- hash ^= bytes[i];
2080
- hash = Math.imul(hash, FNV_PRIME);
2081
- }
2082
- return hash >>> 0;
2083
- }
2084
- function fnv1aUint32s(values) {
2085
- let hash = FNV_OFFSET_BASIS;
2086
- for (const value of values) {
2087
- for (let shift = 24; shift >= 0; shift -= 8) {
2088
- hash ^= value >>> shift & 255;
2089
- hash = Math.imul(hash, FNV_PRIME);
2090
- }
2091
- }
2092
- return hash >>> 0;
2093
- }
2094
- function fnv1a(input) {
2095
- return fnv1aBytes(encoder.encode(input));
2096
- }
2097
-
2098
- // src/partitioning/manager/nested-values.ts
2099
- function setNestedValue(obj, path, value) {
2100
- if (!path.includes(".")) {
2101
- obj[path] = value;
2102
- return;
2103
- }
2104
- const segments = path.split(".");
2105
- let current = obj;
2106
- for (let i = 0; i < segments.length - 1; i++) {
2107
- let next = current[segments[i]];
2108
- if (next === null || next === void 0 || typeof next !== "object") {
2109
- next = {};
2110
- current[segments[i]] = next;
2111
- }
2112
- current = next;
2113
- }
2114
- current[segments[segments.length - 1]] = value;
2115
- }
2116
-
2117
- export { DEG_TO_RAD, MAX_FILTER_DEPTH, MIN_POLYGON_POINTS, VINCENTY_CONVERGENCE_THRESHOLD, VINCENTY_MAX_ITERATIONS, applyPagination, applyProjection, applySorting, bitsetAnd, bitsetFromSet, bitsetHas, bitsetNot, bitsetOr, bitsetSet, boundedLevenshtein, clampRowCount, clearNormalizationCache, cloneProjected, collectQueryTermStats, computeFacets, configureNormalizationCache, countsWithoutScores, createBitSet, decodePageCursor, encodePageCursor, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, fnv1a, fnv1aBytes, fnv1aUint32s, fulltextMatches, fuzzyTermMatches, getAllInternalDocIds, getFieldValueByInternalId, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, kWayMerge, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partitionsIn, polygonContainment, projectionKeepsField, pruneStatsToQueryTerms, readSortValues, requireMatchingCursor, requirePolygonRing, requireSortableFields, requireValidFilter, requireValidOperands, requireWithinResultWindow, resetNormalizationCache, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, textFieldsChanged, tokenize, tokenizeIterator, tokenizeOptions, undeclaredFieldOnStrictIndex, visitTokens };