@delali/narsil 0.2.4-next.59 → 0.3.1-next.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (165) hide show
  1. package/README.md +9 -8
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +1 -1
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
  8. package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
  9. package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
  10. package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
  11. package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
  12. package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
  13. package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
  14. package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
  15. package/dist/browser/chunk-RHG4O77A.mjs +206 -0
  16. package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
  17. package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
  18. package/dist/browser/module-CBB2ETYP.mjs +3111 -0
  19. package/dist/browser/release-76MXRPGV.mjs +33 -0
  20. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  21. package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
  22. package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
  23. package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
  24. package/dist/browser/write-7D6VIK6O.mjs +208 -0
  25. package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
  26. package/dist/chunk-2RIRGOUL.mjs +113 -0
  27. package/dist/chunk-37CD3OKS.mjs +87 -0
  28. package/dist/chunk-44W3CBNG.mjs +7 -0
  29. package/dist/chunk-46D6JQIG.mjs +426 -0
  30. package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
  31. package/dist/chunk-747OHOUA.mjs +11 -0
  32. package/dist/chunk-75XIEF6H.mjs +126 -0
  33. package/dist/chunk-7X5JST5L.mjs +114 -0
  34. package/dist/chunk-BOFBLJFZ.mjs +209 -0
  35. package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
  36. package/dist/chunk-C7SLA6CM.mjs +19 -0
  37. package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
  38. package/dist/chunk-CVXLLEQ3.mjs +1011 -0
  39. package/dist/chunk-DJHHF2L5.mjs +17 -0
  40. package/dist/chunk-E74YLAIG.mjs +24 -0
  41. package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
  42. package/dist/chunk-EWZLIYHV.mjs +57 -0
  43. package/dist/chunk-GBEEG5AN.mjs +35 -0
  44. package/dist/chunk-GES7U2PM.mjs +68 -0
  45. package/dist/chunk-HJSN7UYG.mjs +10 -0
  46. package/dist/chunk-HLPRHOSO.mjs +17 -0
  47. package/dist/chunk-HMFNFVP7.mjs +14 -0
  48. package/dist/chunk-J5QP52HA.mjs +1751 -0
  49. package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
  50. package/dist/chunk-KOKM4PQN.mjs +52 -0
  51. package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
  52. package/dist/chunk-KZ3NX2P7.mjs +21 -0
  53. package/dist/chunk-L5ORED7V.mjs +25 -0
  54. package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
  55. package/dist/chunk-LKOB5COA.mjs +117 -0
  56. package/dist/chunk-LPQOEU5M.mjs +96 -0
  57. package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
  58. package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
  59. package/dist/chunk-OM362BRO.mjs +948 -0
  60. package/dist/chunk-OZDFA4WC.mjs +67 -0
  61. package/dist/chunk-P36MUFEJ.mjs +10 -0
  62. package/dist/chunk-PC2G76VW.mjs +2540 -0
  63. package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
  64. package/dist/chunk-PNFKUCG3.mjs +1130 -0
  65. package/dist/chunk-Q6XWVUZX.mjs +44 -0
  66. package/dist/chunk-R27EMH5M.mjs +617 -0
  67. package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
  68. package/dist/chunk-RF7PLG43.mjs +106 -0
  69. package/dist/chunk-RFBLQLLZ.mjs +16 -0
  70. package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
  71. package/dist/chunk-S7FL6UAH.mjs +521 -0
  72. package/dist/chunk-S7GNDRPS.mjs +651 -0
  73. package/dist/chunk-SP3NM3ZG.mjs +3537 -0
  74. package/dist/chunk-U476IXKQ.mjs +10 -0
  75. package/dist/chunk-UA3AUGNV.mjs +462 -0
  76. package/dist/chunk-UTG5YNXV.mjs +507 -0
  77. package/dist/chunk-VI4R6W25.mjs +8484 -0
  78. package/dist/chunk-WQBCBKY3.mjs +8 -0
  79. package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
  80. package/dist/chunk-X5CV2E2B.mjs +1 -0
  81. package/dist/chunk-XADZPPZI.mjs +614 -0
  82. package/dist/chunk-YEEQHT4V.mjs +212 -0
  83. package/dist/chunk-YTJCGKM4.mjs +56 -0
  84. package/dist/chunk-YYQIODMF.mjs +275 -0
  85. package/dist/chunk-ZI34TVOA.mjs +144 -0
  86. package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
  87. package/dist/chunk-ZM7KU4YW.mjs +35 -0
  88. package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
  89. package/dist/client.d.ts +11 -10
  90. package/dist/client.mjs +209 -45
  91. package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
  92. package/dist/config-DffQlmRq.d.ts +421 -0
  93. package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
  94. package/dist/distribution/coordinator/etcd.d.ts +79 -18
  95. package/dist/distribution/coordinator/etcd.mjs +553 -109
  96. package/dist/distribution/coordinator/in-memory.d.ts +4 -3
  97. package/dist/distribution/coordinator/in-memory.mjs +22 -3
  98. package/dist/distribution/transport/grpc.d.ts +2 -2
  99. package/dist/distribution/transport/grpc.mjs +79 -14
  100. package/dist/distribution/transport/in-memory.d.ts +2 -2
  101. package/dist/distribution/transport/in-memory.mjs +12 -7
  102. package/dist/distribution/transport/tcp.d.ts +2 -2
  103. package/dist/distribution/transport/tcp.mjs +50 -28
  104. package/dist/distribution.d.ts +144 -23
  105. package/dist/distribution.mjs +14479 -8173
  106. package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
  107. package/dist/embeddings/openai.d.ts +2 -2
  108. package/dist/embeddings/openai.mjs +22 -133
  109. package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
  110. package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
  111. package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
  112. package/dist/index.browser.mjs +4 -32180
  113. package/dist/index.d.ts +91 -20
  114. package/dist/index.mjs +33 -16
  115. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  116. package/dist/invalidation/filesystem.d.ts +1 -1
  117. package/dist/invalidation/filesystem.mjs +2 -1
  118. package/dist/invalidation/noop.d.ts +1 -1
  119. package/dist/module-3S65MVCM.mjs +2151 -0
  120. package/dist/partial-words-DTIHO8G4.d.ts +19 -0
  121. package/dist/partial-words.d.ts +22 -0
  122. package/dist/partial-words.mjs +17 -0
  123. package/dist/pattern-Cledccbl.d.ts +18 -0
  124. package/dist/pattern.d.ts +22 -0
  125. package/dist/pattern.mjs +12 -0
  126. package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
  127. package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
  128. package/dist/react.d.ts +279 -183
  129. package/dist/react.mjs +201 -48
  130. package/dist/release-BH3DKW3T.mjs +33 -0
  131. package/dist/replace-CEO5KJJ3.mjs +57 -0
  132. package/dist/rerank-K7C4WZFG.mjs +60 -0
  133. package/dist/rerankers/http.d.ts +50 -0
  134. package/dist/rerankers/http.mjs +104 -0
  135. package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
  136. package/dist/runtime-6Y4ZYXOG.mjs +177 -0
  137. package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
  138. package/dist/server.d.ts +13 -12
  139. package/dist/server.mjs +611 -80
  140. package/dist/snapshot-QGZPODM5.mjs +799 -0
  141. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  142. package/dist/synonyms.d.ts +22 -0
  143. package/dist/synonyms.mjs +12 -0
  144. package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
  145. package/dist/thread-N3IUGUS3.mjs +697 -0
  146. package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
  147. package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
  148. package/dist/vector/search-worker.d.ts +46 -4
  149. package/dist/vector/search-worker.mjs +21 -11
  150. package/dist/workers/entry.mjs +9 -6
  151. package/dist/write-U5RI7MAX.mjs +241 -0
  152. package/package.json +55 -9
  153. package/dist/chunk-3L4U6EJQ.mjs +0 -49
  154. package/dist/chunk-IFBKLBGB.mjs +0 -7
  155. package/dist/chunk-ORBLD3EL.mjs +0 -6
  156. package/dist/chunk-U7KYW4BA.mjs +0 -135
  157. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  158. package/dist/chunk-ZNZBXMP5.mjs +0 -2117
  159. package/dist/chunk-ZZT3EAUC.mjs +0 -11168
  160. package/dist/config-BhldZgaK.d.ts +0 -194
  161. package/dist/direct-executor-ALAVE7FC.mjs +0 -12
  162. package/dist/serialization/crc32-worker.d.ts +0 -20
  163. package/dist/serialization/crc32-worker.mjs +0 -62
  164. package/dist/simd-DW5AtJ6B.d.ts +0 -41
  165. package/dist/thread-DYHEQRVY.mjs +0 -402
@@ -0,0 +1,426 @@
1
+ import { parseSynonymRules, encodeSynonymRules, decodeSynonymRules, SYNONYM_EXPANSION_CACHE_ENTRIES, SYNONYMS_ADDED_PER_QUERY } from './chunk-7X5JST5L.mjs';
2
+ import { encodeStringBlob, tokenize, encodeLookupKey } from './chunk-UTG5YNXV.mjs';
3
+ import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
4
+ import { NarsilError, ErrorCodes } from './chunk-BWJVNZPR.mjs';
5
+
6
+ // src/analysis/registry.ts
7
+ var tokenizers = /* @__PURE__ */ new Map();
8
+ var stopWordSets = /* @__PURE__ */ new Map();
9
+ function requireName(name, kind) {
10
+ if (typeof name !== "string" || name.trim().length === 0) {
11
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `A registered ${kind} needs a non-empty name`, { name });
12
+ }
13
+ }
14
+ function registerTokenizer(name, tokenizer) {
15
+ requireName(name, "tokenizer");
16
+ if (typeof tokenizer?.tokenize !== "function") {
17
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Tokenizer "${name}" needs a tokenize method`, { tokenizer: name });
18
+ }
19
+ tokenizers.set(name, tokenizer);
20
+ }
21
+ function registerStopWords(name, stopWords) {
22
+ requireName(name, "stop word set");
23
+ if (!(stopWords instanceof Set) && typeof stopWords !== "function") {
24
+ throw new NarsilError(
25
+ ErrorCodes.CONFIG_INVALID,
26
+ `Stop word set "${name}" needs a Set of words or a function returning one`,
27
+ { stopWords: name }
28
+ );
29
+ }
30
+ stopWordSets.set(name, stopWords);
31
+ }
32
+ function hasTokenizer(name) {
33
+ return tokenizers.has(name);
34
+ }
35
+ function hasStopWords(name) {
36
+ return stopWordSets.has(name);
37
+ }
38
+ function getTokenizer(name) {
39
+ const tokenizer = tokenizers.get(name);
40
+ if (!tokenizer) {
41
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Tokenizer "${name}" is not registered on this engine`, {
42
+ tokenizer: name,
43
+ available: [...tokenizers.keys()]
44
+ });
45
+ }
46
+ return tokenizer;
47
+ }
48
+ function getStopWords(name) {
49
+ const stopWords = stopWordSets.get(name);
50
+ if (!stopWords) {
51
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Stop word set "${name}" is not registered on this engine`, {
52
+ stopWords: name,
53
+ available: [...stopWordSets.keys()]
54
+ });
55
+ }
56
+ return stopWords;
57
+ }
58
+ function resolveIndexAnalysis(config) {
59
+ return {
60
+ stopWords: typeof config.stopWords === "string" ? getStopWords(config.stopWords) : config.stopWords,
61
+ customTokenizer: typeof config.tokenizer === "string" ? getTokenizer(config.tokenizer) : config.tokenizer
62
+ };
63
+ }
64
+
65
+ // src/core/synonyms/combined-postings.ts
66
+ var NO_DELETED = { size: 0, has: () => false };
67
+ var FIELD_SLOTS = 65536;
68
+ function combinePostingLists(lists) {
69
+ if (lists.length === 1) return lists[0];
70
+ const rowOf = /* @__PURE__ */ new Map();
71
+ const docIds = [];
72
+ const fieldNameIndices = [];
73
+ const frequencies = [];
74
+ const documents = /* @__PURE__ */ new Set();
75
+ let totalTermFrequency = 0;
76
+ for (const list of lists) {
77
+ const deleted = list.deletedDocs;
78
+ const hasDeleted = deleted.size > 0;
79
+ for (let row = 0; row < list.length; row++) {
80
+ const docId = list.docIds[row];
81
+ if (hasDeleted && deleted.has(docId)) continue;
82
+ const field = list.fieldNameIndices[row];
83
+ const frequency = list.termFrequencies[row];
84
+ totalTermFrequency += frequency;
85
+ const key = docId * FIELD_SLOTS + field;
86
+ const existing = rowOf.get(key);
87
+ if (existing !== void 0) {
88
+ frequencies[existing] += frequency;
89
+ continue;
90
+ }
91
+ rowOf.set(key, docIds.length);
92
+ docIds.push(docId);
93
+ fieldNameIndices.push(field);
94
+ frequencies.push(frequency);
95
+ documents.add(docId);
96
+ }
97
+ }
98
+ return {
99
+ length: docIds.length,
100
+ docIds: Uint32Array.from(docIds),
101
+ termFrequencies: Float64Array.from(frequencies),
102
+ fieldNameIndices: Uint16Array.from(fieldNameIndices),
103
+ positions: null,
104
+ docIdSet: documents,
105
+ deletedDocs: NO_DELETED,
106
+ totalTermFrequency,
107
+ structureRevision: 0,
108
+ ordered: false
109
+ };
110
+ }
111
+
112
+ // src/core/synonyms/expand.ts
113
+ function groupSynonymsOf(tokens) {
114
+ return {
115
+ tokens,
116
+ has: (token) => tokens.includes(token),
117
+ documentFrequency(frequencyOf) {
118
+ let most = 0;
119
+ for (const token of tokens) most = Math.max(most, frequencyOf(token));
120
+ return most;
121
+ },
122
+ postings(lookup, own) {
123
+ const lists = [];
124
+ for (const token of [own, ...tokens]) {
125
+ const list = lookup(token);
126
+ if (list !== void 0) lists.push(list);
127
+ }
128
+ return lists.length === 0 ? void 0 : combinePostingLists(lists);
129
+ }
130
+ };
131
+ }
132
+ function expandQuerySynonyms(lookup, queryTokens) {
133
+ const withSynonyms = [];
134
+ const seen = /* @__PURE__ */ new Set();
135
+ for (const token of queryTokens) {
136
+ if (seen.has(token)) continue;
137
+ seen.add(token);
138
+ const synonyms = lookup.synonymsOf(token);
139
+ if (synonyms.length > 0) withSynonyms.push({ token, synonyms });
140
+ if (withSynonyms.length === SYNONYMS_ADDED_PER_QUERY) break;
141
+ }
142
+ const count = withSynonyms.length;
143
+ if (count === 0) return void 0;
144
+ const share = Math.floor(SYNONYMS_ADDED_PER_QUERY / count);
145
+ const extra = SYNONYMS_ADDED_PER_QUERY % count;
146
+ const expanded = /* @__PURE__ */ new Map();
147
+ const added = /* @__PURE__ */ new Set();
148
+ for (let i = 0; i < count; i++) {
149
+ const quota = share + (i < extra ? 1 : 0);
150
+ const { token, synonyms } = withSynonyms[i];
151
+ const kept = synonyms.length <= quota ? synonyms : synonyms.slice(0, quota);
152
+ expanded.set(token, kept);
153
+ for (const synonym of kept) added.add(synonym);
154
+ }
155
+ return {
156
+ tokens: [...added],
157
+ addTo(groupOf) {
158
+ for (const [token, synonyms] of expanded) {
159
+ const group = groupOf(token);
160
+ group.synonyms = groupSynonymsOf(synonyms);
161
+ group.members = group.members.filter((member) => member.token === token || !synonyms.includes(member.token));
162
+ }
163
+ }
164
+ };
165
+ }
166
+
167
+ // src/core/synonyms/lookup.ts
168
+ var NO_SOURCE = -1;
169
+ var NO_SYNONYMS = Object.freeze([]);
170
+ var decoder = new TextDecoder();
171
+ function analyseWord(word, analysis, cache) {
172
+ const cached = cache.get(word);
173
+ if (cached !== void 0) return cached;
174
+ const { tokens } = tokenize(word, analysis.language, {
175
+ stem: true,
176
+ removeStopWords: true,
177
+ customTokenizer: analysis.customTokenizer,
178
+ stopWordOverride: analysis.stopWords
179
+ });
180
+ const result = tokens.length === 1 ? tokens[0].token : tokens.length;
181
+ cache.set(word, result);
182
+ return result;
183
+ }
184
+ function intern(interned, token) {
185
+ let id = interned.ids.get(token);
186
+ if (id === void 0) {
187
+ id = interned.tokens.length;
188
+ interned.tokens.push(token);
189
+ interned.ids.set(token, id);
190
+ }
191
+ return id;
192
+ }
193
+ function sourceWordOf(rule) {
194
+ return "word" in rule ? rule.word : void 0;
195
+ }
196
+ function targetWordsOf(rule) {
197
+ return "words" in rule ? rule.words : rule.synonyms;
198
+ }
199
+ function packedLookup(tokens, ruleStart, ruleTokenIds, ruleSource) {
200
+ const order = tokens.map((_, id) => id).sort((a, b) => compareCodePoints(tokens[a], tokens[b]));
201
+ const rank = new Uint32Array(tokens.length);
202
+ for (let position = 0; position < order.length; position++) rank[order[position]] = position;
203
+ const { blob, offsets } = encodeStringBlob(order.map((id) => tokens[id]));
204
+ const ruleTokens = Uint32Array.from(ruleTokenIds, (id) => rank[id]);
205
+ const sources = Int32Array.from(ruleSource, (id) => id === NO_SOURCE ? NO_SOURCE : rank[id]);
206
+ const ruleCount = sources.length;
207
+ const tokenRuleStart = new Uint32Array(tokens.length + 1);
208
+ for (let rule = 0; rule < ruleCount; rule++) {
209
+ if (sources[rule] !== NO_SOURCE) tokenRuleStart[sources[rule] + 1]++;
210
+ else for (let i = ruleStart[rule]; i < ruleStart[rule + 1]; i++) tokenRuleStart[ruleTokens[i] + 1]++;
211
+ }
212
+ for (let token = 0; token < tokens.length; token++) tokenRuleStart[token + 1] += tokenRuleStart[token];
213
+ const tokenRules = new Uint32Array(tokenRuleStart[tokens.length]);
214
+ const fill = tokenRuleStart.slice(0, tokens.length);
215
+ for (let rule = 0; rule < ruleCount; rule++) {
216
+ if (sources[rule] !== NO_SOURCE) tokenRules[fill[sources[rule]]++] = rule;
217
+ else for (let i = ruleStart[rule]; i < ruleStart[rule + 1]; i++) tokenRules[fill[ruleTokens[i]]++] = rule;
218
+ }
219
+ const cache = /* @__PURE__ */ new Map();
220
+ const bytes = blob.byteLength + offsets.byteLength + ruleStart.byteLength + ruleTokens.byteLength + sources.byteLength + tokenRuleStart.byteLength + tokenRules.byteLength;
221
+ function rankOf(token) {
222
+ const key = encodeLookupKey(token);
223
+ let low = 0;
224
+ let high = offsets.length - 2;
225
+ while (low <= high) {
226
+ const middle = low + high >>> 1;
227
+ const start = offsets[middle];
228
+ const end = offsets[middle + 1];
229
+ let order2 = 0;
230
+ const shorter = Math.min(end - start, key.length);
231
+ for (let i = 0; i < shorter && order2 === 0; i++) order2 = blob[start + i] - key[i];
232
+ if (order2 === 0) order2 = end - start - key.length;
233
+ if (order2 === 0) return middle;
234
+ if (order2 < 0) low = middle + 1;
235
+ else high = middle - 1;
236
+ }
237
+ return -1;
238
+ }
239
+ function collect(token) {
240
+ const found = [];
241
+ for (let i = tokenRuleStart[token]; i < tokenRuleStart[token + 1]; i++) {
242
+ const rule = tokenRules[i];
243
+ for (let j = ruleStart[rule]; j < ruleStart[rule + 1]; j++) {
244
+ if (ruleTokens[j] !== token) found.push(ruleTokens[j]);
245
+ }
246
+ }
247
+ if (found.length === 0) return NO_SYNONYMS;
248
+ found.sort((a, b) => a - b);
249
+ const synonyms = [];
250
+ let previous = -1;
251
+ for (const id of found) {
252
+ if (synonyms.length === SYNONYMS_ADDED_PER_QUERY) break;
253
+ if (id === previous) continue;
254
+ previous = id;
255
+ synonyms.push(decoder.decode(blob.subarray(offsets[id], offsets[id + 1])));
256
+ }
257
+ return Object.freeze(synonyms);
258
+ }
259
+ const lookup = {
260
+ tokenCount: tokens.length,
261
+ bytes,
262
+ expand: (queryTokens) => expandQuerySynonyms(lookup, queryTokens),
263
+ synonymsOf(token) {
264
+ const cached = cache.get(token);
265
+ if (cached !== void 0) {
266
+ cache.delete(token);
267
+ cache.set(token, cached);
268
+ return cached;
269
+ }
270
+ const rankOfToken = rankOf(token);
271
+ if (rankOfToken < 0) return NO_SYNONYMS;
272
+ const synonyms = collect(rankOfToken);
273
+ if (cache.size >= SYNONYM_EXPANSION_CACHE_ENTRIES) {
274
+ const oldest = cache.keys().next();
275
+ if (oldest.done !== true) cache.delete(oldest.value);
276
+ }
277
+ cache.set(token, synonyms);
278
+ return synonyms;
279
+ }
280
+ };
281
+ return lookup;
282
+ }
283
+ function buildSynonymLookup(rules, analysis, raise) {
284
+ const analysed = /* @__PURE__ */ new Map();
285
+ const interned = { ids: /* @__PURE__ */ new Map(), tokens: [] };
286
+ const ruleStart = new Uint32Array(rules.length + 1);
287
+ const ruleTokenIds = [];
288
+ const ruleSource = new Int32Array(rules.length).fill(NO_SOURCE);
289
+ const kept = new Uint8Array(rules.length);
290
+ function tokenOf(word, rule) {
291
+ const result = analyseWord(word, analysis, analysed);
292
+ if (typeof result === "string") return result;
293
+ if (raise !== null) {
294
+ const reason = result === 0 ? "to no token, as a stop word does" : `to ${result} tokens, as a phrase does`;
295
+ raise(`Synonym rule ${rule} holds a word that the index analyses ${reason}`, { rule, tokens: result });
296
+ }
297
+ return null;
298
+ }
299
+ for (let rule = 0; rule < rules.length; rule++) {
300
+ const source = sourceWordOf(rules[rule]);
301
+ const sourceToken = source === void 0 ? null : tokenOf(source, rule);
302
+ const seen = /* @__PURE__ */ new Set();
303
+ for (const word of targetWordsOf(rules[rule])) {
304
+ const token = tokenOf(word, rule);
305
+ if (token === null) continue;
306
+ const id = intern(interned, token);
307
+ if (seen.has(id)) continue;
308
+ seen.add(id);
309
+ ruleTokenIds.push(id);
310
+ }
311
+ if (source === void 0) kept[rule] = 1;
312
+ else if (sourceToken !== null) {
313
+ ruleSource[rule] = intern(interned, sourceToken);
314
+ kept[rule] = 1;
315
+ }
316
+ if (kept[rule] === 0) ruleTokenIds.length = ruleStart[rule];
317
+ ruleStart[rule + 1] = ruleTokenIds.length;
318
+ }
319
+ if (ruleTokenIds.length === 0) return null;
320
+ return packedLookup(interned.tokens, ruleStart, ruleTokenIds, ruleSource);
321
+ }
322
+
323
+ // src/core/synonyms/prepare.ts
324
+ function invalidIn(source) {
325
+ return (message, details) => {
326
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, `${source}${message}`, details);
327
+ };
328
+ }
329
+ var raiseConfigInvalid = invalidIn("");
330
+ function synonymAnalysisOf(config, language) {
331
+ return { language, ...resolveIndexAnalysis(config) };
332
+ }
333
+ function prepareSynonyms(value, config, language, strictWords, raise = raiseConfigInvalid) {
334
+ const report = typeof raise === "string" ? invalidIn(raise) : raise;
335
+ const rules = parseSynonymRules(value, report);
336
+ if (rules.length === 0) return null;
337
+ const lookup = buildSynonymLookup(rules, synonymAnalysisOf(config, language), strictWords ? report : null);
338
+ return { encoded: encodeSynonymRules(rules), lookup };
339
+ }
340
+ function storedSynonymLookup(encoded, config, language) {
341
+ return buildSynonymLookup(decodeSynonymRules(encoded, raiseConfigInvalid), synonymAnalysisOf(config, language), null);
342
+ }
343
+ function storedSynonymRules(encoded) {
344
+ return decodeSynonymRules(encoded, raiseConfigInvalid);
345
+ }
346
+
347
+ // src/core/synonyms/operations.ts
348
+ var replacingByCore = /* @__PURE__ */ new WeakMap();
349
+ async function withOpenIndex(core, indexName, action) {
350
+ core.guardShutdown();
351
+ core.requireIndex(indexName);
352
+ const release = await core.indexState.acquire(indexName);
353
+ try {
354
+ return await action();
355
+ } finally {
356
+ release();
357
+ }
358
+ }
359
+ async function oneAtATime(core, indexName, action) {
360
+ let chains = replacingByCore.get(core);
361
+ if (chains === void 0) {
362
+ chains = /* @__PURE__ */ new Map();
363
+ replacingByCore.set(core, chains);
364
+ }
365
+ const replacing = chains;
366
+ const earlier = replacing.get(indexName) ?? Promise.resolve();
367
+ const current = earlier.catch(() => void 0).then(action);
368
+ replacing.set(indexName, current);
369
+ try {
370
+ return await current;
371
+ } finally {
372
+ if (replacing.get(indexName) === current) replacing.delete(indexName);
373
+ }
374
+ }
375
+ async function replaceNow(core, indexName, value, strictWords) {
376
+ const entry = core.requireIndex(indexName);
377
+ const manager = core.requireManager(indexName);
378
+ const prepared = prepareSynonyms(value, entry.config, entry.language, strictWords);
379
+ const previous = entry.synonyms;
380
+ entry.synonyms = prepared?.encoded ?? null;
381
+ try {
382
+ if (core.durability) await core.durability.manager.persistMetadata(indexName);
383
+ } catch (err) {
384
+ entry.synonyms = previous;
385
+ throw err;
386
+ }
387
+ manager.synonyms = prepared?.lookup ?? null;
388
+ await core.orchestrator.replicateToWorkers({
389
+ type: "replaceSynonyms",
390
+ indexName,
391
+ requestId: `replace-synonyms-${indexName}`,
392
+ ...prepared !== null ? { synonyms: prepared.encoded } : {}
393
+ });
394
+ await core.orchestrator.awaitWrites(indexName);
395
+ }
396
+ function replaceSynonymSet(core, indexName, value, strictWords) {
397
+ return withOpenIndex(
398
+ core,
399
+ indexName,
400
+ () => oneAtATime(core, indexName, () => replaceNow(core, indexName, value, strictWords))
401
+ );
402
+ }
403
+ function applyLoadedSynonyms(core, indexName, load) {
404
+ return withOpenIndex(
405
+ core,
406
+ indexName,
407
+ () => oneAtATime(core, indexName, async () => {
408
+ const rules = await load();
409
+ if (rules === null) return false;
410
+ await replaceNow(core, indexName, rules, false);
411
+ return true;
412
+ })
413
+ );
414
+ }
415
+
416
+ // src/core/synonyms/module.ts
417
+ var synonymSearchModule = {
418
+ name: "synonyms",
419
+ prepare: prepareSynonyms,
420
+ storedLookup: storedSynonymLookup,
421
+ decodeRules: storedSynonymRules,
422
+ replace: replaceSynonymSet,
423
+ applyLoaded: applyLoadedSynonyms
424
+ };
425
+
426
+ export { getStopWords, getTokenizer, hasStopWords, hasTokenizer, registerStopWords, registerTokenizer, resolveIndexAnalysis, synonymSearchModule };