@delali/narsil 0.3.1-next.90 → 0.3.1-next.94

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-4JTBL63P.mjs} +13 -5
  4. package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
  5. package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
  6. package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-HC6DMR4T.mjs} +2445 -2107
  7. package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-PP6N5CCS.mjs} +2 -2
  8. package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
  9. package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
  10. package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
  11. package/dist/browser/release-76MXRPGV.mjs +33 -0
  12. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  13. package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
  14. package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-PPTZ565G.mjs} +368 -103
  15. package/dist/browser/{write-6SBYTS62.mjs → write-2ZAA7UDH.mjs} +4 -4
  16. package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-xsJfdFoQ.d.ts} +11 -3
  17. package/dist/chunk-2GZGBOQI.mjs +426 -0
  18. package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
  19. package/dist/chunk-7KC56574.mjs +507 -0
  20. package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
  21. package/dist/chunk-7X5JST5L.mjs +114 -0
  22. package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
  23. package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
  24. package/dist/{chunk-R6BMMQXH.mjs → chunk-C45FUYPI.mjs} +193 -79
  25. package/dist/{chunk-EMXAYPTQ.mjs → chunk-CHOGQSHK.mjs} +2326 -3503
  26. package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
  27. package/dist/{chunk-VY4DXZ5P.mjs → chunk-EWBXO45L.mjs} +3 -3
  28. package/dist/{chunk-QAMUC2BJ.mjs → chunk-EX62BQT6.mjs} +10 -9
  29. package/dist/{chunk-MWJIAREY.mjs → chunk-EZD4MWOU.mjs} +4 -4
  30. package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
  31. package/dist/{chunk-URGLSNKR.mjs → chunk-GJZHU4FC.mjs} +202 -16
  32. package/dist/{chunk-S4LQDSUL.mjs → chunk-J53IHLG7.mjs} +7 -7
  33. package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
  34. package/dist/chunk-KOKM4PQN.mjs +52 -0
  35. package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
  36. package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
  37. package/dist/{chunk-V7YUSKKZ.mjs → chunk-QWWCLRXX.mjs} +10 -6
  38. package/dist/chunk-QY7GC2MW.mjs +33 -0
  39. package/dist/{chunk-DCK4P5BI.mjs → chunk-R36MIGBU.mjs} +100 -59
  40. package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
  41. package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
  42. package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
  43. package/dist/{chunk-XSZVS4S3.mjs → chunk-RUIXODYU.mjs} +9 -4
  44. package/dist/{chunk-A3UB2MXL.mjs → chunk-SBCEHIPL.mjs} +4 -507
  45. package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
  46. package/dist/chunk-UAUZGXJP.mjs +3591 -0
  47. package/dist/{chunk-CL4OM7GB.mjs → chunk-WBS3EYC4.mjs} +12 -8
  48. package/dist/{chunk-J2BPXTD7.mjs → chunk-WLOF6HSP.mjs} +3 -3
  49. package/dist/chunk-X5CV2E2B.mjs +1 -0
  50. package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
  51. package/dist/chunk-ZHS44YKF.mjs +35 -0
  52. package/dist/{client-BMK53IR2.d.ts → client-CgayPw2R.d.ts} +27 -3
  53. package/dist/client.d.ts +9 -8
  54. package/dist/client.mjs +16 -2
  55. package/dist/{config-UJx1dyX1.d.ts → config-3kBsQ0Pi.d.ts} +2 -2
  56. package/dist/{config-BUR-7eVQ.d.ts → config-B-RnO6dr.d.ts} +2 -2
  57. package/dist/direct-executor-IJTRU3YN.mjs +22 -0
  58. package/dist/distribution/coordinator/etcd.d.ts +3 -2
  59. package/dist/distribution/coordinator/etcd.mjs +5 -1
  60. package/dist/distribution/coordinator/in-memory.d.ts +3 -2
  61. package/dist/distribution/coordinator/in-memory.mjs +4 -0
  62. package/dist/distribution/transport/grpc.mjs +3 -3
  63. package/dist/distribution/transport/in-memory.mjs +1 -1
  64. package/dist/distribution/transport/tcp.mjs +2 -2
  65. package/dist/distribution.d.ts +43 -9
  66. package/dist/distribution.mjs +4620 -4293
  67. package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-Y55SCPFC.mjs} +524 -93
  68. package/dist/embeddings/openai.mjs +1 -1
  69. package/dist/{engine-9X6W5naj.d.ts → engine-DSUfY2F0.d.ts} +53 -17
  70. package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
  71. package/dist/index.browser.mjs +2 -2
  72. package/dist/index.d.ts +44 -25
  73. package/dist/index.mjs +22 -18
  74. package/dist/{memory-CV9NAxLY.d.ts → memory-wuXcYOD1.d.ts} +99 -79
  75. package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
  76. package/dist/partial-words.mjs +5 -4
  77. package/dist/pattern.mjs +3 -3
  78. package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
  79. package/dist/persistence/durability/checkpoint-worker.mjs +27 -23
  80. package/dist/react.d.ts +8 -7
  81. package/dist/react.mjs +1 -0
  82. package/dist/release-BH3DKW3T.mjs +33 -0
  83. package/dist/replace-CEO5KJJ3.mjs +57 -0
  84. package/dist/restore-queue-4SG45BLF.mjs +35 -0
  85. package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
  86. package/dist/server.d.ts +11 -10
  87. package/dist/server.mjs +38 -11
  88. package/dist/snapshot-XBKE7PUG.mjs +738 -0
  89. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  90. package/dist/synonyms.d.ts +22 -0
  91. package/dist/synonyms.mjs +12 -0
  92. package/dist/{tasks-BGaZJyoY.d.ts → tasks-Bj9WBlFG.d.ts} +4 -1
  93. package/dist/{thread-ZP3UMIZT.mjs → thread-U7XYJWGG.mjs} +17 -12
  94. package/dist/vector/search-worker.d.ts +2 -1
  95. package/dist/vector/search-worker.mjs +6 -6
  96. package/dist/workers/entry.mjs +6 -6
  97. package/dist/{write-PXDKPG2R.mjs → write-BMNV7P2H.mjs} +24 -20
  98. package/package.json +15 -7
  99. package/dist/chunk-PR4O7CAP.mjs +0 -18
  100. package/dist/chunk-X4SEXL4U.mjs +0 -2310
  101. package/dist/chunk-Y2F66IQJ.mjs +0 -74
  102. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  103. package/dist/direct-executor-EGESD26S.mjs +0 -18
  104. package/dist/snapshot-22CUEYYW.mjs +0 -469
@@ -1,2310 +0,0 @@
1
- import { PATTERN_EXPRESSION_OPERATORS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, requireTextRangeField, isFiniteNumber, invalid, requireFieldKind, kindOfValue, requireListLength, isObjectRecord, patternSearchModule, DEFAULT_PATTERN_WORK_CAP, bitsetIsEmpty, createBitSet, bitsetSet, PATTERN_WORK_FLUSH_UNITS, MATCHER_STATE_BUDGET_BYTES } from './chunk-T6CNO72J.mjs';
2
- import { GEO_UNIT_METRES, MAX_FILTER_DEPTH, MAX_FILTER_TESTS, FILTER_OPERAND_COPY_DEPTH } from './chunk-WQBCBKY3.mjs';
3
- import { getFlatSchema, tokenize, partialWordSearchModule } from './chunk-A3UB2MXL.mjs';
4
- import { isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES, MIN_POLYGON_POINTS, partialFieldsOf, schemaFieldsOf, flattenSchema, EARTH_RADIUS_METERS, RADIUS_BOX_MARGIN_RATIO, RADIUS_BOX_MARGIN_RADIANS, FULL_TURN_DEGREES, ANTIMERIDIAN_SPAN_DEGREES, DEG_TO_RAD, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS } from './chunk-CVQWQVFX.mjs';
5
- import { MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, MAX_CURSOR_ANCHOR_CODE_POINTS, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PREFIX_MAX_EXPANSIONS } from './chunk-FGMGUDHX.mjs';
6
- import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
7
- import { parsedTypeOf, isStringValuedBase, withSortableOption, isWordIndexedFieldType } from './chunk-BVOJIYRV.mjs';
8
- import { singleFoldTable, appendFoldedCodePoint, sortModeOf, defaultSortMode, SORT_VALUE_MAX_CODE_POINTS, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, isSortMode, SORT_MODES, compareSortValues } from './chunk-R6A773WK.mjs';
9
- import { compareCodePoints, exceedsCodePoints, codePointCount, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
10
- import { isGeopointOrVectorType, PROTOTYPE_POLLUTION_KEYS } from './chunk-L5ORED7V.mjs';
11
- import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
12
-
13
- // src/core/partition/partial-words/platform.ts
14
- function platformPartialWordSearch() {
15
- return partialWordSearchModule;
16
- }
17
-
18
- // src/core/partition/partial-words/registry.ts
19
- var registered = null;
20
- var MODULE_FUNCTIONS = [
21
- "createGrowingSlot",
22
- "createFrozenSlot",
23
- "buildTable",
24
- "expand",
25
- "countSpelling",
26
- "encodeSpellings",
27
- "spellingCountsOf",
28
- "highlightMatcher",
29
- "addTermMembers",
30
- "highlightWords"
31
- ];
32
- function isPartialWordSearchModule(value) {
33
- if (value === null || typeof value !== "object" || value.name !== "partialWords") return false;
34
- return MODULE_FUNCTIONS.every((name) => typeof Reflect.get(value, name) === "function");
35
- }
36
- function registerPartialWordSearch(module) {
37
- if (!isPartialWordSearchModule(module)) {
38
- throw new NarsilError(
39
- ErrorCodes.CONFIG_INVALID,
40
- 'Pass registerPartialWordSearch the partialWordSearch object that "@delali/narsil/partial-words" exports'
41
- );
42
- }
43
- registered = module;
44
- }
45
- function availablePartialWordSearch() {
46
- return registered ?? platformPartialWordSearch();
47
- }
48
- var PARTIAL_WORD_IMPORT_ADVICE = `Add import { partialWordSearch } from '@delali/narsil/partial-words', then call registerPartialWordSearch(partialWordSearch) before you create or load an index with a partial field`;
49
- function partialWordModule() {
50
- const module = availablePartialWordSearch();
51
- if (module !== null) return module;
52
- throw new NarsilError(
53
- ErrorCodes.CONFIG_INVALID,
54
- `This bundle contains no partial-word search code. ${PARTIAL_WORD_IMPORT_ADVICE}`
55
- );
56
- }
57
- function requirePartialWordSearchFor(schema) {
58
- const fields = partialFieldsOf(schema);
59
- if (fields.length === 0 || availablePartialWordSearch() !== null) return;
60
- throw new NarsilError(
61
- ErrorCodes.CONFIG_INVALID,
62
- `The type of field "${fields[0]}" includes "partial", but this bundle contains no partial-word search code. ${PARTIAL_WORD_IMPORT_ADVICE}`,
63
- { field: fields[0] }
64
- );
65
- }
66
- var EMPTY_GROWING_SLOT = {
67
- readerFor: () => null,
68
- spellings: () => null,
69
- bytes: () => 0,
70
- afterDocument: () => void 0,
71
- afterMerge: () => void 0,
72
- forgetSpellings: () => void 0,
73
- clear: () => void 0
74
- };
75
- var EMPTY_FROZEN_SLOT = {
76
- readerFor: () => null,
77
- spellings: () => null,
78
- bytes: () => 0,
79
- afterRemoval: () => void 0,
80
- tableOfEveryDocument: () => null,
81
- countKeptSpellings: () => void 0
82
- };
83
- function createGrowingPartialWords(view) {
84
- return availablePartialWordSearch()?.createGrowingSlot(view) ?? EMPTY_GROWING_SLOT;
85
- }
86
- function createFrozenPartialWords(initial, tokenTable, documents) {
87
- return availablePartialWordSearch()?.createFrozenSlot(initial, tokenTable, documents) ?? EMPTY_FROZEN_SLOT;
88
- }
89
- function buildPartialWordTableFrom(source, tokenTable) {
90
- return availablePartialWordSearch()?.buildTable(source, tokenTable) ?? null;
91
- }
92
- function expandPartialWords(subs, analysis, requests) {
93
- const module = availablePartialWordSearch();
94
- return module === null ? requests.map(() => []) : module.expand(subs, analysis, requests);
95
- }
96
- function countPartialSpelling(counts, token, spelling, change) {
97
- partialWordModule().countSpelling(counts, token, spelling, change);
98
- }
99
- function encodePartialSpellingsOf(fields, counts) {
100
- return partialWordModule().encodeSpellings(fields, counts);
101
- }
102
- function partialSpellingsOfDocuments(documents, analysis) {
103
- const module = partialWordModule();
104
- return module.encodeSpellings(analysis.fields, module.spellingCountsOf(documents, analysis));
105
- }
106
- function partialWordHighlighter(text, language, partial) {
107
- return partialWordModule().highlightMatcher(text, language, partial);
108
- }
109
- function addPartialWordMembers(query, expand, groupOf) {
110
- partialWordModule().addTermMembers(query, expand, groupOf);
111
- }
112
- function partialWordsToHighlight(queryTokens, written) {
113
- return partialWordModule().highlightWords(queryTokens, written);
114
- }
115
-
116
- // src/core/pattern-index/platform.ts
117
- function platformPatternSearch() {
118
- return patternSearchModule;
119
- }
120
-
121
- // src/core/pattern-index/registry.ts
122
- var registered2 = null;
123
- var MODULE_FUNCTIONS2 = [
124
- "createIndex",
125
- "readArrays",
126
- "mergeArrays",
127
- "matchInto",
128
- "checkExpression",
129
- "expressionFootprint",
130
- "largestFootprint",
131
- "checkOperands",
132
- "evaluateTests",
133
- "serializeIndex",
134
- "decodeIndex"
135
- ];
136
- function isPatternSearchModule(value) {
137
- if (value === null || typeof value !== "object" || value.name !== "pattern") return false;
138
- return MODULE_FUNCTIONS2.every((name) => typeof Reflect.get(value, name) === "function");
139
- }
140
- function registerPatternSearch(module) {
141
- if (!isPatternSearchModule(module)) {
142
- throw new NarsilError(
143
- ErrorCodes.CONFIG_INVALID,
144
- 'registerPatternSearch takes the patternSearch object that "@delali/narsil/pattern" exports'
145
- );
146
- }
147
- registered2 = module;
148
- }
149
- function availablePatternSearch() {
150
- return registered2 ?? platformPatternSearch();
151
- }
152
- function requirePatternSearchFor(schema) {
153
- for (const { path, pattern } of schemaFieldsOf(schema)) {
154
- if (pattern) requirePatternSearch(path);
155
- }
156
- }
157
- function requirePatternSearch(fieldPath) {
158
- const module = availablePatternSearch();
159
- if (module !== null) return module;
160
- throw new NarsilError(
161
- ErrorCodes.CONFIG_INVALID,
162
- `Field "${fieldPath}" is a pattern field, but this bundle contains no pattern search code. Add import { patternSearch } from '@delali/narsil/pattern', then call registerPatternSearch(patternSearch) before you create or load the index`,
163
- { field: fieldPath }
164
- );
165
- }
166
-
167
- // src/core/pattern-index/work-meter.ts
168
- var COUNTER_BYTES = 8;
169
- function capExceeded(cap) {
170
- return new NarsilError(
171
- ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
172
- `The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
173
- { cap }
174
- );
175
- }
176
- function localMeter(cap, spent, matcherStateBytes, flushUnits) {
177
- let total = spent;
178
- let sharedCount = null;
179
- const spentSoFar = () => sharedCount === null ? total : Number(Atomics.load(sharedCount, 0));
180
- const meter = {
181
- cap,
182
- matcherStateBytes,
183
- flushUnits,
184
- add(units) {
185
- if (units <= 0) return;
186
- if (sharedCount === null) total += units;
187
- else total = Number(Atomics.add(sharedCount, 0, BigInt(units))) + units;
188
- if (total > cap) throw capExceeded(cap);
189
- },
190
- remaining() {
191
- return cap - spentSoFar();
192
- },
193
- fork(workerCount) {
194
- const copies = Math.max(1, workerCount);
195
- const seed = spentSoFar();
196
- let counter = null;
197
- if (typeof SharedArrayBuffer === "function") {
198
- counter = new SharedArrayBuffer(COUNTER_BYTES);
199
- new BigInt64Array(counter)[0] = BigInt(seed);
200
- }
201
- const shared = {
202
- cap,
203
- spent: seed,
204
- counter,
205
- matcherStateBytes: Math.floor(matcherStateBytes / copies),
206
- flushUnits: Math.max(1, Math.floor(flushUnits / copies))
207
- };
208
- return {
209
- shared,
210
- settle(reportedUnits) {
211
- if (counter === null) {
212
- meter.add(reportedUnits);
213
- return;
214
- }
215
- const unitsSinceFork = spentSoFar() - seed;
216
- const view = new BigInt64Array(counter);
217
- if (unitsSinceFork > 0) Atomics.add(view, 0, BigInt(unitsSinceFork));
218
- sharedCount = view;
219
- total = spentSoFar();
220
- if (total > cap) throw capExceeded(cap);
221
- }
222
- };
223
- }
224
- };
225
- return meter;
226
- }
227
- function createPatternWorkMeter(cap = DEFAULT_PATTERN_WORK_CAP) {
228
- return localMeter(cap, 0, MATCHER_STATE_BUDGET_BYTES, PATTERN_WORK_FLUSH_UNITS);
229
- }
230
- function sharedMeter(work, counter) {
231
- const view = new BigInt64Array(counter);
232
- return {
233
- matcherStateBytes: work.matcherStateBytes,
234
- flushUnits: work.flushUnits,
235
- add(units) {
236
- if (units <= 0) return;
237
- const total = Number(Atomics.add(view, 0, BigInt(units))) + units;
238
- if (total > work.cap) throw capExceeded(work.cap);
239
- },
240
- remaining() {
241
- return work.cap - Number(Atomics.load(view, 0));
242
- }
243
- };
244
- }
245
- function joinPatternWorkMeter(work) {
246
- const meter = work.counter === null ? localMeter(work.cap, work.spent, work.matcherStateBytes, work.flushUnits) : sharedMeter(work, work.counter);
247
- let spent = 0;
248
- return {
249
- matcherStateBytes: meter.matcherStateBytes,
250
- flushUnits: meter.flushUnits,
251
- add(units) {
252
- if (units <= 0) return;
253
- spent += units;
254
- meter.add(units);
255
- },
256
- remaining: () => meter.remaining(),
257
- spent: () => spent
258
- };
259
- }
260
-
261
- // src/partitioning/distributed-scoring.ts
262
- function analysedQueryTokens(term, language, analysis) {
263
- const { tokens } = tokenize(term, language, {
264
- stem: true,
265
- removeStopWords: true,
266
- customTokenizer: analysis.customTokenizer,
267
- stopWordOverride: analysis.stopWords
268
- });
269
- const unique = /* @__PURE__ */ new Set();
270
- for (const { token } of tokens) {
271
- unique.add(token);
272
- }
273
- return unique;
274
- }
275
- function copyNumericRecord(source) {
276
- const copy = /* @__PURE__ */ Object.create(null);
277
- for (const [key, value] of Object.entries(source)) {
278
- if (typeof value === "number" && Number.isFinite(value)) {
279
- copy[key] = value;
280
- }
281
- }
282
- return copy;
283
- }
284
- function sanitizeGlobalStats(stats) {
285
- return {
286
- totalDocuments: typeof stats.totalDocuments === "number" && Number.isFinite(stats.totalDocuments) ? stats.totalDocuments : 0,
287
- docFrequencies: copyNumericRecord(stats.docFrequencies),
288
- totalFieldLengths: copyNumericRecord(stats.totalFieldLengths),
289
- averageFieldLengths: copyNumericRecord(stats.averageFieldLengths)
290
- };
291
- }
292
- function pruneStatsToQueryTerms(stats, term, language, analysis) {
293
- const docFrequencies = /* @__PURE__ */ Object.create(null);
294
- for (const token of analysedQueryTokens(term, language, analysis)) {
295
- if (Object.hasOwn(stats.docFrequencies, token)) {
296
- docFrequencies[token] = stats.docFrequencies[token];
297
- }
298
- }
299
- return {
300
- totalDocuments: stats.totalDocuments,
301
- docFrequencies,
302
- totalFieldLengths: stats.totalFieldLengths,
303
- averageFieldLengths: stats.averageFieldLengths
304
- };
305
- }
306
- function collectQueryTermStats(manager, term, language, analysis, partitionIds) {
307
- const queryTokens = analysedQueryTokens(term, language, analysis);
308
- let totalDocuments = 0;
309
- const docFrequencies = /* @__PURE__ */ Object.create(null);
310
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
311
- const statsPartitions = partitionIds === void 0 ? manager.getAllPartitions() : partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
312
- for (const partition of statsPartitions) {
313
- const stats = partition.stats;
314
- totalDocuments += stats.totalDocuments;
315
- for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
316
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
317
- }
318
- for (const token of queryTokens) {
319
- if (Object.hasOwn(stats.docFrequencies, token)) {
320
- docFrequencies[token] = (docFrequencies[token] ?? 0) + stats.docFrequencies[token];
321
- }
322
- }
323
- }
324
- const averageFieldLengths = /* @__PURE__ */ Object.create(null);
325
- if (totalDocuments > 0) {
326
- for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
327
- averageFieldLengths[field] = totalLength / totalDocuments;
328
- }
329
- }
330
- return { totalDocuments, docFrequencies, totalFieldLengths, averageFieldLengths };
331
- }
332
- function mergePartitionStats(statsArray) {
333
- let totalDocuments = 0;
334
- const docFrequencies = /* @__PURE__ */ Object.create(null);
335
- const totalFieldLengths = /* @__PURE__ */ Object.create(null);
336
- for (const stats of statsArray) {
337
- totalDocuments += stats.totalDocuments;
338
- for (const [term, freq] of Object.entries(stats.docFrequencies)) {
339
- docFrequencies[term] = (docFrequencies[term] ?? 0) + freq;
340
- }
341
- for (const [field, length] of Object.entries(stats.totalFieldLengths)) {
342
- totalFieldLengths[field] = (totalFieldLengths[field] ?? 0) + length;
343
- }
344
- }
345
- const averageFieldLengths = /* @__PURE__ */ Object.create(null);
346
- if (totalDocuments > 0) {
347
- for (const [field, totalLength] of Object.entries(totalFieldLengths)) {
348
- averageFieldLengths[field] = totalLength / totalDocuments;
349
- }
350
- }
351
- return {
352
- totalDocuments,
353
- docFrequencies,
354
- totalFieldLengths,
355
- averageFieldLengths
356
- };
357
- }
358
-
359
- // src/core/partition/scored-merge.ts
360
- function kWayMerge(arrays) {
361
- const nonEmpty = arrays.filter((a) => a.length > 0);
362
- if (nonEmpty.length === 0) return [];
363
- if (nonEmpty.length === 1) return nonEmpty[0];
364
- if (nonEmpty.length <= 4) {
365
- return sequentialMerge(nonEmpty);
366
- }
367
- return heapMerge(nonEmpty);
368
- }
369
- function sequentialMerge(arrays) {
370
- let merged = arrays[0];
371
- for (let i = 1; i < arrays.length; i++) {
372
- merged = mergeTwoSorted(merged, arrays[i]);
373
- }
374
- return merged;
375
- }
376
- function mergeTwoSorted(a, b) {
377
- const result = new Array(a.length + b.length);
378
- let ai = 0;
379
- let bi = 0;
380
- let ri = 0;
381
- while (ai < a.length && bi < b.length) {
382
- if (a[ai].score > b[bi].score || a[ai].score === b[bi].score && compareCodePoints(a[ai].docId, b[bi].docId) <= 0) {
383
- result[ri++] = a[ai++];
384
- } else {
385
- result[ri++] = b[bi++];
386
- }
387
- }
388
- while (ai < a.length) {
389
- result[ri++] = a[ai++];
390
- }
391
- while (bi < b.length) {
392
- result[ri++] = b[bi++];
393
- }
394
- return result;
395
- }
396
- function heapMerge(arrays) {
397
- const heap = [];
398
- let totalSize = 0;
399
- for (let i = 0; i < arrays.length; i++) {
400
- if (arrays[i].length > 0) {
401
- totalSize += arrays[i].length;
402
- heapPush(heap, {
403
- score: arrays[i][0].score,
404
- docId: arrays[i][0].docId,
405
- partitionIdx: i,
406
- resultIdx: 0
407
- });
408
- }
409
- }
410
- const result = new Array(totalSize);
411
- let writeIdx = 0;
412
- while (heap.length > 0) {
413
- const top = heapPop(heap);
414
- result[writeIdx++] = arrays[top.partitionIdx][top.resultIdx];
415
- const nextIdx = top.resultIdx + 1;
416
- if (nextIdx < arrays[top.partitionIdx].length) {
417
- const nextDoc = arrays[top.partitionIdx][nextIdx];
418
- heapPush(heap, {
419
- score: nextDoc.score,
420
- docId: nextDoc.docId,
421
- partitionIdx: top.partitionIdx,
422
- resultIdx: nextIdx
423
- });
424
- }
425
- }
426
- return result;
427
- }
428
- function heapNodeGreater(a, b) {
429
- if (a.score !== b.score) return a.score > b.score;
430
- return compareCodePoints(a.docId, b.docId) < 0;
431
- }
432
- function heapPush(heap, node) {
433
- heap.push(node);
434
- let idx = heap.length - 1;
435
- while (idx > 0) {
436
- const parentIdx = idx - 1 >> 1;
437
- if (heapNodeGreater(heap[idx], heap[parentIdx])) {
438
- const tmp = heap[idx];
439
- heap[idx] = heap[parentIdx];
440
- heap[parentIdx] = tmp;
441
- idx = parentIdx;
442
- } else {
443
- break;
444
- }
445
- }
446
- }
447
- function heapPop(heap) {
448
- const top = heap[0];
449
- const last = heap.pop();
450
- if (heap.length > 0) {
451
- heap[0] = last;
452
- let idx = 0;
453
- while (true) {
454
- const left = 2 * idx + 1;
455
- const right = 2 * idx + 2;
456
- let largest = idx;
457
- if (left < heap.length && heapNodeGreater(heap[left], heap[largest])) {
458
- largest = left;
459
- }
460
- if (right < heap.length && heapNodeGreater(heap[right], heap[largest])) {
461
- largest = right;
462
- }
463
- if (largest !== idx) {
464
- const tmp = heap[idx];
465
- heap[idx] = heap[largest];
466
- heap[largest] = tmp;
467
- idx = largest;
468
- } else {
469
- break;
470
- }
471
- }
472
- }
473
- return top;
474
- }
475
-
476
- // src/core/partition/facets.ts
477
- var UNDECLARED_FIELD_TYPE = "";
478
- function keptFacetValues(limit) {
479
- return limit !== void 0 && limit > 0 ? Math.floor(limit) : void 0;
480
- }
481
- function ordinalBitsetOf(state, matched) {
482
- if ("ordinalBitset" in matched) return matched.ordinalBitset;
483
- const bitset = createBitSet(state.docStore.internalIdCapacity());
484
- for (const docId of matched) {
485
- const ordinal = state.docStore.getInternalId(docId);
486
- if (ordinal !== void 0) bitsetSet(bitset, ordinal);
487
- }
488
- return bitset;
489
- }
490
- function* matchedFieldValues(state, fieldPath, matched) {
491
- const read = state.docStore.fieldReader(fieldPath);
492
- if ("ordinalBitset" in matched) {
493
- const bits = matched.ordinalBitset;
494
- for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
495
- let word = bits[wordIndex];
496
- if (word === 0) continue;
497
- const base = wordIndex << 5;
498
- while (word !== 0) {
499
- const trailingZeros = Math.clz32(word & -word) ^ 31;
500
- yield read(base + trailingZeros);
501
- word &= word - 1;
502
- }
503
- }
504
- return;
505
- }
506
- for (const docId of matched) {
507
- const internalId = state.docStore.getInternalId(docId);
508
- yield internalId === void 0 ? void 0 : read(internalId);
509
- }
510
- }
511
- function countValuesFromDocuments(state, fieldPath, matched) {
512
- const counts = /* @__PURE__ */ new Map();
513
- for (const value of matchedFieldValues(state, fieldPath, matched)) {
514
- if (value === void 0 || value === null) continue;
515
- if (Array.isArray(value)) {
516
- for (const item of value) {
517
- const key = String(item);
518
- counts.set(key, (counts.get(key) ?? 0) + 1);
519
- }
520
- } else {
521
- const key = String(value);
522
- counts.set(key, (counts.get(key) ?? 0) + 1);
523
- }
524
- }
525
- return counts;
526
- }
527
- function countRangeFromDocuments(state, fieldPath, fieldType, range, matched) {
528
- let count = 0;
529
- for (const value of matchedFieldValues(state, fieldPath, matched)) {
530
- if ((fieldType === "number[]" || fieldType === UNDECLARED_FIELD_TYPE) && Array.isArray(value)) {
531
- for (const v of value) {
532
- if (v >= range.from && v < range.to) {
533
- count++;
534
- break;
535
- }
536
- }
537
- } else if (typeof value === "number" && value >= range.from && value < range.to) {
538
- count++;
539
- }
540
- }
541
- return count;
542
- }
543
- function countNumericRanges(state, fieldPath, fieldType, ranges, bitset, matched) {
544
- const counts = /* @__PURE__ */ new Map();
545
- const index = state.numericIndexes.get(fieldPath);
546
- for (const range of ranges) {
547
- const label = `${range.from}-${range.to}`;
548
- if (index === void 0) {
549
- counts.set(label, countRangeFromDocuments(state, fieldPath, fieldType, range, matched));
550
- } else if (Number.isNaN(range.from) || Number.isNaN(range.to)) {
551
- counts.set(label, 0);
552
- } else {
553
- counts.set(label, index.facetRangeCount(range.from, range.to, bitset()));
554
- }
555
- }
556
- return counts;
557
- }
558
- function countFieldValues(state, fieldPath, fieldType, bitset, matched) {
559
- if (fieldType === "enum" || fieldType === "enum[]") {
560
- const index = state.enumIndexes.get(fieldPath);
561
- if (index !== void 0) return index.facetCounts(bitset());
562
- } else if (fieldType === "boolean" || fieldType === "boolean[]") {
563
- const index = state.booleanIndexes.get(fieldPath);
564
- if (index !== void 0) {
565
- const { trueCount, falseCount } = index.facetCounts(bitset());
566
- const counts = /* @__PURE__ */ new Map();
567
- if (trueCount > 0) counts.set("true", trueCount);
568
- if (falseCount > 0) counts.set("false", falseCount);
569
- return counts;
570
- }
571
- } else if (fieldType === "number" || fieldType === "number[]") {
572
- const index = state.numericIndexes.get(fieldPath);
573
- if (index !== void 0) {
574
- const counts = /* @__PURE__ */ new Map();
575
- for (const [value, count] of index.facetValueCounts(bitset())) {
576
- counts.set(String(value), count);
577
- }
578
- return counts;
579
- }
580
- }
581
- return countValuesFromDocuments(state, fieldPath, matched);
582
- }
583
- function computeFacets(state, matched, config, schema) {
584
- const result = {};
585
- const flatSchema = getFlatSchema(state, schema);
586
- let matchedBitset = null;
587
- const bitset = () => {
588
- if (matchedBitset === null) matchedBitset = ordinalBitsetOf(state, matched);
589
- return matchedBitset;
590
- };
591
- for (const [fieldPath, facetOpts] of Object.entries(config)) {
592
- const fieldType = flatSchema[fieldPath] ?? UNDECLARED_FIELD_TYPE;
593
- const valueCounts = facetOpts.ranges && (fieldType === "number" || fieldType === "number[]" || fieldType === UNDECLARED_FIELD_TYPE) ? countNumericRanges(state, fieldPath, fieldType, facetOpts.ranges, bitset, matched) : countFieldValues(state, fieldPath, fieldType, bitset, matched);
594
- const ordered = Array.from(valueCounts.entries());
595
- const sortDir = facetOpts.sort ?? "desc";
596
- ordered.sort((a, b) => (sortDir === "asc" ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
597
- const keep = Math.min(keptFacetValues(facetOpts.limit) ?? ordered.length, ordered.length);
598
- const entries = ordered.slice(0, keep);
599
- let errorBound = 0;
600
- for (let index = keep; index < ordered.length; index++) {
601
- if (ordered[index][1] > errorBound) errorBound = ordered[index][1];
602
- }
603
- const values = {};
604
- for (const [key, count] of entries) {
605
- values[key] = count;
606
- }
607
- result[fieldPath] = { values, count: entries.length, errorBound };
608
- }
609
- return result;
610
- }
611
-
612
- // src/search/oversample.ts
613
- function oversampledShardSize(size) {
614
- return Math.ceil(size * 1.5) + 10;
615
- }
616
-
617
- // src/search/facets.ts
618
- function keptValueCount(fieldConfig) {
619
- return keptFacetValues(fieldConfig?.limit);
620
- }
621
- function everyValueFacetConfig(config) {
622
- const widened = {};
623
- for (const [field, fieldConfig] of Object.entries(config)) {
624
- widened[field] = { ...fieldConfig, limit: void 0 };
625
- }
626
- return widened;
627
- }
628
- function oversampledFacetConfig(config) {
629
- const widened = {};
630
- for (const [field, fieldConfig] of Object.entries(config)) {
631
- if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
632
- widened[field] = { ...fieldConfig, limit: void 0 };
633
- continue;
634
- }
635
- const kept = keptValueCount(fieldConfig);
636
- widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
637
- }
638
- return widened;
639
- }
640
- function mergeFacets(partitionFacets, config) {
641
- const merged = /* @__PURE__ */ new Map();
642
- const bounds = /* @__PURE__ */ new Map();
643
- for (const partition of partitionFacets) {
644
- for (const [field, facetResult] of Object.entries(partition)) {
645
- let fieldMap = merged.get(field);
646
- if (!fieldMap) {
647
- fieldMap = /* @__PURE__ */ new Map();
648
- merged.set(field, fieldMap);
649
- }
650
- bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
651
- for (const [value, count] of Object.entries(facetResult.values)) {
652
- fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
653
- }
654
- }
655
- }
656
- const result = {};
657
- for (const [field, valueMap] of merged) {
658
- const fieldConfig = config[field];
659
- const ascending = fieldConfig?.sort === "asc";
660
- const ordered = Array.from(valueMap.entries());
661
- ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
662
- const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
663
- let largestLeftOut = 0;
664
- for (let index = kept; index < ordered.length; index++) {
665
- if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
666
- }
667
- const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
668
- const values = {};
669
- for (let index = 0; index < kept; index++) {
670
- values[ordered[index][0]] = ordered[index][1];
671
- }
672
- result[field] = { values, count: kept, errorBound };
673
- }
674
- return result;
675
- }
676
-
677
- // src/search/cursor-codec.ts
678
- function encodeCursorText(text) {
679
- if (typeof Buffer !== "undefined") return Buffer.from(text, "utf-8").toString("base64");
680
- const bytes = new TextEncoder().encode(text);
681
- let binary = "";
682
- for (const byte of bytes) binary += String.fromCharCode(byte);
683
- return btoa(binary);
684
- }
685
- function decodeCursorText(encoded) {
686
- if (typeof Buffer !== "undefined") return Buffer.from(encoded, "base64").toString("utf-8");
687
- const binary = atob(encoded);
688
- const bytes = new Uint8Array(binary.length);
689
- for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
690
- return new TextDecoder().decode(bytes);
691
- }
692
-
693
- // src/filters/keys.ts
694
- var FILTER_EXPRESSION_KEYS = ["fields", "and", "or", "not"];
695
- var FIELD_FILTER_OPERATORS = [
696
- "eq",
697
- "ne",
698
- "gt",
699
- "lt",
700
- "gte",
701
- "lte",
702
- "between",
703
- "in",
704
- "nin",
705
- "startsWith",
706
- "endsWith",
707
- "contains",
708
- "wildcard",
709
- "regex",
710
- "caseFold",
711
- "tolerance",
712
- "containsAll",
713
- "matchesAny",
714
- "size",
715
- "exists",
716
- "notExists",
717
- "isEmpty",
718
- "isNotEmpty",
719
- "radius",
720
- "polygon"
721
- ];
722
-
723
- // src/geo/cells.ts
724
- var AXIS_CELLS = 2 ** LOCATION_CODE_BITS_PER_AXIS;
725
- function axisCell(value, lowest, span) {
726
- const cell = Math.floor((value - lowest) / span * AXIS_CELLS);
727
- if (!(cell > 0)) return 0;
728
- return cell >= AXIS_CELLS ? AXIS_CELLS - 1 : cell;
729
- }
730
- function latitudeCell(lat) {
731
- return axisCell(lat, -MAX_LATITUDE_DEGREES, 2 * MAX_LATITUDE_DEGREES);
732
- }
733
- function longitudeCell(lon) {
734
- return axisCell(lon, -MAX_LONGITUDE_DEGREES, FULL_TURN_DEGREES);
735
- }
736
- function interleave(row, column, bits) {
737
- let code = 0;
738
- for (let bit = bits - 1; bit >= 0; bit--) code = code * 4 + (row >>> bit & 1) * 2 + (column >>> bit & 1);
739
- return code;
740
- }
741
- function locationCode(lat, lon) {
742
- return interleave(latitudeCell(lat), longitudeCell(lon), LOCATION_CODE_BITS_PER_AXIS);
743
- }
744
- function appendBoxRanges(box, out) {
745
- const south = latitudeCell(box.south);
746
- const north = latitudeCell(box.north);
747
- const west = longitudeCell(box.west);
748
- const east = longitudeCell(box.east);
749
- let level = LOCATION_CODE_BITS_PER_AXIS;
750
- while (level > 0) {
751
- const shift2 = LOCATION_CODE_BITS_PER_AXIS - level;
752
- const rows = (north >>> shift2) - (south >>> shift2) + 1;
753
- const columns = (east >>> shift2) - (west >>> shift2) + 1;
754
- if (rows * columns <= MAX_COVERING_CELLS) break;
755
- level--;
756
- }
757
- const shift = LOCATION_CODE_BITS_PER_AXIS - level;
758
- const codesPerCell = 4 ** shift;
759
- for (let row = south >>> shift; row <= north >>> shift; row++) {
760
- for (let column = west >>> shift; column <= east >>> shift; column++) {
761
- const from = interleave(row, column, level) * codesPerCell;
762
- out.push({ from, to: from + codesPerCell });
763
- }
764
- }
765
- }
766
- function coveringRanges(boxes) {
767
- const ranges = [];
768
- for (const box of boxes) appendBoxRanges(box, ranges);
769
- ranges.sort((left, right) => left.from - right.from);
770
- const merged = [];
771
- for (const range of ranges) {
772
- const last = merged[merged.length - 1];
773
- if (last !== void 0 && range.from <= last.to) {
774
- if (range.to > last.to) last.to = range.to;
775
- } else {
776
- merged.push({ from: range.from, to: range.to });
777
- }
778
- }
779
- return merged;
780
- }
781
- function longitudeBoxes(south, north, west, east) {
782
- if (east - west >= FULL_TURN_DEGREES) {
783
- return [{ south, north, west: -MAX_LONGITUDE_DEGREES, east: MAX_LONGITUDE_DEGREES }];
784
- }
785
- if (west < -MAX_LONGITUDE_DEGREES) {
786
- return [
787
- { south, north, west: west + FULL_TURN_DEGREES, east: MAX_LONGITUDE_DEGREES },
788
- { south, north, west: -MAX_LONGITUDE_DEGREES, east }
789
- ];
790
- }
791
- if (east > MAX_LONGITUDE_DEGREES) {
792
- return [
793
- { south, north, west, east: MAX_LONGITUDE_DEGREES },
794
- { south, north, west: -MAX_LONGITUDE_DEGREES, east: east - FULL_TURN_DEGREES }
795
- ];
796
- }
797
- return [{ south, north, west, east }];
798
- }
799
- function radiusBoxes(lat, lon, distanceMeters) {
800
- const angle = distanceMeters / EARTH_RADIUS_METERS * RADIUS_BOX_MARGIN_RATIO + RADIUS_BOX_MARGIN_RADIANS;
801
- const angleDegrees = angle / DEG_TO_RAD;
802
- const south = lat - angleDegrees;
803
- const north = lat + angleDegrees;
804
- if (angle >= Math.PI || south <= -MAX_LATITUDE_DEGREES || north >= MAX_LATITUDE_DEGREES) {
805
- return [
806
- {
807
- south: Math.max(south, -MAX_LATITUDE_DEGREES),
808
- north: Math.min(north, MAX_LATITUDE_DEGREES),
809
- west: -MAX_LONGITUDE_DEGREES,
810
- east: MAX_LONGITUDE_DEGREES
811
- }
812
- ];
813
- }
814
- const spread = Math.asin(Math.sin(angle) / Math.cos(lat * DEG_TO_RAD)) / DEG_TO_RAD;
815
- return longitudeBoxes(south, north, lon - spread, lon + spread);
816
- }
817
-
818
- // src/geo/edge-tree.ts
819
- function buildLongitudeEdgeTree(lons) {
820
- const n = lons.length;
821
- const edges = [];
822
- for (let i = 0; i < n; i++) {
823
- if (lons[i] !== lons[i === 0 ? n - 1 : i - 1]) edges.push(i);
824
- }
825
- const lowOf = (i) => Math.min(lons[i], lons[i === 0 ? n - 1 : i - 1]);
826
- edges.sort((left, right) => lowOf(left) - lowOf(right));
827
- const count = edges.length;
828
- const tree = {
829
- from: new Int32Array(count),
830
- to: new Int32Array(count),
831
- low: new Float64Array(count),
832
- high: new Float64Array(count),
833
- subtreeHigh: new Float64Array(count)
834
- };
835
- edges.forEach((i, at) => {
836
- const j = i === 0 ? n - 1 : i - 1;
837
- tree.from[at] = i;
838
- tree.to[at] = j;
839
- tree.low[at] = Math.min(lons[i], lons[j]);
840
- tree.high[at] = Math.max(lons[i], lons[j]);
841
- });
842
- const fillSubtreeHigh = (start, end) => {
843
- if (start >= end) return Number.NEGATIVE_INFINITY;
844
- const mid = start + end >>> 1;
845
- const highest = Math.max(tree.high[mid], fillSubtreeHigh(start, mid), fillSubtreeHigh(mid + 1, end));
846
- tree.subtreeHigh[mid] = highest;
847
- return highest;
848
- };
849
- fillSubtreeHigh(0, count);
850
- return tree;
851
- }
852
- function forEachEdgeSpanning(tree, lon, visit) {
853
- const walk = (start, end) => {
854
- if (start >= end) return;
855
- const mid = start + end >>> 1;
856
- if (tree.subtreeHigh[mid] <= lon) return;
857
- walk(start, mid);
858
- if (tree.low[mid] > lon) return;
859
- if (lon < tree.high[mid]) visit(tree.from[mid], tree.to[mid]);
860
- walk(mid + 1, end);
861
- };
862
- walk(0, tree.from.length);
863
- }
864
-
865
- // src/geo/polygon.ts
866
- function requirePolygonRing(points) {
867
- if (!Array.isArray(points) || points.length < MIN_POLYGON_POINTS) {
868
- throw new NarsilError(
869
- ErrorCodes.SEARCH_INVALID_FILTER,
870
- `A polygon encloses an area only with at least ${MIN_POLYGON_POINTS} points`,
871
- { points: Array.isArray(points) ? points.length : typeof points, minimum: MIN_POLYGON_POINTS }
872
- );
873
- }
874
- points.forEach((point, position) => {
875
- const { lat, lon } = point ?? {};
876
- if (typeof lat !== "number" || !Number.isFinite(lat) || typeof lon !== "number" || !Number.isFinite(lon)) {
877
- throw new NarsilError(
878
- ErrorCodes.SEARCH_INVALID_FILTER,
879
- `A polygon point carries a finite "lat" and "lon", and the point at position ${position} does not`,
880
- { position }
881
- );
882
- }
883
- if (isLatitude(lat) && isLongitude(lon)) return;
884
- throw new NarsilError(
885
- ErrorCodes.SEARCH_INVALID_FILTER,
886
- `A polygon point must hold a "lat" from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES} and a "lon" from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, while the point at position ${position} holds ${lat}, ${lon}`,
887
- { position, lat, lon }
888
- );
889
- });
890
- }
891
- function signedArea(polygon) {
892
- let area = 0;
893
- for (let i = 0, j = polygon.length - 1; i < polygon.length; j = i++) {
894
- area += polygon[j].lon * polygon[i].lat - polygon[i].lon * polygon[j].lat;
895
- }
896
- return area;
897
- }
898
- function crossesAntimeridian(polygon) {
899
- let west = Number.POSITIVE_INFINITY;
900
- let east = Number.NEGATIVE_INFINITY;
901
- for (const point of polygon) {
902
- if (point.lon < west) west = point.lon;
903
- if (point.lon > east) east = point.lon;
904
- }
905
- return east - west >= ANTIMERIDIAN_SPAN_DEGREES && signedArea(polygon) < 0;
906
- }
907
- function eastward(lon) {
908
- return lon < 0 ? lon + FULL_TURN_DEGREES : lon;
909
- }
910
- function polygonContainment(polygon) {
911
- const n = polygon.length;
912
- if (n < MIN_POLYGON_POINTS) return () => false;
913
- const unwrap = crossesAntimeridian(polygon);
914
- const lats = new Float64Array(n);
915
- const lons = new Float64Array(n);
916
- for (let i = 0; i < n; i++) {
917
- lats[i] = polygon[i].lat;
918
- lons[i] = unwrap ? eastward(polygon[i].lon) : polygon[i].lon;
919
- }
920
- const edges = buildLongitudeEdgeTree(lons);
921
- return (lat, rawLon) => {
922
- const lon = unwrap ? eastward(rawLon) : rawLon;
923
- let inside = false;
924
- forEachEdgeSpanning(edges, lon, (i, j) => {
925
- const intersectLat = lats[i] + (lats[j] - lats[i]) / (lons[j] - lons[i]) * (lon - lons[i]);
926
- if (lat < intersectLat) inside = !inside;
927
- });
928
- return inside;
929
- };
930
- }
931
- function polygonBoxes(polygon) {
932
- if (polygon.length < MIN_POLYGON_POINTS) return [];
933
- const unwrap = crossesAntimeridian(polygon);
934
- let south = Number.POSITIVE_INFINITY;
935
- let north = Number.NEGATIVE_INFINITY;
936
- let west = Number.POSITIVE_INFINITY;
937
- let east = Number.NEGATIVE_INFINITY;
938
- for (const point of polygon) {
939
- const lon = unwrap ? eastward(point.lon) : point.lon;
940
- south = Math.min(south, point.lat);
941
- north = Math.max(north, point.lat);
942
- west = Math.min(west, lon);
943
- east = Math.max(east, lon);
944
- }
945
- if (west >= MAX_LONGITUDE_DEGREES)
946
- return longitudeBoxes(south, north, west - FULL_TURN_DEGREES, east - FULL_TURN_DEGREES);
947
- return longitudeBoxes(south, north, west, east);
948
- }
949
-
950
- // src/filters/operands/geo.ts
951
- function requireFlag(fieldPath, operator, flag, value) {
952
- if (value === void 0 || typeof value === "boolean") return;
953
- invalid(`"${operator}.${flag}" on field "${fieldPath}" must be true or false`, fieldPath, operator);
954
- }
955
- function requireRadius(fieldPath, radius) {
956
- const circle = radius;
957
- if (circle === null || typeof circle !== "object") {
958
- invalid(`"radius" on field "${fieldPath}" must be an object`, fieldPath, "radius");
959
- }
960
- for (const key of ["lat", "lon", "distance"]) {
961
- if (!isFiniteNumber(circle[key])) {
962
- invalid(`"radius.${key}" on field "${fieldPath}" must be a finite number`, fieldPath, "radius");
963
- }
964
- }
965
- if (typeof circle.unit !== "string" || !Object.hasOwn(GEO_UNIT_METRES, circle.unit)) {
966
- invalid(
967
- `"radius.unit" on field "${fieldPath}" must be one of ${Object.keys(GEO_UNIT_METRES).join(", ")}, and this filter sets ${String(circle.unit)}`,
968
- fieldPath,
969
- "radius"
970
- );
971
- }
972
- requireFlag(fieldPath, "radius", "inside", circle.inside);
973
- requireFlag(fieldPath, "radius", "highPrecision", circle.highPrecision);
974
- if (circle.distance < 0) {
975
- invalid(
976
- `"radius.distance" on field "${fieldPath}" must be zero or more, and this filter sets ${String(circle.distance)}`,
977
- fieldPath,
978
- "radius"
979
- );
980
- }
981
- if (!isLatitude(circle.lat)) {
982
- invalid(
983
- `"radius.lat" on field "${fieldPath}" must be from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES}, and this filter sets ${String(circle.lat)}`,
984
- fieldPath,
985
- "radius"
986
- );
987
- }
988
- if (!isLongitude(circle.lon)) {
989
- invalid(
990
- `"radius.lon" on field "${fieldPath}" must be from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, and this filter sets ${String(circle.lon)}`,
991
- fieldPath,
992
- "radius"
993
- );
994
- }
995
- }
996
- function requireGeoOperands(fieldPath, filter) {
997
- if (filter.radius !== void 0) requireRadius(fieldPath, filter.radius);
998
- if (filter.polygon === void 0) return;
999
- if (!isObjectRecord(filter.polygon)) {
1000
- invalid(`"polygon" on field "${fieldPath}" must be an object`, fieldPath, "polygon");
1001
- }
1002
- requireFlag(fieldPath, "polygon", "inside", filter.polygon.inside);
1003
- const points = filter.polygon.points;
1004
- if (Array.isArray(points)) requireListLength(fieldPath, "polygon", points);
1005
- requirePolygonRing(points);
1006
- }
1007
-
1008
- // src/filters/operands/lists.ts
1009
- var WHOLE_VALUE_OPERATORS = [
1010
- "eq",
1011
- "ne",
1012
- "in",
1013
- "nin",
1014
- "startsWith",
1015
- "endsWith",
1016
- "matchesAny",
1017
- "containsAll"
1018
- ];
1019
- var ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
1020
- var PRESENCE_OPERATORS = ["exists", "notExists", "isEmpty", "isNotEmpty"];
1021
- var SIZE_OPERAND_KEYS = ["size"];
1022
- var SIZE_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
1023
- var SIZE_OPERATOR_SET = new Set(SIZE_OPERATORS);
1024
- var ELEMENT_KIND_BY_BASE = {
1025
- "number[]": "number",
1026
- "boolean[]": "boolean",
1027
- "enum[]": "text",
1028
- "string[]": "text",
1029
- "verbatim[]": "text"
1030
- };
1031
- function requireWholeValueIndex(fieldPath, filter, type) {
1032
- const parsed = parsedTypeOf(type);
1033
- if (parsed === void 0 || parsed.pattern || parsed.base !== "string" && parsed.base !== "string[]") return;
1034
- for (const operator of WHOLE_VALUE_OPERATORS) {
1035
- if (filter[operator] === void 0) continue;
1036
- invalid(
1037
- `"${operator}" compares whole values of field "${fieldPath}", which the engine finds only through a pattern index. A "${parsed.base}" field keeps a pattern index where its type includes "pattern", so declare the field as "${parsed.base}:pattern" in a new index to filter it this way`,
1038
- fieldPath,
1039
- operator
1040
- );
1041
- }
1042
- }
1043
- function requirePresenceOperands(fieldPath, filter) {
1044
- for (const operator of PRESENCE_OPERATORS) {
1045
- const operand = filter[operator];
1046
- if (operand !== void 0 && typeof operand !== "boolean") {
1047
- invalid(`"${operator}" on field "${fieldPath}" must be true or false`, fieldPath, operator);
1048
- }
1049
- }
1050
- }
1051
- function listElementKind(fieldPath, operator, type) {
1052
- const base = parsedTypeOf(type)?.base;
1053
- const kind = base === void 0 ? void 0 : ELEMENT_KIND_BY_BASE[base];
1054
- if (kind !== void 0) return kind;
1055
- return invalid(
1056
- `"${operator}" tests the elements of a list field, while the schema declares field "${fieldPath}" as "${type}"`,
1057
- fieldPath,
1058
- operator
1059
- );
1060
- }
1061
- function requireElementOperands(fieldPath, filter, type) {
1062
- for (const operator of ELEMENT_OPERATORS) {
1063
- const values = filter[operator];
1064
- if (values === void 0) continue;
1065
- if (!Array.isArray(values)) invalid(`"${operator}" on field "${fieldPath}" must be a list`, fieldPath, operator);
1066
- requireListLength(fieldPath, operator, values);
1067
- const kind = listElementKind(fieldPath, operator, type);
1068
- if (values.some((value) => kindOfValue(value) !== kind)) {
1069
- invalid(
1070
- `Every value of "${operator}" on field "${fieldPath}" must be a ${kind === "text" ? "string" : kind === "number" ? "finite number" : "boolean"}, because the field holds a list of them`,
1071
- fieldPath,
1072
- operator
1073
- );
1074
- }
1075
- }
1076
- }
1077
- function requireSizeOperands(fieldPath, filter, type) {
1078
- const size = filter.size;
1079
- if (size === void 0) return;
1080
- if (!isObjectRecord(size)) invalid(`"size" on field "${fieldPath}" must be a comparison object`, fieldPath, "size");
1081
- listElementKind(fieldPath, "size", type);
1082
- for (const [operator, operand] of Object.entries(size)) {
1083
- if (!SIZE_OPERATOR_SET.has(operator)) {
1084
- invalid(
1085
- `"size" on field "${fieldPath}" takes ${SIZE_OPERATORS.join(", ")}, while "${operator}" is none of them`,
1086
- fieldPath,
1087
- "size"
1088
- );
1089
- }
1090
- const bounds = operator === "between" ? operand : [operand];
1091
- if (!Array.isArray(bounds) || bounds.length !== (operator === "between" ? 2 : 1) || !bounds.every(isFiniteNumber)) {
1092
- invalid(
1093
- `"size.${operator}" on field "${fieldPath}" must be ${operator === "between" ? "two finite numbers, the lower bound first" : "a finite number"}`,
1094
- fieldPath,
1095
- "size"
1096
- );
1097
- }
1098
- }
1099
- }
1100
-
1101
- // src/core/pattern-index/host.ts
1102
- var PATTERN_HOST = { appendFoldedCodePoint, singleFolds: singleFoldTable };
1103
- function patternIndexOf(indexes, fieldPath) {
1104
- let index = indexes.get(fieldPath);
1105
- if (index === void 0) {
1106
- index = requirePatternSearch(fieldPath).createIndex(PATTERN_HOST);
1107
- indexes.set(fieldPath, index);
1108
- }
1109
- return index;
1110
- }
1111
-
1112
- // src/filters/operands/pattern.ts
1113
- function requirePatternField(fieldPath, operator, type) {
1114
- if (parsedTypeOf(type)?.pattern === true) return;
1115
- const declared = type === void 0 ? `The schema declares no field "${fieldPath}"` : `Field "${fieldPath}" has type "${type}"`;
1116
- invalid(
1117
- `"${operator}" applies only to a pattern field, because only a pattern field has a pattern index. A pattern field is a "verbatim" field, or a text field whose type includes "pattern". ${declared}`,
1118
- fieldPath,
1119
- operator
1120
- );
1121
- }
1122
- var PATTERN_OPERAND_KEYS = [
1123
- "contains",
1124
- "caseFold",
1125
- "tolerance",
1126
- ...PATTERN_EXPRESSION_OPERATORS,
1127
- ...PATTERN_TEST_OPERATORS,
1128
- ...PATTERN_ELEMENT_OPERATORS
1129
- ];
1130
- function requirePatternOperands(fieldPath, filter, type) {
1131
- if (filter.contains !== void 0) {
1132
- if (typeof filter.contains !== "string") {
1133
- invalid(`"contains" on field "${fieldPath}" must be a string`, fieldPath, "contains");
1134
- }
1135
- requirePatternField(fieldPath, "contains", type);
1136
- }
1137
- if (filter.caseFold !== void 0) {
1138
- if (typeof filter.caseFold !== "boolean") {
1139
- invalid(`"caseFold" on field "${fieldPath}" must be true or false`, fieldPath, "caseFold");
1140
- }
1141
- requirePatternField(fieldPath, "caseFold", type);
1142
- }
1143
- for (const operator of PATTERN_EXPRESSION_OPERATORS) {
1144
- const pattern = filter[operator];
1145
- if (pattern === void 0) continue;
1146
- if (typeof pattern !== "string")
1147
- invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
1148
- requirePatternField(fieldPath, operator, type);
1149
- }
1150
- if (filter.tolerance !== void 0) requirePatternField(fieldPath, "tolerance", type);
1151
- if (parsedTypeOf(type)?.pattern !== true) return;
1152
- requirePatternSearch(fieldPath).checkOperands(fieldPath, filter, PATTERN_HOST);
1153
- }
1154
- function requireWellFormedOperand(fieldPath, operator, text) {
1155
- if (typeof text !== "string" || !holdsUnpairedSurrogate(text)) return;
1156
- invalid(
1157
- `The text of "${operator}" on field "${fieldPath}" holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so no stored value can hold it`,
1158
- fieldPath,
1159
- operator
1160
- );
1161
- }
1162
- function requireWellFormedText(fieldPath, operator, operand) {
1163
- if (!Array.isArray(operand)) {
1164
- requireWellFormedOperand(fieldPath, operator, operand);
1165
- return;
1166
- }
1167
- for (const text of operand) requireWellFormedOperand(fieldPath, operator, text);
1168
- }
1169
-
1170
- // src/filters/operands/index.ts
1171
- var RANGE_OPERATORS = ["gt", "lt", "gte", "lte"];
1172
- var TEXT_OPERATORS = ["startsWith", "endsWith"];
1173
- var EQUALITY_OPERATORS = ["eq", "ne"];
1174
- var LIST_OPERATORS = ["in", "nin"];
1175
- var GEO_OPERATORS = ["radius", "polygon"];
1176
- var TEST_MODIFIERS = /* @__PURE__ */ new Set(["caseFold", "tolerance"]);
1177
- var EXPRESSION_KEYS = new Set(FILTER_EXPRESSION_KEYS);
1178
- var OPERATOR_KEYS = new Set(FIELD_FILTER_OPERATORS);
1179
- var PROTOTYPE_KEY = "__proto__";
1180
- var validatedFilters = /* @__PURE__ */ new WeakMap();
1181
- function invalidShape(message) {
1182
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message);
1183
- }
1184
- function invalidKey(message, key) {
1185
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message, { key });
1186
- }
1187
- function undeclaredFieldOnStrictIndex(field, option) {
1188
- return new NarsilError(
1189
- ErrorCodes.SEARCH_INVALID_FIELD,
1190
- `The index is strict and its schema declares no field "${field}", so no stored document has a value in that field for this ${option}`,
1191
- { field, option }
1192
- );
1193
- }
1194
- function undeclaredFilterField(field) {
1195
- return new NarsilError(
1196
- ErrorCodes.SEARCH_INVALID_FIELD,
1197
- `The schema declares no field "${field}", while the engine answers a filter only from the indexes that it keeps for declared fields. Declare the field in the schema of a new index to filter by it`,
1198
- { field, option: "filter" }
1199
- );
1200
- }
1201
- function requireRangeOperands(fieldPath, filter, type) {
1202
- for (const operator of RANGE_OPERATORS) {
1203
- const bound = filter[operator];
1204
- if (bound === void 0) continue;
1205
- if (typeof bound === "string") {
1206
- requireTextRangeField(fieldPath, operator, type);
1207
- continue;
1208
- }
1209
- if (!isFiniteNumber(bound)) {
1210
- invalid(`"${operator}" on field "${fieldPath}" must be a finite number or a string`, fieldPath, operator);
1211
- }
1212
- requireFieldKind(fieldPath, operator, type, ["number"]);
1213
- }
1214
- if (filter.between === void 0) return;
1215
- const bounds = filter.between;
1216
- const isPair = Array.isArray(bounds) && bounds.length === 2;
1217
- if (isPair && typeof bounds[0] === "string" && typeof bounds[1] === "string") {
1218
- requireTextRangeField(fieldPath, "between", type);
1219
- return;
1220
- }
1221
- if (!isPair || !isFiniteNumber(bounds[0]) || !isFiniteNumber(bounds[1])) {
1222
- invalid(
1223
- `"between" on field "${fieldPath}" must be two finite numbers or two strings, the lower bound first`,
1224
- fieldPath,
1225
- "between"
1226
- );
1227
- }
1228
- requireFieldKind(fieldPath, "between", type, ["number"]);
1229
- }
1230
- function requireValueOperands(fieldPath, filter, type) {
1231
- for (const operator of TEXT_OPERATORS) {
1232
- if (filter[operator] === void 0) continue;
1233
- if (typeof filter[operator] !== "string") {
1234
- invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
1235
- }
1236
- requireFieldKind(fieldPath, operator, type, ["text"]);
1237
- }
1238
- for (const operator of EQUALITY_OPERATORS) {
1239
- if (filter[operator] === void 0) continue;
1240
- const kind = kindOfValue(filter[operator]);
1241
- if (kind === void 0) {
1242
- invalid(
1243
- `"${operator}" on field "${fieldPath}" must be a string, a finite number, or a boolean`,
1244
- fieldPath,
1245
- operator
1246
- );
1247
- }
1248
- requireFieldKind(fieldPath, operator, type, [kind]);
1249
- }
1250
- for (const operator of LIST_OPERATORS) {
1251
- if (filter[operator] === void 0) continue;
1252
- const values = filter[operator];
1253
- if (!Array.isArray(values) || values.some((value) => typeof value !== "string")) {
1254
- invalid(`"${operator}" on field "${fieldPath}" must be a list of strings`, fieldPath, operator);
1255
- }
1256
- requireListLength(fieldPath, operator, values);
1257
- requireFieldKind(fieldPath, operator, type, ["text"]);
1258
- }
1259
- for (const operator of GEO_OPERATORS) {
1260
- if (filter[operator] !== void 0) requireFieldKind(fieldPath, operator, type, ["geopoint"]);
1261
- }
1262
- }
1263
- var OPERAND_CHECKS = [
1264
- [[...RANGE_OPERATORS, "between"], requireRangeOperands],
1265
- [[...TEXT_OPERATORS, ...EQUALITY_OPERATORS, ...LIST_OPERATORS, ...GEO_OPERATORS], requireValueOperands],
1266
- [WHOLE_VALUE_OPERATORS, requireWholeValueIndex],
1267
- [PRESENCE_OPERATORS, requirePresenceOperands],
1268
- [ELEMENT_OPERATORS, requireElementOperands],
1269
- [SIZE_OPERAND_KEYS, requireSizeOperands],
1270
- [PATTERN_OPERAND_KEYS, requirePatternOperands],
1271
- [GEO_OPERATORS, requireGeoOperands]
1272
- ];
1273
- var CHECKS_BY_OPERATOR = checksByOperator();
1274
- function checksByOperator() {
1275
- const checks = /* @__PURE__ */ new Map();
1276
- for (let at = 0; at < OPERAND_CHECKS.length; at++) {
1277
- for (const operator of OPERAND_CHECKS[at][0]) checks.set(operator, (checks.get(operator) ?? 0) | 1 << at);
1278
- }
1279
- return checks;
1280
- }
1281
- function testsOfOperator(operator, operand) {
1282
- if (TEST_MODIFIERS.has(operator)) return 0;
1283
- return operator === "size" && isObjectRecord(operand) ? Math.max(1, Object.keys(operand).length) : 1;
1284
- }
1285
- function requireValidFieldTests(fieldPath, filter, type, walk) {
1286
- const operators = Object.keys(filter);
1287
- let wanted = 0;
1288
- let tests = 0;
1289
- for (const operator of operators) {
1290
- if (walk.unknownKey === null && !OPERATOR_KEYS.has(operator)) {
1291
- walk.unknownKey = {
1292
- message: `Field "${fieldPath}" is tested with "${operator}", which is no filter operator`,
1293
- key: operator
1294
- };
1295
- }
1296
- tests += testsOfOperator(operator, filter[operator]);
1297
- wanted |= CHECKS_BY_OPERATOR.get(operator) ?? 0;
1298
- }
1299
- countTests(walk, Math.max(1, tests));
1300
- for (const operator of operators) requireWellFormedText(fieldPath, operator, filter[operator]);
1301
- for (let at = 0; wanted !== 0; at++, wanted >>>= 1) {
1302
- if ((wanted & 1) !== 0) OPERAND_CHECKS[at][1](fieldPath, filter, type);
1303
- }
1304
- }
1305
- function filterTestCount(expression) {
1306
- if (!isObjectRecord(expression)) return 0;
1307
- const fieldFilters = isObjectRecord(expression.fields) ? Object.values(expression.fields) : [];
1308
- let tests = fieldFilters.length === 0 && !holdsNestedExpressions(expression.and, expression.or, expression.not) ? 1 : 0;
1309
- for (const filter of fieldFilters) tests += isObjectRecord(filter) ? testsIn(filter) : 1;
1310
- for (const list of [expression.and, expression.or]) {
1311
- if (Array.isArray(list)) for (const nested of list) tests += filterTestCount(nested);
1312
- }
1313
- return tests + filterTestCount(expression.not);
1314
- }
1315
- function testsIn(filter) {
1316
- let tests = 0;
1317
- for (const [operator, operand] of Object.entries(filter)) tests += testsOfOperator(operator, operand);
1318
- return Math.max(1, tests);
1319
- }
1320
- function countTests(walk, tests) {
1321
- walk.tests += tests;
1322
- if (walk.tests > MAX_FILTER_TESTS) {
1323
- invalidShape(`A filter holds at most ${MAX_FILTER_TESTS} tests across all of its fields and nested expressions`);
1324
- }
1325
- }
1326
- function holdsNestedExpressions(and, or, not) {
1327
- return Array.isArray(and) && and.length > 0 || Array.isArray(or) && or.length > 0 || not !== void 0;
1328
- }
1329
- function noteUnknownExpressionKey(expression, walk) {
1330
- for (const key in expression) {
1331
- if (!Object.hasOwn(expression, key) || EXPRESSION_KEYS.has(key)) continue;
1332
- walk.unknownKey = {
1333
- message: `A filter expression takes "fields", "and", "or", and "not", and "${key}" is none of them. A per-field test such as { ${key}: { eq: ... } } belongs under "fields"`,
1334
- key
1335
- };
1336
- return;
1337
- }
1338
- }
1339
- function setOwn(record, key, value) {
1340
- if (key === PROTOTYPE_KEY) {
1341
- Object.defineProperty(record, key, { value, enumerable: true, writable: true, configurable: true });
1342
- } else {
1343
- record[key] = value;
1344
- }
1345
- }
1346
- function ownedOperandOf(value, fieldPath, operator, depth) {
1347
- if (typeof value !== "object" || value === null || depth > FILTER_OPERAND_COPY_DEPTH) return value;
1348
- if (Array.isArray(value)) {
1349
- requireListLength(fieldPath, operator, value);
1350
- const owned2 = [];
1351
- for (let at = 0; at < value.length; at++) owned2.push(ownedOperandOf(value[at], fieldPath, operator, depth + 1));
1352
- return owned2;
1353
- }
1354
- const record = value;
1355
- const owned = {};
1356
- for (const key of Object.keys(record)) setOwn(owned, key, ownedOperandOf(record[key], fieldPath, operator, depth + 1));
1357
- return owned;
1358
- }
1359
- function ownedFieldFilterOf(fieldPath, filter) {
1360
- const owned = {};
1361
- for (const operator of Object.keys(filter)) {
1362
- setOwn(owned, operator, ownedOperandOf(filter[operator], fieldPath, operator, 1));
1363
- }
1364
- return owned;
1365
- }
1366
- function ownedFieldFiltersOf(fields, walk) {
1367
- const owned = {};
1368
- for (const fieldPath of Object.keys(fields)) {
1369
- const type = walk.fieldTypes[fieldPath];
1370
- if (!Object.hasOwn(walk.fieldTypes, fieldPath) || type === void 0) throw undeclaredFilterField(fieldPath);
1371
- const filter = fields[fieldPath];
1372
- if (!isObjectRecord(filter)) invalidShape(`The filter on field "${fieldPath}" must be an object of tests`);
1373
- const ownedFilter = ownedFieldFilterOf(fieldPath, filter);
1374
- requireValidFieldTests(fieldPath, ownedFilter, type, walk);
1375
- setOwn(owned, fieldPath, ownedFilter);
1376
- }
1377
- return owned;
1378
- }
1379
- function ownedClauseOf(clause, list, walk, depth) {
1380
- if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
1381
- const owned = [];
1382
- for (let at = 0; at < list.length; at++) owned.push(ownedExpressionOf(list[at], walk, depth + 1));
1383
- return owned;
1384
- }
1385
- function ownedExpressionOf(expression, walk, depth) {
1386
- if (depth > MAX_FILTER_DEPTH) invalidShape(`A filter expression nests at most ${MAX_FILTER_DEPTH} levels deep`);
1387
- if (!isObjectRecord(expression)) invalidShape("Each filter expression must be an object");
1388
- if (walk.unknownKey === null) noteUnknownExpressionKey(expression, walk);
1389
- const { fields, and, or, not } = expression;
1390
- if (fields !== void 0 && !isObjectRecord(fields)) {
1391
- invalidShape('"fields" in a filter expression must be an object keyed by field path');
1392
- }
1393
- const owned = {};
1394
- const fieldCount = fields === void 0 ? 0 : Object.keys(fields).length;
1395
- if (fieldCount === 0 && !holdsNestedExpressions(and, or, not)) countTests(walk, 1);
1396
- if (fields !== void 0) owned.fields = ownedFieldFiltersOf(fields, walk);
1397
- if (and !== void 0) owned.and = ownedClauseOf("and", and, walk, depth);
1398
- if (or !== void 0) owned.or = ownedClauseOf("or", or, walk, depth);
1399
- if (not !== void 0) owned.not = ownedExpressionOf(not, walk, depth + 1);
1400
- return owned;
1401
- }
1402
- function validatedFilterOf(expression, fieldTypes) {
1403
- if (validatedFilters.get(expression) === fieldTypes) return expression;
1404
- const walk = { fieldTypes, tests: 0, unknownKey: null };
1405
- const validated = ownedExpressionOf(expression, walk, 0);
1406
- if (walk.unknownKey !== null) invalidKey(walk.unknownKey.message, walk.unknownKey.key);
1407
- validatedFilters.set(validated, fieldTypes);
1408
- return validated;
1409
- }
1410
-
1411
- // src/search/sorting.ts
1412
- function isFieldList(sort) {
1413
- return Array.isArray(sort);
1414
- }
1415
- function normalizeSort(sort) {
1416
- if (sort === void 0) return [];
1417
- if (isFieldList(sort)) {
1418
- return sort.map(
1419
- (entry) => entry.mode === void 0 ? { field: entry.field, direction: entry.direction } : { field: entry.field, direction: entry.direction, mode: entry.mode }
1420
- );
1421
- }
1422
- return Object.entries(sort).map(([field, direction]) => ({ field, direction }));
1423
- }
1424
- function sortModesOf(fields) {
1425
- return fields.map(sortModeOf);
1426
- }
1427
- function sortSignatureEntry(entry) {
1428
- const mode = sortModeOf(entry);
1429
- return mode === defaultSortMode(entry.direction) ? [entry.field, entry.direction] : [entry.field, entry.direction, mode];
1430
- }
1431
- function requireSortMode(entry, fieldType) {
1432
- if (entry.mode === void 0) return;
1433
- if (!isSortMode(entry.mode)) {
1434
- throw new NarsilError(
1435
- ErrorCodes.SEARCH_INVALID_MODE,
1436
- `A sort mode is one of ${SORT_MODES.map((mode) => `"${mode}"`).join(", ")}, and the mode on "${entry.field}" is "${String(entry.mode)}"`,
1437
- { field: entry.field, mode: String(entry.mode) }
1438
- );
1439
- }
1440
- const averages = entry.mode === "avg" || entry.mode === "median";
1441
- if (averages && fieldType !== void 0 && fieldType !== "number" && fieldType !== "number[]") {
1442
- throw new NarsilError(
1443
- ErrorCodes.SEARCH_INVALID_FIELD,
1444
- `The engine applies the "${entry.mode}" mode to a number field alone, and "${entry.field}" is a ${fieldType} field`,
1445
- { field: entry.field, fieldType, mode: entry.mode }
1446
- );
1447
- }
1448
- }
1449
- function requireSortableFields(sort, schema, strict = false) {
1450
- const fields = normalizeSort(sort);
1451
- if (fields.length === 0) return;
1452
- const flatSchema = flattenSchema(schema);
1453
- for (const entry of fields) {
1454
- const fieldType = flatSchema[entry.field];
1455
- if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(entry.field, "sort");
1456
- if (entry.direction !== "asc" && entry.direction !== "desc") {
1457
- throw new NarsilError(
1458
- ErrorCodes.SEARCH_INVALID_MODE,
1459
- `A sort direction is "asc" or "desc", and the direction on "${entry.field}" is "${String(entry.direction)}"`,
1460
- { field: entry.field, direction: String(entry.direction) }
1461
- );
1462
- }
1463
- requireSortMode(entry, fieldType);
1464
- const parsed = parsedTypeOf(fieldType);
1465
- if (parsed !== void 0 && isStringValuedBase(parsed.base) && !parsed.sortable) {
1466
- throw new NarsilError(
1467
- ErrorCodes.SEARCH_INVALID_FIELD,
1468
- `The engine sorts by text field "${entry.field}" only where its type includes "sortable", as in "${withSortableOption(parsed)}", because it keeps up to the first ${SORT_VALUE_MAX_CODE_POINTS} code points of every document's value in memory to sort by a text field`,
1469
- { field: entry.field, fieldType }
1470
- );
1471
- }
1472
- if (isGeopointOrVectorType(fieldType)) {
1473
- throw new NarsilError(
1474
- ErrorCodes.SEARCH_INVALID_FIELD,
1475
- `The engine sorts by a number, boolean, enum, or sortable text field, and "${entry.field}" is a ${fieldType} field`,
1476
- { field: entry.field, fieldType }
1477
- );
1478
- }
1479
- }
1480
- }
1481
- function readSortValues(document, fields, modes) {
1482
- if (!document) return fields.map(() => null);
1483
- return fields.map((field, index) => toReducedSortValue(readSortField(document, field), modes[index] ?? "min"));
1484
- }
1485
- function applySorting(hits, sort, sortKeyOf) {
1486
- const normalized = normalizeSort(sort);
1487
- if (normalized.length === 0) return hits;
1488
- const directions = normalized.map((entry) => entry.direction);
1489
- const keyCache = /* @__PURE__ */ new Map();
1490
- for (const hit of hits) {
1491
- keyCache.set(hit.id, sortKeyOf(hit.id));
1492
- }
1493
- const sorted = hits.slice();
1494
- sorted.sort(
1495
- (a, b) => compareComparableKeys(keyCache.get(a.id) ?? [], keyCache.get(b.id) ?? [], directions) || compareCodePoints(a.id, b.id)
1496
- );
1497
- return sorted;
1498
- }
1499
-
1500
- // src/search/cursor.ts
1501
- var CURSOR_VERSION = 4;
1502
- function invalidCursor(cursor, reason) {
1503
- const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
1504
- return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
1505
- }
1506
- function sortSignatureOf(sort) {
1507
- const fields = normalizeSort(sort);
1508
- if (fields.length === 0) return null;
1509
- if (fields.length > MAX_SORT_FIELDS) {
1510
- throw new NarsilError(
1511
- ErrorCodes.SEARCH_INVALID_FIELD,
1512
- `A sort names at most ${MAX_SORT_FIELDS} fields, because the paging cursor carries one value per field`,
1513
- { count: fields.length, limit: MAX_SORT_FIELDS }
1514
- );
1515
- }
1516
- for (const entry of fields) {
1517
- if (entry.field.length > MAX_SORT_FIELD_NAME_LENGTH) {
1518
- throw new NarsilError(
1519
- ErrorCodes.SEARCH_INVALID_FIELD,
1520
- `A sort field name holds at most ${MAX_SORT_FIELD_NAME_LENGTH} characters, because the paging cursor carries the sort with every page`,
1521
- { field: entry.field.slice(0, 64), length: entry.field.length, limit: MAX_SORT_FIELD_NAME_LENGTH }
1522
- );
1523
- }
1524
- }
1525
- return JSON.stringify(fields.map(sortSignatureEntry));
1526
- }
1527
- function encodePageCursor(cursor) {
1528
- const payload = { v: CURSOR_VERSION, a: cursor.anchor };
1529
- if (cursor.sortKey !== null && cursor.sortSignature !== null) {
1530
- payload.k = cursor.sortKey.map(toComparableSortValue);
1531
- payload.o = cursor.sortSignature;
1532
- } else if (cursor.score !== null) {
1533
- payload.s = cursor.score;
1534
- }
1535
- if (cursor.depth !== void 0 && cursor.depth > 0) payload.d = cursor.depth;
1536
- payload.q = cursor.binding;
1537
- return encodeCursorText(JSON.stringify(payload));
1538
- }
1539
- function decodeSortKey(value, cursor) {
1540
- if (!Array.isArray(value)) throw invalidCursor(cursor, '"k" must be an array');
1541
- if (value.length > MAX_SORT_FIELDS) throw invalidCursor(cursor, '"k" carries too many sort values');
1542
- const key = [];
1543
- for (const entry of value) {
1544
- if (entry === null || typeof entry === "boolean") {
1545
- key.push(entry);
1546
- continue;
1547
- }
1548
- if (typeof entry === "number") {
1549
- if (!Number.isFinite(entry)) throw invalidCursor(cursor, '"k" holds a number that is not finite');
1550
- key.push(entry);
1551
- continue;
1552
- }
1553
- if (typeof entry === "string") {
1554
- if (exceedsCodePoints(entry, SORT_VALUE_MAX_CODE_POINTS)) {
1555
- throw invalidCursor(cursor, '"k" holds an oversized sort value');
1556
- }
1557
- key.push(entry);
1558
- continue;
1559
- }
1560
- throw invalidCursor(cursor, '"k" accepts a string, a finite number, a boolean, or null');
1561
- }
1562
- return key;
1563
- }
1564
- function decodePageCursor(cursor) {
1565
- if (cursor.length > MAX_CURSOR_LENGTH) throw invalidCursor(cursor, "the cursor is too long");
1566
- let json;
1567
- try {
1568
- json = decodeCursorText(cursor);
1569
- } catch {
1570
- throw invalidCursor(cursor, "invalid base64 encoding");
1571
- }
1572
- let parsed;
1573
- try {
1574
- parsed = JSON.parse(json);
1575
- } catch {
1576
- throw invalidCursor(cursor, "invalid JSON");
1577
- }
1578
- if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
1579
- throw invalidCursor(cursor, "expected an object");
1580
- }
1581
- const { v, a, s, k, o, d, q } = parsed;
1582
- if (v !== CURSOR_VERSION) throw invalidCursor(cursor, `unsupported cursor version ${String(v)}`);
1583
- if (typeof q !== "string" || !/^[0-9a-f]{8}$/.test(q)) {
1584
- throw invalidCursor(cursor, '"q" must be 8 lowercase hex digits');
1585
- }
1586
- if (typeof a !== "string" || a.length === 0) throw invalidCursor(cursor, '"a" must be a non-empty string');
1587
- if (exceedsCodePoints(a, MAX_CURSOR_ANCHOR_CODE_POINTS)) {
1588
- throw invalidCursor(cursor, '"a" is too long to be a document id');
1589
- }
1590
- if (s !== void 0 && (typeof s !== "number" || !Number.isFinite(s))) {
1591
- throw invalidCursor(cursor, '"s" must be a finite number');
1592
- }
1593
- if (s !== void 0 && k !== void 0) throw invalidCursor(cursor, 'a cursor carries "s" or "k", never both');
1594
- if (d !== void 0 && (typeof d !== "number" || !Number.isSafeInteger(d) || d < 1)) {
1595
- throw invalidCursor(cursor, '"d" must be a positive integer');
1596
- }
1597
- const sortKey = k === void 0 ? null : decodeSortKey(k, cursor);
1598
- if (o !== void 0 && typeof o !== "string") throw invalidCursor(cursor, '"o" must be a string');
1599
- if (sortKey === null !== (o === void 0)) {
1600
- throw invalidCursor(cursor, '"k" and "o" have to arrive together');
1601
- }
1602
- return {
1603
- anchor: a,
1604
- score: typeof s === "number" ? s : null,
1605
- sortKey,
1606
- sortSignature: typeof o === "string" ? o : null,
1607
- depth: typeof d === "number" ? d : 0,
1608
- binding: q
1609
- };
1610
- }
1611
- function requireMatchingCursor(cursor, encoded, signature, scoreAnchored, binding) {
1612
- if (cursor.binding !== binding) {
1613
- throw invalidCursor(encoded, "the cursor belongs to a different query");
1614
- }
1615
- if (signature !== null) {
1616
- if (cursor.sortSignature !== signature) {
1617
- throw invalidCursor(encoded, "the cursor belongs to a different sort order");
1618
- }
1619
- return;
1620
- }
1621
- if (cursor.sortSignature !== null) {
1622
- throw invalidCursor(encoded, "the cursor belongs to a sorted request");
1623
- }
1624
- if (scoreAnchored && cursor.score === null) {
1625
- throw invalidCursor(encoded, "the cursor carries no score to seek from");
1626
- }
1627
- if (!scoreAnchored && cursor.score !== null) {
1628
- throw invalidCursor(encoded, "the cursor belongs to a search rather than a listing");
1629
- }
1630
- }
1631
-
1632
- // src/search/pagination.ts
1633
- function clampRowCount(value, fallback) {
1634
- if (value === void 0 || !Number.isFinite(value)) return fallback;
1635
- return Math.max(0, Math.floor(value));
1636
- }
1637
- function requireWithinResultWindow(limit, offset) {
1638
- const depth = offset + limit;
1639
- if (depth <= RESULT_WINDOW) return;
1640
- throw new NarsilError(
1641
- ErrorCodes.SEARCH_RESULT_WINDOW_EXCEEDED,
1642
- `A request reaches the first ${RESULT_WINDOW} results, and offset + limit is ${depth}. Page past that with the cursor each result carries`,
1643
- { limit, offset, window: RESULT_WINDOW }
1644
- );
1645
- }
1646
- function lastOrganicResult(sliced, pinnedIds) {
1647
- for (let i = sliced.length - 1; i >= 0; i--) {
1648
- if (pinnedIds === void 0 || !pinnedIds.has(sliced[i].id)) return sliced[i];
1649
- }
1650
- return void 0;
1651
- }
1652
- function ordersAfterAnchor(result, anchor, sort) {
1653
- if (sort !== void 0 && anchor.sortKey !== null) {
1654
- const comparison = compareSortValues(sort.sortKeyOf(result.id), anchor.sortKey, sort.directions) || compareCodePoints(result.id, anchor.anchor);
1655
- return comparison > 0;
1656
- }
1657
- if (anchor.score === null) return true;
1658
- if (result.score === void 0) return true;
1659
- if (result.score < anchor.score) return true;
1660
- return result.score === anchor.score && compareCodePoints(result.id, anchor.anchor) > 0;
1661
- }
1662
- function applyPagination(results, limit, offset, binding, cursor, sort, pinnedIds) {
1663
- const decoded = cursor ? decodePageCursor(cursor) : null;
1664
- if (decoded !== null && cursor !== void 0) {
1665
- requireMatchingCursor(decoded, cursor, sort?.signature ?? null, true, binding);
1666
- }
1667
- if (limit <= 0) {
1668
- return { paginated: [] };
1669
- }
1670
- let startIndex = 0;
1671
- if (decoded !== null) {
1672
- startIndex = results.length;
1673
- for (let i = 0; i < results.length; i++) {
1674
- if (ordersAfterAnchor(results[i], decoded, sort)) {
1675
- startIndex = i;
1676
- break;
1677
- }
1678
- }
1679
- }
1680
- const afterOffset = startIndex + offset;
1681
- const sliced = results.slice(afterOffset, afterOffset + limit);
1682
- let nextCursor;
1683
- const hasMore = afterOffset + limit < results.length;
1684
- const lastResult = lastOrganicResult(sliced, pinnedIds);
1685
- const depth = afterOffset + sliced.length;
1686
- if (hasMore && lastResult !== void 0) {
1687
- nextCursor = sort !== void 0 ? encodePageCursor({
1688
- anchor: lastResult.id,
1689
- score: null,
1690
- sortKey: sort.sortKeyOf(lastResult.id).map(toComparableSortValue),
1691
- sortSignature: sort.signature,
1692
- depth,
1693
- binding
1694
- }) : encodePageCursor({
1695
- anchor: lastResult.id,
1696
- score: lastResult.score ?? null,
1697
- sortKey: null,
1698
- sortSignature: null,
1699
- depth,
1700
- binding
1701
- });
1702
- }
1703
- return { paginated: sliced, nextCursor };
1704
- }
1705
-
1706
- // src/search/term-groups.ts
1707
- function writtenWordsByPosition(term, language, analysis) {
1708
- const unstemmed = tokenize(term, language, {
1709
- stem: false,
1710
- removeStopWords: true,
1711
- customTokenizer: analysis?.customTokenizer,
1712
- stopWordOverride: analysis?.stopWords
1713
- });
1714
- const written = /* @__PURE__ */ new Map();
1715
- for (const token of unstemmed.tokens) {
1716
- if (!written.has(token.position)) written.set(token.position, token.token);
1717
- }
1718
- return written;
1719
- }
1720
- function groupFor(groups, token, typoMatches) {
1721
- let group = groups.get(token);
1722
- if (group === void 0) {
1723
- group = { token, members: [{ token, factor: 1, partialFactor: 1 }], typoMatches };
1724
- groups.set(token, group);
1725
- }
1726
- return group;
1727
- }
1728
- function addCompletions(groups, partition, lastToken, typedPrefix) {
1729
- if (typedPrefix === void 0 || typedPrefix.length === 0) return;
1730
- const group = groupFor(groups, lastToken.token, false);
1731
- const ownLength = codePointCount(lastToken.token);
1732
- for (const completion of partition.expandTermPrefix(typedPrefix, lastToken.token, PREFIX_MAX_EXPANSIONS)) {
1733
- if (completion === lastToken.token) continue;
1734
- const factor = Math.min(1, ownLength / codePointCount(completion));
1735
- group.members.push({ token: completion, factor, partialFactor: factor });
1736
- }
1737
- }
1738
- function partialWordAnalysisOf(fields, language, analysis) {
1739
- return {
1740
- fields,
1741
- language,
1742
- options: {
1743
- stopWordOverride: analysis?.stopWords,
1744
- customTokenizer: analysis?.customTokenizer,
1745
- collectSurfaces: true
1746
- }
1747
- };
1748
- }
1749
- function searchesAPartialField(params, partialFields) {
1750
- if (params.fields === void 0) return true;
1751
- for (const field of params.fields) {
1752
- if (partialFields.includes(field)) return true;
1753
- }
1754
- return false;
1755
- }
1756
- function resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, analysis) {
1757
- const term = params.term;
1758
- if (term === void 0 || params.exact === true) return {};
1759
- const usesPrefix = params.prefix === true;
1760
- const partialFields = params.partialWords === true ? partialFieldsOf(schema) : [];
1761
- const usesPartialWords = partialFields.length > 0 && searchesAPartialField(params, partialFields);
1762
- if (!usesPrefix && !usesPartialWords) return {};
1763
- const written = writtenWordsByPosition(term, language, analysis);
1764
- const groups = /* @__PURE__ */ new Map();
1765
- if (usesPrefix) addCompletions(groups, partition, lastToken, written.get(lastToken.position));
1766
- const prefixToken = groups.has(lastToken.token) ? lastToken.token : void 0;
1767
- if (usesPartialWords) {
1768
- addPartialWordMembers(
1769
- {
1770
- queryTokens,
1771
- written,
1772
- prefixToken,
1773
- tolerance: params.tolerance ?? 0,
1774
- prefixLength: params.prefixLength,
1775
- analysis: partialWordAnalysisOf(partialFields, language, analysis)
1776
- },
1777
- (partialAnalysis, requests) => partition.expandPartialWords(partialAnalysis, requests),
1778
- (token, typoMatches) => groupFor(groups, token, typoMatches)
1779
- );
1780
- }
1781
- const termGroups = [...groups.values()];
1782
- if (termGroups.length === 0) return {};
1783
- return usesPartialWords ? { termGroups, partialFields: [...partialFields] } : { termGroups };
1784
- }
1785
-
1786
- // src/search/fulltext.ts
1787
- function prepareFulltextQuery(partition, params, language, schema, options) {
1788
- if (!params.term || params.term.trim().length === 0) return null;
1789
- if (params.fields && params.fields.length === 0) return null;
1790
- const flatSchema = flattenSchema(schema);
1791
- if (params.fields) {
1792
- validateSearchFields(params.fields, flatSchema);
1793
- }
1794
- const queryTokenResult = tokenize(params.term, language, {
1795
- stem: true,
1796
- removeStopWords: true,
1797
- customTokenizer: options?.customTokenizer,
1798
- stopWordOverride: options?.stopWords
1799
- });
1800
- if (queryTokenResult.tokens.length === 0) return null;
1801
- const queryTokens = deduplicateTokens(queryTokenResult.tokens);
1802
- let filterBitset;
1803
- if (params.filters) {
1804
- filterBitset = partition.applyFiltersBitset(
1805
- params.filters,
1806
- schema,
1807
- options?.patternWork ?? createPatternWorkMeter()
1808
- );
1809
- if (bitsetIsEmpty(filterBitset)) return null;
1810
- }
1811
- const lastToken = queryTokenResult.tokens[queryTokenResult.tokens.length - 1];
1812
- const groups = resolveTermGroups(partition, params, language, schema, queryTokens, lastToken, options);
1813
- return { queryTokens, ...groups, filterBitset };
1814
- }
1815
- function fulltextMatches(partition, params, language, schema, options) {
1816
- const prepared = prepareFulltextQuery(partition, params, language, schema, options);
1817
- if (prepared === null) return null;
1818
- return partition.searchFulltextMatches({
1819
- queryTokens: prepared.queryTokens,
1820
- termGroups: prepared.termGroups,
1821
- partialFields: prepared.partialFields,
1822
- fields: params.fields,
1823
- tolerance: params.tolerance ?? 0,
1824
- prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
1825
- exact: params.exact ?? false,
1826
- filterBitset: prepared.filterBitset
1827
- });
1828
- }
1829
- function fulltextSearch(partition, params, language, schema, options) {
1830
- const prepared = prepareFulltextQuery(partition, params, language, schema, options);
1831
- if (prepared === null) {
1832
- return { scored: [], totalMatched: 0 };
1833
- }
1834
- const { queryTokens, termGroups, partialFields, filterBitset } = prepared;
1835
- const needsAllResults = params.minScore !== void 0 || params.termMatch !== void 0 && params.termMatch !== "any" || params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
1836
- const maxResults = needsAllResults ? void 0 : clampRowCount(params.limit, DEFAULT_PAGE_SIZE) + clampRowCount(params.offset, 0) + 1;
1837
- const collectMatchedSet = params.facets !== void 0 && !needsAllResults ? "ordinals" : void 0;
1838
- const collectComponents = params.includeScoreComponents === true || params.termMatch !== void 0 && params.termMatch !== "any";
1839
- const rawResult = partition.searchFulltext({
1840
- queryTokens,
1841
- termGroups,
1842
- partialFields,
1843
- fields: params.fields,
1844
- boost: params.boost,
1845
- tolerance: params.tolerance ?? 0,
1846
- prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
1847
- exact: params.exact ?? false,
1848
- bm25Params: options?.bm25Params,
1849
- globalStats: options?.globalStats,
1850
- maxResults,
1851
- termMatch: params.termMatch,
1852
- filterBitset,
1853
- collectComponents,
1854
- collectMatchedSet
1855
- });
1856
- let scored = rawResult.scored;
1857
- const termMatch = params.termMatch ?? "any";
1858
- if (termMatch !== "any") {
1859
- scored = filterByTermCoverage(
1860
- scored,
1861
- queryTokens,
1862
- termMatch,
1863
- params.tolerance ?? 0,
1864
- params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
1865
- params.exact ?? false,
1866
- termGroups,
1867
- partialFields ?? []
1868
- );
1869
- }
1870
- if (params.minScore !== void 0 && params.minScore > 0) {
1871
- const threshold = params.minScore;
1872
- scored = scored.filter((doc) => doc.score >= threshold);
1873
- }
1874
- const totalMatched = needsAllResults ? scored.length : rawResult.totalMatched;
1875
- if (params.facets === void 0) {
1876
- return { scored, totalMatched };
1877
- }
1878
- if (needsAllResults) {
1879
- return { scored, totalMatched, matchedIds: scored.map((doc) => doc.docId) };
1880
- }
1881
- return { scored, totalMatched, matchedOrdinalBitset: rawResult.matchedOrdinalBitset };
1882
- }
1883
- function validateSearchFields(fields, flatSchema) {
1884
- for (const field of fields) {
1885
- const fieldType = flatSchema[field];
1886
- if (!fieldType) {
1887
- throw new NarsilError(ErrorCodes.SEARCH_INVALID_FIELD, `Field "${field}" does not exist in the schema`, { field });
1888
- }
1889
- if (!isWordIndexedFieldType(fieldType)) {
1890
- throw new NarsilError(
1891
- ErrorCodes.SEARCH_INVALID_FIELD,
1892
- `Field "${field}" has type "${fieldType}" which cannot be used for full-text search`,
1893
- { field, fieldType }
1894
- );
1895
- }
1896
- }
1897
- }
1898
- function deduplicateTokens(tokens) {
1899
- const seen = /* @__PURE__ */ new Set();
1900
- const result = [];
1901
- for (const t of tokens) {
1902
- if (!seen.has(t.token)) {
1903
- seen.add(t.token);
1904
- result.push(t);
1905
- }
1906
- }
1907
- return result;
1908
- }
1909
- function filterByTermCoverage(scored, queryTokens, policy, tolerance, prefixLength, exact, termGroups, partialFields) {
1910
- const requiredCount = policy === "all" ? queryTokens.length : policy;
1911
- if (requiredCount <= 0) return scored;
1912
- if (requiredCount > queryTokens.length) return [];
1913
- const groups = /* @__PURE__ */ new Map();
1914
- for (const group of termGroups ?? []) {
1915
- const partialOnly = /* @__PURE__ */ new Set();
1916
- for (const member of group.members) {
1917
- if (member.factor === 0) partialOnly.add(member.token);
1918
- }
1919
- groups.set(group.token, {
1920
- members: new Set(group.members.map((member) => member.token)),
1921
- partialOnly,
1922
- typoMatches: group.typoMatches
1923
- });
1924
- }
1925
- return scored.filter((doc) => {
1926
- const matched = countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields);
1927
- return matched >= requiredCount;
1928
- });
1929
- }
1930
- function countDocTermMatches(doc, queryTokens, tolerance, prefixLength, exact, groups, partialFields) {
1931
- const indexTokens = Object.keys(doc.idf);
1932
- if (indexTokens.length === 0) return 0;
1933
- let count = 0;
1934
- for (const qt of queryTokens) {
1935
- const group = groups.get(qt.token);
1936
- const satisfied = group === void 0 ? queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact) : groupSatisfied(group, doc, indexTokens, partialFields) || group.typoMatches && queryTermSatisfied(qt.token, indexTokens, tolerance, prefixLength, exact);
1937
- if (satisfied) count++;
1938
- }
1939
- return count;
1940
- }
1941
- function matchedInPartialField(doc, token, partialFields) {
1942
- for (const field of partialFields) {
1943
- if (doc.termFrequencies[`${field}:${token}`] !== void 0) return true;
1944
- }
1945
- return false;
1946
- }
1947
- function groupSatisfied(group, doc, indexTokens, partialFields) {
1948
- for (const indexToken of indexTokens) {
1949
- if (!group.members.has(indexToken)) continue;
1950
- if (!group.partialOnly.has(indexToken) || matchedInPartialField(doc, indexToken, partialFields)) return true;
1951
- }
1952
- return false;
1953
- }
1954
- function queryTermSatisfied(queryToken, indexTokens, tolerance, prefixLength, exact) {
1955
- if (exact || tolerance === 0) {
1956
- return indexTokens.includes(queryToken);
1957
- }
1958
- for (const indexToken of indexTokens) {
1959
- if (fuzzyTermMatches(queryToken, indexToken, tolerance, prefixLength)) {
1960
- return true;
1961
- }
1962
- }
1963
- return false;
1964
- }
1965
-
1966
- // src/partitioning/partition-selection.ts
1967
- function partitionsIn(manager, partitionIds) {
1968
- if (partitionIds === void 0) {
1969
- return manager.getAllPartitions();
1970
- }
1971
- const partitions = [];
1972
- for (const partitionId of partitionIds) {
1973
- const partition = manager.partitionAt(partitionId);
1974
- if (partition !== void 0) {
1975
- partitions.push(partition);
1976
- }
1977
- }
1978
- return partitions;
1979
- }
1980
-
1981
- // src/partitioning/fan-out.ts
1982
- async function fanOutQuery(manager, params, language, schema, config, searchOptions) {
1983
- const partitions = partitionsIn(manager, config.partitionIds);
1984
- if (partitions.length === 0) {
1985
- return { scored: [], totalMatched: 0 };
1986
- }
1987
- let globalStats;
1988
- const effectiveMode = resolveEffectiveMode(config);
1989
- if (effectiveMode === "dfs") {
1990
- if (params.term !== void 0 && params.term.trim().length > 0) {
1991
- globalStats = collectQueryTermStats(
1992
- manager,
1993
- params.term,
1994
- language,
1995
- {
1996
- stopWords: searchOptions?.stopWords,
1997
- customTokenizer: searchOptions?.customTokenizer
1998
- },
1999
- config.partitionIds
2000
- );
2001
- }
2002
- } else if (effectiveMode === "broadcast") {
2003
- globalStats = config.globalStats;
2004
- }
2005
- const options = buildSearchOptions(searchOptions, globalStats);
2006
- if (partitions.length === 1 && !config.dispatcher) {
2007
- const result = dispatchSinglePartition(partitions[0], params, language, schema, options);
2008
- let facets2;
2009
- if (params.facets) {
2010
- facets2 = partitions[0].computeFacets(facetMatchSetOf(result), params.facets, schema);
2011
- }
2012
- return { scored: result.scored, totalMatched: result.totalMatched, facets: facets2 };
2013
- }
2014
- let outcomes;
2015
- let facets;
2016
- if (config.dispatcher) {
2017
- outcomes = await dispatchWithDispatcher(partitions, params, language, schema, options, config.dispatcher);
2018
- if (params.facets) {
2019
- facets = collectAndMergeFacets(outcomes, params.facets, schema);
2020
- }
2021
- } else {
2022
- outcomes = [];
2023
- const partitionFacets = [];
2024
- const everyValue = params.facets ? everyValueFacetConfig(params.facets) : void 0;
2025
- for (const partition of partitions) {
2026
- const result = dispatchSinglePartition(partition, params, language, schema, options);
2027
- if (everyValue) {
2028
- partitionFacets.push(partition.computeFacets(facetMatchSetOf(result), everyValue, schema));
2029
- }
2030
- outcomes.push({ result, partition });
2031
- }
2032
- if (params.facets) {
2033
- facets = mergeFacets(partitionFacets, params.facets);
2034
- }
2035
- }
2036
- const allScoredArrays = outcomes.map((o) => o.result.scored);
2037
- const merged = kWayMerge(allScoredArrays);
2038
- let totalMatched = 0;
2039
- for (const outcome of outcomes) {
2040
- totalMatched += outcome.result.totalMatched;
2041
- }
2042
- return { scored: merged, totalMatched, facets };
2043
- }
2044
- function facetMatchSetOf(result) {
2045
- if (result.matchedOrdinalBitset !== void 0) {
2046
- return { ordinalBitset: result.matchedOrdinalBitset };
2047
- }
2048
- return new Set(result.matchedIds ?? []);
2049
- }
2050
- function resolveEffectiveMode(config) {
2051
- if (config.scoringMode === "broadcast" && !config.globalStats) {
2052
- return "dfs";
2053
- }
2054
- return config.scoringMode;
2055
- }
2056
- function buildSearchOptions(base, globalStats) {
2057
- return {
2058
- ...base,
2059
- globalStats: globalStats ?? base?.globalStats,
2060
- patternWork: base?.patternWork ?? createPatternWorkMeter()
2061
- };
2062
- }
2063
- function dispatchWithDispatcher(partitions, params, language, schema, options, dispatcher) {
2064
- const promises = partitions.map(
2065
- (partition) => dispatcher.dispatch(partition, params, language, schema, options).then((result) => ({ result, partition }))
2066
- );
2067
- return Promise.all(promises);
2068
- }
2069
- function dispatchSinglePartition(partition, params, language, schema, options) {
2070
- return fulltextSearch(partition, params, language, schema, options);
2071
- }
2072
- function collectAndMergeFacets(outcomes, facetConfig, schema) {
2073
- const everyValue = everyValueFacetConfig(facetConfig);
2074
- const partitionFacets = [];
2075
- for (const outcome of outcomes) {
2076
- partitionFacets.push(outcome.partition.computeFacets(facetMatchSetOf(outcome.result), everyValue, schema));
2077
- }
2078
- return mergeFacets(partitionFacets, facetConfig);
2079
- }
2080
-
2081
- // src/partitioning/match-count.ts
2082
- function countsWithoutScores(params) {
2083
- return params.minScore === void 0 && (params.termMatch === void 0 || params.termMatch === "any");
2084
- }
2085
- function fanOutMatchCount(manager, params, language, schema, options) {
2086
- let count = 0;
2087
- const searchOptions = {
2088
- ...options?.searchOptions,
2089
- patternWork: options?.searchOptions?.patternWork ?? createPatternWorkMeter()
2090
- };
2091
- for (const partition of partitionsIn(manager, options?.partitionIds)) {
2092
- const matches = fulltextMatches(partition, params, language, schema, searchOptions);
2093
- if (matches !== null) count += matches.count;
2094
- }
2095
- return count;
2096
- }
2097
-
2098
- // src/core/projection.ts
2099
- function requireFieldPathList(paths, option) {
2100
- if (paths === void 0) return;
2101
- if (!Array.isArray(paths)) {
2102
- throw new NarsilError(
2103
- ErrorCodes.SEARCH_INVALID_FIELD,
2104
- `The "document.${option}" option takes an array of field paths`,
2105
- { option: `document.${option}`, received: paths === null ? "null" : typeof paths }
2106
- );
2107
- }
2108
- for (let position = 0; position < paths.length; position++) {
2109
- const entry = paths[position];
2110
- if (typeof entry === "string") continue;
2111
- throw new NarsilError(
2112
- ErrorCodes.SEARCH_INVALID_FIELD,
2113
- `The "document.${option}" option takes field paths as strings, and entry ${position} is not a string`,
2114
- { option: `document.${option}`, position, received: entry === null ? "null" : typeof entry }
2115
- );
2116
- }
2117
- }
2118
- function splitPaths(paths) {
2119
- if (!paths) return [];
2120
- const split = [];
2121
- for (const path of paths) {
2122
- const segments = path.split(".").filter((segment) => segment.length > 0);
2123
- if (segments.length === 0) continue;
2124
- if (segments.some((segment) => PROTOTYPE_POLLUTION_KEYS.has(segment))) continue;
2125
- split.push(segments);
2126
- }
2127
- return split;
2128
- }
2129
- function overlaps(a, b) {
2130
- const shared = Math.min(a.length, b.length);
2131
- for (let i = 0; i < shared; i++) {
2132
- if (a[i] !== b[i]) return false;
2133
- }
2134
- return true;
2135
- }
2136
- function covers(outer, inner) {
2137
- if (outer.length > inner.length) return false;
2138
- for (let i = 0; i < outer.length; i++) {
2139
- if (outer[i] !== inner[i]) return false;
2140
- }
2141
- return true;
2142
- }
2143
- function readProjectedPath(source, path) {
2144
- let current = source;
2145
- for (const segment of path) {
2146
- if (typeof current !== "object" || current === null || Array.isArray(current))
2147
- return { found: false, value: void 0 };
2148
- const record = current;
2149
- if (!Object.hasOwn(record, segment)) return { found: false, value: void 0 };
2150
- current = record[segment];
2151
- }
2152
- return { found: true, value: current };
2153
- }
2154
- function writePath(target, path, value) {
2155
- let current = target;
2156
- for (let i = 0; i < path.length - 1; i++) {
2157
- const segment = path[i];
2158
- const existing = current[segment];
2159
- if (typeof existing === "object" && existing !== null && !Array.isArray(existing)) {
2160
- current = existing;
2161
- continue;
2162
- }
2163
- const created = {};
2164
- current[segment] = created;
2165
- current = created;
2166
- }
2167
- current[path[path.length - 1]] = value;
2168
- }
2169
- function omitPath(source, path) {
2170
- const segment = path[0];
2171
- if (!Object.hasOwn(source, segment)) return source;
2172
- const copy = { ...source };
2173
- if (path.length === 1) {
2174
- delete copy[segment];
2175
- return copy;
2176
- }
2177
- const child = copy[segment];
2178
- if (typeof child !== "object" || child === null || Array.isArray(child)) return copy;
2179
- copy[segment] = omitPath(child, path.slice(1));
2180
- return copy;
2181
- }
2182
- function cloneExcept(value, path, exclude) {
2183
- const reachesDeeper = exclude.some((excluded) => excluded.length > path.length && covers(path, excluded));
2184
- if (!reachesDeeper) return structuredClone(value);
2185
- if (typeof value !== "object" || value === null || Array.isArray(value)) return structuredClone(value);
2186
- const source = value;
2187
- const kept = {};
2188
- for (const key of Object.keys(source)) {
2189
- const childPath = [...path, key];
2190
- if (exclude.some((excluded) => covers(excluded, childPath))) continue;
2191
- Object.defineProperty(kept, key, {
2192
- value: cloneExcept(source[key], childPath, exclude),
2193
- enumerable: true,
2194
- writable: true,
2195
- configurable: true
2196
- });
2197
- }
2198
- return kept;
2199
- }
2200
- function resolveProjection(projection) {
2201
- if (projection === void 0 || projection === true) return { kind: "full" };
2202
- if (projection === false) return { kind: "none" };
2203
- if (typeof projection !== "object" || projection === null) return { kind: "full" };
2204
- requireFieldPathList(projection.include, "include");
2205
- requireFieldPathList(projection.exclude, "exclude");
2206
- const askedInclude = Array.isArray(projection.include) && projection.include.length > 0;
2207
- const askedExclude = Array.isArray(projection.exclude) && projection.exclude.length > 0;
2208
- if (!askedInclude && !askedExclude) return { kind: "full" };
2209
- return {
2210
- kind: "fields",
2211
- include: askedInclude ? splitPaths(projection.include) : null,
2212
- exclude: splitPaths(projection.exclude)
2213
- };
2214
- }
2215
- function projectionKeepsField(projection, fieldPath) {
2216
- if (projection.kind === "full") return true;
2217
- if (projection.kind === "none") return false;
2218
- const path = fieldPath.split(".").filter((segment) => segment.length > 0);
2219
- if (path.length === 0) return false;
2220
- if (projection.include && !projection.include.some((included) => overlaps(included, path))) return false;
2221
- return !projection.exclude.some((excluded) => covers(excluded, path));
2222
- }
2223
- function cloneProjected(document, projection) {
2224
- if (projection === void 0 || projection.kind === "full") return structuredClone(document);
2225
- if (projection.kind === "none") return {};
2226
- if (projection.include === null) return cloneExcept(document, [], projection.exclude);
2227
- return projectIncludedPaths(projection.include, projection.exclude, (path) => readProjectedPath(document, path));
2228
- }
2229
- function projectIncludedPaths(include, exclude, read) {
2230
- const projected = {};
2231
- for (const path of include) {
2232
- if (exclude.some((excluded) => covers(excluded, path))) continue;
2233
- const { found, value } = read(path);
2234
- if (!found) continue;
2235
- writePath(projected, path, cloneExcept(value, path, exclude));
2236
- }
2237
- return projected;
2238
- }
2239
- function applyProjection(document, projection) {
2240
- if (projection.kind === "full") return document;
2241
- if (projection.kind === "none") return {};
2242
- let projected;
2243
- if (projection.include) {
2244
- projected = {};
2245
- for (const path of projection.include) {
2246
- const { found, value } = readProjectedPath(document, path);
2247
- if (found) writePath(projected, path, value);
2248
- }
2249
- } else {
2250
- projected = document;
2251
- }
2252
- for (const path of projection.exclude) {
2253
- projected = omitPath(projected, path);
2254
- }
2255
- return projected;
2256
- }
2257
-
2258
- // src/core/hash.ts
2259
- var FNV_OFFSET_BASIS = 2166136261;
2260
- var FNV_PRIME = 16777619;
2261
- var encoder = new TextEncoder();
2262
- function fnv1aBytes(bytes) {
2263
- let hash = FNV_OFFSET_BASIS;
2264
- for (let i = 0; i < bytes.length; i++) {
2265
- hash ^= bytes[i];
2266
- hash = Math.imul(hash, FNV_PRIME);
2267
- }
2268
- return hash >>> 0;
2269
- }
2270
- function fnv1aUint32s(values) {
2271
- let hash = FNV_OFFSET_BASIS;
2272
- for (const value of values) {
2273
- for (let shift = 24; shift >= 0; shift -= 8) {
2274
- hash ^= value >>> shift & 255;
2275
- hash = Math.imul(hash, FNV_PRIME);
2276
- }
2277
- }
2278
- return hash >>> 0;
2279
- }
2280
- function fnv1a(input) {
2281
- return fnv1aBytes(encoder.encode(input));
2282
- }
2283
-
2284
- // src/partitioning/manager/nested-values.ts
2285
- function setNestedValue(obj, path, value) {
2286
- if (!path.includes(".")) {
2287
- obj[path] = value;
2288
- return;
2289
- }
2290
- const segments = path.split(".");
2291
- let current = obj;
2292
- for (let i = 0; i < segments.length - 1; i++) {
2293
- let next = current[segments[i]];
2294
- if (next === null || next === void 0 || typeof next !== "object") {
2295
- next = {};
2296
- current[segments[i]] = next;
2297
- }
2298
- current = next;
2299
- }
2300
- current[segments[segments.length - 1]] = value;
2301
- }
2302
- function copyVectorsInto(document, docId, vectorIndexes, projection) {
2303
- for (const [fieldPath, vectorIndex] of vectorIndexes) {
2304
- if (projection !== void 0 && !projectionKeepsField(projection, fieldPath)) continue;
2305
- const vector = vectorIndex.getVector(docId);
2306
- if (vector) setNestedValue(document, fieldPath, vector);
2307
- }
2308
- }
2309
-
2310
- export { FIELD_FILTER_OPERATORS, FILTER_EXPRESSION_KEYS, PATTERN_HOST, applyPagination, applyProjection, applySorting, availablePatternSearch, buildPartialWordTableFrom, clampRowCount, cloneProjected, collectQueryTermStats, computeFacets, copyVectorsInto, countPartialSpelling, countsWithoutScores, coveringRanges, createFrozenPartialWords, createGrowingPartialWords, createPatternWorkMeter, decodePageCursor, encodePageCursor, encodePartialSpellingsOf, everyValueFacetConfig, expandPartialWords, fanOutMatchCount, fanOutQuery, filterTestCount, fnv1a, fnv1aBytes, fnv1aUint32s, fulltextMatches, joinPatternWorkMeter, kWayMerge, locationCode, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partialSpellingsOfDocuments, partialWordHighlighter, partialWordsToHighlight, partitionsIn, patternIndexOf, polygonBoxes, polygonContainment, projectIncludedPaths, projectionKeepsField, pruneStatsToQueryTerms, radiusBoxes, readProjectedPath, readSortValues, registerPartialWordSearch, registerPatternSearch, requireMatchingCursor, requirePartialWordSearchFor, requirePatternSearch, requirePatternSearchFor, requireSortableFields, requireWithinResultWindow, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, undeclaredFieldOnStrictIndex, undeclaredFilterField, validatedFilterOf };