@delali/narsil 0.3.1-next.70 → 0.3.1-next.72

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/{capabilities-Dt-7B4Wz.d.ts → capabilities-D6gjjQyA.d.ts} +2 -2
  4. package/dist/{chunk-P4DSQ6M4.mjs → chunk-2X2ZQOAB.mjs} +69 -28
  5. package/dist/chunk-3HQPUCCT.mjs +785 -0
  6. package/dist/{chunk-VNMTB3KC.mjs → chunk-7QMTA7AA.mjs} +194 -59
  7. package/dist/{chunk-FVLQ63B7.mjs → chunk-CVPA5Q7R.mjs} +24 -3
  8. package/dist/{chunk-2STVUTNA.mjs → chunk-EDMCZLSJ.mjs} +4 -3
  9. package/dist/{chunk-M76V3OGJ.mjs → chunk-HVJYTKK7.mjs} +940 -647
  10. package/dist/{chunk-HWQLTLNG.mjs → chunk-KZTI25D3.mjs} +1 -1
  11. package/dist/{chunk-YJDQ7NLA.mjs → chunk-LCCL6EEH.mjs} +3 -3
  12. package/dist/{chunk-OI42GWLR.mjs → chunk-LJYTW7GT.mjs} +57 -11
  13. package/dist/{chunk-653BNMAF.mjs → chunk-M6XF5HDE.mjs} +57 -17
  14. package/dist/{chunk-YNVIUGF5.mjs → chunk-P2ZZN32C.mjs} +1 -1
  15. package/dist/{chunk-LMO5TJUS.mjs → chunk-QR4OKRVY.mjs} +637 -196
  16. package/dist/{chunk-USKFU7J5.mjs → chunk-RV77FR2F.mjs} +1 -0
  17. package/dist/{chunk-GIVAGD3Q.mjs → chunk-XZ4PAREY.mjs} +1 -1
  18. package/dist/{chunk-3JGWULDF.mjs → chunk-Y52WZWAU.mjs} +5 -5
  19. package/dist/chunk-ZKRV5M2P.mjs +75 -0
  20. package/dist/{client-COek5GXK.d.ts → client-C2TiW29y.d.ts} +3 -3
  21. package/dist/client.d.ts +8 -8
  22. package/dist/client.mjs +1 -0
  23. package/dist/{config-rBpbgTw8.d.ts → config-BLguB-om.d.ts} +2 -2
  24. package/dist/{config-Dfanyqqt.d.ts → config-C7Ip8Ife.d.ts} +17 -2
  25. package/dist/direct-executor-SVCHXR74.mjs +14 -0
  26. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  27. package/dist/distribution/coordinator/etcd.mjs +2 -2
  28. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  29. package/dist/distribution/coordinator/in-memory.mjs +1 -1
  30. package/dist/distribution/transport/grpc.mjs +1 -1
  31. package/dist/distribution.d.ts +8 -8
  32. package/dist/distribution.mjs +58 -35
  33. package/dist/embeddings/openai.mjs +1 -1
  34. package/dist/{engine-CXKfmK7I.d.ts → engine-Dm4446j1.d.ts} +104 -62
  35. package/dist/index.browser.mjs +2222 -1100
  36. package/dist/index.d.ts +27 -8
  37. package/dist/index.mjs +14 -12
  38. package/dist/invalidation/filesystem.mjs +1 -1
  39. package/dist/{memory-C7UMCMH3.d.ts → memory-CKHTU1CL.d.ts} +66 -25
  40. package/dist/pattern-Cledccbl.d.ts +18 -0
  41. package/dist/pattern.d.ts +22 -0
  42. package/dist/pattern.mjs +7 -0
  43. package/dist/persistence/durability/checkpoint-worker.d.ts +17 -1
  44. package/dist/persistence/durability/checkpoint-worker.mjs +14 -12
  45. package/dist/react.d.ts +277 -182
  46. package/dist/react.mjs +190 -45
  47. package/dist/{schema-Bgzr4g7b.d.ts → schema-BBOCdhND.d.ts} +9 -7
  48. package/dist/server.d.ts +10 -10
  49. package/dist/server.mjs +13 -12
  50. package/dist/{tasks-BZt_zes6.d.ts → tasks-BECd0b__.d.ts} +1 -1
  51. package/dist/{thread-JEVAYETR.mjs → thread-IBSIZZ6W.mjs} +22 -13
  52. package/dist/{types-gOUqhs2F.d.ts → types-BiUmaiWT.d.ts} +1 -1
  53. package/dist/vector/search-worker.d.ts +1 -1
  54. package/dist/vector/search-worker.mjs +4 -4
  55. package/dist/workers/entry.mjs +4 -4
  56. package/package.json +15 -7
  57. package/dist/direct-executor-GYJEKNV5.mjs +0 -12
@@ -33,6 +33,7 @@ var ErrorCodes = {
33
33
  SEARCH_INVALID_MODE: "SEARCH_INVALID_MODE",
34
34
  SEARCH_INVALID_CURSOR: "SEARCH_INVALID_CURSOR",
35
35
  SEARCH_RESULT_WINDOW_EXCEEDED: "SEARCH_RESULT_WINDOW_EXCEEDED",
36
+ SEARCH_WORK_CAP_EXCEEDED: "SEARCH_WORK_CAP_EXCEEDED",
36
37
  LANGUAGE_NOT_SUPPORTED: "LANGUAGE_NOT_SUPPORTED",
37
38
  ENVELOPE_VERSION_MISMATCH: "ENVELOPE_VERSION_MISMATCH",
38
39
  ENVELOPE_INVALID_MAGIC: "ENVELOPE_INVALID_MAGIC",
@@ -209,6 +210,288 @@ function resolveIndexAnalysis(config) {
209
210
  };
210
211
  }
211
212
 
213
+ // src/core/pattern-index/platform.browser.ts
214
+ function platformPatternSearch() {
215
+ return null;
216
+ }
217
+
218
+ // src/schema/constants.ts
219
+ var MAX_NESTING_DEPTH = 4;
220
+ var MAX_DOCUMENT_NESTING_DEPTH = 32;
221
+ var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
222
+ var MIN_PATTERN_VALUE_LIMIT = 1;
223
+ var MAX_PATTERN_VALUE_LIMIT = 65536;
224
+
225
+ // src/schema/validator/shared.ts
226
+ var VECTOR_PATTERN = /^vector\[(\d+)]$/;
227
+ function isGeopointOrVectorType(fieldType) {
228
+ return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
229
+ }
230
+ var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
231
+ var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
232
+ var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
233
+ function isPlainObject(value) {
234
+ return typeof value === "object" && value !== null && !Array.isArray(value);
235
+ }
236
+
237
+ // src/schema/validator/field-type.ts
238
+ var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
239
+ var OPTIONS_BY_BASE = {
240
+ string: FIELD_OPTION_ORDER,
241
+ "string[]": FIELD_OPTION_ORDER,
242
+ verbatim: ["sortable"],
243
+ "verbatim[]": ["sortable"],
244
+ number: [],
245
+ "number[]": [],
246
+ boolean: [],
247
+ "boolean[]": [],
248
+ enum: [],
249
+ "enum[]": [],
250
+ geopoint: [],
251
+ vector: []
252
+ };
253
+ var TYPE_SEPARATOR = ":";
254
+ function isNamedBase(name) {
255
+ return name !== "vector" && Object.hasOwn(OPTIONS_BY_BASE, name);
256
+ }
257
+ function describeAllowedOptions(baseName, allowed) {
258
+ if (allowed.length === 0) return `the "${baseName}" type has no options`;
259
+ const quoted = allowed.map((option) => `"${option}"`);
260
+ if (quoted.length === 1) return `the only option of the "${baseName}" type is ${quoted[0]}`;
261
+ return `the options of the "${baseName}" type are ${quoted.slice(0, -1).join(", ")}, and ${quoted[quoted.length - 1]}`;
262
+ }
263
+ function invalidType(path, type, message) {
264
+ throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, message, { field: path, type });
265
+ }
266
+ function parseBase(baseName, path, type) {
267
+ if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
268
+ const vectorMatch = VECTOR_PATTERN.exec(baseName);
269
+ if (vectorMatch === null) {
270
+ invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
271
+ }
272
+ const dimension = Number.parseInt(vectorMatch[1], 10);
273
+ if (dimension <= 0) {
274
+ throw new NarsilError(
275
+ ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
276
+ `Vector field "${path}" must have a positive dimension, got ${dimension}`,
277
+ { field: path, dimension }
278
+ );
279
+ }
280
+ return { base: "vector", dimension };
281
+ }
282
+ function parseFieldType(type, path) {
283
+ const separatorIndex = type.indexOf(TYPE_SEPARATOR);
284
+ const baseName = separatorIndex === -1 ? type : type.slice(0, separatorIndex);
285
+ const { base, dimension } = parseBase(baseName, path, type);
286
+ const allowed = OPTIONS_BY_BASE[base];
287
+ const options = /* @__PURE__ */ new Set();
288
+ if (separatorIndex !== -1) {
289
+ for (const option of type.slice(separatorIndex + 1).split(TYPE_SEPARATOR)) {
290
+ if (!allowed.includes(option)) {
291
+ invalidType(
292
+ path,
293
+ type,
294
+ `Field "${path}" has type "${type}" with the option "${option}", but ${describeAllowedOptions(baseName, allowed)}`
295
+ );
296
+ }
297
+ if (options.has(option)) {
298
+ invalidType(path, type, `Field "${path}" has the "${option}" option twice in type "${type}"`);
299
+ }
300
+ options.add(option);
301
+ }
302
+ }
303
+ const ordered = FIELD_OPTION_ORDER.filter((option) => options.has(option));
304
+ const verbatim = base === "verbatim" || base === "verbatim[]";
305
+ return {
306
+ type: [baseName, ...ordered].join(TYPE_SEPARATOR),
307
+ base,
308
+ sortable: options.has("sortable"),
309
+ pattern: verbatim || options.has("pattern"),
310
+ partial: options.has("partial"),
311
+ dimension
312
+ };
313
+ }
314
+ function parsedTypeOf(type) {
315
+ if (type === void 0) return void 0;
316
+ try {
317
+ return parseFieldType(type, "");
318
+ } catch {
319
+ return void 0;
320
+ }
321
+ }
322
+ function withSortableOption(parsed) {
323
+ if (parsed.sortable) return parsed.type;
324
+ const separatorIndex = parsed.type.indexOf(TYPE_SEPARATOR);
325
+ const baseName = separatorIndex === -1 ? parsed.type : parsed.type.slice(0, separatorIndex);
326
+ return parseFieldType(`${baseName}${TYPE_SEPARATOR}sortable${parsed.type.slice(baseName.length)}`, "").type;
327
+ }
328
+ function isListBase(base) {
329
+ return base.endsWith("[]");
330
+ }
331
+ function isWordIndexedBase(base) {
332
+ return base === "string" || base === "string[]";
333
+ }
334
+ function isSingleStringFieldType(fieldType) {
335
+ return parsedTypeOf(fieldType)?.base === "string";
336
+ }
337
+ function isWordIndexedFieldType(fieldType) {
338
+ const parsed = parsedTypeOf(fieldType);
339
+ return parsed !== void 0 && isWordIndexedBase(parsed.base);
340
+ }
341
+ function isStringValuedBase(base) {
342
+ return base === "string" || base === "string[]" || base === "verbatim" || base === "verbatim[]";
343
+ }
344
+
345
+ // src/schema/validator/schema.ts
346
+ function validateSchemaFields(schema, depth, prefix) {
347
+ if (depth > MAX_NESTING_DEPTH) {
348
+ throw new NarsilError(
349
+ ErrorCodes.SCHEMA_DEPTH_EXCEEDED,
350
+ `Schema nesting exceeds the maximum depth of ${MAX_NESTING_DEPTH} at "${prefix}"`,
351
+ { path: prefix, maxDepth: MAX_NESTING_DEPTH }
352
+ );
353
+ }
354
+ let canonical = null;
355
+ const replace = (field, value) => {
356
+ if (canonical === null) canonical = { ...schema };
357
+ canonical[field] = value;
358
+ };
359
+ for (const [field, type] of Object.entries(schema)) {
360
+ const path = prefix ? `${prefix}.${field}` : field;
361
+ if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
362
+ throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field name "${field}" is not allowed in a schema`, {
363
+ field,
364
+ path
365
+ });
366
+ }
367
+ if (!FIELD_NAME_PATTERN.test(field)) {
368
+ throw new NarsilError(
369
+ ErrorCodes.SCHEMA_INVALID_TYPE,
370
+ `Field name "${field}" contains characters that are not allowed; use letters, digits, and underscores only`,
371
+ { field, path }
372
+ );
373
+ }
374
+ if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
375
+ throw new NarsilError(
376
+ ErrorCodes.SCHEMA_INVALID_TYPE,
377
+ `Field "${field}" is reserved and cannot be defined in a schema`,
378
+ { field, path }
379
+ );
380
+ }
381
+ if (isPlainObject(type)) {
382
+ const nested = validateSchemaFields(type, depth + 1, path);
383
+ if (nested !== type) replace(field, nested);
384
+ continue;
385
+ }
386
+ if (typeof type !== "string") {
387
+ throw new NarsilError(
388
+ ErrorCodes.SCHEMA_INVALID_TYPE,
389
+ `Field "${path}" has an invalid type definition: ${String(type)}`,
390
+ {
391
+ field: path,
392
+ type: String(type)
393
+ }
394
+ );
395
+ }
396
+ const parsed = parseFieldType(type, path);
397
+ if (parsed.type !== type) replace(field, parsed.type);
398
+ }
399
+ return canonical ?? schema;
400
+ }
401
+ function validateSchema(schema) {
402
+ if (!isPlainObject(schema)) {
403
+ throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must be a plain object", {
404
+ received: typeof schema
405
+ });
406
+ }
407
+ if (Object.keys(schema).length === 0) {
408
+ throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
409
+ }
410
+ return validateSchemaFields(schema, 1, "");
411
+ }
412
+ function requireValidFieldTypes(flatSchema) {
413
+ for (const [path, type] of Object.entries(flatSchema)) {
414
+ if (typeof type !== "string") {
415
+ throw new NarsilError(
416
+ ErrorCodes.SCHEMA_INVALID_TYPE,
417
+ `Field "${path}" has an invalid type definition: ${String(type)}`,
418
+ { field: path, type: String(type) }
419
+ );
420
+ }
421
+ parseFieldType(type, path);
422
+ }
423
+ }
424
+ function flattenRecursive(schema, prefix, result) {
425
+ for (const [field, type] of Object.entries(schema)) {
426
+ const path = prefix ? `${prefix}.${field}` : field;
427
+ if (isPlainObject(type)) {
428
+ flattenRecursive(type, path, result);
429
+ } else {
430
+ result[path] = type;
431
+ }
432
+ }
433
+ }
434
+ var flattenedSchemas = /* @__PURE__ */ new WeakMap();
435
+ function flattenSchema(schema) {
436
+ const cached = flattenedSchemas.get(schema);
437
+ if (cached !== void 0) return cached;
438
+ const result = /* @__PURE__ */ Object.create(null);
439
+ flattenRecursive(schema, "", result);
440
+ flattenedSchemas.set(schema, Object.freeze(result));
441
+ return result;
442
+ }
443
+ var parsedSchemas = /* @__PURE__ */ new WeakMap();
444
+ function schemaFieldsOf(schema) {
445
+ const cached = parsedSchemas.get(schema);
446
+ if (cached !== void 0) return cached;
447
+ const fields = [];
448
+ for (const [path, type] of Object.entries(flattenSchema(schema))) {
449
+ fields.push({ ...parseFieldType(type, path), path });
450
+ }
451
+ const frozen = Object.freeze(fields);
452
+ parsedSchemas.set(schema, frozen);
453
+ return frozen;
454
+ }
455
+ function extractVectorFieldsFromSchema(schema) {
456
+ const result = /* @__PURE__ */ new Map();
457
+ for (const field of schemaFieldsOf(schema)) {
458
+ if (field.base === "vector") result.set(field.path, field.dimension);
459
+ }
460
+ return result;
461
+ }
462
+
463
+ // src/core/pattern-index/registry.ts
464
+ var registered = null;
465
+ function isPatternSearchModule(value) {
466
+ return value !== null && typeof value === "object" && value.name === "pattern" && "createIndex" in value && typeof value.createIndex === "function" && "readArrays" in value && typeof value.readArrays === "function" && "mergeArrays" in value && typeof value.mergeArrays === "function" && "matchInto" in value && typeof value.matchInto === "function";
467
+ }
468
+ function registerPatternSearch(module) {
469
+ if (!isPatternSearchModule(module)) {
470
+ throw new NarsilError(
471
+ ErrorCodes.CONFIG_INVALID,
472
+ 'registerPatternSearch takes the patternSearch object that "@delali/narsil/pattern" exports'
473
+ );
474
+ }
475
+ registered = module;
476
+ }
477
+ function availablePatternSearch() {
478
+ return registered ?? platformPatternSearch();
479
+ }
480
+ function requirePatternSearchFor(schema) {
481
+ for (const { path, pattern } of schemaFieldsOf(schema)) {
482
+ if (pattern) requirePatternSearch(path);
483
+ }
484
+ }
485
+ function requirePatternSearch(fieldPath) {
486
+ const module = availablePatternSearch();
487
+ if (module !== null) return module;
488
+ throw new NarsilError(
489
+ ErrorCodes.CONFIG_INVALID,
490
+ `Field "${fieldPath}" is a pattern field, but this bundle contains no pattern search code. Add import { patternSearch } from '@delali/narsil/pattern', then call registerPatternSearch(patternSearch) before you create or load the index`,
491
+ { field: fieldPath }
492
+ );
493
+ }
494
+
212
495
  // src/core/tokenizer/constants.ts
213
496
  var TOKEN_CACHE_SIZE_FLOOR = 5e4;
214
497
  var TOKEN_CACHE_SIZE_CEILING = 2e6;
@@ -1820,7 +2103,8 @@ var NARSIL_CONFIG_KEYS = {
1820
2103
  embeddingAdapters: true,
1821
2104
  durability: true,
1822
2105
  analysis: true,
1823
- lifecycle: true
2106
+ lifecycle: true,
2107
+ patternWorkCap: true
1824
2108
  };
1825
2109
  var DURABILITY_CONFIG_KEYS = {
1826
2110
  tier: true,
@@ -1867,170 +2151,65 @@ function requireNamedPlugins(plugins) {
1867
2151
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The configuration sets "plugins" to something other than a list');
1868
2152
  }
1869
2153
  plugins.forEach((plugin, position) => {
1870
- const name = plugin?.name;
1871
- if (typeof name !== "string" || name.trim().length === 0) {
1872
- throw new NarsilError(
1873
- ErrorCodes.CONFIG_INVALID,
1874
- `The plugin at position ${position} carries no name, and the engine names each plugin in its errors and event payloads`,
1875
- { position }
1876
- );
1877
- }
1878
- });
1879
- }
1880
- function requireKnownConfig(config) {
1881
- if (config === void 0) return;
1882
- requireKnownKeys("", config, NARSIL_CONFIG_KEYS);
1883
- requireKnownKeys("durability.", config.durability, DURABILITY_CONFIG_KEYS);
1884
- requireKnownKeys("workers.", config.workers, WORKER_CONFIG_KEYS);
1885
- requireKnownKeys("lifecycle.", config.lifecycle, LIFECYCLE_CONFIG_KEYS);
1886
- requireKnownKeys("analysis.", config.analysis, ANALYSIS_CONFIG_KEYS);
1887
- requireNamedPlugins(config.plugins);
1888
- }
1889
-
1890
- // src/core/id-generator.ts
1891
- function generateId() {
1892
- const bytes = new Uint8Array(16);
1893
- crypto.getRandomValues(bytes);
1894
- const now2 = Date.now();
1895
- bytes[0] = Math.floor(now2 / 2 ** 40);
1896
- bytes[1] = Math.floor(now2 / 2 ** 32) % 256;
1897
- bytes[2] = Math.floor(now2 / 2 ** 24) % 256;
1898
- bytes[3] = Math.floor(now2 / 2 ** 16) % 256;
1899
- bytes[4] = Math.floor(now2 / 2 ** 8) % 256;
1900
- bytes[5] = now2 % 256;
1901
- bytes[6] = bytes[6] & 15 | 112;
1902
- bytes[8] = bytes[8] & 63 | 128;
1903
- const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
1904
- return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
1905
- }
1906
-
1907
- // src/schema/constants.ts
1908
- var MAX_NESTING_DEPTH = 4;
1909
- var MAX_DOCUMENT_NESTING_DEPTH = 32;
1910
- var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
1911
- var MIN_PATTERN_VALUE_LIMIT = 1;
1912
- var MAX_PATTERN_VALUE_LIMIT = 65536;
1913
-
1914
- // src/schema/validator/shared.ts
1915
- var VECTOR_PATTERN = /^vector\[(\d+)]$/;
1916
- function isGeopointOrVectorType(fieldType) {
1917
- return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
1918
- }
1919
- var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
1920
- var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
1921
- var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
1922
- function isPlainObject(value) {
1923
- return typeof value === "object" && value !== null && !Array.isArray(value);
1924
- }
1925
-
1926
- // src/schema/validator/field-type.ts
1927
- var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
1928
- var OPTIONS_BY_BASE = {
1929
- string: FIELD_OPTION_ORDER,
1930
- "string[]": FIELD_OPTION_ORDER,
1931
- verbatim: ["sortable"],
1932
- "verbatim[]": ["sortable"],
1933
- number: [],
1934
- "number[]": [],
1935
- boolean: [],
1936
- "boolean[]": [],
1937
- enum: [],
1938
- "enum[]": [],
1939
- geopoint: [],
1940
- vector: []
1941
- };
1942
- var TYPE_SEPARATOR = ":";
1943
- function isNamedBase(name) {
1944
- return name !== "vector" && Object.hasOwn(OPTIONS_BY_BASE, name);
1945
- }
1946
- function describeAllowedOptions(baseName, allowed) {
1947
- if (allowed.length === 0) return `the "${baseName}" type has no options`;
1948
- const quoted = allowed.map((option) => `"${option}"`);
1949
- if (quoted.length === 1) return `the only option of the "${baseName}" type is ${quoted[0]}`;
1950
- return `the options of the "${baseName}" type are ${quoted.slice(0, -1).join(", ")}, and ${quoted[quoted.length - 1]}`;
1951
- }
1952
- function invalidType(path, type, message) {
1953
- throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, message, { field: path, type });
1954
- }
1955
- function parseBase(baseName, path, type) {
1956
- if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
1957
- const vectorMatch = VECTOR_PATTERN.exec(baseName);
1958
- if (vectorMatch === null) {
1959
- invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
1960
- }
1961
- const dimension = Number.parseInt(vectorMatch[1], 10);
1962
- if (dimension <= 0) {
1963
- throw new NarsilError(
1964
- ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
1965
- `Vector field "${path}" must have a positive dimension, got ${dimension}`,
1966
- { field: path, dimension }
1967
- );
1968
- }
1969
- return { base: "vector", dimension };
1970
- }
1971
- function parseFieldType(type, path) {
1972
- const separatorIndex = type.indexOf(TYPE_SEPARATOR);
1973
- const baseName = separatorIndex === -1 ? type : type.slice(0, separatorIndex);
1974
- const { base, dimension } = parseBase(baseName, path, type);
1975
- const allowed = OPTIONS_BY_BASE[base];
1976
- const options = /* @__PURE__ */ new Set();
1977
- if (separatorIndex !== -1) {
1978
- for (const option of type.slice(separatorIndex + 1).split(TYPE_SEPARATOR)) {
1979
- if (!allowed.includes(option)) {
1980
- invalidType(
1981
- path,
1982
- type,
1983
- `Field "${path}" has type "${type}" with the option "${option}", but ${describeAllowedOptions(baseName, allowed)}`
1984
- );
1985
- }
1986
- if (options.has(option)) {
1987
- invalidType(path, type, `Field "${path}" has the "${option}" option twice in type "${type}"`);
1988
- }
1989
- options.add(option);
1990
- }
1991
- }
1992
- const ordered = FIELD_OPTION_ORDER.filter((option) => options.has(option));
1993
- const verbatim = base === "verbatim" || base === "verbatim[]";
1994
- return {
1995
- type: [baseName, ...ordered].join(TYPE_SEPARATOR),
1996
- base,
1997
- sortable: options.has("sortable"),
1998
- pattern: verbatim || options.has("pattern"),
1999
- partial: options.has("partial"),
2000
- dimension
2001
- };
2002
- }
2003
- function parsedTypeOf(type) {
2004
- if (type === void 0) return void 0;
2005
- try {
2006
- return parseFieldType(type, "");
2007
- } catch {
2008
- return void 0;
2009
- }
2010
- }
2011
- function withSortableOption(parsed) {
2012
- if (parsed.sortable) return parsed.type;
2013
- const separatorIndex = parsed.type.indexOf(TYPE_SEPARATOR);
2014
- const baseName = separatorIndex === -1 ? parsed.type : parsed.type.slice(0, separatorIndex);
2015
- return parseFieldType(`${baseName}${TYPE_SEPARATOR}sortable${parsed.type.slice(baseName.length)}`, "").type;
2016
- }
2017
- function isListBase(base) {
2018
- return base.endsWith("[]");
2019
- }
2020
- function isWordIndexedBase(base) {
2021
- return base === "string" || base === "string[]";
2154
+ const name = plugin?.name;
2155
+ if (typeof name !== "string" || name.trim().length === 0) {
2156
+ throw new NarsilError(
2157
+ ErrorCodes.CONFIG_INVALID,
2158
+ `The plugin at position ${position} carries no name, and the engine names each plugin in its errors and event payloads`,
2159
+ { position }
2160
+ );
2161
+ }
2162
+ });
2022
2163
  }
2023
- function isSingleStringFieldType(fieldType) {
2024
- return parsedTypeOf(fieldType)?.base === "string";
2164
+ function requireKnownConfig(config) {
2165
+ if (config === void 0) return;
2166
+ requireKnownKeys("", config, NARSIL_CONFIG_KEYS);
2167
+ requireKnownKeys("durability.", config.durability, DURABILITY_CONFIG_KEYS);
2168
+ requireKnownKeys("workers.", config.workers, WORKER_CONFIG_KEYS);
2169
+ requireKnownKeys("lifecycle.", config.lifecycle, LIFECYCLE_CONFIG_KEYS);
2170
+ requireKnownKeys("analysis.", config.analysis, ANALYSIS_CONFIG_KEYS);
2171
+ requireNamedPlugins(config.plugins);
2172
+ requireValidPatternWorkCap(config.patternWorkCap);
2025
2173
  }
2026
- function isWordIndexedFieldType(fieldType) {
2027
- const parsed = parsedTypeOf(fieldType);
2028
- return parsed !== void 0 && isWordIndexedBase(parsed.base);
2174
+ function requireValidPatternWorkCap(cap) {
2175
+ if (cap === void 0) return;
2176
+ if (typeof cap === "number" && Number.isSafeInteger(cap) && cap >= 1) return;
2177
+ throw new NarsilError(
2178
+ ErrorCodes.CONFIG_INVALID,
2179
+ `The configuration sets "patternWorkCap" to ${String(cap)}, while the cap must be a whole number from 1 to ${Number.MAX_SAFE_INTEGER}`,
2180
+ { setting: "patternWorkCap" }
2181
+ );
2029
2182
  }
2030
- function isStringValuedBase(base) {
2031
- return base === "string" || base === "string[]" || base === "verbatim" || base === "verbatim[]";
2183
+
2184
+ // src/core/id-generator.ts
2185
+ function generateId() {
2186
+ const bytes = new Uint8Array(16);
2187
+ crypto.getRandomValues(bytes);
2188
+ const now2 = Date.now();
2189
+ bytes[0] = Math.floor(now2 / 2 ** 40);
2190
+ bytes[1] = Math.floor(now2 / 2 ** 32) % 256;
2191
+ bytes[2] = Math.floor(now2 / 2 ** 24) % 256;
2192
+ bytes[3] = Math.floor(now2 / 2 ** 16) % 256;
2193
+ bytes[4] = Math.floor(now2 / 2 ** 8) % 256;
2194
+ bytes[5] = now2 % 256;
2195
+ bytes[6] = bytes[6] & 15 | 112;
2196
+ bytes[8] = bytes[8] & 63 | 128;
2197
+ const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
2198
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
2032
2199
  }
2033
2200
 
2201
+ // src/core/pattern-index/constants.ts
2202
+ var DEFAULT_PATTERN_WORK_CAP = 25e6;
2203
+ var MAX_PATTERN_TEXT_CODE_POINTS = 1024;
2204
+ var MAX_PATTERN_POSITIONS = 2048;
2205
+ var MATCHER_STATE_BUDGET_BYTES = 1e6;
2206
+ var MATCHER_CODE_POINT_BYTES = 8;
2207
+ var MATCHER_SIMULATION_BYTES_PER_POSITION = 16;
2208
+ var MATCHER_STATE_OVERHEAD_BYTES = 96;
2209
+ var PATTERN_WORK_FLUSH_UNITS = 65536;
2210
+ var PATTERN_RUN_CODE_POINTS = 3;
2211
+ var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "contains"];
2212
+
2034
2213
  // src/core/ordering/code-points.ts
2035
2214
  function exceedsCodePoints(value, maximum) {
2036
2215
  if (value.length <= maximum) return false;
@@ -2048,6 +2227,31 @@ function exceedsCodePoints(value, maximum) {
2048
2227
  }
2049
2228
  return false;
2050
2229
  }
2230
+ var SURROGATE_UNIT = /[\ud800-\udfff]/;
2231
+ function holdsUnpairedSurrogate(value) {
2232
+ if (!SURROGATE_UNIT.test(value)) return false;
2233
+ for (let i = 0; i < value.length; i++) {
2234
+ const unit = value.charCodeAt(i);
2235
+ if (unit < 55296 || unit >= 57344) continue;
2236
+ if (unit >= 56320) return true;
2237
+ const low = value.charCodeAt(i + 1);
2238
+ if (!(low >= 56320 && low < 57344)) return true;
2239
+ i++;
2240
+ }
2241
+ return false;
2242
+ }
2243
+ function codePointCount(value) {
2244
+ let counted = 0;
2245
+ for (let i = 0; i < value.length; i++) {
2246
+ const unit = value.charCodeAt(i);
2247
+ if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
2248
+ const low = value.charCodeAt(i + 1);
2249
+ if (low >= 56320 && low < 57344) i++;
2250
+ }
2251
+ counted++;
2252
+ }
2253
+ return counted;
2254
+ }
2051
2255
  function adjustUtf16Unit(unit) {
2052
2256
  if (unit < 55296) return unit;
2053
2257
  if (unit < 57344) return unit + 8192;
@@ -2061,6 +2265,29 @@ function compareCodePoints(a, b) {
2061
2265
  return adjustUtf16Unit(a.charCodeAt(i)) - adjustUtf16Unit(b.charCodeAt(i));
2062
2266
  }
2063
2267
 
2268
+ // src/geo/constants.ts
2269
+ var DEG_TO_RAD = Math.PI / 180;
2270
+ var MAX_LATITUDE_DEGREES = 90;
2271
+ var MAX_LONGITUDE_DEGREES = 180;
2272
+ var MIN_POLYGON_POINTS = 3;
2273
+ var ANTIMERIDIAN_SPAN_DEGREES = 180;
2274
+ var FULL_TURN_DEGREES = 360;
2275
+ var EARTH_RADIUS_METERS = 63710088e-1;
2276
+ var LOCATION_CODE_BITS_PER_AXIS = 26;
2277
+ var MAX_COVERING_CELLS = 16;
2278
+ var RADIUS_BOX_MARGIN_RATIO = 1.02;
2279
+ var RADIUS_BOX_MARGIN_RADIANS = 1e-9;
2280
+ var VINCENTY_MAX_ITERATIONS = 200;
2281
+ var VINCENTY_CONVERGENCE_THRESHOLD = 1e-12;
2282
+
2283
+ // src/geo/coordinates.ts
2284
+ function isLatitude(value) {
2285
+ return value >= -MAX_LATITUDE_DEGREES && value <= MAX_LATITUDE_DEGREES;
2286
+ }
2287
+ function isLongitude(value) {
2288
+ return value >= -MAX_LONGITUDE_DEGREES && value <= MAX_LONGITUDE_DEGREES;
2289
+ }
2290
+
2064
2291
  // src/schema/validator/field-values.ts
2065
2292
  function validateGeopoint(path, value) {
2066
2293
  if (!isPlainObject(value)) {
@@ -2080,7 +2307,7 @@ function validateGeopoint(path, value) {
2080
2307
  lon: Number.isNaN(value.lon) ? "NaN" : typeof value.lon
2081
2308
  });
2082
2309
  }
2083
- if (value.lat < -90 || value.lat > 90) {
2310
+ if (!isLatitude(value.lat)) {
2084
2311
  throw new NarsilError(
2085
2312
  ErrorCodes.SCHEMA_INVALID_GEOPOINT,
2086
2313
  `Field "${path}" geopoint lat must be between -90 and 90`,
@@ -2090,7 +2317,7 @@ function validateGeopoint(path, value) {
2090
2317
  }
2091
2318
  );
2092
2319
  }
2093
- if (value.lon < -180 || value.lon > 180) {
2320
+ if (!isLongitude(value.lon)) {
2094
2321
  throw new NarsilError(
2095
2322
  ErrorCodes.SCHEMA_INVALID_GEOPOINT,
2096
2323
  `Field "${path}" geopoint lon must be between -180 and 180`,
@@ -2120,7 +2347,7 @@ function validateVector(path, value, expectedDimension) {
2120
2347
  );
2121
2348
  }
2122
2349
  for (let i = 0; i < arr.length; i++) {
2123
- if (typeof arr[i] !== "number" || Number.isNaN(arr[i])) {
2350
+ if (typeof arr[i] !== "number" || !Number.isFinite(arr[i])) {
2124
2351
  throw new NarsilError(
2125
2352
  ErrorCodes.DOC_VALIDATION_FAILED,
2126
2353
  `Field "${path}" vector element at index ${i} is not a valid number`,
@@ -2145,11 +2372,12 @@ function validateTypedArray(path, value, elementType) {
2145
2372
  { field: path, index: i, expected: elementType, received: typeof el }
2146
2373
  );
2147
2374
  }
2148
- if (elementType === "number" && Number.isNaN(el)) {
2149
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" array element at index ${i} is NaN`, {
2150
- field: path,
2151
- index: i
2152
- });
2375
+ if (elementType === "number" && !Number.isFinite(el)) {
2376
+ throw new NarsilError(
2377
+ ErrorCodes.DOC_VALIDATION_FAILED,
2378
+ `Field "${path}" array element at index ${i} is ${String(el)}, while a number field holds finite numbers alone`,
2379
+ { field: path, index: i }
2380
+ );
2153
2381
  }
2154
2382
  }
2155
2383
  }
@@ -2180,11 +2408,11 @@ function validateFieldValue(path, value, type, patternValueLimit) {
2180
2408
  if (type.pattern) validatePatternValue(path, value, patternValueLimit);
2181
2409
  return;
2182
2410
  case "number":
2183
- if (typeof value !== "number" || Number.isNaN(value)) {
2184
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a valid number`, {
2411
+ if (typeof value !== "number" || !Number.isFinite(value)) {
2412
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a finite number`, {
2185
2413
  field: path,
2186
2414
  expected: "number",
2187
- received: Number.isNaN(value) ? "NaN" : typeof value
2415
+ received: typeof value === "number" ? String(value) : typeof value
2188
2416
  });
2189
2417
  }
2190
2418
  return;
@@ -2242,6 +2470,7 @@ function validateFieldValue(path, value, type, patternValueLimit) {
2242
2470
 
2243
2471
  // src/schema/validator/storable.ts
2244
2472
  var RESERVED_OBJECT_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
2473
+ var UNPAIRED_SURROGATE_REASON = "holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8";
2245
2474
  function reject(path, reason) {
2246
2475
  throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
2247
2476
  field: path
@@ -2254,8 +2483,11 @@ function isPlainObject2(value) {
2254
2483
  function walkStorable(value, path, depth, ancestors, skip) {
2255
2484
  if (value === null || value === void 0) return;
2256
2485
  const kind = typeof value;
2257
- if (kind === "string" || kind === "boolean") return;
2258
- if (kind === "number") return;
2486
+ if (kind === "string") {
2487
+ if (holdsUnpairedSurrogate(value)) reject(path, UNPAIRED_SURROGATE_REASON);
2488
+ return;
2489
+ }
2490
+ if (kind === "boolean" || kind === "number") return;
2259
2491
  if (kind === "bigint" || kind === "symbol" || kind === "function") {
2260
2492
  reject(path, `holds a ${kind}, which cannot be stored or recovered`);
2261
2493
  }
@@ -2287,6 +2519,7 @@ function walkStorable(value, path, depth, ancestors, skip) {
2287
2519
  if (RESERVED_OBJECT_KEYS.has(key)) {
2288
2520
  reject(keyPath, "uses a reserved object key");
2289
2521
  }
2522
+ if (holdsUnpairedSurrogate(key)) reject(keyPath, `has a name that ${UNPAIRED_SURROGATE_REASON}`);
2290
2523
  if (skip.has(keyPath)) continue;
2291
2524
  walkStorable(record[key], keyPath, depth + 1, ancestors, skip);
2292
2525
  }
@@ -2353,9 +2586,20 @@ function validateDocument(document2, schema, patternValueLimit = DEFAULT_PATTERN
2353
2586
  received: typeof document2
2354
2587
  });
2355
2588
  }
2589
+ requireIdFieldKey(document2);
2356
2590
  assertStorableDocument(document2, vectorFieldPaths(schema));
2357
2591
  validateDocumentFields(document2, schema, "", patternValueLimit);
2358
2592
  }
2593
+ function requireIdFieldKey(document2) {
2594
+ if (!Object.hasOwn(document2, "id")) return;
2595
+ const id = document2.id;
2596
+ if (typeof id === "string" && id.length > 0) return;
2597
+ throw new NarsilError(
2598
+ ErrorCodes.DOC_VALIDATION_FAILED,
2599
+ `Field "id" holds ${typeof id === "string" ? "an empty string" : id === null ? "null" : `a ${typeof id}`}, while the engine stores a document under its id field, so that field must hold a non-empty string`,
2600
+ { field: "id", received: id === null ? "null" : typeof id }
2601
+ );
2602
+ }
2359
2603
  function collectExtraFields(doc, schema, prefix, extras) {
2360
2604
  for (const key of Object.keys(doc)) {
2361
2605
  if (key === "id") continue;
@@ -2403,124 +2647,6 @@ function validateRequiredFields(document2, required) {
2403
2647
  }
2404
2648
  }
2405
2649
 
2406
- // src/schema/validator/schema.ts
2407
- function validateSchemaFields(schema, depth, prefix) {
2408
- if (depth > MAX_NESTING_DEPTH) {
2409
- throw new NarsilError(
2410
- ErrorCodes.SCHEMA_DEPTH_EXCEEDED,
2411
- `Schema nesting exceeds the maximum depth of ${MAX_NESTING_DEPTH} at "${prefix}"`,
2412
- { path: prefix, maxDepth: MAX_NESTING_DEPTH }
2413
- );
2414
- }
2415
- let canonical = null;
2416
- const replace = (field, value) => {
2417
- if (canonical === null) canonical = { ...schema };
2418
- canonical[field] = value;
2419
- };
2420
- for (const [field, type] of Object.entries(schema)) {
2421
- const path = prefix ? `${prefix}.${field}` : field;
2422
- if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
2423
- throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field name "${field}" is not allowed in a schema`, {
2424
- field,
2425
- path
2426
- });
2427
- }
2428
- if (!FIELD_NAME_PATTERN.test(field)) {
2429
- throw new NarsilError(
2430
- ErrorCodes.SCHEMA_INVALID_TYPE,
2431
- `Field name "${field}" contains characters that are not allowed; use letters, digits, and underscores only`,
2432
- { field, path }
2433
- );
2434
- }
2435
- if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
2436
- throw new NarsilError(
2437
- ErrorCodes.SCHEMA_INVALID_TYPE,
2438
- `Field "${field}" is reserved and cannot be defined in a schema`,
2439
- { field, path }
2440
- );
2441
- }
2442
- if (isPlainObject(type)) {
2443
- const nested = validateSchemaFields(type, depth + 1, path);
2444
- if (nested !== type) replace(field, nested);
2445
- continue;
2446
- }
2447
- if (typeof type !== "string") {
2448
- throw new NarsilError(
2449
- ErrorCodes.SCHEMA_INVALID_TYPE,
2450
- `Field "${path}" has an invalid type definition: ${String(type)}`,
2451
- {
2452
- field: path,
2453
- type: String(type)
2454
- }
2455
- );
2456
- }
2457
- const parsed = parseFieldType(type, path);
2458
- if (parsed.type !== type) replace(field, parsed.type);
2459
- }
2460
- return canonical ?? schema;
2461
- }
2462
- function validateSchema(schema) {
2463
- if (!isPlainObject(schema)) {
2464
- throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must be a plain object", {
2465
- received: typeof schema
2466
- });
2467
- }
2468
- if (Object.keys(schema).length === 0) {
2469
- throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
2470
- }
2471
- return validateSchemaFields(schema, 1, "");
2472
- }
2473
- function requireValidFieldTypes(flatSchema) {
2474
- for (const [path, type] of Object.entries(flatSchema)) {
2475
- if (typeof type !== "string") {
2476
- throw new NarsilError(
2477
- ErrorCodes.SCHEMA_INVALID_TYPE,
2478
- `Field "${path}" has an invalid type definition: ${String(type)}`,
2479
- { field: path, type: String(type) }
2480
- );
2481
- }
2482
- parseFieldType(type, path);
2483
- }
2484
- }
2485
- function flattenRecursive(schema, prefix, result) {
2486
- for (const [field, type] of Object.entries(schema)) {
2487
- const path = prefix ? `${prefix}.${field}` : field;
2488
- if (isPlainObject(type)) {
2489
- flattenRecursive(type, path, result);
2490
- } else {
2491
- result[path] = type;
2492
- }
2493
- }
2494
- }
2495
- var flattenedSchemas = /* @__PURE__ */ new WeakMap();
2496
- function flattenSchema(schema) {
2497
- const cached = flattenedSchemas.get(schema);
2498
- if (cached !== void 0) return cached;
2499
- const result = /* @__PURE__ */ Object.create(null);
2500
- flattenRecursive(schema, "", result);
2501
- flattenedSchemas.set(schema, Object.freeze(result));
2502
- return result;
2503
- }
2504
- var parsedSchemas = /* @__PURE__ */ new WeakMap();
2505
- function schemaFieldsOf(schema) {
2506
- const cached = parsedSchemas.get(schema);
2507
- if (cached !== void 0) return cached;
2508
- const fields = [];
2509
- for (const [path, type] of Object.entries(flattenSchema(schema))) {
2510
- fields.push({ ...parseFieldType(type, path), path });
2511
- }
2512
- const frozen = Object.freeze(fields);
2513
- parsedSchemas.set(schema, frozen);
2514
- return frozen;
2515
- }
2516
- function extractVectorFieldsFromSchema(schema) {
2517
- const result = /* @__PURE__ */ new Map();
2518
- for (const field of schemaFieldsOf(schema)) {
2519
- if (field.base === "vector") result.set(field.path, field.dimension);
2520
- }
2521
- return result;
2522
- }
2523
-
2524
2650
  // src/vector/hnsw/constants.ts
2525
2651
  var MAX_LAYER_CAP = 32;
2526
2652
  var MAX_M = 512;
@@ -2868,6 +2994,44 @@ function sanitizeSurfaceForms(raw) {
2868
2994
  }
2869
2995
  return count > 0 ? result : void 0;
2870
2996
  }
2997
+ function patternIndexesToWire(pattern) {
2998
+ if (pattern === void 0) return {};
2999
+ const wire = /* @__PURE__ */ Object.create(null);
3000
+ for (const [field, index] of Object.entries(pattern)) wire[field] = { doc_ids: index.docIds, runs: index.runs };
3001
+ return { pattern: wire };
3002
+ }
3003
+ function isWireRecord(value) {
3004
+ return value !== null && typeof value === "object" && !Array.isArray(value);
3005
+ }
3006
+ function corruptPatternEntry(field) {
3007
+ return new NarsilError(
3008
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
3009
+ `The pattern entry of field "${field}" in this partition must hold a list of document IDs and a map of runs`,
3010
+ { field }
3011
+ );
3012
+ }
3013
+ function patternIndexesFromWire(raw) {
3014
+ if (raw === void 0 || raw === null) return {};
3015
+ if (!isWireRecord(raw)) {
3016
+ throw new NarsilError(
3017
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
3018
+ "The pattern entry of this partition must be a map of fields"
3019
+ );
3020
+ }
3021
+ const pattern = /* @__PURE__ */ Object.create(null);
3022
+ for (const [field, index] of Object.entries(raw)) {
3023
+ if (!isWireRecord(index) || !isWireRecord(index.runs) || !Array.isArray(index.doc_ids)) {
3024
+ throw corruptPatternEntry(field);
3025
+ }
3026
+ const docIds = [];
3027
+ for (const docId of index.doc_ids) {
3028
+ if (typeof docId !== "string") throw corruptPatternEntry(field);
3029
+ docIds.push(docId);
3030
+ }
3031
+ pattern[field] = { docIds, runs: index.runs };
3032
+ }
3033
+ return { pattern };
3034
+ }
2871
3035
  function validateHnswMetric(value) {
2872
3036
  if (value === void 0 || value === null) return void 0;
2873
3037
  if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
@@ -2939,7 +3103,8 @@ function wireToPartition(raw) {
2939
3103
  numeric,
2940
3104
  boolean,
2941
3105
  enum: raw.field_indexes?.enum ?? {},
2942
- geopoint
3106
+ geopoint,
3107
+ ...patternIndexesFromWire(raw.field_indexes?.pattern)
2943
3108
  },
2944
3109
  surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
2945
3110
  vectorData,
@@ -3046,7 +3211,8 @@ function deserializePayloadV2(data) {
3046
3211
  numeric,
3047
3212
  boolean,
3048
3213
  enum: raw.field_indexes?.enum ?? {},
3049
- geopoint
3214
+ geopoint,
3215
+ ...patternIndexesFromWire(raw.field_indexes?.pattern)
3050
3216
  },
3051
3217
  surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
3052
3218
  vectorData,
@@ -3096,6 +3262,10 @@ function multiFoldTable() {
3096
3262
 
3097
3263
  // src/core/ordering/fold-compare.ts
3098
3264
  var EXHAUSTED = -1;
3265
+ function foldOf(codePoint, single, multi) {
3266
+ if (codePoint < 128) return codePoint >= 65 && codePoint <= 90 ? codePoint + 32 : codePoint;
3267
+ return single.get(codePoint) ?? multi.get(codePoint) ?? codePoint;
3268
+ }
3099
3269
  function nextFoldedCodePoint(text, cursor, single, multi) {
3100
3270
  if (cursor.pending !== null) {
3101
3271
  const cp2 = cursor.pending[cursor.pendingIndex];
@@ -3109,9 +3279,6 @@ function nextFoldedCodePoint(text, cursor, single, multi) {
3109
3279
  if (cursor.index >= text.length) return EXHAUSTED;
3110
3280
  let cp = text.charCodeAt(cursor.index);
3111
3281
  cursor.index++;
3112
- if (cp < 128) {
3113
- return cp >= 65 && cp <= 90 ? cp + 32 : cp;
3114
- }
3115
3282
  if (cp >= 55296 && cp < 56320 && cursor.index < text.length) {
3116
3283
  const low = text.charCodeAt(cursor.index);
3117
3284
  if (low >= 56320 && low < 57344) {
@@ -3119,15 +3286,13 @@ function nextFoldedCodePoint(text, cursor, single, multi) {
3119
3286
  cursor.index++;
3120
3287
  }
3121
3288
  }
3122
- const mapped = single.get(cp);
3123
- if (mapped !== void 0) return mapped;
3124
- const expansion = multi.get(cp);
3125
- if (expansion === void 0) return cp;
3126
- if (expansion.length > 1) {
3127
- cursor.pending = expansion;
3289
+ const folded = foldOf(cp, single, multi);
3290
+ if (typeof folded === "number") return folded;
3291
+ if (folded.length > 1) {
3292
+ cursor.pending = folded;
3128
3293
  cursor.pendingIndex = 1;
3129
3294
  }
3130
- return expansion[0];
3295
+ return folded[0];
3131
3296
  }
3132
3297
  function compareCaseFolded(a, b) {
3133
3298
  const shorter = a.length < b.length ? a.length : b.length;
@@ -3154,6 +3319,28 @@ function compareSortStrings(a, b) {
3154
3319
  if (folded !== 0) return folded;
3155
3320
  return compareCodePoints(a, b);
3156
3321
  }
3322
+ function appendFoldedCodePoint(codePoint, out) {
3323
+ const folded = foldOf(codePoint, singleFoldTable(), multiFoldTable());
3324
+ if (typeof folded === "number") {
3325
+ out.push(folded);
3326
+ return;
3327
+ }
3328
+ for (const foldedCodePoint of folded) out.push(foldedCodePoint);
3329
+ }
3330
+ function appendFoldedCodePoints(text, out) {
3331
+ const single = singleFoldTable();
3332
+ const multi = multiFoldTable();
3333
+ const cursor = { index: 0, pending: null, pendingIndex: 0 };
3334
+ for (let codePoint = nextFoldedCodePoint(text, cursor, single, multi); codePoint !== EXHAUSTED; ) {
3335
+ out.push(codePoint);
3336
+ codePoint = nextFoldedCodePoint(text, cursor, single, multi);
3337
+ }
3338
+ }
3339
+ function foldedCodePointCount(text) {
3340
+ const folded = [];
3341
+ appendFoldedCodePoints(text, folded);
3342
+ return folded.length;
3343
+ }
3157
3344
 
3158
3345
  // src/core/ordering/constants.ts
3159
3346
  var SORT_VALUE_MAX_CODE_POINTS = 512;
@@ -4281,17 +4468,18 @@ function bitsetAnd(a, b) {
4281
4468
  }
4282
4469
  return result;
4283
4470
  }
4284
- function bitsetOr(a, b) {
4285
- const longer = a.length >= b.length ? a : b;
4286
- const shorter = a.length >= b.length ? b : a;
4287
- const result = new Uint32Array(longer.length);
4288
- for (let i = 0; i < shorter.length; i++) {
4289
- result[i] = longer[i] | shorter[i];
4290
- }
4291
- for (let i = shorter.length; i < longer.length; i++) {
4292
- result[i] = longer[i];
4293
- }
4294
- return result;
4471
+ function bitsetAndInto(target, other) {
4472
+ const shared2 = Math.min(target.length, other.length);
4473
+ for (let i = 0; i < shared2; i++) target[i] &= other[i];
4474
+ target.fill(0, shared2);
4475
+ }
4476
+ function bitsetOrInto(target, other) {
4477
+ const shared2 = Math.min(target.length, other.length);
4478
+ for (let i = 0; i < shared2; i++) target[i] |= other[i];
4479
+ }
4480
+ function bitsetAndNotInto(target, removed) {
4481
+ const shared2 = Math.min(target.length, removed.length);
4482
+ for (let i = 0; i < shared2; i++) target[i] &= ~removed[i];
4295
4483
  }
4296
4484
  function bitsetNot(bs, capacity) {
4297
4485
  const result = new Uint32Array(bs.length);
@@ -4385,13 +4573,6 @@ function getFieldValueByInternalId(docStore, internalId, fieldPath) {
4385
4573
  if (externalId === void 0) return void 0;
4386
4574
  return getFieldValueForDoc(docStore, externalId, fieldPath);
4387
4575
  }
4388
- function getAllInternalDocIds(docStore) {
4389
- const ids = /* @__PURE__ */ new Set();
4390
- for (const internalId of docStore.allInternalIds()) {
4391
- ids.add(internalId);
4392
- }
4393
- return ids;
4394
- }
4395
4576
 
4396
4577
  // src/core/partition/facets.ts
4397
4578
  var UNDECLARED_FIELD_TYPE = "";
@@ -4527,16 +4708,113 @@ function computeFacets(state, matched, config, schema) {
4527
4708
  return result;
4528
4709
  }
4529
4710
 
4530
- // src/geo/constants.ts
4531
- var DEG_TO_RAD = Math.PI / 180;
4532
- var MIN_POLYGON_POINTS = 3;
4533
- var ANTIMERIDIAN_SPAN_DEGREES = 180;
4534
- var FULL_TURN_DEGREES = 360;
4535
- var VINCENTY_MAX_ITERATIONS = 200;
4536
- var VINCENTY_CONVERGENCE_THRESHOLD = 1e-12;
4711
+ // src/core/pattern-index/host.ts
4712
+ var PATTERN_HOST = { appendFoldedCodePoint };
4713
+ function patternIndexOf(indexes, fieldPath) {
4714
+ let index = indexes.get(fieldPath);
4715
+ if (index === void 0) {
4716
+ index = requirePatternSearch(fieldPath).createIndex(PATTERN_HOST);
4717
+ indexes.set(fieldPath, index);
4718
+ }
4719
+ return index;
4720
+ }
4721
+
4722
+ // src/geo/cells.ts
4723
+ var AXIS_CELLS = 2 ** LOCATION_CODE_BITS_PER_AXIS;
4724
+ function axisCell(value, lowest, span) {
4725
+ const cell = Math.floor((value - lowest) / span * AXIS_CELLS);
4726
+ if (!(cell > 0)) return 0;
4727
+ return cell >= AXIS_CELLS ? AXIS_CELLS - 1 : cell;
4728
+ }
4729
+ function latitudeCell(lat) {
4730
+ return axisCell(lat, -MAX_LATITUDE_DEGREES, 2 * MAX_LATITUDE_DEGREES);
4731
+ }
4732
+ function longitudeCell(lon) {
4733
+ return axisCell(lon, -MAX_LONGITUDE_DEGREES, FULL_TURN_DEGREES);
4734
+ }
4735
+ function interleave(row, column, bits) {
4736
+ let code = 0;
4737
+ for (let bit = bits - 1; bit >= 0; bit--) code = code * 4 + (row >>> bit & 1) * 2 + (column >>> bit & 1);
4738
+ return code;
4739
+ }
4740
+ function locationCode(lat, lon) {
4741
+ return interleave(latitudeCell(lat), longitudeCell(lon), LOCATION_CODE_BITS_PER_AXIS);
4742
+ }
4743
+ function appendBoxRanges(box, out) {
4744
+ const south = latitudeCell(box.south);
4745
+ const north = latitudeCell(box.north);
4746
+ const west = longitudeCell(box.west);
4747
+ const east = longitudeCell(box.east);
4748
+ let level2 = LOCATION_CODE_BITS_PER_AXIS;
4749
+ while (level2 > 0) {
4750
+ const shift2 = LOCATION_CODE_BITS_PER_AXIS - level2;
4751
+ const rows = (north >>> shift2) - (south >>> shift2) + 1;
4752
+ const columns = (east >>> shift2) - (west >>> shift2) + 1;
4753
+ if (rows * columns <= MAX_COVERING_CELLS) break;
4754
+ level2--;
4755
+ }
4756
+ const shift = LOCATION_CODE_BITS_PER_AXIS - level2;
4757
+ const codesPerCell = 4 ** shift;
4758
+ for (let row = south >>> shift; row <= north >>> shift; row++) {
4759
+ for (let column = west >>> shift; column <= east >>> shift; column++) {
4760
+ const from = interleave(row, column, level2) * codesPerCell;
4761
+ out.push({ from, to: from + codesPerCell });
4762
+ }
4763
+ }
4764
+ }
4765
+ function coveringRanges(boxes) {
4766
+ const ranges = [];
4767
+ for (const box of boxes) appendBoxRanges(box, ranges);
4768
+ ranges.sort((left, right) => left.from - right.from);
4769
+ const merged = [];
4770
+ for (const range of ranges) {
4771
+ const last = merged[merged.length - 1];
4772
+ if (last !== void 0 && range.from <= last.to) {
4773
+ if (range.to > last.to) last.to = range.to;
4774
+ } else {
4775
+ merged.push({ from: range.from, to: range.to });
4776
+ }
4777
+ }
4778
+ return merged;
4779
+ }
4780
+ function longitudeBoxes(south, north, west, east) {
4781
+ if (east - west >= FULL_TURN_DEGREES) {
4782
+ return [{ south, north, west: -MAX_LONGITUDE_DEGREES, east: MAX_LONGITUDE_DEGREES }];
4783
+ }
4784
+ if (west < -MAX_LONGITUDE_DEGREES) {
4785
+ return [
4786
+ { south, north, west: west + FULL_TURN_DEGREES, east: MAX_LONGITUDE_DEGREES },
4787
+ { south, north, west: -MAX_LONGITUDE_DEGREES, east }
4788
+ ];
4789
+ }
4790
+ if (east > MAX_LONGITUDE_DEGREES) {
4791
+ return [
4792
+ { south, north, west, east: MAX_LONGITUDE_DEGREES },
4793
+ { south, north, west: -MAX_LONGITUDE_DEGREES, east: east - FULL_TURN_DEGREES }
4794
+ ];
4795
+ }
4796
+ return [{ south, north, west, east }];
4797
+ }
4798
+ function radiusBoxes(lat, lon, distanceMeters) {
4799
+ const angle = distanceMeters / EARTH_RADIUS_METERS * RADIUS_BOX_MARGIN_RATIO + RADIUS_BOX_MARGIN_RADIANS;
4800
+ const angleDegrees = angle / DEG_TO_RAD;
4801
+ const south = lat - angleDegrees;
4802
+ const north = lat + angleDegrees;
4803
+ if (angle >= Math.PI || south <= -MAX_LATITUDE_DEGREES || north >= MAX_LATITUDE_DEGREES) {
4804
+ return [
4805
+ {
4806
+ south: Math.max(south, -MAX_LATITUDE_DEGREES),
4807
+ north: Math.min(north, MAX_LATITUDE_DEGREES),
4808
+ west: -MAX_LONGITUDE_DEGREES,
4809
+ east: MAX_LONGITUDE_DEGREES
4810
+ }
4811
+ ];
4812
+ }
4813
+ const spread = Math.asin(Math.sin(angle) / Math.cos(lat * DEG_TO_RAD)) / DEG_TO_RAD;
4814
+ return longitudeBoxes(south, north, lon - spread, lon + spread);
4815
+ }
4537
4816
 
4538
4817
  // src/geo/haversine.ts
4539
- var EARTH_RADIUS_METERS = 63710088e-1;
4540
4818
  function haversineDistance(lat1, lon1, lat2, lon2) {
4541
4819
  const phi1 = lat1 * DEG_TO_RAD;
4542
4820
  const phi2 = lat2 * DEG_TO_RAD;
@@ -4549,6 +4827,53 @@ function haversineDistance(lat1, lon1, lat2, lon2) {
4549
4827
  return EARTH_RADIUS_METERS * c;
4550
4828
  }
4551
4829
 
4830
+ // src/geo/edge-tree.ts
4831
+ function buildLongitudeEdgeTree(lons) {
4832
+ const n = lons.length;
4833
+ const edges = [];
4834
+ for (let i = 0; i < n; i++) {
4835
+ if (lons[i] !== lons[i === 0 ? n - 1 : i - 1]) edges.push(i);
4836
+ }
4837
+ const lowOf = (i) => Math.min(lons[i], lons[i === 0 ? n - 1 : i - 1]);
4838
+ edges.sort((left, right) => lowOf(left) - lowOf(right));
4839
+ const count = edges.length;
4840
+ const tree = {
4841
+ from: new Int32Array(count),
4842
+ to: new Int32Array(count),
4843
+ low: new Float64Array(count),
4844
+ high: new Float64Array(count),
4845
+ subtreeHigh: new Float64Array(count)
4846
+ };
4847
+ edges.forEach((i, at) => {
4848
+ const j = i === 0 ? n - 1 : i - 1;
4849
+ tree.from[at] = i;
4850
+ tree.to[at] = j;
4851
+ tree.low[at] = Math.min(lons[i], lons[j]);
4852
+ tree.high[at] = Math.max(lons[i], lons[j]);
4853
+ });
4854
+ const fillSubtreeHigh = (start, end) => {
4855
+ if (start >= end) return Number.NEGATIVE_INFINITY;
4856
+ const mid = start + end >>> 1;
4857
+ const highest = Math.max(tree.high[mid], fillSubtreeHigh(start, mid), fillSubtreeHigh(mid + 1, end));
4858
+ tree.subtreeHigh[mid] = highest;
4859
+ return highest;
4860
+ };
4861
+ fillSubtreeHigh(0, count);
4862
+ return tree;
4863
+ }
4864
+ function forEachEdgeSpanning(tree, lon, visit) {
4865
+ const walk = (start, end) => {
4866
+ if (start >= end) return;
4867
+ const mid = start + end >>> 1;
4868
+ if (tree.subtreeHigh[mid] <= lon) return;
4869
+ walk(start, mid);
4870
+ if (tree.low[mid] > lon) return;
4871
+ if (lon < tree.high[mid]) visit(tree.from[mid], tree.to[mid]);
4872
+ walk(mid + 1, end);
4873
+ };
4874
+ walk(0, tree.from.length);
4875
+ }
4876
+
4552
4877
  // src/geo/polygon.ts
4553
4878
  function requirePolygonRing(points) {
4554
4879
  if (!Array.isArray(points) || points.length < MIN_POLYGON_POINTS) {
@@ -4560,11 +4885,18 @@ function requirePolygonRing(points) {
4560
4885
  }
4561
4886
  points.forEach((point, position) => {
4562
4887
  const { lat, lon } = point ?? {};
4563
- if (typeof lat === "number" && Number.isFinite(lat) && typeof lon === "number" && Number.isFinite(lon)) return;
4888
+ if (typeof lat !== "number" || !Number.isFinite(lat) || typeof lon !== "number" || !Number.isFinite(lon)) {
4889
+ throw new NarsilError(
4890
+ ErrorCodes.SEARCH_INVALID_FILTER,
4891
+ `A polygon point carries a finite "lat" and "lon", and the point at position ${position} does not`,
4892
+ { position }
4893
+ );
4894
+ }
4895
+ if (isLatitude(lat) && isLongitude(lon)) return;
4564
4896
  throw new NarsilError(
4565
4897
  ErrorCodes.SEARCH_INVALID_FILTER,
4566
- `A polygon point carries a finite "lat" and "lon", and the point at position ${position} does not`,
4567
- { position }
4898
+ `A polygon point must hold a "lat" from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES} and a "lon" from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, while the point at position ${position} holds ${lat}, ${lon}`,
4899
+ { position, lat, lon }
4568
4900
  );
4569
4901
  });
4570
4902
  }
@@ -4597,20 +4929,35 @@ function polygonContainment(polygon) {
4597
4929
  lats[i] = polygon[i].lat;
4598
4930
  lons[i] = unwrap ? eastward(polygon[i].lon) : polygon[i].lon;
4599
4931
  }
4932
+ const edges = buildLongitudeEdgeTree(lons);
4600
4933
  return (lat, rawLon) => {
4601
4934
  const lon = unwrap ? eastward(rawLon) : rawLon;
4602
4935
  let inside = false;
4603
- for (let i = 0, j = n - 1; i < n; j = i++) {
4604
- const lonI = lons[i];
4605
- const lonJ = lons[j];
4606
- if (lonI > lon !== lonJ > lon) {
4607
- const intersectLat = lats[i] + (lats[j] - lats[i]) / (lonJ - lonI) * (lon - lonI);
4608
- if (lat < intersectLat) inside = !inside;
4609
- }
4610
- }
4936
+ forEachEdgeSpanning(edges, lon, (i, j) => {
4937
+ const intersectLat = lats[i] + (lats[j] - lats[i]) / (lons[j] - lons[i]) * (lon - lons[i]);
4938
+ if (lat < intersectLat) inside = !inside;
4939
+ });
4611
4940
  return inside;
4612
4941
  };
4613
4942
  }
4943
+ function polygonBoxes(polygon) {
4944
+ if (polygon.length < MIN_POLYGON_POINTS) return [];
4945
+ const unwrap = crossesAntimeridian(polygon);
4946
+ let south = Number.POSITIVE_INFINITY;
4947
+ let north = Number.NEGATIVE_INFINITY;
4948
+ let west = Number.POSITIVE_INFINITY;
4949
+ let east = Number.NEGATIVE_INFINITY;
4950
+ for (const point of polygon) {
4951
+ const lon = unwrap ? eastward(point.lon) : point.lon;
4952
+ south = Math.min(south, point.lat);
4953
+ north = Math.max(north, point.lat);
4954
+ west = Math.min(west, lon);
4955
+ east = Math.max(east, lon);
4956
+ }
4957
+ if (west >= MAX_LONGITUDE_DEGREES)
4958
+ return longitudeBoxes(south, north, west - FULL_TURN_DEGREES, east - FULL_TURN_DEGREES);
4959
+ return longitudeBoxes(south, north, west, east);
4960
+ }
4614
4961
 
4615
4962
  // src/geo/vincenty.ts
4616
4963
  var WGS84_A = 6378137;
@@ -4662,61 +5009,94 @@ function vincentyDistance(lat1, lon1, lat2, lon2) {
4662
5009
  }
4663
5010
 
4664
5011
  // src/geo/geo-index.ts
5012
+ function firstAtOrAfter(entries, code) {
5013
+ let lo = 0;
5014
+ let hi = entries.length;
5015
+ while (lo < hi) {
5016
+ const mid = lo + hi >>> 1;
5017
+ if (entries[mid].code < code) lo = mid + 1;
5018
+ else hi = mid;
5019
+ }
5020
+ return lo;
5021
+ }
4665
5022
  function createGeoIndex() {
4666
5023
  const entries = [];
4667
- return {
4668
- get entries() {
4669
- return entries;
4670
- },
5024
+ const byDocument = /* @__PURE__ */ new Map();
5025
+ let sorted = true;
5026
+ function ensureSorted() {
5027
+ if (sorted) return;
5028
+ entries.sort((left, right) => left.code - right.code || left.docId - right.docId);
5029
+ sorted = true;
5030
+ }
5031
+ function removeEntry(internalId) {
5032
+ const entry = byDocument.get(internalId);
5033
+ if (entry === void 0) return;
5034
+ byDocument.delete(internalId);
5035
+ ensureSorted();
5036
+ for (let at = firstAtOrAfter(entries, entry.code); at < entries.length; at++) {
5037
+ if (entries[at] !== entry) continue;
5038
+ entries.splice(at, 1);
5039
+ return;
5040
+ }
5041
+ }
5042
+ function matching(boxes, passes, inside) {
5043
+ ensureSorted();
5044
+ const within = /* @__PURE__ */ new Set();
5045
+ for (const range of coveringRanges(boxes)) {
5046
+ for (let at = firstAtOrAfter(entries, range.from); at < entries.length && entries[at].code < range.to; at++) {
5047
+ if (passes(entries[at])) within.add(entries[at].docId);
5048
+ }
5049
+ }
5050
+ if (inside) return within;
5051
+ const outside = /* @__PURE__ */ new Set();
5052
+ for (const entry of entries) {
5053
+ if (!within.has(entry.docId)) outside.add(entry.docId);
5054
+ }
5055
+ return outside;
5056
+ }
5057
+ const index = {
4671
5058
  insert(internalId, lat, lon) {
4672
- entries.push({ lat, lon, docId: internalId });
5059
+ removeEntry(internalId);
5060
+ const entry = { lat, lon, docId: internalId, code: locationCode(lat, lon) };
5061
+ entries.push(entry);
5062
+ byDocument.set(internalId, entry);
5063
+ sorted = false;
4673
5064
  },
4674
5065
  remove(internalId) {
4675
- for (let i = entries.length - 1; i >= 0; i--) {
4676
- if (entries[i].docId === internalId) {
4677
- entries.splice(i, 1);
4678
- return;
4679
- }
4680
- }
5066
+ removeEntry(internalId);
4681
5067
  },
4682
5068
  radiusQuery(lat, lon, distanceMeters, inside, highPrecision) {
4683
- const result = /* @__PURE__ */ new Set();
4684
5069
  const distanceFn = highPrecision ? vincentyDistance : haversineDistance;
4685
- for (let i = 0; i < entries.length; i++) {
4686
- const entry = entries[i];
4687
- const dist = distanceFn(lat, lon, entry.lat, entry.lon);
4688
- const withinRadius = dist <= distanceMeters;
4689
- if (inside ? withinRadius : !withinRadius) {
4690
- result.add(entry.docId);
4691
- }
4692
- }
4693
- return result;
5070
+ return matching(
5071
+ radiusBoxes(lat, lon, distanceMeters),
5072
+ (entry) => distanceFn(lat, lon, entry.lat, entry.lon) <= distanceMeters,
5073
+ inside
5074
+ );
4694
5075
  },
4695
5076
  polygonQuery(points, inside) {
4696
- const result = /* @__PURE__ */ new Set();
4697
5077
  const contains = polygonContainment(points);
4698
- for (let i = 0; i < entries.length; i++) {
4699
- const entry = entries[i];
4700
- const withinPolygon = contains(entry.lat, entry.lon);
4701
- if (inside ? withinPolygon : !withinPolygon) {
4702
- result.add(entry.docId);
4703
- }
4704
- }
4705
- return result;
5078
+ return matching(polygonBoxes(points), (entry) => contains(entry.lat, entry.lon), inside);
5079
+ },
5080
+ documentsBitset(capacity) {
5081
+ const bits = createBitSet(capacity);
5082
+ for (const entry of entries) bitsetSet(bits, entry.docId);
5083
+ return bits;
4706
5084
  },
4707
5085
  clear() {
4708
5086
  entries.length = 0;
5087
+ byDocument.clear();
5088
+ sorted = true;
4709
5089
  },
4710
5090
  serialize() {
4711
- return entries.map((e) => ({ ...e }));
5091
+ ensureSorted();
5092
+ return entries.map((entry) => ({ lat: entry.lat, lon: entry.lon, docId: entry.docId }));
4712
5093
  },
4713
5094
  deserialize(data) {
4714
- entries.length = 0;
4715
- for (const entry of data) {
4716
- entries.push({ lat: entry.lat, lon: entry.lon, docId: entry.docId });
4717
- }
5095
+ index.clear();
5096
+ for (const entry of data) index.insert(entry.docId, entry.lat, entry.lon);
4718
5097
  }
4719
5098
  };
5099
+ return index;
4720
5100
  }
4721
5101
 
4722
5102
  // src/core/field-index/boolean.ts
@@ -4790,9 +5170,90 @@ function createBooleanIndex() {
4790
5170
  };
4791
5171
  }
4792
5172
 
5173
+ // src/core/field-index/affixes.ts
5174
+ function compareUnits(left, right) {
5175
+ if (left < right) return -1;
5176
+ return left > right ? 1 : 0;
5177
+ }
5178
+ function reversedUnits(value) {
5179
+ let reversed = "";
5180
+ for (let at = value.length - 1; at >= 0; at--) reversed += value[at];
5181
+ return reversed;
5182
+ }
5183
+ function firstAtOrAfter2(sorted, key) {
5184
+ let lo = 0;
5185
+ let hi = sorted.length;
5186
+ while (lo < hi) {
5187
+ const mid = lo + hi >>> 1;
5188
+ if (sorted[mid] < key) lo = mid + 1;
5189
+ else hi = mid;
5190
+ }
5191
+ return lo;
5192
+ }
5193
+ function settledColumn(column, holds) {
5194
+ const { keys, values } = column;
5195
+ const order = [];
5196
+ for (let at = 0; at < keys.length; at++) if (holds(values[at])) order.push(at);
5197
+ order.sort((left, right) => compareUnits(keys[left], keys[right]) || compareUnits(values[left], values[right]));
5198
+ const settled2 = { keys: [], values: [] };
5199
+ for (const at of order) {
5200
+ const last = settled2.values.length - 1;
5201
+ if (last >= 0 && settled2.keys[last] === keys[at] && settled2.values[last] === values[at]) continue;
5202
+ settled2.keys.push(keys[at]);
5203
+ settled2.values.push(values[at]);
5204
+ }
5205
+ return settled2;
5206
+ }
5207
+ var ValueAffixes = class {
5208
+ constructor(holds, values) {
5209
+ this.holds = holds;
5210
+ for (const value of values) this.add(value);
5211
+ }
5212
+ forward = { keys: [], values: [] };
5213
+ backward = { keys: [], values: [] };
5214
+ settled = true;
5215
+ add(value) {
5216
+ this.forward.keys.push(value);
5217
+ this.forward.values.push(value);
5218
+ this.backward.keys.push(reversedUnits(value));
5219
+ this.backward.values.push(value);
5220
+ this.settled = false;
5221
+ }
5222
+ forget() {
5223
+ this.settled = false;
5224
+ }
5225
+ withPrefix(prefix) {
5226
+ this.settle();
5227
+ return this.matching(this.forward, prefix);
5228
+ }
5229
+ withSuffix(suffix) {
5230
+ this.settle();
5231
+ return this.matching(this.backward, reversedUnits(suffix));
5232
+ }
5233
+ settle() {
5234
+ if (this.settled) return;
5235
+ this.forward = settledColumn(this.forward, this.holds);
5236
+ this.backward = settledColumn(this.backward, this.holds);
5237
+ this.settled = true;
5238
+ }
5239
+ matching(column, start) {
5240
+ const found = [];
5241
+ for (let at = firstAtOrAfter2(column.keys, start); at < column.keys.length; at++) {
5242
+ if (!column.keys[at].startsWith(start)) break;
5243
+ found.push(column.values[at]);
5244
+ }
5245
+ return found;
5246
+ }
5247
+ };
5248
+
4793
5249
  // src/core/field-index/enum.ts
4794
5250
  function createEnumIndex() {
4795
5251
  const valueMap = /* @__PURE__ */ new Map();
5252
+ let affixes = null;
5253
+ function affixesOf() {
5254
+ affixes ??= new ValueAffixes((value) => valueMap.has(value), valueMap.keys());
5255
+ return affixes;
5256
+ }
4796
5257
  function allDocs() {
4797
5258
  const result = /* @__PURE__ */ new Set();
4798
5259
  for (const docSet of valueMap.values()) {
@@ -4806,6 +5267,7 @@ function createEnumIndex() {
4806
5267
  if (!docSet) {
4807
5268
  docSet = /* @__PURE__ */ new Set();
4808
5269
  valueMap.set(value, docSet);
5270
+ affixes?.add(value);
4809
5271
  }
4810
5272
  docSet.add(internalId);
4811
5273
  },
@@ -4813,7 +5275,15 @@ function createEnumIndex() {
4813
5275
  const docSet = valueMap.get(value);
4814
5276
  if (!docSet) return;
4815
5277
  docSet.delete(internalId);
4816
- if (docSet.size === 0) valueMap.delete(value);
5278
+ if (docSet.size > 0) return;
5279
+ valueMap.delete(value);
5280
+ affixes?.forget();
5281
+ },
5282
+ queryPrefixBitset(prefix, capacity) {
5283
+ return this.queryInBitset(affixesOf().withPrefix(prefix), capacity);
5284
+ },
5285
+ querySuffixBitset(suffix, capacity) {
5286
+ return this.queryInBitset(affixesOf().withSuffix(suffix), capacity);
4817
5287
  },
4818
5288
  queryEq(value) {
4819
5289
  const docSet = valueMap.get(value);
@@ -4889,6 +5359,7 @@ function createEnumIndex() {
4889
5359
  },
4890
5360
  clear() {
4891
5361
  valueMap.clear();
5362
+ affixes = null;
4892
5363
  },
4893
5364
  serialize() {
4894
5365
  const result = /* @__PURE__ */ Object.create(null);
@@ -4899,6 +5370,7 @@ function createEnumIndex() {
4899
5370
  },
4900
5371
  deserialize(data) {
4901
5372
  valueMap.clear();
5373
+ affixes = null;
4902
5374
  for (const value of Object.keys(data)) {
4903
5375
  valueMap.set(value, new Set(data[value]));
4904
5376
  }
@@ -5080,6 +5552,44 @@ function createNumericIndex() {
5080
5552
  };
5081
5553
  }
5082
5554
 
5555
+ // src/core/partition/value-sizes.ts
5556
+ function keepsValueSize(base) {
5557
+ return base === "string" || base === "verbatim" || base.endsWith("[]");
5558
+ }
5559
+ function valueSizeOf(value) {
5560
+ if (Array.isArray(value)) return value.length;
5561
+ return value === "" ? 0 : 1;
5562
+ }
5563
+ function sizeIndexOf(indexes, fieldPath) {
5564
+ let index = indexes.get(fieldPath);
5565
+ if (index === void 0) {
5566
+ index = createNumericIndex();
5567
+ indexes.set(fieldPath, index);
5568
+ }
5569
+ return index;
5570
+ }
5571
+ function recordValueSize(indexes, internalId, fieldPath, base, value) {
5572
+ if (value === void 0 || value === null || !keepsValueSize(base)) return;
5573
+ sizeIndexOf(indexes, fieldPath).insert(internalId, valueSizeOf(value));
5574
+ }
5575
+ function forgetValueSize(indexes, internalId, fieldPath, base, value) {
5576
+ if (value === void 0 || value === null || !keepsValueSize(base)) return;
5577
+ indexes.get(fieldPath)?.remove(internalId, valueSizeOf(value));
5578
+ }
5579
+ function rebuildSizeIndexes(state, schema) {
5580
+ state.sizeIndexes.clear();
5581
+ const fields = schemaFieldsOf(schema).filter((field) => keepsValueSize(field.base));
5582
+ if (fields.length === 0) return;
5583
+ for (const [docId, stored] of state.docStore.all()) {
5584
+ const internalId = state.docStore.getInternalId(docId);
5585
+ if (internalId === void 0) continue;
5586
+ for (const { path, base } of fields) {
5587
+ const value = getNestedValue(stored.fields, path);
5588
+ recordValueSize(state.sizeIndexes, internalId, path, base, value);
5589
+ }
5590
+ }
5591
+ }
5592
+
5083
5593
  // src/core/partition/indexing.ts
5084
5594
  function ensureFieldIndex(state, fieldPath, base) {
5085
5595
  if ((base === "number" || base === "number[]") && !state.numericIndexes.has(fieldPath)) {
@@ -5218,10 +5728,12 @@ function indexDocument(state, docId, document2, fields, language, options) {
5218
5728
  const fieldLengths = {};
5219
5729
  const tokensByField = {};
5220
5730
  const internalId = resolveInternalId(state, docId);
5221
- for (const { path: fieldPath, base } of fields) {
5731
+ for (const { path: fieldPath, base, pattern } of fields) {
5222
5732
  const value = getNestedValue(document2, fieldPath);
5223
5733
  if (value === void 0 || value === null) continue;
5734
+ recordValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
5224
5735
  ensureFieldIndex(state, fieldPath, base);
5736
+ if (pattern) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
5225
5737
  if (base === "string") {
5226
5738
  indexStringField(state, internalId, fieldPath, value, language, options, fieldLengths, tokensByField);
5227
5739
  } else if (base === "number") {
@@ -5265,9 +5777,11 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
5265
5777
  const fields = storedDoc.fields;
5266
5778
  const internalId = resolveInternalId(state, docId);
5267
5779
  const opts = tokenizeOptions(options);
5268
- for (const { path: fieldPath, base } of schemaFields) {
5780
+ for (const { path: fieldPath, base, pattern } of schemaFields) {
5269
5781
  const value = getNestedValue(fields, fieldPath);
5270
5782
  if (value === void 0 || value === null) continue;
5783
+ forgetValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
5784
+ if (pattern) state.patternIndexes.get(fieldPath)?.remove(internalId, value);
5271
5785
  if (base === "string") {
5272
5786
  const result = tokenize(value, language, opts);
5273
5787
  fieldLengths[fieldPath] = result.tokens.length;
@@ -5339,16 +5853,24 @@ function resolveInternalId2(state, docId) {
5339
5853
  }
5340
5854
  function updateFieldIndexOnly(state, docId, oldFields, newDoc, fields) {
5341
5855
  const internalId = resolveInternalId2(state, docId);
5342
- for (const { path: fieldPath, base: fieldType } of fields) {
5856
+ for (const { path: fieldPath, base: fieldType, pattern } of fields) {
5343
5857
  if (isWordIndexedBase(fieldType)) continue;
5344
5858
  const oldVal = getNestedValue(oldFields, fieldPath);
5345
5859
  const newVal = getNestedValue(newDoc, fieldPath);
5346
5860
  if (oldVal === newVal) continue;
5347
5861
  if (oldVal !== void 0 && oldVal !== null && newVal !== void 0 && newVal !== null && typeof oldVal === typeof newVal && JSON.stringify(oldVal) === JSON.stringify(newVal))
5348
5862
  continue;
5863
+ forgetValueSize(state.sizeIndexes, internalId, fieldPath, fieldType, oldVal);
5864
+ recordValueSize(state.sizeIndexes, internalId, fieldPath, fieldType, newVal);
5349
5865
  if (newVal !== void 0 && newVal !== null) {
5350
5866
  ensureFieldIndex(state, fieldPath, fieldType);
5351
5867
  }
5868
+ if (pattern) {
5869
+ if (oldVal !== void 0 && oldVal !== null) state.patternIndexes.get(fieldPath)?.remove(internalId, oldVal);
5870
+ if (newVal !== void 0 && newVal !== null) {
5871
+ patternIndexOf(state.patternIndexes, fieldPath).add(internalId, newVal);
5872
+ }
5873
+ }
5352
5874
  if (fieldType === "number") {
5353
5875
  const numIdx = state.numericIndexes.get(fieldPath);
5354
5876
  if (numIdx) {
@@ -5410,27 +5932,224 @@ function updateFieldIndexOnly(state, docId, oldFields, newDoc, fields) {
5410
5932
  }
5411
5933
  }
5412
5934
 
5413
- // src/filters/combinators.ts
5414
- function applyAndBitset(bitsets) {
5415
- if (bitsets.length === 0) return new Uint32Array(0);
5416
- if (bitsets.length === 1) return bitsets[0];
5417
- let result = bitsets[0];
5418
- for (let i = 1; i < bitsets.length; i++) {
5419
- result = bitsetAnd(result, bitsets[i]);
5935
+ // src/filters/constants.ts
5936
+ var MAX_FILTER_DEPTH = 30;
5937
+ var MAX_FILTER_TESTS = 1024;
5938
+ var MAX_FILTER_ARRAY_SIZE = 65536;
5939
+ var GEO_UNIT_METRES = { km: 1e3, mi: 1609.344, m: 1 };
5940
+
5941
+ // src/filters/pattern-tests.ts
5942
+ function excluding(context, fieldPath, excluded) {
5943
+ const universe = bitsetAnd(context.presentBitset(fieldPath), context.allDocIdsBitset);
5944
+ return bitsetAnd(universe, bitsetNot(excluded, context.capacity));
5945
+ }
5946
+ function textsOf(operand) {
5947
+ const texts = /* @__PURE__ */ new Set();
5948
+ if (!Array.isArray(operand)) return texts;
5949
+ for (const text of operand) if (typeof text === "string") texts.add(text);
5950
+ return texts;
5951
+ }
5952
+ function evaluatePatternTests(fieldPath, filter, context, bitsets) {
5953
+ const caseFold = filter.caseFold === true;
5954
+ const matching = (operator, texts) => {
5955
+ const into = createBitSet(context.capacity);
5956
+ for (const text of texts) context.matchPattern(fieldPath, operator, text, caseFold, into);
5957
+ return into;
5958
+ };
5959
+ if (typeof filter.eq === "string") bitsets.push(matching("eq", [filter.eq]));
5960
+ if (typeof filter.ne === "string") bitsets.push(excluding(context, fieldPath, matching("eq", [filter.ne])));
5961
+ if (filter.in !== void 0) bitsets.push(matching("eq", textsOf(filter.in)));
5962
+ if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq", textsOf(filter.nin))));
5963
+ if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
5964
+ if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
5965
+ if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
5966
+ }
5967
+
5968
+ // src/filters/field-tests.ts
5969
+ var COMPARISON_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
5970
+ var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
5971
+ var FieldBits = class {
5972
+ constructor(context, fieldPath) {
5973
+ this.context = context;
5974
+ this.fieldPath = fieldPath;
5420
5975
  }
5421
- return result;
5976
+ none() {
5977
+ return createBitSet(this.context.capacity);
5978
+ }
5979
+ present() {
5980
+ return this.context.presentBitset(this.fieldPath);
5981
+ }
5982
+ holdingSomething() {
5983
+ const empty = this.context.sizeIndex(this.fieldPath)?.queryEqBitset(0, this.context.capacity);
5984
+ return empty === void 0 ? this.present() : this.without(this.present(), empty);
5985
+ }
5986
+ without(from, removed) {
5987
+ const kept = from.slice();
5988
+ bitsetAndNotInto(kept, removed);
5989
+ return kept;
5990
+ }
5991
+ complement(bits) {
5992
+ return this.without(this.context.allDocIdsBitset, bits);
5993
+ }
5994
+ };
5995
+ function comparisonBits(index, operator, operand, bits, capacity) {
5996
+ if (operator === "between") {
5997
+ const [low, high] = operand;
5998
+ return index?.queryBetweenBitset(low, high, capacity) ?? bits.none();
5999
+ }
6000
+ const value = operand;
6001
+ if (operator === "ne") return bits.without(bits.present(), index?.queryEqBitset(value, capacity) ?? bits.none());
6002
+ if (index === void 0) return bits.none();
6003
+ if (operator === "eq") return index.queryEqBitset(value, capacity);
6004
+ if (operator === "gt") return index.queryGtBitset(value, capacity);
6005
+ if (operator === "gte") return index.queryGteBitset(value, capacity);
6006
+ if (operator === "lt") return index.queryLtBitset(value, capacity);
6007
+ return index.queryLteBitset(value, capacity);
6008
+ }
6009
+ function elementBits(context, fieldPath, parsed, value) {
6010
+ const { capacity } = context;
6011
+ if (parsed.pattern) {
6012
+ const matches = createBitSet(capacity);
6013
+ context.matchPattern(fieldPath, "eq", value, false, matches);
6014
+ return matches;
6015
+ }
6016
+ if (typeof value === "number")
6017
+ return context.numericIndex(fieldPath)?.queryEqBitset(value, capacity) ?? createBitSet(capacity);
6018
+ if (typeof value === "boolean")
6019
+ return context.booleanIndex(fieldPath)?.queryEqBitset(value, capacity) ?? createBitSet(capacity);
6020
+ return context.enumIndex(fieldPath)?.queryEqBitset(value, capacity) ?? createBitSet(capacity);
6021
+ }
6022
+ function anyElementBits(context, fieldPath, parsed, values) {
6023
+ const { capacity } = context;
6024
+ const union = createBitSet(capacity);
6025
+ if (parsed.pattern) {
6026
+ for (const value of values) context.matchPattern(fieldPath, "eq", value, false, union);
6027
+ return union;
6028
+ }
6029
+ if (parsed.base === "enum[]") {
6030
+ return context.enumIndex(fieldPath)?.queryInBitset(values, capacity) ?? union;
6031
+ }
6032
+ const numbers = context.numericIndex(fieldPath);
6033
+ const booleans = context.booleanIndex(fieldPath);
6034
+ for (const value of values) {
6035
+ if (typeof value === "number") for (const docId of numbers?.queryEq(value) ?? []) bitsetSet(union, docId);
6036
+ if (typeof value === "boolean") for (const docId of booleans?.queryEq(value) ?? []) bitsetSet(union, docId);
6037
+ }
6038
+ return union;
6039
+ }
6040
+ function distinctScalars(values) {
6041
+ return [...new Set(values)];
5422
6042
  }
5423
- function applyOrBitset(bitsets) {
5424
- if (bitsets.length === 0) return new Uint32Array(0);
5425
- if (bitsets.length === 1) return bitsets[0];
5426
- let result = bitsets[0];
5427
- for (let i = 1; i < bitsets.length; i++) {
5428
- result = bitsetOr(result, bitsets[i]);
6043
+ function narrowByElements(fieldPath, f, parsed, context, bits, narrow) {
6044
+ if (f.matchesAny !== void 0) narrow(anyElementBits(context, fieldPath, parsed, distinctScalars(f.matchesAny)));
6045
+ if (f.containsAll !== void 0) {
6046
+ const all = bits.present().slice();
6047
+ for (const value of distinctScalars(f.containsAll)) {
6048
+ if (bitsetIsEmpty(all)) break;
6049
+ bitsetAndInto(all, elementBits(context, fieldPath, parsed, value));
6050
+ }
6051
+ narrow(all);
5429
6052
  }
5430
- return result;
6053
+ if (f.size !== void 0) {
6054
+ const sizes = context.sizeIndex(fieldPath);
6055
+ const comparison = f.size;
6056
+ for (const operator of COMPARISON_OPERATORS) {
6057
+ if (comparison[operator] === void 0) continue;
6058
+ narrow(comparisonBits(sizes, operator, comparison[operator], bits, context.capacity));
6059
+ }
6060
+ }
6061
+ }
6062
+ function narrowByValue(fieldPath, f, parsed, context, bits, narrow) {
6063
+ const { capacity } = context;
6064
+ const base = parsed.base;
6065
+ if (base === "number" || base === "number[]") {
6066
+ const index = context.numericIndex(fieldPath);
6067
+ for (const operator of COMPARISON_OPERATORS) {
6068
+ const operand = f[operator];
6069
+ if (operand === void 0 || typeof operand === "string") continue;
6070
+ if (operator === "between" && typeof operand[0] === "string") continue;
6071
+ narrow(comparisonBits(index, operator, operand, bits, capacity));
6072
+ }
6073
+ return;
6074
+ }
6075
+ if (base === "boolean" || base === "boolean[]") {
6076
+ const index = context.booleanIndex(fieldPath);
6077
+ const equal = (value) => index?.queryEqBitset(value, capacity) ?? bits.none();
6078
+ if (typeof f.eq === "boolean") narrow(equal(f.eq));
6079
+ if (typeof f.ne === "boolean") narrow(bits.without(bits.present(), equal(f.ne)));
6080
+ return;
6081
+ }
6082
+ if (base === "enum" || base === "enum[]") {
6083
+ const index = context.enumIndex(fieldPath);
6084
+ const equal = (value) => index?.queryEqBitset(value, capacity) ?? bits.none();
6085
+ const anyOf = (values) => index?.queryInBitset(values, capacity) ?? bits.none();
6086
+ if (typeof f.eq === "string") narrow(equal(f.eq));
6087
+ if (typeof f.ne === "string") narrow(bits.without(bits.present(), equal(f.ne)));
6088
+ if (Array.isArray(f.in)) narrow(anyOf(f.in));
6089
+ if (Array.isArray(f.nin)) narrow(bits.without(bits.present(), anyOf(f.nin)));
6090
+ if (typeof f.startsWith === "string") narrow(index?.queryPrefixBitset(f.startsWith, capacity) ?? bits.none());
6091
+ if (typeof f.endsWith === "string") narrow(index?.querySuffixBitset(f.endsWith, capacity) ?? bits.none());
6092
+ }
6093
+ }
6094
+ function narrowByLocation(fieldPath, f, context, narrow) {
6095
+ const index = context.geoIndex(fieldPath);
6096
+ if (f.radius !== void 0) {
6097
+ const circle = f.radius;
6098
+ const found = index?.radiusQuery(
6099
+ circle.lat,
6100
+ circle.lon,
6101
+ circle.distance * GEO_UNIT_METRES[circle.unit],
6102
+ circle.inside ?? true,
6103
+ circle.highPrecision ?? false
6104
+ );
6105
+ narrow(bitsetFromSet(found ?? /* @__PURE__ */ new Set(), context.capacity));
6106
+ }
6107
+ if (f.polygon !== void 0) {
6108
+ const polygon = f.polygon;
6109
+ narrow(bitsetFromSet(index?.polygonQuery(polygon.points, polygon.inside ?? true) ?? /* @__PURE__ */ new Set(), context.capacity));
6110
+ }
6111
+ }
6112
+ function narrowByPresence(f, bits, narrow) {
6113
+ if (typeof f.exists === "boolean") narrow(f.exists ? bits.present() : bits.complement(bits.present()));
6114
+ if (typeof f.notExists === "boolean") narrow(f.notExists ? bits.complement(bits.present()) : bits.present());
6115
+ if (typeof f.isNotEmpty === "boolean") {
6116
+ const holding = bits.holdingSomething();
6117
+ narrow(f.isNotEmpty ? holding : bits.complement(holding));
6118
+ }
6119
+ if (typeof f.isEmpty === "boolean") {
6120
+ const holding = bits.holdingSomething();
6121
+ narrow(f.isEmpty ? bits.complement(holding) : holding);
6122
+ }
6123
+ }
6124
+ function textBoundsOf(operator, bound) {
6125
+ if (operator === "gt" || operator === "gte") {
6126
+ return { lower: bound, upper: null, lowerInclusive: operator === "gte", upperInclusive: false };
6127
+ }
6128
+ return { lower: null, upper: bound, upperInclusive: operator === "lte", lowerInclusive: false };
5431
6129
  }
5432
- function applyNotBitset(excluded, capacity) {
5433
- return bitsetNot(excluded, capacity);
6130
+ function narrowByTextRange(fieldPath, f, context, narrow) {
6131
+ for (const operator of TEXT_RANGE_OPERATORS) {
6132
+ const bound = f[operator];
6133
+ if (typeof bound === "string") narrow(context.textRangeBitset(fieldPath, textBoundsOf(operator, bound)));
6134
+ }
6135
+ if (!Array.isArray(f.between)) return;
6136
+ const [low, high] = f.between;
6137
+ if (typeof low !== "string" || typeof high !== "string") return;
6138
+ narrow(context.textRangeBitset(fieldPath, { lower: low, lowerInclusive: true, upper: high, upperInclusive: true }));
6139
+ }
6140
+ function narrowByFieldTests(fieldPath, f, parsed, context, narrow) {
6141
+ const bits = new FieldBits(context, fieldPath);
6142
+ narrowByPresence(f, bits, narrow);
6143
+ narrowByLocation(fieldPath, f, context, narrow);
6144
+ narrowByTextRange(fieldPath, f, context, narrow);
6145
+ if (parsed.pattern) {
6146
+ const patternBits = [];
6147
+ evaluatePatternTests(fieldPath, f, context, patternBits);
6148
+ for (const matched of patternBits) narrow(matched);
6149
+ } else {
6150
+ narrowByValue(fieldPath, f, parsed, context, bits, narrow);
6151
+ }
6152
+ narrowByElements(fieldPath, f, parsed, context, bits, narrow);
5434
6153
  }
5435
6154
 
5436
6155
  // src/filters/keys.ts
@@ -5447,6 +6166,8 @@ var FIELD_FILTER_OPERATORS = [
5447
6166
  "nin",
5448
6167
  "startsWith",
5449
6168
  "endsWith",
6169
+ "contains",
6170
+ "caseFold",
5450
6171
  "containsAll",
5451
6172
  "matchesAny",
5452
6173
  "size",
@@ -5458,14 +6179,7 @@ var FIELD_FILTER_OPERATORS = [
5458
6179
  "polygon"
5459
6180
  ];
5460
6181
 
5461
- // src/filters/constants.ts
5462
- var MAX_FILTER_DEPTH = 30;
5463
-
5464
- // src/filters/operands.ts
5465
- var RANGE_OPERATORS = ["gt", "lt", "gte", "lte"];
5466
- var TEXT_OPERATORS = ["startsWith", "endsWith"];
5467
- var EQUALITY_OPERATORS = ["eq", "ne"];
5468
- var LIST_OPERATORS = ["in", "nin"];
6182
+ // src/filters/operands/checks.ts
5469
6183
  function operandKindOf(type) {
5470
6184
  if (type === void 0) return void 0;
5471
6185
  if (type === "number" || type === "number[]") return "number";
@@ -5483,6 +6197,9 @@ function kindOfValue(value) {
5483
6197
  if (typeof value === "string") return "text";
5484
6198
  return void 0;
5485
6199
  }
6200
+ function isObjectRecord(value) {
6201
+ return value !== null && typeof value === "object" && !Array.isArray(value);
6202
+ }
5486
6203
  function invalid(message, fieldPath, operator) {
5487
6204
  throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message, { fieldPath, operator });
5488
6205
  }
@@ -5505,6 +6222,20 @@ function requireTextRangeField(fieldPath, operator, type) {
5505
6222
  operator
5506
6223
  );
5507
6224
  }
6225
+ function requireListLength(fieldPath, operator, values) {
6226
+ if (values.length <= MAX_FILTER_ARRAY_SIZE) return;
6227
+ invalid(
6228
+ `"${operator}" on field "${fieldPath}" holds ${values.length} values, while a filter list holds at most ${MAX_FILTER_ARRAY_SIZE}`,
6229
+ fieldPath,
6230
+ operator
6231
+ );
6232
+ }
6233
+
6234
+ // src/filters/operands/geo.ts
6235
+ function requireFlag(fieldPath, operator, flag, value) {
6236
+ if (value === void 0 || typeof value === "boolean") return;
6237
+ invalid(`"${operator}.${flag}" on field "${fieldPath}" must be true or false`, fieldPath, operator);
6238
+ }
5508
6239
  function requireRadius(fieldPath, radius) {
5509
6240
  const circle = radius;
5510
6241
  if (circle === null || typeof circle !== "object") {
@@ -5515,6 +6246,15 @@ function requireRadius(fieldPath, radius) {
5515
6246
  invalid(`"radius.${key}" on field "${fieldPath}" must be a finite number`, fieldPath, "radius");
5516
6247
  }
5517
6248
  }
6249
+ if (typeof circle.unit !== "string" || !Object.hasOwn(GEO_UNIT_METRES, circle.unit)) {
6250
+ invalid(
6251
+ `"radius.unit" on field "${fieldPath}" must be one of ${Object.keys(GEO_UNIT_METRES).join(", ")}, and this filter sets ${String(circle.unit)}`,
6252
+ fieldPath,
6253
+ "radius"
6254
+ );
6255
+ }
6256
+ requireFlag(fieldPath, "radius", "inside", circle.inside);
6257
+ requireFlag(fieldPath, "radius", "highPrecision", circle.highPrecision);
5518
6258
  if (circle.distance < 0) {
5519
6259
  invalid(
5520
6260
  `"radius.distance" on field "${fieldPath}" must be zero or more, and this filter sets ${String(circle.distance)}`,
@@ -5522,13 +6262,197 @@ function requireRadius(fieldPath, radius) {
5522
6262
  "radius"
5523
6263
  );
5524
6264
  }
6265
+ if (!isLatitude(circle.lat)) {
6266
+ invalid(
6267
+ `"radius.lat" on field "${fieldPath}" must be from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES}, and this filter sets ${String(circle.lat)}`,
6268
+ fieldPath,
6269
+ "radius"
6270
+ );
6271
+ }
6272
+ if (!isLongitude(circle.lon)) {
6273
+ invalid(
6274
+ `"radius.lon" on field "${fieldPath}" must be from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, and this filter sets ${String(circle.lon)}`,
6275
+ fieldPath,
6276
+ "radius"
6277
+ );
6278
+ }
6279
+ }
6280
+ function requireGeoOperands(fieldPath, filter) {
6281
+ if (filter.radius !== void 0) requireRadius(fieldPath, filter.radius);
6282
+ if (filter.polygon === void 0) return;
6283
+ if (!isObjectRecord(filter.polygon)) {
6284
+ invalid(`"polygon" on field "${fieldPath}" must be an object`, fieldPath, "polygon");
6285
+ }
6286
+ requireFlag(fieldPath, "polygon", "inside", filter.polygon.inside);
6287
+ const points = filter.polygon.points;
6288
+ if (Array.isArray(points)) requireListLength(fieldPath, "polygon", points);
6289
+ requirePolygonRing(points);
6290
+ }
6291
+
6292
+ // src/filters/operands/lists.ts
6293
+ var WHOLE_VALUE_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "matchesAny", "containsAll"];
6294
+ var ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
6295
+ var PRESENCE_OPERATORS = ["exists", "notExists", "isEmpty", "isNotEmpty"];
6296
+ var SIZE_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
6297
+ var SIZE_OPERATOR_SET = new Set(SIZE_OPERATORS);
6298
+ var ELEMENT_KIND_BY_BASE = {
6299
+ "number[]": "number",
6300
+ "boolean[]": "boolean",
6301
+ "enum[]": "text",
6302
+ "string[]": "text",
6303
+ "verbatim[]": "text"
6304
+ };
6305
+ function requireWholeValueIndex(fieldPath, filter, type) {
6306
+ const parsed = parsedTypeOf(type);
6307
+ if (parsed === void 0 || parsed.pattern || parsed.base !== "string" && parsed.base !== "string[]") return;
6308
+ for (const operator of WHOLE_VALUE_OPERATORS) {
6309
+ if (filter[operator] === void 0) continue;
6310
+ invalid(
6311
+ `"${operator}" compares whole values of field "${fieldPath}", which the engine finds only through a pattern index. A "${parsed.base}" field keeps a pattern index where its type includes "pattern", so declare the field as "${parsed.base}:pattern" in a new index to filter it this way`,
6312
+ fieldPath,
6313
+ operator
6314
+ );
6315
+ }
5525
6316
  }
6317
+ function requirePresenceOperands(fieldPath, filter) {
6318
+ for (const operator of PRESENCE_OPERATORS) {
6319
+ const operand = filter[operator];
6320
+ if (operand !== void 0 && typeof operand !== "boolean") {
6321
+ invalid(`"${operator}" on field "${fieldPath}" must be true or false`, fieldPath, operator);
6322
+ }
6323
+ }
6324
+ }
6325
+ function listElementKind(fieldPath, operator, type) {
6326
+ const base = parsedTypeOf(type)?.base;
6327
+ const kind = base === void 0 ? void 0 : ELEMENT_KIND_BY_BASE[base];
6328
+ if (kind !== void 0) return kind;
6329
+ return invalid(
6330
+ `"${operator}" tests the elements of a list field, while the schema declares field "${fieldPath}" as "${type}"`,
6331
+ fieldPath,
6332
+ operator
6333
+ );
6334
+ }
6335
+ function requireElementOperands(fieldPath, filter, type) {
6336
+ for (const operator of ELEMENT_OPERATORS) {
6337
+ const values = filter[operator];
6338
+ if (values === void 0) continue;
6339
+ if (!Array.isArray(values)) invalid(`"${operator}" on field "${fieldPath}" must be a list`, fieldPath, operator);
6340
+ requireListLength(fieldPath, operator, values);
6341
+ const kind = listElementKind(fieldPath, operator, type);
6342
+ if (values.some((value) => kindOfValue(value) !== kind)) {
6343
+ invalid(
6344
+ `Every value of "${operator}" on field "${fieldPath}" must be a ${kind === "text" ? "string" : kind === "number" ? "finite number" : "boolean"}, because the field holds a list of them`,
6345
+ fieldPath,
6346
+ operator
6347
+ );
6348
+ }
6349
+ }
6350
+ }
6351
+ function requireSizeOperands(fieldPath, filter, type) {
6352
+ const size = filter.size;
6353
+ if (size === void 0) return;
6354
+ if (!isObjectRecord(size)) invalid(`"size" on field "${fieldPath}" must be a comparison object`, fieldPath, "size");
6355
+ listElementKind(fieldPath, "size", type);
6356
+ for (const [operator, operand] of Object.entries(size)) {
6357
+ if (!SIZE_OPERATOR_SET.has(operator)) {
6358
+ invalid(
6359
+ `"size" on field "${fieldPath}" takes ${SIZE_OPERATORS.join(", ")}, while "${operator}" is none of them`,
6360
+ fieldPath,
6361
+ "size"
6362
+ );
6363
+ }
6364
+ const bounds = operator === "between" ? operand : [operand];
6365
+ if (!Array.isArray(bounds) || bounds.length !== (operator === "between" ? 2 : 1) || !bounds.every(isFiniteNumber)) {
6366
+ invalid(
6367
+ `"size.${operator}" on field "${fieldPath}" must be ${operator === "between" ? "two finite numbers, the lower bound first" : "a finite number"}`,
6368
+ fieldPath,
6369
+ "size"
6370
+ );
6371
+ }
6372
+ }
6373
+ }
6374
+
6375
+ // src/filters/operands/pattern.ts
6376
+ function requirePatternField(fieldPath, operator, type) {
6377
+ if (parsedTypeOf(type)?.pattern === true) return;
6378
+ const declared = type === void 0 ? `The schema declares no field "${fieldPath}"` : `Field "${fieldPath}" has type "${type}"`;
6379
+ invalid(
6380
+ `"${operator}" applies only to a pattern field, because only a pattern field has a pattern index. A pattern field is a "verbatim" field, or a text field whose type includes "pattern". ${declared}`,
6381
+ fieldPath,
6382
+ operator
6383
+ );
6384
+ }
6385
+ function requirePatternText(fieldPath, operator, text, caseFold) {
6386
+ if (exceedsCodePoints(text, MAX_PATTERN_TEXT_CODE_POINTS)) {
6387
+ invalid(
6388
+ `The text of "${operator}" on pattern field "${fieldPath}" holds more than ${MAX_PATTERN_TEXT_CODE_POINTS} code points`,
6389
+ fieldPath,
6390
+ operator
6391
+ );
6392
+ }
6393
+ if (caseFold && foldedCodePointCount(text) > MAX_PATTERN_POSITIONS) {
6394
+ invalid(
6395
+ `The text of "${operator}" on pattern field "${fieldPath}" has a case fold of more than ${MAX_PATTERN_POSITIONS} code points, while a pattern holds at most ${MAX_PATTERN_POSITIONS} positions`,
6396
+ fieldPath,
6397
+ operator
6398
+ );
6399
+ }
6400
+ }
6401
+ var ELEMENT_OPERATORS2 = ["matchesAny", "containsAll"];
6402
+ function requirePatternOperands(fieldPath, filter, type) {
6403
+ if (filter.contains !== void 0) {
6404
+ if (typeof filter.contains !== "string") {
6405
+ invalid(`"contains" on field "${fieldPath}" must be a string`, fieldPath, "contains");
6406
+ }
6407
+ requirePatternField(fieldPath, "contains", type);
6408
+ }
6409
+ if (filter.caseFold !== void 0) {
6410
+ if (typeof filter.caseFold !== "boolean") {
6411
+ invalid(`"caseFold" on field "${fieldPath}" must be true or false`, fieldPath, "caseFold");
6412
+ }
6413
+ requirePatternField(fieldPath, "caseFold", type);
6414
+ }
6415
+ if (parsedTypeOf(type)?.pattern !== true) return;
6416
+ const caseFold = filter.caseFold === true;
6417
+ for (const operator of PATTERN_TEST_OPERATORS) {
6418
+ const operand = filter[operator];
6419
+ if (typeof operand === "string") requirePatternText(fieldPath, operator, operand, caseFold);
6420
+ if (!Array.isArray(operand)) continue;
6421
+ for (const text of operand) {
6422
+ if (typeof text === "string") requirePatternText(fieldPath, operator, text, caseFold);
6423
+ }
6424
+ }
6425
+ for (const operator of ELEMENT_OPERATORS2) {
6426
+ const operand = filter[operator];
6427
+ if (!Array.isArray(operand)) continue;
6428
+ for (const text of operand) {
6429
+ if (typeof text === "string") requirePatternText(fieldPath, operator, text, false);
6430
+ }
6431
+ }
6432
+ }
6433
+ function requireWellFormedText(fieldPath, filter) {
6434
+ for (const [operator, operand] of Object.entries(filter)) {
6435
+ const texts = Array.isArray(operand) ? operand : [operand];
6436
+ for (const text of texts) {
6437
+ if (typeof text !== "string" || !holdsUnpairedSurrogate(text)) continue;
6438
+ invalid(
6439
+ `The text of "${operator}" on field "${fieldPath}" holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so no stored value can hold it`,
6440
+ fieldPath,
6441
+ operator
6442
+ );
6443
+ }
6444
+ }
6445
+ }
6446
+
6447
+ // src/filters/operands/index.ts
6448
+ var RANGE_OPERATORS = ["gt", "lt", "gte", "lte"];
6449
+ var TEXT_OPERATORS = ["startsWith", "endsWith"];
6450
+ var EQUALITY_OPERATORS = ["eq", "ne"];
6451
+ var LIST_OPERATORS = ["in", "nin"];
6452
+ var GEO_OPERATORS = ["radius", "polygon"];
5526
6453
  function invalidShape(message) {
5527
6454
  throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message);
5528
6455
  }
5529
- function isObjectRecord(value) {
5530
- return value !== null && typeof value === "object" && !Array.isArray(value);
5531
- }
5532
6456
  function undeclaredFieldOnStrictIndex(field, option) {
5533
6457
  return new NarsilError(
5534
6458
  ErrorCodes.SEARCH_INVALID_FIELD,
@@ -5536,34 +6460,14 @@ function undeclaredFieldOnStrictIndex(field, option) {
5536
6460
  { field, option }
5537
6461
  );
5538
6462
  }
5539
- function requireValidExpression(expression, rules, depth) {
5540
- if (depth > MAX_FILTER_DEPTH) invalidShape(`A filter expression nests at most ${MAX_FILTER_DEPTH} levels deep`);
5541
- if (!isObjectRecord(expression)) invalidShape("Each filter expression must be an object");
5542
- const { fields, and, or, not } = expression;
5543
- if (fields !== void 0) {
5544
- if (!isObjectRecord(fields)) invalidShape('"fields" in a filter expression must be an object keyed by field path');
5545
- for (const [fieldPath, filter] of Object.entries(fields)) {
5546
- if (rules.declaredOnly && !Object.hasOwn(rules.fieldTypes, fieldPath)) {
5547
- throw undeclaredFieldOnStrictIndex(fieldPath, "filter");
5548
- }
5549
- if (isObjectRecord(filter)) requireValidOperands(fieldPath, filter, rules.fieldTypes[fieldPath]);
5550
- }
5551
- }
5552
- for (const [clause, list] of [
5553
- ["and", and],
5554
- ["or", or]
5555
- ]) {
5556
- if (list === void 0) continue;
5557
- if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
5558
- for (const nested of list) requireValidExpression(nested, rules, depth + 1);
5559
- }
5560
- if (not !== void 0) requireValidExpression(not, rules, depth + 1);
5561
- }
5562
- function requireValidFilter(expression, fieldTypes, declaredOnly = false) {
5563
- if (expression === void 0 || expression === null) return;
5564
- requireValidExpression(expression, { fieldTypes, declaredOnly }, 0);
6463
+ function undeclaredFilterField(field) {
6464
+ return new NarsilError(
6465
+ ErrorCodes.SEARCH_INVALID_FIELD,
6466
+ `The schema declares no field "${field}", while the engine answers a filter only from the indexes that it keeps for declared fields. Declare the field in the schema of a new index to filter by it`,
6467
+ { field, option: "filter" }
6468
+ );
5565
6469
  }
5566
- function requireValidOperands(fieldPath, filter, type) {
6470
+ function requireRangeOperands(fieldPath, filter, type) {
5567
6471
  for (const operator of RANGE_OPERATORS) {
5568
6472
  const bound = filter[operator];
5569
6473
  if (bound === void 0) continue;
@@ -5576,22 +6480,23 @@ function requireValidOperands(fieldPath, filter, type) {
5576
6480
  }
5577
6481
  requireFieldKind(fieldPath, operator, type, ["number"]);
5578
6482
  }
5579
- if (filter.between !== void 0) {
5580
- const bounds = filter.between;
5581
- const isPair = Array.isArray(bounds) && bounds.length === 2;
5582
- if (isPair && typeof bounds[0] === "string" && typeof bounds[1] === "string") {
5583
- requireTextRangeField(fieldPath, "between", type);
5584
- } else {
5585
- if (!isPair || !isFiniteNumber(bounds[0]) || !isFiniteNumber(bounds[1])) {
5586
- invalid(
5587
- `"between" on field "${fieldPath}" must be two finite numbers or two strings, the lower bound first`,
5588
- fieldPath,
5589
- "between"
5590
- );
5591
- }
5592
- requireFieldKind(fieldPath, "between", type, ["number"]);
5593
- }
6483
+ if (filter.between === void 0) return;
6484
+ const bounds = filter.between;
6485
+ const isPair = Array.isArray(bounds) && bounds.length === 2;
6486
+ if (isPair && typeof bounds[0] === "string" && typeof bounds[1] === "string") {
6487
+ requireTextRangeField(fieldPath, "between", type);
6488
+ return;
6489
+ }
6490
+ if (!isPair || !isFiniteNumber(bounds[0]) || !isFiniteNumber(bounds[1])) {
6491
+ invalid(
6492
+ `"between" on field "${fieldPath}" must be two finite numbers or two strings, the lower bound first`,
6493
+ fieldPath,
6494
+ "between"
6495
+ );
5594
6496
  }
6497
+ requireFieldKind(fieldPath, "between", type, ["number"]);
6498
+ }
6499
+ function requireValueOperands(fieldPath, filter, type) {
5595
6500
  for (const operator of TEXT_OPERATORS) {
5596
6501
  if (filter[operator] === void 0) continue;
5597
6502
  if (typeof filter[operator] !== "string") {
@@ -5617,247 +6522,74 @@ function requireValidOperands(fieldPath, filter, type) {
5617
6522
  if (!Array.isArray(values) || values.some((value) => typeof value !== "string")) {
5618
6523
  invalid(`"${operator}" on field "${fieldPath}" must be a list of strings`, fieldPath, operator);
5619
6524
  }
6525
+ requireListLength(fieldPath, operator, values);
5620
6526
  requireFieldKind(fieldPath, operator, type, ["text"]);
5621
6527
  }
5622
- if (filter.size !== void 0) {
5623
- const size = filter.size;
5624
- if (size === null || typeof size !== "object") {
5625
- invalid(`"size" on field "${fieldPath}" must be a comparison object`, fieldPath, "size");
5626
- }
5627
- requireValidOperands(`${fieldPath}.size`, size, "number");
5628
- }
5629
- if (filter.radius !== void 0) requireRadius(fieldPath, filter.radius);
5630
- if (filter.polygon !== void 0) {
5631
- if (!isObjectRecord(filter.polygon)) {
5632
- invalid(`"polygon" on field "${fieldPath}" must be an object`, fieldPath, "polygon");
5633
- }
5634
- requirePolygonRing(filter.polygon.points);
5635
- }
5636
- }
5637
-
5638
- // src/filters/operators/shared.ts
5639
- var METERS_PER_KM = 1e3;
5640
- var METERS_PER_MI = 1609.344;
5641
- function convertToMeters(distance, unit) {
5642
- if (unit === "km") return distance * METERS_PER_KM;
5643
- if (unit === "mi") return distance * METERS_PER_MI;
5644
- if (unit === "m") return distance;
5645
- throw new NarsilError(
5646
- ErrorCodes.SEARCH_INVALID_FILTER,
5647
- `The engine measures a radius in "km", "mi", or "m", and "${String(unit)}" is none of them`,
5648
- { unit: String(unit) }
5649
- );
5650
- }
5651
- function matchesNumericComparison(value, filter) {
5652
- if (filter.eq !== void 0 && value !== filter.eq) return false;
5653
- if (filter.ne !== void 0 && value === filter.ne) return false;
5654
- if (filter.gt !== void 0 && !(value > filter.gt)) return false;
5655
- if (filter.lt !== void 0 && !(value < filter.lt)) return false;
5656
- if (filter.gte !== void 0 && !(value >= filter.gte)) return false;
5657
- if (filter.lte !== void 0 && !(value <= filter.lte)) return false;
5658
- if (filter.between !== void 0 && !(value >= filter.between[0] && value <= filter.between[1])) return false;
5659
- return true;
5660
- }
5661
- function holdsValue(stored, value) {
5662
- return Array.isArray(stored) ? stored.includes(value) : stored === value;
5663
- }
5664
- function lacksValue(stored, value) {
5665
- if (stored === void 0 || stored === null) return false;
5666
- return Array.isArray(stored) ? !stored.includes(value) : stored !== value;
5667
- }
5668
- function isStringIn(values) {
5669
- return (element) => typeof element === "string" && values.has(element);
5670
- }
5671
- function holdsStringIn(stored, values) {
5672
- return Array.isArray(stored) ? stored.some(isStringIn(values)) : isStringIn(values)(stored);
5673
- }
5674
- function holdsNoStringIn(stored, values) {
5675
- if (Array.isArray(stored)) return !stored.some(isStringIn(values));
5676
- return typeof stored === "string" && !values.has(stored);
5677
- }
5678
- function holdsStringWhere(stored, test) {
5679
- const matches = (element) => typeof element === "string" && test(element);
5680
- return Array.isArray(stored) ? stored.some(matches) : matches(stored);
5681
- }
5682
-
5683
- // src/filters/operators/bitset.ts
5684
- function resolveBitset(bs) {
5685
- return typeof bs === "function" ? bs() : bs;
5686
- }
5687
- function scanToBitset(docIds, capacity, getValue, predicate) {
5688
- const resolved = resolveBitset(docIds);
5689
- const result = createBitSet(capacity);
5690
- for (let wi = 0; wi < resolved.length; wi++) {
5691
- let word = resolved[wi];
5692
- if (word === 0) continue;
5693
- const base = wi << 5;
5694
- while (word !== 0) {
5695
- const tz = Math.clz32(word & -word) ^ 31;
5696
- const id = base + tz;
5697
- if (predicate(getValue(id))) {
5698
- bitsetSet(result, id);
5699
- }
5700
- word &= word - 1;
5701
- }
5702
- }
5703
- return result;
5704
- }
5705
- function applyEqBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
5706
- if (fieldIndex?.type === "numeric" && typeof value === "number") {
5707
- return fieldIndex.index.eqBitset(value, capacity);
5708
- }
5709
- if (fieldIndex?.type === "boolean" && typeof value === "boolean") {
5710
- return value ? fieldIndex.index.getTrueBitset(capacity) : fieldIndex.index.getFalseBitset(capacity);
5711
- }
5712
- if (fieldIndex?.type === "enum" && typeof value === "string") {
5713
- return fieldIndex.index.getDocIdsBitset(value, capacity);
5714
- }
5715
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsValue(v, value));
5716
- }
5717
- function applyNeBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
5718
- if (fieldIndex?.type === "numeric" && typeof value === "number") {
5719
- const all = fieldIndex.index.allDocIdsBitset(capacity);
5720
- const eq = fieldIndex.index.eqBitset(value, capacity);
5721
- return bitsetAnd(all, bitsetNot(eq, capacity));
5722
- }
5723
- if (fieldIndex?.type === "boolean" && typeof value === "boolean") {
5724
- return value ? fieldIndex.index.getFalseBitset(capacity) : fieldIndex.index.getTrueBitset(capacity);
5725
- }
5726
- if (fieldIndex?.type === "enum" && typeof value === "string") {
5727
- const all = fieldIndex.index.allDocIdsBitset(capacity);
5728
- const eq = fieldIndex.index.getDocIdsBitset(value, capacity);
5729
- return bitsetAnd(all, bitsetNot(eq, capacity));
5730
- }
5731
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => lacksValue(v, value));
5732
- }
5733
- function applyGtBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
5734
- if (fieldIndex?.type === "numeric") {
5735
- return fieldIndex.index.gtBitset(value, capacity);
5736
- }
5737
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v > value);
5738
- }
5739
- function applyLtBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
5740
- if (fieldIndex?.type === "numeric") {
5741
- return fieldIndex.index.ltBitset(value, capacity);
5742
- }
5743
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v < value);
5744
- }
5745
- function applyGteBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
5746
- if (fieldIndex?.type === "numeric") {
5747
- return fieldIndex.index.gteBitset(value, capacity);
5748
- }
5749
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v >= value);
5750
- }
5751
- function applyLteBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
5752
- if (fieldIndex?.type === "numeric") {
5753
- return fieldIndex.index.lteBitset(value, capacity);
5754
- }
5755
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v <= value);
5756
- }
5757
- function applyBetweenBitset(range, allDocsBitset, capacity, getValue, fieldIndex) {
5758
- if (fieldIndex?.type === "numeric") {
5759
- return fieldIndex.index.betweenBitset(range[0], range[1], capacity);
6528
+ for (const operator of GEO_OPERATORS) {
6529
+ if (filter[operator] !== void 0) requireFieldKind(fieldPath, operator, type, ["geopoint"]);
5760
6530
  }
5761
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v >= range[0] && v <= range[1]);
5762
6531
  }
5763
- function applyInBitset(values, allDocsBitset, capacity, getValue, fieldIndex) {
5764
- if (fieldIndex?.type === "enum") {
5765
- return fieldIndex.index.getDocIdsInBitset(values, capacity);
6532
+ function requireValidOperands(fieldPath, filter, type) {
6533
+ requireWellFormedText(fieldPath, filter);
6534
+ requireRangeOperands(fieldPath, filter, type);
6535
+ requireValueOperands(fieldPath, filter, type);
6536
+ requireWholeValueIndex(fieldPath, filter, type);
6537
+ requirePresenceOperands(fieldPath, filter);
6538
+ requireElementOperands(fieldPath, filter, type);
6539
+ requireSizeOperands(fieldPath, filter, type);
6540
+ requirePatternOperands(fieldPath, filter, type);
6541
+ requireGeoOperands(fieldPath, filter);
6542
+ }
6543
+ function testsIn(filter) {
6544
+ let tests = 0;
6545
+ for (const [operator, operand] of Object.entries(filter)) {
6546
+ if (operator === "caseFold") continue;
6547
+ tests += operator === "size" && isObjectRecord(operand) ? Math.max(1, Object.keys(operand).length) : 1;
6548
+ }
6549
+ return Math.max(1, tests);
6550
+ }
6551
+ function countTests(walk, tests) {
6552
+ walk.tests += tests;
6553
+ if (walk.tests > MAX_FILTER_TESTS) {
6554
+ invalidShape(`A filter holds at most ${MAX_FILTER_TESTS} tests across all of its fields and nested expressions`);
6555
+ }
6556
+ }
6557
+ function holdsNestedExpressions(expression) {
6558
+ const { and, or, not } = expression;
6559
+ return Array.isArray(and) && and.length > 0 || Array.isArray(or) && or.length > 0 || not !== void 0;
6560
+ }
6561
+ function requireValidExpression(expression, walk, depth) {
6562
+ if (depth > MAX_FILTER_DEPTH) invalidShape(`A filter expression nests at most ${MAX_FILTER_DEPTH} levels deep`);
6563
+ if (!isObjectRecord(expression)) invalidShape("Each filter expression must be an object");
6564
+ const { fields, and, or, not } = expression;
6565
+ if (fields !== void 0 && !isObjectRecord(fields)) {
6566
+ invalidShape('"fields" in a filter expression must be an object keyed by field path');
6567
+ }
6568
+ const fieldFilters = Object.entries(fields ?? {});
6569
+ if (fieldFilters.length === 0 && !holdsNestedExpressions(expression)) countTests(walk, 1);
6570
+ for (const [fieldPath, filter] of fieldFilters) {
6571
+ const type = walk.fieldTypes[fieldPath];
6572
+ if (!Object.hasOwn(walk.fieldTypes, fieldPath) || type === void 0) throw undeclaredFilterField(fieldPath);
6573
+ if (!isObjectRecord(filter)) invalidShape(`The filter on field "${fieldPath}" must be an object of tests`);
6574
+ countTests(walk, testsIn(filter));
6575
+ requireValidOperands(fieldPath, filter, type);
5766
6576
  }
5767
- const valSet = new Set(values);
5768
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsStringIn(v, valSet));
5769
- }
5770
- function applyNinBitset(values, allDocsBitset, capacity, getValue, fieldIndex) {
5771
- if (fieldIndex?.type === "enum") {
5772
- const all = fieldIndex.index.allDocIdsBitset(capacity);
5773
- const matched = fieldIndex.index.getDocIdsInBitset(values, capacity);
5774
- return bitsetAnd(all, bitsetNot(matched, capacity));
6577
+ for (const [clause, list] of [
6578
+ ["and", and],
6579
+ ["or", or]
6580
+ ]) {
6581
+ if (list === void 0) continue;
6582
+ if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
6583
+ for (const nested of list) requireValidExpression(nested, walk, depth + 1);
5775
6584
  }
5776
- const valSet = new Set(values);
5777
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsNoStringIn(v, valSet));
5778
- }
5779
- function applyStartsWithBitset(prefix, allDocsBitset, capacity, getValue) {
5780
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsStringWhere(v, (text) => text.startsWith(prefix)));
5781
- }
5782
- function applyEndsWithBitset(suffix, allDocsBitset, capacity, getValue) {
5783
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsStringWhere(v, (text) => text.endsWith(suffix)));
5784
- }
5785
- function applyContainsAllBitset(values, allDocsBitset, capacity, getValue) {
5786
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
5787
- if (!Array.isArray(v)) return false;
5788
- const arr = v;
5789
- for (const val of values) {
5790
- if (!arr.includes(val)) return false;
5791
- }
5792
- return true;
5793
- });
5794
- }
5795
- function applyMatchesAnyBitset(values, allDocsBitset, capacity, getValue) {
5796
- const valSet = new Set(values);
5797
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
5798
- if (!Array.isArray(v)) return false;
5799
- for (const item of v) {
5800
- if (valSet.has(item)) return true;
5801
- }
5802
- return false;
5803
- });
5804
- }
5805
- function applySizeBitset(sizeFilter, allDocsBitset, capacity, getValue) {
5806
- return scanToBitset(
5807
- allDocsBitset,
5808
- capacity,
5809
- getValue,
5810
- (v) => Array.isArray(v) && matchesNumericComparison(v.length, sizeFilter)
5811
- );
5812
- }
5813
- function applyExistsBitset(allDocsBitset, capacity, getValue) {
5814
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => v !== void 0 && v !== null);
5815
- }
5816
- function applyNotExistsBitset(allDocsBitset, capacity, getValue) {
5817
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => v === void 0 || v === null);
5818
- }
5819
- function applyIsEmptyBitset(allDocsBitset, capacity, getValue) {
5820
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
5821
- if (v === void 0 || v === null) return true;
5822
- if (typeof v === "string" && v === "") return true;
5823
- if (Array.isArray(v) && v.length === 0) return true;
5824
- return false;
5825
- });
5826
- }
5827
- function applyIsNotEmptyBitset(allDocsBitset, capacity, getValue) {
5828
- return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
5829
- if (v === void 0 || v === null) return false;
5830
- if (typeof v === "string" && v === "") return false;
5831
- if (Array.isArray(v) && v.length === 0) return false;
5832
- return true;
5833
- });
5834
- }
5835
- function applyGeoRadiusBitset(filter, geoIndex, capacity) {
5836
- const distanceMeters = convertToMeters(filter.distance, filter.unit);
5837
- const setResult = geoIndex.radiusQuery(
5838
- filter.lat,
5839
- filter.lon,
5840
- distanceMeters,
5841
- filter.inside ?? true,
5842
- filter.highPrecision ?? false
5843
- );
5844
- return bitsetFromSet(setResult, capacity);
6585
+ if (not !== void 0) requireValidExpression(not, walk, depth + 1);
5845
6586
  }
5846
- function applyGeoPolygonBitset(filter, geoIndex, capacity) {
5847
- requirePolygonRing(filter.points);
5848
- const setResult = geoIndex.polygonQuery(filter.points, filter.inside ?? true);
5849
- return bitsetFromSet(setResult, capacity);
6587
+ function requireValidFilter(expression, fieldTypes) {
6588
+ if (expression === void 0 || expression === null) return;
6589
+ requireValidExpression(expression, { fieldTypes, tests: 0 }, 0);
5850
6590
  }
5851
6591
 
5852
6592
  // src/filters/evaluator.ts
5853
- var UNBOUNDED = { lower: null, upper: null };
5854
- function textBoundsOf(operator, bound) {
5855
- if (operator === "gt" || operator === "gte") {
5856
- return { ...UNBOUNDED, lower: bound, lowerInclusive: operator === "gte", upperInclusive: false };
5857
- }
5858
- return { ...UNBOUNDED, upper: bound, upperInclusive: operator === "lte", lowerInclusive: false };
5859
- }
5860
- var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
5861
6593
  var EXPRESSION_KEYS = new Set(FILTER_EXPRESSION_KEYS);
5862
6594
  var OPERATOR_KEYS = new Set(FIELD_FILTER_OPERATORS);
5863
6595
  function requireKnownKeys2(value, known, describe) {
@@ -5867,160 +6599,66 @@ function requireKnownKeys2(value, known, describe) {
5867
6599
  }
5868
6600
  }
5869
6601
  }
6602
+ var Narrowing = class {
6603
+ bits = null;
6604
+ narrow = (matched) => {
6605
+ if (this.bits === null) this.bits = matched.slice();
6606
+ else bitsetAndInto(this.bits, matched);
6607
+ };
6608
+ value(universe) {
6609
+ return this.bits ?? universe().slice();
6610
+ }
6611
+ };
6612
+ function narrowByField(fieldPath, filter, context, narrow) {
6613
+ requireKnownKeys2(
6614
+ filter,
6615
+ OPERATOR_KEYS,
6616
+ (key) => `Field "${fieldPath}" is tested with "${key}", which is no filter operator`
6617
+ );
6618
+ const type = context.fieldTypes[fieldPath];
6619
+ const parsed = parsedTypeOf(type);
6620
+ if (type === void 0 || parsed === void 0) throw undeclaredFilterField(fieldPath);
6621
+ const f = filter;
6622
+ requireValidOperands(fieldPath, f, type);
6623
+ narrowByFieldTests(fieldPath, f, parsed, context, narrow.narrow);
6624
+ }
5870
6625
  function evaluateFilters(expression, context) {
5871
6626
  requireKnownKeys2(
5872
6627
  expression,
5873
6628
  EXPRESSION_KEYS,
5874
6629
  (key) => `A filter expression takes "fields", "and", "or", and "not", and "${key}" is none of them. A per-field test such as { ${key}: { eq: ... } } belongs under "fields"`
5875
6630
  );
5876
- const bitsets = [];
6631
+ const narrowing = new Narrowing();
5877
6632
  if (expression.fields) {
5878
6633
  for (const [fieldPath, filter] of Object.entries(expression.fields)) {
5879
- bitsets.push(evaluateFieldFilter(fieldPath, filter, context));
6634
+ narrowByField(fieldPath, filter, context, narrowing);
5880
6635
  }
5881
6636
  }
5882
6637
  if (expression.and?.length) {
5883
- const andBitsets = expression.and.map((expr) => evaluateFilters(expr, context));
5884
- bitsets.push(applyAndBitset(andBitsets));
6638
+ for (const nested of expression.and) narrowing.narrow(evaluateFilters(nested, context));
5885
6639
  }
5886
6640
  if (expression.or?.length) {
5887
- const orBitsets = expression.or.map((expr) => evaluateFilters(expr, context));
5888
- bitsets.push(applyOrBitset(orBitsets));
6641
+ const union = createBitSet(context.capacity);
6642
+ for (const nested of expression.or) bitsetOrInto(union, evaluateFilters(nested, context));
6643
+ narrowing.narrow(union);
5889
6644
  }
5890
6645
  if (expression.not) {
5891
6646
  const excluded = evaluateFilters(expression.not, context);
5892
- const universe = context.allDocIdsBitset;
5893
- bitsets.push(applyAndBitset([universe, applyNotBitset(excluded, context.capacity)]));
6647
+ const kept = context.allDocIdsBitset.slice();
6648
+ bitsetAndNotInto(kept, excluded);
6649
+ narrowing.narrow(kept);
5894
6650
  }
5895
- if (bitsets.length === 0) return context.allDocIdsBitset;
5896
- if (bitsets.length === 1) return bitsets[0];
5897
- return applyAndBitset(bitsets);
5898
- }
5899
- function evaluateFieldFilter(fieldPath, filter, context) {
5900
- requireKnownKeys2(
5901
- filter,
5902
- OPERATOR_KEYS,
5903
- (key) => `Field "${fieldPath}" is tested with "${key}", which is no filter operator`
5904
- );
5905
- const f = filter;
5906
- requireValidOperands(fieldPath, f, context.fieldTypes[fieldPath]);
5907
- const fieldIndex = context.fieldIndexes[fieldPath];
5908
- const getValue = (internalId) => context.getFieldValue(internalId, fieldPath);
5909
- const bitsets = [];
5910
- const { capacity } = context;
5911
- let allDocsBitset = null;
5912
- const getAllDocsBitset = () => {
5913
- if (allDocsBitset === null) {
5914
- allDocsBitset = context.allDocIdsBitset;
5915
- }
5916
- return allDocsBitset;
5917
- };
5918
- if ("radius" in f && f.radius) {
5919
- const geoIndex = resolveGeoIndex(fieldPath, fieldIndex);
5920
- bitsets.push(applyGeoRadiusBitset(filter.radius, geoIndex, capacity));
5921
- }
5922
- if ("polygon" in f && f.polygon) {
5923
- const geoIndex = resolveGeoIndex(fieldPath, fieldIndex);
5924
- bitsets.push(applyGeoPolygonBitset(filter.polygon, geoIndex, capacity));
5925
- }
5926
- if ("eq" in f && f.eq !== void 0) {
5927
- bitsets.push(applyEqBitset(f.eq, getAllDocsBitset, capacity, getValue, fieldIndex));
5928
- }
5929
- if ("ne" in f && f.ne !== void 0) {
5930
- bitsets.push(applyNeBitset(f.ne, getAllDocsBitset, capacity, getValue, fieldIndex));
5931
- }
5932
- for (const operator of TEXT_RANGE_OPERATORS) {
5933
- const bound = f[operator];
5934
- if (typeof bound === "string") bitsets.push(context.textRangeBitset(fieldPath, textBoundsOf(operator, bound)));
5935
- }
5936
- if (typeof f.gt === "number") {
5937
- bitsets.push(applyGtBitset(f.gt, getAllDocsBitset, capacity, getValue, fieldIndex));
5938
- }
5939
- if (typeof f.lt === "number") {
5940
- bitsets.push(applyLtBitset(f.lt, getAllDocsBitset, capacity, getValue, fieldIndex));
5941
- }
5942
- if (typeof f.gte === "number") {
5943
- bitsets.push(applyGteBitset(f.gte, getAllDocsBitset, capacity, getValue, fieldIndex));
5944
- }
5945
- if (typeof f.lte === "number") {
5946
- bitsets.push(applyLteBitset(f.lte, getAllDocsBitset, capacity, getValue, fieldIndex));
5947
- }
5948
- if (Array.isArray(f.between)) {
5949
- const [low, high] = f.between;
5950
- if (typeof low === "string" && typeof high === "string") {
5951
- bitsets.push(
5952
- context.textRangeBitset(fieldPath, { lower: low, lowerInclusive: true, upper: high, upperInclusive: true })
5953
- );
5954
- } else {
5955
- bitsets.push(
5956
- applyBetweenBitset([low, high], getAllDocsBitset, capacity, getValue, fieldIndex)
5957
- );
5958
- }
5959
- }
5960
- if ("in" in f && f.in !== void 0) {
5961
- bitsets.push(applyInBitset(f.in, getAllDocsBitset, capacity, getValue, fieldIndex));
5962
- }
5963
- if ("nin" in f && f.nin !== void 0) {
5964
- bitsets.push(applyNinBitset(f.nin, getAllDocsBitset, capacity, getValue, fieldIndex));
5965
- }
5966
- if ("startsWith" in f && f.startsWith !== void 0) {
5967
- bitsets.push(applyStartsWithBitset(f.startsWith, getAllDocsBitset, capacity, getValue));
5968
- }
5969
- if ("endsWith" in f && f.endsWith !== void 0) {
5970
- bitsets.push(applyEndsWithBitset(f.endsWith, getAllDocsBitset, capacity, getValue));
5971
- }
5972
- if ("containsAll" in f && f.containsAll !== void 0) {
5973
- bitsets.push(
5974
- applyContainsAllBitset(f.containsAll, getAllDocsBitset, capacity, getValue)
5975
- );
5976
- }
5977
- if ("matchesAny" in f && f.matchesAny !== void 0) {
5978
- bitsets.push(
5979
- applyMatchesAnyBitset(f.matchesAny, getAllDocsBitset, capacity, getValue)
5980
- );
5981
- }
5982
- if ("size" in f && f.size !== void 0) {
5983
- bitsets.push(applySizeBitset(f.size, getAllDocsBitset, capacity, getValue));
5984
- }
5985
- if ("exists" in f && f.exists !== void 0) {
5986
- bitsets.push(
5987
- f.exists ? applyExistsBitset(getAllDocsBitset, capacity, getValue) : applyNotExistsBitset(getAllDocsBitset, capacity, getValue)
5988
- );
5989
- }
5990
- if ("notExists" in f && f.notExists !== void 0) {
5991
- bitsets.push(
5992
- f.notExists ? applyNotExistsBitset(getAllDocsBitset, capacity, getValue) : applyExistsBitset(getAllDocsBitset, capacity, getValue)
5993
- );
5994
- }
5995
- if ("isEmpty" in f && f.isEmpty !== void 0) {
5996
- bitsets.push(
5997
- f.isEmpty ? applyIsEmptyBitset(getAllDocsBitset, capacity, getValue) : applyIsNotEmptyBitset(getAllDocsBitset, capacity, getValue)
5998
- );
5999
- }
6000
- if ("isNotEmpty" in f && f.isNotEmpty !== void 0) {
6001
- bitsets.push(
6002
- f.isNotEmpty ? applyIsNotEmptyBitset(getAllDocsBitset, capacity, getValue) : applyIsEmptyBitset(getAllDocsBitset, capacity, getValue)
6003
- );
6004
- }
6005
- if (bitsets.length === 0) return context.allDocIdsBitset;
6006
- if (bitsets.length === 1) return bitsets[0];
6007
- return applyAndBitset(bitsets);
6008
- }
6009
- function resolveGeoIndex(fieldPath, fieldIndex) {
6010
- if (fieldIndex?.type !== "geopoint") {
6011
- throw new NarsilError(
6012
- ErrorCodes.SEARCH_INVALID_FILTER,
6013
- `Field "${fieldPath}" requires a geopoint index for geo filters`,
6014
- { fieldPath }
6015
- );
6016
- }
6017
- return fieldIndex.index;
6651
+ return narrowing.value(() => context.allDocIdsBitset);
6018
6652
  }
6019
6653
 
6020
6654
  // src/core/partition/constants.ts
6021
6655
  var TARGET_BLOCK_ENTRIES = 128;
6022
6656
  var PRUNING_REJECTION_SLACK = 1e-12;
6023
6657
  var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
6658
+ var SEGMENT_TOKEN_TABLE_START = 1 << 12;
6659
+ var SEGMENT_POSTING_LOG_START = 1 << 14;
6660
+ var SEGMENT_POSITION_POOL_START = 1 << 16;
6661
+ var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
6024
6662
  var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
6025
6663
  var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
6026
6664
 
@@ -6637,99 +7275,72 @@ function sortColumnBytes(state) {
6637
7275
  }
6638
7276
 
6639
7277
  // src/core/partition/filters.ts
6640
- var EMPTY_GEO_INDEX = {
6641
- radiusQuery: () => /* @__PURE__ */ new Set(),
6642
- polygonQuery: () => /* @__PURE__ */ new Set()
6643
- };
6644
- function buildFilterContext(state, schema) {
7278
+ function presentBitsetOf(state, fieldPath, type, capacity) {
7279
+ const base = parsedTypeOf(type)?.base;
7280
+ let present;
7281
+ if (base === "number") present = state.numericIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
7282
+ else if (base === "boolean") present = state.booleanIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
7283
+ else if (base === "enum") present = state.enumIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
7284
+ else if (base === "geopoint") present = state.geoIndexes.get(fieldPath)?.documentsBitset(capacity);
7285
+ else if (base !== void 0 && keepsValueSize(base)) {
7286
+ present = state.sizeIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
7287
+ }
7288
+ return present ?? createBitSet(capacity);
7289
+ }
7290
+ function liveDocumentsBitset(state, capacity) {
7291
+ const bits = createBitSet(capacity);
7292
+ for (const internalId of state.docStore.allInternalIds()) bitsetSet(bits, internalId);
7293
+ return bits;
7294
+ }
7295
+ function buildFilterContext(state, schema, meter) {
6645
7296
  const flat = getFlatSchema(state, schema);
6646
- const fieldIndexes = {};
6647
7297
  const capacity = state.docStore.internalIdCapacity();
6648
- for (const { path: fieldPath, base: fieldType } of schemaFieldsOf(schema)) {
6649
- if (fieldType === "number" || fieldType === "number[]") {
6650
- const numIdx = state.numericIndexes.get(fieldPath);
6651
- if (numIdx) {
6652
- fieldIndexes[fieldPath] = {
6653
- type: "numeric",
6654
- index: {
6655
- eq: (v) => numIdx.queryEq(v),
6656
- gt: (v) => numIdx.queryGt(v),
6657
- gte: (v) => numIdx.queryGte(v),
6658
- lt: (v) => numIdx.queryLt(v),
6659
- lte: (v) => numIdx.queryLte(v),
6660
- between: (min, max) => numIdx.queryBetween(min, max),
6661
- allDocIds: () => numIdx.getAllDocIds(),
6662
- eqBitset: (v, cap) => numIdx.queryEqBitset(v, cap),
6663
- gtBitset: (v, cap) => numIdx.queryGtBitset(v, cap),
6664
- gteBitset: (v, cap) => numIdx.queryGteBitset(v, cap),
6665
- ltBitset: (v, cap) => numIdx.queryLtBitset(v, cap),
6666
- lteBitset: (v, cap) => numIdx.queryLteBitset(v, cap),
6667
- betweenBitset: (min, max, cap) => numIdx.queryBetweenBitset(min, max, cap),
6668
- allDocIdsBitset: (cap) => numIdx.getAllDocIdsBitset(cap)
6669
- }
6670
- };
6671
- }
6672
- } else if (fieldType === "boolean" || fieldType === "boolean[]") {
6673
- const boolIdx = state.booleanIndexes.get(fieldPath);
6674
- if (boolIdx) {
6675
- fieldIndexes[fieldPath] = {
6676
- type: "boolean",
6677
- index: {
6678
- getTrue: () => boolIdx.queryEq(true),
6679
- getFalse: () => boolIdx.queryEq(false),
6680
- allDocIds: () => boolIdx.getAllDocIds(),
6681
- getTrueBitset: (cap) => boolIdx.queryEqBitset(true, cap),
6682
- getFalseBitset: (cap) => boolIdx.queryEqBitset(false, cap),
6683
- allDocIdsBitset: (cap) => boolIdx.getAllDocIdsBitset(cap)
6684
- }
6685
- };
6686
- }
6687
- } else if (fieldType === "enum" || fieldType === "enum[]") {
6688
- const enumIdx = state.enumIndexes.get(fieldPath);
6689
- if (enumIdx) {
6690
- fieldIndexes[fieldPath] = {
6691
- type: "enum",
6692
- index: {
6693
- getDocIds: (v) => enumIdx.queryEq(v),
6694
- allDocIds: () => enumIdx.getAllDocIds(),
6695
- getDocIdsBitset: (v, cap) => enumIdx.queryEqBitset(v, cap),
6696
- getDocIdsInBitset: (values, cap) => enumIdx.queryInBitset(values, cap),
6697
- allDocIdsBitset: (cap) => enumIdx.getAllDocIdsBitset(cap)
6698
- }
6699
- };
6700
- }
6701
- } else if (fieldType === "geopoint") {
6702
- const geoIdx = state.geoIndexes.get(fieldPath);
6703
- fieldIndexes[fieldPath] = {
6704
- type: "geopoint",
6705
- index: geoIdx === void 0 ? EMPTY_GEO_INDEX : geoIdx
6706
- };
6707
- }
7298
+ const patternFields = /* @__PURE__ */ new Set();
7299
+ for (const { path: fieldPath, pattern } of schemaFieldsOf(schema)) {
7300
+ if (pattern) patternFields.add(fieldPath);
6708
7301
  }
6709
- let cachedAllDocIds = null;
7302
+ const getFieldValue = (internalId, fieldPath) => getFieldValueByInternalId(state.docStore, internalId, fieldPath);
6710
7303
  let cachedAllDocIdsBitset = null;
7304
+ const cachedPresentBitsets = /* @__PURE__ */ new Map();
6711
7305
  return {
6712
7306
  fieldTypes: flat,
6713
- fieldIndexes,
6714
- getFieldValue: (internalId, fieldPath) => getFieldValueByInternalId(state.docStore, internalId, fieldPath),
7307
+ numericIndex: (fieldPath) => state.numericIndexes.get(fieldPath),
7308
+ booleanIndex: (fieldPath) => state.booleanIndexes.get(fieldPath),
7309
+ enumIndex: (fieldPath) => state.enumIndexes.get(fieldPath),
7310
+ geoIndex: (fieldPath) => state.geoIndexes.get(fieldPath),
7311
+ sizeIndex: (fieldPath) => state.sizeIndexes.get(fieldPath),
6715
7312
  textRangeBitset: (fieldPath, bounds) => textRangeBitsetOf(state, fieldPath, flat[fieldPath], bounds, capacity),
6716
- get allDocIds() {
6717
- if (!cachedAllDocIds) {
6718
- cachedAllDocIds = getAllInternalDocIds(state.docStore);
6719
- }
6720
- return cachedAllDocIds;
7313
+ isPatternField: (fieldPath) => patternFields.has(fieldPath),
7314
+ matchPattern: (fieldPath, operator, text, caseFold, into) => {
7315
+ const index = state.patternIndexes.get(fieldPath);
7316
+ if (index === void 0) return;
7317
+ requirePatternSearch(fieldPath).matchInto({
7318
+ operator,
7319
+ text,
7320
+ caseFold,
7321
+ index,
7322
+ matches: into,
7323
+ meter,
7324
+ host: PATTERN_HOST,
7325
+ valueOf: (internalId) => getFieldValue(internalId, fieldPath)
7326
+ });
7327
+ },
7328
+ presentBitset: (fieldPath) => {
7329
+ const cached = cachedPresentBitsets.get(fieldPath);
7330
+ if (cached !== void 0) return cached;
7331
+ const present = presentBitsetOf(state, fieldPath, flat[fieldPath], capacity);
7332
+ cachedPresentBitsets.set(fieldPath, present);
7333
+ return present;
6721
7334
  },
6722
7335
  capacity,
6723
7336
  get allDocIdsBitset() {
6724
- if (!cachedAllDocIdsBitset) {
6725
- cachedAllDocIdsBitset = bitsetFromSet(this.allDocIds, capacity);
6726
- }
7337
+ cachedAllDocIdsBitset ??= liveDocumentsBitset(state, capacity);
6727
7338
  return cachedAllDocIdsBitset;
6728
7339
  }
6729
7340
  };
6730
7341
  }
6731
- function applyPartitionFilters(state, filters, schema) {
6732
- const context = buildFilterContext(state, schema);
7342
+ function applyPartitionFilters(state, filters, schema, meter) {
7343
+ const context = buildFilterContext(state, schema, meter);
6733
7344
  const resultBitset = evaluateFilters(filters, context);
6734
7345
  const resolver = state.docStore.resolver();
6735
7346
  const externalResult = /* @__PURE__ */ new Set();
@@ -6749,12 +7360,12 @@ function applyPartitionFilters(state, filters, schema) {
6749
7360
  }
6750
7361
  return externalResult;
6751
7362
  }
6752
- function applyPartitionFiltersBitset(state, filters, schema) {
6753
- const context = buildFilterContext(state, schema);
7363
+ function applyPartitionFiltersBitset(state, filters, schema, meter) {
7364
+ const context = buildFilterContext(state, schema, meter);
6754
7365
  return evaluateFilters(filters, context);
6755
7366
  }
6756
- function partitionFilterMatches(state, filters, schema) {
6757
- const accepted = applyPartitionFiltersBitset(state, filters, schema);
7367
+ function partitionFilterMatches(state, filters, schema, meter) {
7368
+ const accepted = applyPartitionFiltersBitset(state, filters, schema, meter);
6758
7369
  const docStore = state.docStore;
6759
7370
  const resolver = docStore.resolver();
6760
7371
  let count = 0;
@@ -6911,6 +7522,8 @@ function estimatePartitionBytes(state) {
6911
7522
  bytes += docCount * state.booleanIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
6912
7523
  bytes += docCount * state.enumIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
6913
7524
  bytes += docCount * state.geoIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
7525
+ bytes += docCount * state.sizeIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
7526
+ for (const index of state.patternIndexes.values()) bytes += index.bytes();
6914
7527
  return bytes + sortColumnBytes(state);
6915
7528
  }
6916
7529
 
@@ -6970,12 +7583,12 @@ function mergeSurfaceForms(target, source) {
6970
7583
  target.surfaceRegistry.add(surface, value[1], value[0]);
6971
7584
  }
6972
7585
  }
6973
- function mergeFieldIndexes(target, source, idMapping) {
6974
- for (const [fieldPath, index] of source.numericIndexes) {
6975
- let targetIndex = target.numericIndexes.get(fieldPath);
7586
+ function mergeNumericIndexes(target, source, idMapping) {
7587
+ for (const [fieldPath, index] of source) {
7588
+ let targetIndex = target.get(fieldPath);
6976
7589
  if (targetIndex === void 0) {
6977
7590
  targetIndex = createNumericIndex();
6978
- target.numericIndexes.set(fieldPath, targetIndex);
7591
+ target.set(fieldPath, targetIndex);
6979
7592
  }
6980
7593
  for (const entry of index.serialize()) {
6981
7594
  const targetInternalId = idMapping.get(entry.docId);
@@ -6983,6 +7596,10 @@ function mergeFieldIndexes(target, source, idMapping) {
6983
7596
  targetIndex.insert(targetInternalId, entry.value);
6984
7597
  }
6985
7598
  }
7599
+ }
7600
+ function mergeFieldIndexes(target, source, idMapping) {
7601
+ mergeNumericIndexes(target.numericIndexes, source.numericIndexes, idMapping);
7602
+ mergeNumericIndexes(target.sizeIndexes, source.sizeIndexes, idMapping);
6986
7603
  for (const [fieldPath, index] of source.booleanIndexes) {
6987
7604
  let targetIndex = target.booleanIndexes.get(fieldPath);
6988
7605
  if (targetIndex === void 0) {
@@ -7024,6 +7641,12 @@ function mergeFieldIndexes(target, source, idMapping) {
7024
7641
  targetIndex.insert(targetInternalId, entry.lat, entry.lon);
7025
7642
  }
7026
7643
  }
7644
+ for (const [fieldPath, index] of source.patternIndexes) {
7645
+ patternIndexOf(target.patternIndexes, fieldPath).addArrays(
7646
+ index.toArrays(fieldPath, null),
7647
+ (sourceInternalId) => idMapping.get(sourceInternalId) ?? -1
7648
+ );
7649
+ }
7027
7650
  }
7028
7651
  function mergeSegmentState(target, source) {
7029
7652
  if (source.stats.totalDocuments === 0 && source.docStore.count() === 0) return;
@@ -8044,19 +8667,23 @@ function searchFulltext(state, params) {
8044
8667
  }
8045
8668
 
8046
8669
  // src/core/partition/segment-field-indexes.ts
8047
- function encodeFieldIndexes(state, remap) {
8048
- const survives = (internalId) => remap[internalId] >= 0;
8049
- const numeric = [];
8050
- for (const [fieldPath, index] of state.numericIndexes) {
8051
- const entries = index.serialize().filter((entry) => survives(entry.docId));
8670
+ function encodeNumericIndexes(indexes, remap) {
8671
+ const encoded = [];
8672
+ for (const [fieldPath, index] of indexes) {
8673
+ const entries = index.serialize().filter((entry) => remap[entry.docId] >= 0);
8052
8674
  const docIds = new Uint32Array(entries.length);
8053
8675
  const values = new Float64Array(entries.length);
8054
8676
  for (let i = 0; i < entries.length; i++) {
8055
8677
  docIds[i] = remap[entries[i].docId];
8056
8678
  values[i] = entries[i].value;
8057
8679
  }
8058
- numeric.push({ fieldPath, docIds, values });
8680
+ encoded.push({ fieldPath, docIds, values });
8059
8681
  }
8682
+ return encoded;
8683
+ }
8684
+ function encodeFieldIndexes(state, remap) {
8685
+ const survives = (internalId) => remap[internalId] >= 0;
8686
+ const numeric = encodeNumericIndexes(state.numericIndexes, remap);
8060
8687
  const booleans = [];
8061
8688
  for (const [fieldPath, index] of state.booleanIndexes) {
8062
8689
  const { trueDocs, falseDocs } = index.serialize();
@@ -8097,7 +8724,9 @@ function encodeFieldIndexes(state, remap) {
8097
8724
  }
8098
8725
  geo.push({ fieldPath, docIds, latitudes, longitudes });
8099
8726
  }
8100
- return { numeric, boolean: booleans, enums, geo };
8727
+ const pattern = [];
8728
+ for (const [fieldPath, index] of state.patternIndexes) pattern.push(index.toArrays(fieldPath, remap));
8729
+ return { numeric, boolean: booleans, enums, geo, pattern, sizes: encodeNumericIndexes(state.sizeIndexes, remap) };
8101
8730
  }
8102
8731
 
8103
8732
  // src/core/partition/segment-payload.ts
@@ -8254,15 +8883,19 @@ function mergePayloadPostings(target, payload, ordinalBase) {
8254
8883
  }
8255
8884
  }
8256
8885
  }
8257
- function mergePayloadFieldIndexes(target, payload, ordinalBase) {
8258
- for (const entry of payload.numeric) {
8259
- let index = target.numericIndexes.get(entry.fieldPath);
8886
+ function mergePayloadNumericIndexes(target, entries, ordinalBase) {
8887
+ for (const entry of entries) {
8888
+ let index = target.get(entry.fieldPath);
8260
8889
  if (index === void 0) {
8261
8890
  index = createNumericIndex();
8262
- target.numericIndexes.set(entry.fieldPath, index);
8891
+ target.set(entry.fieldPath, index);
8263
8892
  }
8264
8893
  for (let i = 0; i < entry.docIds.length; i++) index.insert(ordinalBase + entry.docIds[i], entry.values[i]);
8265
8894
  }
8895
+ }
8896
+ function mergePayloadFieldIndexes(target, payload, ordinalBase) {
8897
+ mergePayloadNumericIndexes(target.numericIndexes, payload.numeric, ordinalBase);
8898
+ mergePayloadNumericIndexes(target.sizeIndexes, payload.sizes, ordinalBase);
8266
8899
  for (const entry of payload.boolean) {
8267
8900
  let index = target.booleanIndexes.get(entry.fieldPath);
8268
8901
  if (index === void 0) {
@@ -8293,6 +8926,9 @@ function mergePayloadFieldIndexes(target, payload, ordinalBase) {
8293
8926
  index.insert(ordinalBase + entry.docIds[i], entry.latitudes[i], entry.longitudes[i]);
8294
8927
  }
8295
8928
  }
8929
+ for (const entry of payload.pattern) {
8930
+ patternIndexOf(target.patternIndexes, entry.fieldPath).addArrays(entry, (ordinal) => ordinalBase + ordinal);
8931
+ }
8296
8932
  }
8297
8933
  function mergeSegmentPayload(target, payload, documents) {
8298
8934
  if (payload.documentCount === 0) return;
@@ -8316,25 +8952,142 @@ function mergeSegmentPayload(target, payload, documents) {
8316
8952
  });
8317
8953
  }
8318
8954
  }
8319
- mergePayloadPostings(target, payload, ordinalBase);
8320
- mergePayloadFieldIndexes(target, payload, ordinalBase);
8321
- target.stats.totalDocuments += payload.documentCount;
8322
- for (const [fieldName, length] of Object.entries(payload.totalFieldLengths)) {
8323
- target.stats.totalFieldLengths[fieldName] = (target.stats.totalFieldLengths[fieldName] ?? 0) + length;
8955
+ mergePayloadPostings(target, payload, ordinalBase);
8956
+ mergePayloadFieldIndexes(target, payload, ordinalBase);
8957
+ target.stats.totalDocuments += payload.documentCount;
8958
+ for (const [fieldName, length] of Object.entries(payload.totalFieldLengths)) {
8959
+ target.stats.totalFieldLengths[fieldName] = (target.stats.totalFieldLengths[fieldName] ?? 0) + length;
8960
+ }
8961
+ for (const [token, frequency] of Object.entries(payload.docFrequencies)) {
8962
+ target.stats.docFrequencies[token] = (target.stats.docFrequencies[token] ?? 0) + frequency;
8963
+ }
8964
+ target.stats.recalculateAverages();
8965
+ if (payload.surfaceForms !== null) {
8966
+ for (const surface of Object.keys(payload.surfaceForms)) {
8967
+ const value = payload.surfaceForms[surface];
8968
+ if (!Array.isArray(value)) continue;
8969
+ target.surfaceRegistry.add(surface, value[1], value[0]);
8970
+ }
8971
+ }
8972
+ target.sortColumns = null;
8973
+ target.scoreBuffer = null;
8974
+ }
8975
+
8976
+ // src/core/partition/pattern-snapshot.ts
8977
+ function loadFailed(fieldPath, problem) {
8978
+ return new NarsilError(
8979
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
8980
+ `The pattern index of field "${fieldPath}" in this partition is corrupt: ${problem}`,
8981
+ { field: fieldPath }
8982
+ );
8983
+ }
8984
+ function serializePatternIndex(state, fieldPath, index) {
8985
+ const resolver = state.docStore.resolver();
8986
+ const arrays = index.toArrays(fieldPath, null);
8987
+ const positionOf = /* @__PURE__ */ new Map();
8988
+ const docIds = [];
8989
+ for (const ordinal of arrays.docIds) {
8990
+ const externalId = resolver.toExternal(ordinal);
8991
+ if (externalId === void 0) continue;
8992
+ positionOf.set(ordinal, docIds.length);
8993
+ docIds.push(externalId);
8994
+ }
8995
+ const runs = /* @__PURE__ */ Object.create(null);
8996
+ for (let run = 0; run + 1 < arrays.offsets.length; run++) {
8997
+ const positions = [];
8998
+ for (let at = arrays.offsets[run]; at < arrays.offsets[run + 1]; at++) {
8999
+ const position = positionOf.get(arrays.postings[at]);
9000
+ if (position !== void 0) positions.push(position);
9001
+ }
9002
+ if (positions.length === 0) continue;
9003
+ const base = run * PATTERN_RUN_CODE_POINTS;
9004
+ runs[String.fromCodePoint(arrays.runs[base], arrays.runs[base + 1], arrays.runs[base + 2])] = positions;
9005
+ }
9006
+ return { docIds, runs };
9007
+ }
9008
+ function serializePatternIndexes(state, schema) {
9009
+ const patternFields = schemaFieldsOf(schema).filter((field) => field.pattern);
9010
+ if (patternFields.length === 0) return void 0;
9011
+ const serialized = /* @__PURE__ */ Object.create(null);
9012
+ for (const { path: fieldPath } of patternFields) {
9013
+ const index = state.patternIndexes.get(fieldPath);
9014
+ serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } : serializePatternIndex(state, fieldPath, index);
9015
+ }
9016
+ return serialized;
9017
+ }
9018
+ function appendRunCodePoints(fieldPath, key, out) {
9019
+ const start = out.length;
9020
+ for (const character of key) out.push(character.codePointAt(0) ?? 0);
9021
+ const count = out.length - start;
9022
+ if (count !== PATTERN_RUN_CODE_POINTS) {
9023
+ throw loadFailed(
9024
+ fieldPath,
9025
+ `the key "${key}" holds ${count} code points, while each key must hold ${PATTERN_RUN_CODE_POINTS}`
9026
+ );
9027
+ }
9028
+ }
9029
+ function requirePositions(fieldPath, key, list, documentCount) {
9030
+ if (!Array.isArray(list)) throw loadFailed(fieldPath, `the run "${key}" must map to a list of positions`);
9031
+ let previous = -1;
9032
+ for (const position of list) {
9033
+ if (!Number.isInteger(position) || position < 0 || position >= documentCount) {
9034
+ throw loadFailed(fieldPath, `the run "${key}" names position ${String(position)} outside its document list`);
9035
+ }
9036
+ if (position <= previous) {
9037
+ throw loadFailed(fieldPath, `the positions of the run "${key}" are out of ascending order`);
9038
+ }
9039
+ previous = position;
9040
+ }
9041
+ return list;
9042
+ }
9043
+ function requireDistinctDocuments(fieldPath, docIds) {
9044
+ const seen = /* @__PURE__ */ new Set();
9045
+ for (const docId of docIds) {
9046
+ if (seen.has(docId)) throw loadFailed(fieldPath, `the document list names "${docId}" twice`);
9047
+ seen.add(docId);
8324
9048
  }
8325
- for (const [token, frequency] of Object.entries(payload.docFrequencies)) {
8326
- target.stats.docFrequencies[token] = (target.stats.docFrequencies[token] ?? 0) + frequency;
9049
+ }
9050
+ function decodePatternEntry(fieldPath, entry) {
9051
+ requireDistinctDocuments(fieldPath, entry.docIds);
9052
+ const runs = [];
9053
+ const offsets = [0];
9054
+ const postings = [];
9055
+ for (const [key, list] of Object.entries(entry.runs)) {
9056
+ appendRunCodePoints(fieldPath, key, runs);
9057
+ for (const position of requirePositions(fieldPath, key, list, entry.docIds.length)) postings.push(position);
9058
+ offsets.push(postings.length);
8327
9059
  }
8328
- target.stats.recalculateAverages();
8329
- if (payload.surfaceForms !== null) {
8330
- for (const surface of Object.keys(payload.surfaceForms)) {
8331
- const value = payload.surfaceForms[surface];
8332
- if (!Array.isArray(value)) continue;
8333
- target.surfaceRegistry.add(surface, value[1], value[0]);
9060
+ const positions = new Uint32Array(entry.docIds.length);
9061
+ for (let position = 0; position < positions.length; position++) positions[position] = position;
9062
+ return {
9063
+ fieldPath,
9064
+ docIds: positions,
9065
+ runs: Uint32Array.from(runs),
9066
+ offsets: Uint32Array.from(offsets),
9067
+ postings: Uint32Array.from(postings)
9068
+ };
9069
+ }
9070
+ function rebuildPatternIndex(state, fieldPath) {
9071
+ for (const [docId, stored] of state.docStore.all()) {
9072
+ const value = getNestedValue(stored.fields, fieldPath);
9073
+ if (value === void 0 || value === null) continue;
9074
+ const internalId = state.docStore.getInternalId(docId);
9075
+ if (internalId !== void 0) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
9076
+ }
9077
+ }
9078
+ function loadPatternIndexes(state, serialized, schema) {
9079
+ const resolver = state.docStore.resolver();
9080
+ for (const { path: fieldPath, pattern } of schemaFieldsOf(schema)) {
9081
+ if (!pattern) continue;
9082
+ const entry = serialized !== void 0 && Object.hasOwn(serialized, fieldPath) ? serialized[fieldPath] : void 0;
9083
+ if (entry === void 0) {
9084
+ rebuildPatternIndex(state, fieldPath);
9085
+ continue;
8334
9086
  }
9087
+ const decoded = decodePatternEntry(fieldPath, entry);
9088
+ const ordinals = entry.docIds.map((docId) => resolver.toInternal(docId) ?? -1);
9089
+ patternIndexOf(state.patternIndexes, fieldPath).addArrays(decoded, (position) => ordinals[position]);
8335
9090
  }
8336
- target.sortColumns = null;
8337
- target.scoreBuffer = null;
8338
9091
  }
8339
9092
 
8340
9093
  // src/core/partition/serialization.ts
@@ -8409,6 +9162,7 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
8409
9162
  }));
8410
9163
  }
8411
9164
  const serializedStats = state.stats.serialize();
9165
+ const serializedPattern = serializePatternIndexes(state, schema);
8412
9166
  return {
8413
9167
  indexName,
8414
9168
  partitionId,
@@ -8423,7 +9177,8 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
8423
9177
  numeric: serializedNumeric,
8424
9178
  boolean: serializedBoolean,
8425
9179
  enum: serializedEnum,
8426
- geopoint: serializedGeo
9180
+ geopoint: serializedGeo,
9181
+ ...serializedPattern === void 0 ? {} : { pattern: serializedPattern }
8427
9182
  },
8428
9183
  surfaceForms: state.surfaceRegistry.serialize(),
8429
9184
  statistics: serializedStats
@@ -8501,6 +9256,8 @@ function deserializePartition(state, data, clearFn, schema) {
8501
9256
  throw new Error(`Schema mismatch: serialized data contains unknown field "${field}" not present in schema`);
8502
9257
  }
8503
9258
  }
9259
+ loadPatternIndexes(state, data.fieldIndexes.pattern, schema);
9260
+ rebuildSizeIndexes(state, schema);
8504
9261
  }
8505
9262
 
8506
9263
  // src/core/partition/suggestions.ts
@@ -8663,7 +9420,8 @@ function serializePartitionToWirePayloadV2(state, partitionId, indexName, totalP
8663
9420
  numeric: wireNumeric,
8664
9421
  boolean: wireBoolean,
8665
9422
  enum: wireEnum,
8666
- geopoint: wireGeo
9423
+ geopoint: wireGeo,
9424
+ ...patternIndexesToWire(serializePatternIndexes(state, schema))
8667
9425
  },
8668
9426
  surface_forms: state.surfaceRegistry.serialize(),
8669
9427
  statistics: {
@@ -8698,6 +9456,8 @@ function createPartitionIndex(partitionId, trackPositions = true) {
8698
9456
  booleanIndexes: /* @__PURE__ */ new Map(),
8699
9457
  enumIndexes: /* @__PURE__ */ new Map(),
8700
9458
  geoIndexes: /* @__PURE__ */ new Map(),
9459
+ patternIndexes: /* @__PURE__ */ new Map(),
9460
+ sizeIndexes: /* @__PURE__ */ new Map(),
8701
9461
  fieldNameTable,
8702
9462
  flatSchemaCache: null,
8703
9463
  lastSchemaRef: null,
@@ -8717,6 +9477,8 @@ function createPartitionIndex(partitionId, trackPositions = true) {
8717
9477
  state.booleanIndexes.clear();
8718
9478
  state.enumIndexes.clear();
8719
9479
  state.geoIndexes.clear();
9480
+ state.patternIndexes.clear();
9481
+ state.sizeIndexes.clear();
8720
9482
  state.stats.deserialize({ totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} });
8721
9483
  state.flatSchemaCache = null;
8722
9484
  state.lastSchemaRef = null;
@@ -8892,14 +9654,14 @@ function createPartitionIndex(partitionId, trackPositions = true) {
8892
9654
  sortValues(docId, fields, fieldTypes, modes) {
8893
9655
  return sortValuesOf(state, docId, fields, fieldTypes, modes);
8894
9656
  },
8895
- applyFilters(filters, schema) {
8896
- return applyPartitionFilters(state, filters, schema);
9657
+ applyFilters(filters, schema, meter) {
9658
+ return applyPartitionFilters(state, filters, schema, meter);
8897
9659
  },
8898
- applyFiltersBitset(filters, schema) {
8899
- return applyPartitionFiltersBitset(state, filters, schema);
9660
+ applyFiltersBitset(filters, schema, meter) {
9661
+ return applyPartitionFiltersBitset(state, filters, schema, meter);
8900
9662
  },
8901
- filterMatches(filters, schema) {
8902
- return partitionFilterMatches(state, filters, schema);
9663
+ filterMatches(filters, schema, meter) {
9664
+ return partitionFilterMatches(state, filters, schema, meter);
8903
9665
  },
8904
9666
  computeFacets(matched, config, schema) {
8905
9667
  return computeFacets(state, matched, config, schema);
@@ -9281,6 +10043,100 @@ function readHeapStatistics() {
9281
10043
  return null;
9282
10044
  }
9283
10045
 
10046
+ // src/core/pattern-index/work-meter.ts
10047
+ var COUNTER_BYTES = 8;
10048
+ function capExceeded(cap) {
10049
+ return new NarsilError(
10050
+ ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
10051
+ `The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
10052
+ { cap }
10053
+ );
10054
+ }
10055
+ function localMeter(cap, spent, matcherStateBytes, flushUnits) {
10056
+ let total = spent;
10057
+ let sharedCount = null;
10058
+ const spentSoFar = () => sharedCount === null ? total : Number(Atomics.load(sharedCount, 0));
10059
+ const meter = {
10060
+ cap,
10061
+ matcherStateBytes,
10062
+ flushUnits,
10063
+ add(units) {
10064
+ if (units <= 0) return;
10065
+ if (sharedCount === null) total += units;
10066
+ else total = Number(Atomics.add(sharedCount, 0, BigInt(units))) + units;
10067
+ if (total > cap) throw capExceeded(cap);
10068
+ },
10069
+ remaining() {
10070
+ return cap - spentSoFar();
10071
+ },
10072
+ fork(workerCount) {
10073
+ const copies = Math.max(1, workerCount);
10074
+ const seed = spentSoFar();
10075
+ let counter = null;
10076
+ if (typeof SharedArrayBuffer === "function") {
10077
+ counter = new SharedArrayBuffer(COUNTER_BYTES);
10078
+ new BigInt64Array(counter)[0] = BigInt(seed);
10079
+ }
10080
+ const shared2 = {
10081
+ cap,
10082
+ spent: seed,
10083
+ counter,
10084
+ matcherStateBytes: Math.floor(matcherStateBytes / copies),
10085
+ flushUnits: Math.max(1, Math.floor(flushUnits / copies))
10086
+ };
10087
+ return {
10088
+ shared: shared2,
10089
+ settle(reportedUnits) {
10090
+ if (counter === null) {
10091
+ meter.add(reportedUnits);
10092
+ return;
10093
+ }
10094
+ const unitsSinceFork = spentSoFar() - seed;
10095
+ const view = new BigInt64Array(counter);
10096
+ if (unitsSinceFork > 0) Atomics.add(view, 0, BigInt(unitsSinceFork));
10097
+ sharedCount = view;
10098
+ total = spentSoFar();
10099
+ if (total > cap) throw capExceeded(cap);
10100
+ }
10101
+ };
10102
+ }
10103
+ };
10104
+ return meter;
10105
+ }
10106
+ function createPatternWorkMeter(cap = DEFAULT_PATTERN_WORK_CAP) {
10107
+ return localMeter(cap, 0, MATCHER_STATE_BUDGET_BYTES, PATTERN_WORK_FLUSH_UNITS);
10108
+ }
10109
+ function sharedMeter(work, counter) {
10110
+ const view = new BigInt64Array(counter);
10111
+ return {
10112
+ matcherStateBytes: work.matcherStateBytes,
10113
+ flushUnits: work.flushUnits,
10114
+ add(units) {
10115
+ if (units <= 0) return;
10116
+ const total = Number(Atomics.add(view, 0, BigInt(units))) + units;
10117
+ if (total > work.cap) throw capExceeded(work.cap);
10118
+ },
10119
+ remaining() {
10120
+ return work.cap - Number(Atomics.load(view, 0));
10121
+ }
10122
+ };
10123
+ }
10124
+ function joinPatternWorkMeter(work) {
10125
+ const meter = work.counter === null ? localMeter(work.cap, work.spent, work.matcherStateBytes, work.flushUnits) : sharedMeter(work, work.counter);
10126
+ let spent = 0;
10127
+ return {
10128
+ matcherStateBytes: meter.matcherStateBytes,
10129
+ flushUnits: meter.flushUnits,
10130
+ add(units) {
10131
+ if (units <= 0) return;
10132
+ spent += units;
10133
+ meter.add(units);
10134
+ },
10135
+ remaining: () => meter.remaining(),
10136
+ spent: () => spent
10137
+ };
10138
+ }
10139
+
9284
10140
  // src/partitioning/distributed-scoring.ts
9285
10141
  function analysedQueryTokens(term, language, analysis) {
9286
10142
  const { tokens } = tokenize(term, language, {
@@ -9938,7 +10794,11 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
9938
10794
  }
9939
10795
  let filterBitset;
9940
10796
  if (params.filters) {
9941
- filterBitset = partition.applyFiltersBitset(params.filters, schema);
10797
+ filterBitset = partition.applyFiltersBitset(
10798
+ params.filters,
10799
+ schema,
10800
+ options?.patternWork ?? createPatternWorkMeter()
10801
+ );
9942
10802
  if (bitsetIsEmpty(filterBitset)) return null;
9943
10803
  }
9944
10804
  return { queryTokens, prefixExpansion, filterBitset };
@@ -10181,7 +11041,8 @@ function resolveEffectiveMode(config) {
10181
11041
  function buildSearchOptions(base, globalStats) {
10182
11042
  return {
10183
11043
  ...base,
10184
- globalStats: globalStats ?? base?.globalStats
11044
+ globalStats: globalStats ?? base?.globalStats,
11045
+ patternWork: base?.patternWork ?? createPatternWorkMeter()
10185
11046
  };
10186
11047
  }
10187
11048
  function dispatchWithDispatcher(partitions, params, language, schema, options, dispatcher) {
@@ -10216,12 +11077,13 @@ function frozenSegmentBytes(segment) {
10216
11077
  bytes += arrays.documentTable.blob.byteLength + arrays.documentTable.offsets.byteLength;
10217
11078
  }
10218
11079
  for (const column of arrays.fieldLengthColumns) bytes += column.byteLength;
10219
- for (const entry of arrays.numeric) bytes += entry.docIds.byteLength + entry.values.byteLength;
11080
+ for (const entry of [...arrays.numeric, ...arrays.sizes]) bytes += entry.docIds.byteLength + entry.values.byteLength;
10220
11081
  for (const entry of arrays.boolean) bytes += entry.trueDocs.byteLength + entry.falseDocs.byteLength;
10221
11082
  for (const entry of arrays.enums) bytes += entry.offsets.byteLength + entry.docIds.byteLength;
10222
11083
  for (const entry of arrays.geo) {
10223
11084
  bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
10224
11085
  }
11086
+ for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
10225
11087
  return bytes;
10226
11088
  }
10227
11089
 
@@ -10299,24 +11161,24 @@ function placeSubBitset(composite, layout, subIndex, subBits) {
10299
11161
  if (words === 0) return;
10300
11162
  composite.set(subBits.length > words ? subBits.subarray(0, words) : subBits, layout.bases[subIndex] >> 5);
10301
11163
  }
10302
- function compositeFiltersBitset(subs, layout, filters, schema) {
11164
+ function compositeFiltersBitset(subs, layout, filters, schema, meter) {
10303
11165
  const composite = new Uint32Array(Math.ceil(layout.totalCapacity / WORD_BITS));
10304
11166
  for (let i = 0; i < subs.length; i++) {
10305
- placeSubBitset(composite, layout, i, applyPartitionFiltersBitset(subs[i], filters, schema));
11167
+ placeSubBitset(composite, layout, i, applyPartitionFiltersBitset(subs[i], filters, schema, meter));
10306
11168
  }
10307
11169
  return composite;
10308
11170
  }
10309
- function compositeFilters(subs, filters, schema) {
11171
+ function compositeFilters(subs, filters, schema, meter) {
10310
11172
  const combined = /* @__PURE__ */ new Set();
10311
11173
  for (const sub of subs) {
10312
- for (const docId of applyPartitionFilters(sub, filters, schema)) {
11174
+ for (const docId of applyPartitionFilters(sub, filters, schema, meter)) {
10313
11175
  combined.add(docId);
10314
11176
  }
10315
11177
  }
10316
11178
  return combined;
10317
11179
  }
10318
- function compositeFilterMatches(subs, layout, filters, schema) {
10319
- const bits = compositeFiltersBitset(subs, layout, filters, schema);
11180
+ function compositeFilterMatches(subs, layout, filters, schema, meter) {
11181
+ const bits = compositeFiltersBitset(subs, layout, filters, schema, meter);
10320
11182
  let count = 0;
10321
11183
  for (let i = 0; i < subs.length; i++) {
10322
11184
  const resolver = subs[i].docStore.resolver();
@@ -10777,14 +11639,14 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
10777
11639
  searchFulltextMatches(params) {
10778
11640
  return compositeSearchMatches(subs(), layout(), params);
10779
11641
  },
10780
- applyFilters(filters, schema) {
10781
- return compositeFilters(subs(), filters, schema);
11642
+ applyFilters(filters, schema, meter) {
11643
+ return compositeFilters(subs(), filters, schema, meter);
10782
11644
  },
10783
- applyFiltersBitset(filters, schema) {
10784
- return compositeFiltersBitset(subs(), layout(), filters, schema);
11645
+ applyFiltersBitset(filters, schema, meter) {
11646
+ return compositeFiltersBitset(subs(), layout(), filters, schema, meter);
10785
11647
  },
10786
- filterMatches(filters, schema) {
10787
- return compositeFilterMatches(subs(), layout(), filters, schema);
11648
+ filterMatches(filters, schema, meter) {
11649
+ return compositeFilterMatches(subs(), layout(), filters, schema, meter);
10788
11650
  },
10789
11651
  computeFacets(matched, config, schema) {
10790
11652
  const everyValue = everyValueFacetConfig(config);
@@ -10954,6 +11816,7 @@ function setNestedValue(obj, path, value) {
10954
11816
 
10955
11817
  // src/partitioning/manager/index.ts
10956
11818
  function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
11819
+ requirePatternSearchFor(config.schema);
10957
11820
  const count = initialPartitionCount ?? 1;
10958
11821
  const analysis = resolveIndexAnalysis(config);
10959
11822
  const trackPositions = config.trackPositions ?? true;
@@ -11265,8 +12128,12 @@ function countsWithoutScores(params) {
11265
12128
  }
11266
12129
  function fanOutMatchCount(manager, params, language, schema, options) {
11267
12130
  let count = 0;
12131
+ const searchOptions = {
12132
+ ...options?.searchOptions,
12133
+ patternWork: options?.searchOptions?.patternWork ?? createPatternWorkMeter()
12134
+ };
11268
12135
  for (const partition of partitionsIn(manager, options?.partitionIds)) {
11269
- const matches = fulltextMatches(partition, params, language, schema, options?.searchOptions);
12136
+ const matches = fulltextMatches(partition, params, language, schema, searchOptions);
11270
12137
  if (matches !== null) count += matches.count;
11271
12138
  }
11272
12139
  return count;
@@ -17951,7 +18818,22 @@ function createFrozenEnumReader(entry) {
17951
18818
  function rangeOf(value) {
17952
18819
  return rangeByValue.get(value) ?? { from: 0, to: 0 };
17953
18820
  }
18821
+ let affixes = null;
18822
+ function affixesOf() {
18823
+ affixes ??= new ValueAffixes(() => true, values);
18824
+ return affixes;
18825
+ }
18826
+ function bitsetOfValues(matching, capacity) {
18827
+ const bits = createBitSet(capacity);
18828
+ for (const value of matching) {
18829
+ const { from, to } = rangeOf(value);
18830
+ for (let i = from; i < to; i++) bitsetSet(bits, docIds[i]);
18831
+ }
18832
+ return bits;
18833
+ }
17954
18834
  return {
18835
+ queryPrefixBitset: (prefix, capacity) => bitsetOfValues(affixesOf().withPrefix(prefix), capacity),
18836
+ querySuffixBitset: (suffix, capacity) => bitsetOfValues(affixesOf().withSuffix(suffix), capacity),
17955
18837
  serialize: () => {
17956
18838
  const byValue = /* @__PURE__ */ Object.create(null);
17957
18839
  for (let i = 0; i < values.length; i++) {
@@ -18396,6 +19278,11 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId) {
18396
19278
  docIds: sharedUint32(entry.docIds),
18397
19279
  values: sharedFloat64(entry.values)
18398
19280
  })),
19281
+ sizes: payload.sizes.map((entry) => ({
19282
+ fieldPath: entry.fieldPath,
19283
+ docIds: sharedUint32(entry.docIds),
19284
+ values: sharedFloat64(entry.values)
19285
+ })),
18399
19286
  boolean: payload.boolean.map((entry) => ({
18400
19287
  fieldPath: entry.fieldPath,
18401
19288
  trueDocs: sharedUint32(entry.trueDocs),
@@ -18413,6 +19300,13 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId) {
18413
19300
  latitudes: sharedFloat64(entry.latitudes),
18414
19301
  longitudes: sharedFloat64(entry.longitudes)
18415
19302
  })),
19303
+ pattern: payload.pattern.map((entry) => ({
19304
+ fieldPath: entry.fieldPath,
19305
+ docIds: sharedUint32(entry.docIds),
19306
+ runs: sharedUint32(entry.runs),
19307
+ offsets: sharedUint32(entry.offsets),
19308
+ postings: sharedUint32(entry.postings)
19309
+ })),
18416
19310
  surfaceTable: sharedSurfaceTable(payload)
18417
19311
  };
18418
19312
  }
@@ -18459,10 +19353,16 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
18459
19353
  const docStore = createFrozenDocTable(source.idTable, lengthColumns, documentSource, tombstones);
18460
19354
  const numericIndexes = /* @__PURE__ */ new Map();
18461
19355
  for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
19356
+ const sizeIndexes = /* @__PURE__ */ new Map();
19357
+ for (const entry of source.sizes) sizeIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
18462
19358
  const booleanIndexes = /* @__PURE__ */ new Map();
18463
19359
  for (const entry of source.boolean) booleanIndexes.set(entry.fieldPath, createFrozenBooleanReader(entry));
18464
19360
  const enumIndexes = /* @__PURE__ */ new Map();
18465
19361
  for (const entry of source.enums) enumIndexes.set(entry.fieldPath, createFrozenEnumReader(entry));
19362
+ const patternIndexes = /* @__PURE__ */ new Map();
19363
+ for (const entry of source.pattern) {
19364
+ patternIndexes.set(entry.fieldPath, requirePatternSearch(entry.fieldPath).readArrays(entry));
19365
+ }
18466
19366
  const fieldNameTable = {
18467
19367
  names: [...source.fieldNames],
18468
19368
  indexMap: new Map(source.fieldNames.map((name, index) => [name, index]))
@@ -18480,6 +19380,8 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
18480
19380
  booleanIndexes,
18481
19381
  enumIndexes,
18482
19382
  geoIndexes: buildGeoReaders(source.geo),
19383
+ patternIndexes,
19384
+ sizeIndexes,
18483
19385
  fieldNameTable,
18484
19386
  trackPositions: source.positionOffsets !== null,
18485
19387
  flatSchemaCache: null,
@@ -18607,12 +19509,8 @@ function remapDocIds(inputs, pick) {
18607
19509
  }
18608
19510
  return out;
18609
19511
  }
18610
- function mergeNumeric(inputs) {
18611
- const byFieldPath = rowsByFieldPath(
18612
- inputs,
18613
- (input) => input.segment.arrays.numeric,
18614
- (entry, at) => [entry.values[at]]
18615
- );
19512
+ function mergeNumeric(inputs, pick) {
19513
+ const byFieldPath = rowsByFieldPath(inputs, pick, (entry, at) => [entry.values[at]]);
18616
19514
  const merged = [];
18617
19515
  for (const [fieldPath, rows] of byFieldPath) {
18618
19516
  rows.sort((a, b) => a.values[0] - b.values[0] || a.docId - b.docId);
@@ -18685,12 +19583,32 @@ function mergeEnums(inputs) {
18685
19583
  }
18686
19584
  return merged;
18687
19585
  }
19586
+ function mergePattern(inputs) {
19587
+ const byFieldPath = /* @__PURE__ */ new Map();
19588
+ for (const input of inputs) {
19589
+ for (const arrays of input.segment.arrays.pattern) {
19590
+ let entries = byFieldPath.get(arrays.fieldPath);
19591
+ if (entries === void 0) {
19592
+ entries = [];
19593
+ byFieldPath.set(arrays.fieldPath, entries);
19594
+ }
19595
+ entries.push({ arrays, remap: input.remap });
19596
+ }
19597
+ }
19598
+ const merged = [];
19599
+ for (const [fieldPath, entries] of byFieldPath) {
19600
+ merged.push(requirePatternSearch(fieldPath).mergeArrays(fieldPath, entries));
19601
+ }
19602
+ return merged;
19603
+ }
18688
19604
  function mergeFieldIndexes2(inputs) {
18689
19605
  return {
18690
- numeric: mergeNumeric(inputs),
19606
+ numeric: mergeNumeric(inputs, (input) => input.segment.arrays.numeric),
18691
19607
  boolean: mergeBoolean(inputs),
18692
19608
  enums: mergeEnums(inputs),
18693
- geo: mergeGeo(inputs)
19609
+ geo: mergeGeo(inputs),
19610
+ pattern: mergePattern(inputs),
19611
+ sizes: mergeNumeric(inputs, (input) => input.segment.arrays.sizes)
18694
19612
  };
18695
19613
  }
18696
19614
 
@@ -19032,54 +19950,7 @@ var GrowableUint32 = class {
19032
19950
  }
19033
19951
  };
19034
19952
 
19035
- // src/core/partition/segment-builder.ts
19036
- var TOKEN_TABLE_START = 1 << 12;
19037
- var POSTING_LOG_START = 1 << 14;
19038
- var POSITION_POOL_START = 1 << 16;
19039
- var FIELD_SCRATCH_START = 1 << 10;
19040
- function indexOf(indexes, fieldPath, create) {
19041
- let index = indexes.get(fieldPath);
19042
- if (index === void 0) {
19043
- index = create();
19044
- indexes.set(fieldPath, index);
19045
- }
19046
- return index;
19047
- }
19048
- function indexFilterValue(indexes, ordinal, fieldPath, fieldType, value) {
19049
- if (fieldType === "number" || fieldType === "number[]") {
19050
- const index = indexOf(indexes.numericIndexes, fieldPath, createNumericIndex);
19051
- for (const entry of fieldType === "number" ? [value] : value) index.insert(ordinal, entry);
19052
- } else if (fieldType === "boolean" || fieldType === "boolean[]") {
19053
- const index = indexOf(indexes.booleanIndexes, fieldPath, createBooleanIndex);
19054
- for (const entry of fieldType === "boolean" ? [value] : value)
19055
- index.insert(ordinal, entry);
19056
- } else if (fieldType === "enum" || fieldType === "enum[]") {
19057
- const index = indexOf(indexes.enumIndexes, fieldPath, createEnumIndex);
19058
- for (const entry of fieldType === "enum" ? [value] : value) index.insert(ordinal, entry);
19059
- } else if (fieldType === "geopoint") {
19060
- const point = value;
19061
- indexOf(indexes.geoIndexes, fieldPath, createGeoIndex).insert(ordinal, point.lat, point.lon);
19062
- }
19063
- }
19064
- function refuseRepeatedDocIds(documents) {
19065
- const seen = /* @__PURE__ */ new Set();
19066
- for (const { docId } of documents) {
19067
- if (seen.has(docId)) {
19068
- throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists in partition 0`, {
19069
- docId,
19070
- partitionId: 0
19071
- });
19072
- }
19073
- seen.add(docId);
19074
- }
19075
- }
19076
- function validateAsAnInsertDoes(documents, schema, options) {
19077
- if (options?.validate === false) return;
19078
- for (const { document: document2 } of documents) {
19079
- validateDocument(document2, schema, options?.patternValueLimit);
19080
- if (options?.strict) validateDocumentStrict(document2, schema);
19081
- }
19082
- }
19953
+ // src/core/partition/segment-postings.ts
19083
19954
  var TextPostings = class {
19084
19955
  constructor(trackPositions) {
19085
19956
  this.trackPositions = trackPositions;
@@ -19088,22 +19959,22 @@ var TextPostings = class {
19088
19959
  fieldNames = [];
19089
19960
  tokenIds = /* @__PURE__ */ new Map();
19090
19961
  fieldNameIndexes = /* @__PURE__ */ new Map();
19091
- postingsPerToken = new GrowableUint32(TOKEN_TABLE_START);
19092
- docFrequencyPerToken = new GrowableUint32(TOKEN_TABLE_START);
19093
- lastDocPerToken = new GrowableUint32(TOKEN_TABLE_START);
19094
- fieldStampPerToken = new GrowableUint32(TOKEN_TABLE_START);
19095
- fieldSlotPerToken = new GrowableUint32(TOKEN_TABLE_START);
19096
- logToken = new GrowableUint32(POSTING_LOG_START);
19097
- logDoc = new GrowableUint32(POSTING_LOG_START);
19098
- logFrequency = new GrowableUint32(POSTING_LOG_START);
19099
- logField = new GrowableUint32(POSTING_LOG_START);
19100
- logPositionStart = new GrowableUint32(POSTING_LOG_START);
19101
- positionPool = new GrowableUint32(POSITION_POOL_START);
19102
- slotToken = new GrowableUint32(FIELD_SCRATCH_START);
19103
- slotCount = new GrowableUint32(FIELD_SCRATCH_START);
19104
- slotCursor = new GrowableUint32(FIELD_SCRATCH_START);
19105
- eventSlot = new GrowableUint32(FIELD_SCRATCH_START);
19106
- eventPosition = new GrowableUint32(FIELD_SCRATCH_START);
19962
+ postingsPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
19963
+ docFrequencyPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
19964
+ lastDocPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
19965
+ fieldStampPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
19966
+ fieldSlotPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
19967
+ logToken = new GrowableUint32(SEGMENT_POSTING_LOG_START);
19968
+ logDoc = new GrowableUint32(SEGMENT_POSTING_LOG_START);
19969
+ logFrequency = new GrowableUint32(SEGMENT_POSTING_LOG_START);
19970
+ logField = new GrowableUint32(SEGMENT_POSTING_LOG_START);
19971
+ logPositionStart = new GrowableUint32(SEGMENT_POSTING_LOG_START);
19972
+ positionPool = new GrowableUint32(SEGMENT_POSITION_POOL_START);
19973
+ slotToken = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
19974
+ slotCount = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
19975
+ slotCursor = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
19976
+ eventSlot = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
19977
+ eventPosition = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
19107
19978
  fieldStamp = 0;
19108
19979
  beginField() {
19109
19980
  this.fieldStamp++;
@@ -19231,11 +20102,56 @@ var TextPostings = class {
19231
20102
  };
19232
20103
  }
19233
20104
  };
20105
+
20106
+ // src/core/partition/segment-builder.ts
20107
+ function indexOf(indexes, fieldPath, create) {
20108
+ let index = indexes.get(fieldPath);
20109
+ if (index === void 0) {
20110
+ index = create();
20111
+ indexes.set(fieldPath, index);
20112
+ }
20113
+ return index;
20114
+ }
20115
+ function indexFilterValue(indexes, ordinal, fieldPath, fieldType, value) {
20116
+ if (fieldType === "number" || fieldType === "number[]") {
20117
+ const index = indexOf(indexes.numericIndexes, fieldPath, createNumericIndex);
20118
+ for (const entry of fieldType === "number" ? [value] : value) index.insert(ordinal, entry);
20119
+ } else if (fieldType === "boolean" || fieldType === "boolean[]") {
20120
+ const index = indexOf(indexes.booleanIndexes, fieldPath, createBooleanIndex);
20121
+ for (const entry of fieldType === "boolean" ? [value] : value)
20122
+ index.insert(ordinal, entry);
20123
+ } else if (fieldType === "enum" || fieldType === "enum[]") {
20124
+ const index = indexOf(indexes.enumIndexes, fieldPath, createEnumIndex);
20125
+ for (const entry of fieldType === "enum" ? [value] : value) index.insert(ordinal, entry);
20126
+ } else if (fieldType === "geopoint") {
20127
+ const point = value;
20128
+ indexOf(indexes.geoIndexes, fieldPath, createGeoIndex).insert(ordinal, point.lat, point.lon);
20129
+ }
20130
+ }
20131
+ function refuseRepeatedDocIds(documents) {
20132
+ const seen = /* @__PURE__ */ new Set();
20133
+ for (const { docId } of documents) {
20134
+ if (seen.has(docId)) {
20135
+ throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists in partition 0`, {
20136
+ docId,
20137
+ partitionId: 0
20138
+ });
20139
+ }
20140
+ seen.add(docId);
20141
+ }
20142
+ }
20143
+ function validateAsAnInsertDoes(documents, schema, options) {
20144
+ if (options?.validate === false) return;
20145
+ for (const { document: document2 } of documents) {
20146
+ validateDocument(document2, schema, options?.patternValueLimit);
20147
+ if (options?.strict) validateDocumentStrict(document2, schema);
20148
+ }
20149
+ }
19234
20150
  var SurfaceCounts = class {
19235
20151
  ids = /* @__PURE__ */ new Map();
19236
20152
  surfaces = [];
19237
20153
  tokens = [];
19238
- counts = new GrowableUint32(TOKEN_TABLE_START);
20154
+ counts = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
19239
20155
  count(surface, token) {
19240
20156
  if (surface === void 0 || surface.length === 0) return;
19241
20157
  const known = this.ids.get(surface);
@@ -19295,7 +20211,9 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
19295
20211
  numericIndexes: /* @__PURE__ */ new Map(),
19296
20212
  booleanIndexes: /* @__PURE__ */ new Map(),
19297
20213
  enumIndexes: /* @__PURE__ */ new Map(),
19298
- geoIndexes: /* @__PURE__ */ new Map()
20214
+ geoIndexes: /* @__PURE__ */ new Map(),
20215
+ patternIndexes: /* @__PURE__ */ new Map(),
20216
+ sizeIndexes: /* @__PURE__ */ new Map()
19299
20217
  };
19300
20218
  let positionOffset = 0;
19301
20219
  const addToken = (token, position, surface) => {
@@ -19304,9 +20222,11 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
19304
20222
  };
19305
20223
  for (let ordinal = 0; ordinal < documents.length; ordinal++) {
19306
20224
  const document2 = documents[ordinal].document;
19307
- for (const { path: fieldPath, base: fieldType } of fields) {
20225
+ for (const { path: fieldPath, base: fieldType, pattern } of fields) {
19308
20226
  const value = getNestedValue(document2, fieldPath);
19309
20227
  if (value === void 0 || value === null) continue;
20228
+ recordValueSize(filters.sizeIndexes, ordinal, fieldPath, fieldType, value);
20229
+ if (pattern) patternIndexOf(filters.patternIndexes, fieldPath).add(ordinal, value);
19310
20230
  if (!isWordIndexedBase(fieldType)) {
19311
20231
  indexFilterValue(filters, ordinal, fieldPath, fieldType, value);
19312
20232
  continue;
@@ -19831,6 +20751,7 @@ function createDirectExecutor(options) {
19831
20751
  }
19832
20752
  case "query": {
19833
20753
  const entry = requireIndex(action.indexName);
20754
+ const patternWork = joinPatternWorkMeter(action.patternWork);
19834
20755
  const result = await fanOutQuery(
19835
20756
  entry.manager,
19836
20757
  action.params,
@@ -19841,15 +20762,17 @@ function createDirectExecutor(options) {
19841
20762
  partitionIds: action.partitionIds,
19842
20763
  ...action.globalStats !== void 0 ? { globalStats: sanitizeGlobalStats(action.globalStats) } : {}
19843
20764
  },
19844
- entry.searchOptions
20765
+ { ...entry.searchOptions, patternWork }
19845
20766
  );
20767
+ result.patternWorkSpent = patternWork.spent();
19846
20768
  return result;
19847
20769
  }
19848
20770
  case "preflight": {
19849
20771
  const entry = requireIndex(action.indexName);
20772
+ const searchOptions = { ...entry.searchOptions, patternWork: joinPatternWorkMeter(action.patternWork) };
19850
20773
  if (countsWithoutScores(action.params)) {
19851
20774
  const count = fanOutMatchCount(entry.manager, action.params, entry.language, entry.config.schema, {
19852
- searchOptions: entry.searchOptions
20775
+ searchOptions
19853
20776
  });
19854
20777
  return { count };
19855
20778
  }
@@ -19859,7 +20782,7 @@ function createDirectExecutor(options) {
19859
20782
  entry.language,
19860
20783
  entry.config.schema,
19861
20784
  { scoringMode: entry.config.defaultScoring ?? "local" },
19862
- entry.searchOptions
20785
+ searchOptions
19863
20786
  );
19864
20787
  return { count: result.totalMatched };
19865
20788
  }
@@ -25245,6 +26168,13 @@ function validateDocId(docId) {
25245
26168
  if (docId.includes("\0")) {
25246
26169
  throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
25247
26170
  }
26171
+ if (holdsUnpairedSurrogate(docId)) {
26172
+ throw new NarsilError(
26173
+ ErrorCodes.DOC_VALIDATION_FAILED,
26174
+ "Document ID holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8",
26175
+ { docId }
26176
+ );
26177
+ }
25248
26178
  }
25249
26179
  function validatePartitionConfig(partitions) {
25250
26180
  const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
@@ -27707,6 +28637,64 @@ function stopIdleSweep(state) {
27707
28637
  state.idleSweep = null;
27708
28638
  }
27709
28639
 
28640
+ // src/core/pattern-index/matcher-budget.ts
28641
+ var POSITIONS_PER_WORD = 32;
28642
+ function positionWords(positions) {
28643
+ return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
28644
+ }
28645
+ function automatonBytes(positions) {
28646
+ return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES;
28647
+ }
28648
+ function smallestMatcherBytes(positions) {
28649
+ return automatonBytes(positions) + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
28650
+ }
28651
+ function widestPatternSplit(positions) {
28652
+ return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / smallestMatcherBytes(positions)));
28653
+ }
28654
+
28655
+ // src/core/pattern-index/query-positions.ts
28656
+ function isRecord5(value) {
28657
+ return value !== null && typeof value === "object" && !Array.isArray(value);
28658
+ }
28659
+ function textPositions(text, caseFold) {
28660
+ if (typeof text !== "string") return 0;
28661
+ return caseFold ? foldedCodePointCount(text) : codePointCount(text);
28662
+ }
28663
+ function larger(left, right) {
28664
+ if (left === null) return right;
28665
+ if (right === null) return left;
28666
+ return Math.max(left, right);
28667
+ }
28668
+ function largestTestOf(filter) {
28669
+ const caseFold = filter.caseFold === true;
28670
+ let largest = null;
28671
+ for (const operator of PATTERN_TEST_OPERATORS) {
28672
+ const operand = filter[operator];
28673
+ if (operand === void 0) continue;
28674
+ const texts = Array.isArray(operand) ? operand : [operand];
28675
+ for (const text of texts) largest = larger(largest, textPositions(text, caseFold));
28676
+ }
28677
+ return largest;
28678
+ }
28679
+ function largestIn(expression, patternFields) {
28680
+ let largest = null;
28681
+ if (isRecord5(expression.fields)) {
28682
+ for (const [fieldPath, filter] of Object.entries(expression.fields)) {
28683
+ if (patternFields.has(fieldPath) && isRecord5(filter)) largest = larger(largest, largestTestOf(filter));
28684
+ }
28685
+ }
28686
+ const nested = [...expression.and ?? [], ...expression.or ?? []];
28687
+ if (expression.not !== void 0) nested.push(expression.not);
28688
+ for (const child of nested) largest = larger(largest, largestIn(child, patternFields));
28689
+ return largest;
28690
+ }
28691
+ function largestPatternTest(expression, schema) {
28692
+ if (expression === void 0) return null;
28693
+ const patternFields = /* @__PURE__ */ new Set();
28694
+ for (const { path, pattern } of schemaFieldsOf(schema)) if (pattern) patternFields.add(path);
28695
+ return patternFields.size === 0 ? null : largestIn(expression, patternFields);
28696
+ }
28697
+
27710
28698
  // src/engine/orchestration/search.ts
27711
28699
  function partitionsPerLease(scope, leaseCount) {
27712
28700
  const assignments = Array.from({ length: leaseCount }, () => []);
@@ -27718,15 +28706,18 @@ function partitionsPerLease(scope, leaseCount) {
27718
28706
  function mergeWorkerResults(results, facetConfig) {
27719
28707
  const merged = kWayMerge(results.map((result) => result.scored));
27720
28708
  let totalMatched = 0;
28709
+ let patternWorkSpent = 0;
27721
28710
  const workerFacets = [];
27722
28711
  for (const result of results) {
27723
28712
  totalMatched += result.totalMatched;
28713
+ patternWorkSpent += result.patternWorkSpent ?? 0;
27724
28714
  if (result.facets !== void 0) workerFacets.push(result.facets);
27725
28715
  }
27726
28716
  return {
27727
28717
  scored: merged,
27728
28718
  totalMatched,
27729
- facets: facetConfig !== void 0 && workerFacets.length > 0 ? mergeFacets(workerFacets, facetConfig) : void 0
28719
+ facets: facetConfig !== void 0 && workerFacets.length > 0 ? mergeFacets(workerFacets, facetConfig) : void 0,
28720
+ patternWorkSpent
27730
28721
  };
27731
28722
  }
27732
28723
  function scoresPerPartition(state, indexName, params, globalStats) {
@@ -27734,26 +28725,45 @@ function scoresPerPartition(state, indexName, params, globalStats) {
27734
28725
  if (mode === "local") return true;
27735
28726
  return mode === "broadcast" && globalStats !== void 0;
27736
28727
  }
27737
- function queryAction(indexName, params, globalStats, partitionIds) {
28728
+ function queryAction(indexName, params, globalStats, partitionIds, patternWork) {
27738
28729
  return {
27739
28730
  type: "query",
27740
28731
  indexName,
27741
28732
  params,
27742
28733
  requestId: createRequestId(),
28734
+ patternWork,
27743
28735
  ...partitionIds !== void 0 ? { partitionIds } : {},
27744
28736
  ...globalStats !== void 0 ? { globalStats } : {}
27745
28737
  };
27746
28738
  }
27747
- async function runSplit(leases, scope, indexName, params, globalStats) {
28739
+ async function runSplit(leases, scope, indexName, params, globalStats, patternWork) {
27748
28740
  const assignments = partitionsPerLease(scope, leases.length);
27749
28741
  const workerParams = params.facets !== void 0 ? { ...params, facets: oversampledFacetConfig(params.facets) } : params;
27750
- const results = await Promise.all(
28742
+ const outcomes = await Promise.allSettled(
27751
28743
  assignments.map(
27752
- (partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds)).finally(() => leases[at].release())
28744
+ (partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds, patternWork)).finally(() => leases[at].release())
27753
28745
  )
27754
28746
  );
28747
+ const results = [];
28748
+ const failures = [];
28749
+ for (const outcome of outcomes) {
28750
+ if (outcome.status === "fulfilled") results.push(outcome.value);
28751
+ else failures.push(outcome.reason);
28752
+ }
28753
+ if (failures.length > 0) throw failures.find(isWorkCapError) ?? failures[0];
27755
28754
  return mergeWorkerResults(results, params.facets);
27756
28755
  }
28756
+ function isWorkCapError(err) {
28757
+ return err instanceof NarsilError && err.code === ErrorCodes.SEARCH_WORK_CAP_EXCEEDED;
28758
+ }
28759
+ function patternSplitWidth(state, indexName, params) {
28760
+ const schema = state.indexRegistry.get(indexName)?.config.schema;
28761
+ if (schema === void 0) return Number.POSITIVE_INFINITY;
28762
+ const largest = largestPatternTest(params.filters, schema);
28763
+ if (largest === null) return Number.POSITIVE_INFINITY;
28764
+ if (typeof SharedArrayBuffer !== "function") return 1;
28765
+ return widestPatternSplit(largest);
28766
+ }
27757
28767
  function takeMainCopyTurn(state) {
27758
28768
  if (state.mainCopyQueries === "none" || state.mainCopyTurnTaken) return false;
27759
28769
  state.mainCopyTurnTaken = true;
@@ -27767,7 +28777,7 @@ function answersFasterOnMainCopy(state, pool, manager) {
27767
28777
  if (manager.countDocuments() > MAIN_COPY_LONE_QUERY_DOCUMENTS) return false;
27768
28778
  return takeMainCopyTurn(state);
27769
28779
  }
27770
- async function searchViaWorker(state, indexName, params, globalStats, partitionIds) {
28780
+ async function searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork) {
27771
28781
  const pool = state.workerPool;
27772
28782
  if (!pool) return null;
27773
28783
  if (!state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName)) return null;
@@ -27775,19 +28785,30 @@ async function searchViaWorker(state, indexName, params, globalStats, partitionI
27775
28785
  if (!manager) return null;
27776
28786
  if (answersFasterOnMainCopy(state, pool, manager)) return null;
27777
28787
  const scope = partitionIds ?? Array.from({ length: manager.partitionCount }, (_, partitionId) => partitionId);
27778
- const idle = pool.leaseIdle(scope.length);
28788
+ const idle = pool.leaseIdle(Math.min(scope.length, patternSplitWidth(state, indexName, params)));
27779
28789
  const leases = [];
28790
+ let fork = null;
27780
28791
  try {
27781
28792
  if (idle.length >= 2 && scoresPerPartition(state, indexName, params, globalStats)) {
27782
28793
  leases.push(...idle);
27783
- return await runSplit(leases, scope, indexName, params, globalStats);
28794
+ fork = patternWork.fork(leases.length);
28795
+ const merged = await runSplit(leases, scope, indexName, params, globalStats, fork.shared);
28796
+ fork.settle(merged.patternWorkSpent ?? 0);
28797
+ return merged;
27784
28798
  }
27785
28799
  for (const lease2 of idle.slice(1)) lease2.release();
27786
28800
  const lease = idle[0] ?? (takeMainCopyTurn(state) ? null : pool.leaseLeastBusy());
27787
28801
  if (lease === null) return null;
27788
28802
  leases.push(lease);
27789
- return await lease.executor.execute(queryAction(indexName, params, globalStats, partitionIds));
28803
+ fork = patternWork.fork(1);
28804
+ const result = await lease.executor.execute(
28805
+ queryAction(indexName, params, globalStats, partitionIds, fork.shared)
28806
+ );
28807
+ fork.settle(result.patternWorkSpent ?? 0);
28808
+ return result;
27790
28809
  } catch (err) {
28810
+ if (isWorkCapError(err)) throw err;
28811
+ fork?.settle(0);
27791
28812
  console.warn("Worker search failed, falling back to local:", err);
27792
28813
  return null;
27793
28814
  } finally {
@@ -28016,7 +29037,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
28016
29037
  segmentBuildConcurrency: (indexName) => segmentBuildConcurrency(state, indexName),
28017
29038
  holdUnbroadcastSegments: (indexName, segmentIds) => holdUnbroadcastSegments(state, indexName, segmentIds),
28018
29039
  releaseUnbroadcastSegments: (indexName, segmentIds) => releaseUnbroadcastSegments(state, indexName, segmentIds),
28019
- searchViaWorker: (indexName, params, globalStats, partitionIds) => searchViaWorker(state, indexName, params, globalStats, partitionIds),
29040
+ searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork),
28020
29041
  hasWorkerPool: () => state.workerPool !== null,
28021
29042
  mainCopyQueries: () => state.mainCopyQueries ?? "lone",
28022
29043
  shareMainThread: () => {
@@ -28040,6 +29061,30 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
28040
29061
  };
28041
29062
  }
28042
29063
 
29064
+ // src/engine/mutations/write-visibility.ts
29065
+ async function awaitWriteVisibility(ctx, indexName) {
29066
+ await ctx.awaitRebalanceReplay(indexName);
29067
+ await ctx.orchestrator.awaitWrites(indexName);
29068
+ }
29069
+
29070
+ // src/engine/write-events.ts
29071
+ var WRITE_EVENT = "write";
29072
+ function emitWriteEvent(source, indexName) {
29073
+ const handlers = source.eventHandlers.get(WRITE_EVENT);
29074
+ if (handlers === void 0 || handlers.size === 0) return;
29075
+ const emit = () => {
29076
+ emitEngineEvent(source.eventHandlers, WRITE_EVENT, { indexName });
29077
+ };
29078
+ awaitWriteVisibility(source.mutationCtx, indexName).then(emit, emit);
29079
+ }
29080
+ async function emitWriteEventAfter(source, indexName, write) {
29081
+ try {
29082
+ return await write;
29083
+ } finally {
29084
+ emitWriteEvent(source, indexName);
29085
+ }
29086
+ }
29087
+
28043
29088
  // src/engine/core.ts
28044
29089
  function createEngineCore(config, hooks, capacityScope = "index") {
28045
29090
  validateWorkerConfig(config?.workers, config?.lifecycle);
@@ -28179,7 +29224,11 @@ function createEngineCore(config, hooks, capacityScope = "index") {
28179
29224
  }
28180
29225
  return names;
28181
29226
  },
28182
- reloadIndex: (indexName) => durability?.manager.reloadIndex?.(indexName) ?? Promise.resolve(),
29227
+ reloadIndex: (indexName) => emitWriteEventAfter(
29228
+ { eventHandlers, mutationCtx },
29229
+ indexName,
29230
+ durability?.manager.reloadIndex?.(indexName) ?? Promise.resolve()
29231
+ ),
28183
29232
  onError(error) {
28184
29233
  if (emitEngineEvent(eventHandlers, "invalidationError", { error }) === 0) {
28185
29234
  console.warn("Invalidation error:", error);
@@ -28277,7 +29326,8 @@ function createEngineCore(config, hooks, capacityScope = "index") {
28277
29326
  analysisRebuild,
28278
29327
  indexState,
28279
29328
  mutationCtx,
28280
- rebalanceCtx
29329
+ rebalanceCtx,
29330
+ patternWorkCap: config?.patternWorkCap ?? DEFAULT_PATTERN_WORK_CAP
28281
29331
  };
28282
29332
  }
28283
29333
 
@@ -28376,6 +29426,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
28376
29426
  }
28377
29427
  const schema = validateSchema(requestedConfig.schema);
28378
29428
  const indexConfig = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
29429
+ requirePatternSearchFor(schema);
28379
29430
  validatePatternValueLimit(indexConfig.patternValueLimit);
28380
29431
  validateVectorPromotion(indexConfig.vectorPromotion);
28381
29432
  validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
@@ -28406,10 +29457,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
28406
29457
  const fields = indexConfig.embedding.fields;
28407
29458
  if (typeof configuredAdapter === "string") {
28408
29459
  embeddingAdapterName = configuredAdapter;
28409
- const registered = core.embeddingAdapters.get(configuredAdapter);
28410
- if (registered !== void 0) {
29460
+ const registered2 = core.embeddingAdapters.get(configuredAdapter);
29461
+ if (registered2 !== void 0) {
28411
29462
  resolvedEmbeddingAdapter = validateEmbeddingConfig(
28412
- { fields, adapter: registered },
29463
+ { fields, adapter: registered2 },
28413
29464
  indexConfig.schema,
28414
29465
  config?.embedding
28415
29466
  );
@@ -28763,9 +29814,30 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
28763
29814
 
28764
29815
  // src/engine/mutations/insert-admission.ts
28765
29816
  function providedDocId(document2) {
29817
+ if (document2 === null || typeof document2 !== "object") return void 0;
28766
29818
  const id = document2.id;
28767
29819
  return typeof id === "string" && id.length > 0 ? id : void 0;
28768
29820
  }
29821
+ function requireDocumentObject(document2) {
29822
+ if (document2 !== null && typeof document2 === "object" && !Array.isArray(document2)) return;
29823
+ const received = Array.isArray(document2) ? "a list" : document2 === null ? "null" : `a ${typeof document2}`;
29824
+ throw new NarsilError(
29825
+ ErrorCodes.DOC_VALIDATION_FAILED,
29826
+ `Each document must be an object, while this entry is ${received}`,
29827
+ {
29828
+ received
29829
+ }
29830
+ );
29831
+ }
29832
+ function requireMatchingDocId(docId, document2) {
29833
+ const embeddedDocId = providedDocId(document2);
29834
+ if (embeddedDocId === void 0 || embeddedDocId === docId) return;
29835
+ throw new NarsilError(
29836
+ ErrorCodes.DOC_VALIDATION_FAILED,
29837
+ `The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
29838
+ { docId, documentId: embeddedDocId }
29839
+ );
29840
+ }
28769
29841
  function asBatchInsertError(err) {
28770
29842
  return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
28771
29843
  }
@@ -28872,25 +29944,12 @@ async function embedChunkDocuments(entry, documents, chunkStart, chunkEnd, signa
28872
29944
  }
28873
29945
  }
28874
29946
 
28875
- // src/engine/mutations/write-visibility.ts
28876
- async function awaitWriteVisibility(ctx, indexName) {
28877
- await ctx.awaitRebalanceReplay(indexName);
28878
- await ctx.orchestrator.awaitWrites(indexName);
28879
- }
28880
-
28881
29947
  // src/engine/mutations/insert.ts
28882
29948
  async function insertDocument(ctx, indexName, document2, docId, options) {
28883
29949
  ctx.guardShutdown();
28884
29950
  const entry = ctx.requireIndex(indexName);
28885
- const embeddedDocId = providedDocId(document2);
28886
- if (docId !== void 0 && embeddedDocId !== void 0 && embeddedDocId !== docId) {
28887
- throw new NarsilError(
28888
- ErrorCodes.DOC_VALIDATION_FAILED,
28889
- `The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
28890
- { docId, documentId: embeddedDocId }
28891
- );
28892
- }
28893
- const resolvedDocId = docId ?? embeddedDocId ?? ctx.idGenerator();
29951
+ if (docId !== void 0) requireMatchingDocId(docId, document2);
29952
+ const resolvedDocId = docId ?? providedDocId(document2) ?? ctx.idGenerator();
28894
29953
  validateDocId(resolvedDocId);
28895
29954
  await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document: document2 });
28896
29955
  if (entry.config.required && entry.config.required.length > 0) {
@@ -29127,6 +30186,7 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
29127
30186
  if (chunkFailedIndexes.has(i)) continue;
29128
30187
  const docId = providedDocId(documents[i]) ?? ctx.idGenerator();
29129
30188
  try {
30189
+ requireDocumentObject(documents[i]);
29130
30190
  validateDocId(docId);
29131
30191
  if (hasBeforeHook) {
29132
30192
  await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents[i] });
@@ -29625,6 +30685,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
29625
30685
  if (chunkFailedIndexes.has(i)) continue;
29626
30686
  const batchDocId = providedDocId(documents[i]) ?? ctx.idGenerator();
29627
30687
  try {
30688
+ requireDocumentObject(documents[i]);
29628
30689
  validateDocId(batchDocId);
29629
30690
  if (hasBeforeHook) {
29630
30691
  await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents[i] });
@@ -29847,6 +30908,7 @@ async function settleUpdate(ctx, indexName, prepared) {
29847
30908
  async function prepareUpdate(ctx, indexName, entry, docId, document2) {
29848
30909
  ctx.guardShutdown();
29849
30910
  validateDocId(docId);
30911
+ requireMatchingDocId(docId, document2);
29850
30912
  const manager = ctx.requireManager(indexName);
29851
30913
  if (ctx.pluginRegistry.hasHooks("beforeUpdate")) {
29852
30914
  await ctx.pluginRegistry.runHook("beforeUpdate", {
@@ -29956,6 +31018,22 @@ async function updateDocument(ctx, indexName, docId, document2, options) {
29956
31018
  ctx.checkHeapPressure(indexName);
29957
31019
  if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
29958
31020
  }
31021
+ function reportedDocIdOf(update) {
31022
+ const docId = update !== null && typeof update === "object" ? update.docId : void 0;
31023
+ return typeof docId === "string" ? docId : "";
31024
+ }
31025
+ function requireUpdateEntry(update) {
31026
+ if (update !== null && typeof update === "object" && !Array.isArray(update)) {
31027
+ const { docId, document: document2 } = update;
31028
+ if (typeof docId === "string" && document2 !== null && typeof document2 === "object" && !Array.isArray(document2)) {
31029
+ return { document: document2 };
31030
+ }
31031
+ }
31032
+ throw new NarsilError(
31033
+ ErrorCodes.DOC_VALIDATION_FAILED,
31034
+ "Each entry of a batch update must be an object whose docId is a string and whose document is an object"
31035
+ );
31036
+ }
29959
31037
  async function updateDocumentBatch(ctx, indexName, updates, options) {
29960
31038
  ctx.guardShutdown();
29961
31039
  const entry = ctx.requireIndex(indexName);
@@ -29968,10 +31046,13 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
29968
31046
  const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
29969
31047
  const prepared = [];
29970
31048
  for (let i = chunkStart; i < chunkEnd; i++) {
31049
+ const update = updates[i];
31050
+ const docId = reportedDocIdOf(update);
29971
31051
  try {
29972
- prepared.push(await prepareUpdate(ctx, indexName, entry, updates[i].docId, updates[i].document));
31052
+ const { document: document2 } = requireUpdateEntry(update);
31053
+ prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document2));
29973
31054
  } catch (err) {
29974
- failed.push({ docId: updates[i].docId, error: asUpdateError(err) });
31055
+ failed.push({ docId, error: asUpdateError(err) });
29975
31056
  }
29976
31057
  }
29977
31058
  const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
@@ -30956,6 +32037,9 @@ function requireValidTypoTolerance(params) {
30956
32037
  }
30957
32038
 
30958
32039
  // src/engine/query/shared.ts
32040
+ function withPatternWork(context) {
32041
+ return { ...context, patternWork: createPatternWorkMeter(context.patternWorkCap) };
32042
+ }
30959
32043
  function partitionsFor(manager, partitionIds) {
30960
32044
  return partitionsIn(manager, partitionIds);
30961
32045
  }
@@ -31002,18 +32086,19 @@ function broadcastStatsForWorker(params, context, scoring) {
31002
32086
  }
31003
32087
  return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, context.manager.analysis);
31004
32088
  }
31005
- function searchOptionsFor(manager) {
32089
+ function searchOptionsFor(manager, patternWork) {
31006
32090
  return {
31007
32091
  bm25Params: manager.config.bm25,
31008
32092
  stopWords: manager.analysis.stopWords,
31009
- customTokenizer: manager.analysis.customTokenizer
32093
+ customTokenizer: manager.analysis.customTokenizer,
32094
+ patternWork
31010
32095
  };
31011
32096
  }
31012
- function collectFilterDocIds(manager, params, schema, partitionIds) {
32097
+ function collectFilterDocIds(manager, params, schema, partitionIds, patternWork) {
31013
32098
  const filterDocIds = /* @__PURE__ */ new Set();
31014
32099
  if (!params.filters) return filterDocIds;
31015
32100
  for (const partition of partitionsFor(manager, partitionIds)) {
31016
- const partitionFiltered = partition.applyFilters(params.filters, schema);
32101
+ const partitionFiltered = partition.applyFilters(params.filters, schema, patternWork);
31017
32102
  for (const docId of partitionFiltered) {
31018
32103
  filterDocIds.add(docId);
31019
32104
  }
@@ -31048,11 +32133,11 @@ function requireValidQueryOptions(params, config) {
31048
32133
  const strict = config.strict === true;
31049
32134
  requireValidTypoTolerance(params);
31050
32135
  requireSortableFields(params.sort, config.schema, strict);
31051
- requireValidQueryFilter(params.filters, config.schema, strict);
32136
+ requireValidQueryFilter(params.filters, config.schema);
31052
32137
  requireUsableQueryFields(params, config.schema, strict);
31053
32138
  }
31054
- function requireValidQueryFilter(filters, schema, strict = false) {
31055
- if (filters !== void 0) requireValidFilter(filters, flattenSchema(schema), strict);
32139
+ function requireValidQueryFilter(filters, schema) {
32140
+ if (filters !== void 0) requireValidFilter(filters, flattenSchema(schema));
31056
32141
  }
31057
32142
  function requireVectorSearchable(params, context, needsVector) {
31058
32143
  if (!needsVector) return;
@@ -31088,8 +32173,9 @@ function firstIdAfter(sorted, anchor) {
31088
32173
  }
31089
32174
  return low;
31090
32175
  }
31091
- function collectFilterMatches(partitions, filters, schema) {
31092
- const perPartition = partitions.map((partition) => partition.filterMatches(filters, schema));
32176
+ function collectFilterMatches(partitions, filters, schema, patternWorkCap) {
32177
+ const meter = createPatternWorkMeter(patternWorkCap);
32178
+ const perPartition = partitions.map((partition) => partition.filterMatches(filters, schema, meter));
31093
32179
  let total = 0;
31094
32180
  for (const matches of perPartition) total += matches.count;
31095
32181
  return { matchesFor: (partitionIndex) => perPartition[partitionIndex] ?? null, total };
@@ -31167,7 +32253,7 @@ function executeListDocuments(params, context) {
31167
32253
  const limit = clampListLimit(params.limit);
31168
32254
  const signature = sortSignatureOf(params.sort);
31169
32255
  requireSortableFields(params.sort, schema, context.strict === true);
31170
- requireValidQueryFilter(params.filters, schema, context.strict === true);
32256
+ requireValidQueryFilter(params.filters, schema);
31171
32257
  const binding = listBindingOf(params.filters);
31172
32258
  let cursor = null;
31173
32259
  if (params.cursor !== void 0) {
@@ -31175,7 +32261,7 @@ function executeListDocuments(params, context) {
31175
32261
  requireMatchingCursor(cursor, params.cursor, signature, false, binding);
31176
32262
  }
31177
32263
  const partitions = context.partitionIds === void 0 ? manager.getAllPartitions() : context.partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
31178
- const filtered = params.filters === void 0 ? null : collectFilterMatches(partitions, params.filters, schema);
32264
+ const filtered = params.filters === void 0 ? null : collectFilterMatches(partitions, params.filters, schema, context.patternWorkCap);
31179
32265
  const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
31180
32266
  const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
31181
32267
  const projection = resolveProjection(params.document);
@@ -31542,7 +32628,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
31542
32628
  const flatSchema = flattenSchema(schema);
31543
32629
  const fieldTypes = fields.map((field) => flatSchema[field]);
31544
32630
  const modes = sortModesOf(normalized);
31545
- const options = searchOptionsFor(manager);
32631
+ const options = searchOptionsFor(manager, context.patternWork);
31546
32632
  const partitions = partitionsFor(manager, context.partitionIds);
31547
32633
  const partitionLimit = offset + limit + 1;
31548
32634
  const merged = [];
@@ -31743,7 +32829,7 @@ async function executeVectorSearch(params, context, limit, offset, cursorDepth =
31743
32829
  let filterDocIds;
31744
32830
  let filterPartitions;
31745
32831
  if (params.filters) {
31746
- filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds);
32832
+ filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds, context.patternWork);
31747
32833
  if (filterDocIds.size === 0) {
31748
32834
  return { scored: [], totalMatched: 0 };
31749
32835
  }
@@ -31771,11 +32857,19 @@ async function textLeg(textOnlyParams, context) {
31771
32857
  indexName,
31772
32858
  textOnlyParams,
31773
32859
  broadcastStatsForWorker(textOnlyParams, context, scoring),
31774
- context.partitionIds
32860
+ context.partitionIds,
32861
+ context.patternWork
31775
32862
  );
31776
32863
  if (viaWorker) return viaWorker;
31777
32864
  }
31778
- return fanOutQuery(manager, textOnlyParams, language, config.schema, scoring, searchOptionsFor(manager));
32865
+ return fanOutQuery(
32866
+ manager,
32867
+ textOnlyParams,
32868
+ language,
32869
+ config.schema,
32870
+ scoring,
32871
+ searchOptionsFor(manager, context.patternWork)
32872
+ );
31779
32873
  }
31780
32874
  async function vectorLeg(vectorConfig, queryVector, context, filterDocIds, k) {
31781
32875
  const { manager } = context;
@@ -31801,7 +32895,7 @@ async function executeHybridSearch(params, context, limit, offset, cursorDepth =
31801
32895
  const fusion = resolveHybridFusion(params.hybrid);
31802
32896
  let filterDocIds;
31803
32897
  if (params.filters) {
31804
- filterDocIds = collectFilterDocIds(manager, params, config.schema, context.partitionIds);
32898
+ filterDocIds = collectFilterDocIds(manager, params, config.schema, context.partitionIds, context.patternWork);
31805
32899
  if (filterDocIds.size === 0) {
31806
32900
  return { scored: [], totalMatched: 0 };
31807
32901
  }
@@ -31845,7 +32939,8 @@ function requireCursorDepth(cursor, signature, binding) {
31845
32939
  requireMatchingCursor(decoded, cursor, signature, true, binding);
31846
32940
  return decoded.depth ?? 0;
31847
32941
  }
31848
- async function executeQuery(params, context) {
32942
+ async function executeQuery(params, callerContext) {
32943
+ const context = withPatternWork(callerContext);
31849
32944
  const { manager, language, config, workerSearch, indexName } = context;
31850
32945
  const startTime = now();
31851
32946
  const limit = clampLimit(params.limit);
@@ -31891,11 +32986,24 @@ async function executeQuery(params, context) {
31891
32986
  fanOutResult = await executeHybridSearch(params, context, limit, offset, depth);
31892
32987
  } else {
31893
32988
  const scoring = scoringConfigFor(params, context);
31894
- const workerResult = workerSearch ? await workerSearch(indexName, params, broadcastStatsForWorker(params, context, scoring), context.partitionIds) : null;
32989
+ const workerResult = workerSearch ? await workerSearch(
32990
+ indexName,
32991
+ params,
32992
+ broadcastStatsForWorker(params, context, scoring),
32993
+ context.partitionIds,
32994
+ context.patternWork
32995
+ ) : null;
31895
32996
  if (workerResult) {
31896
32997
  fanOutResult = workerResult;
31897
32998
  } else {
31898
- fanOutResult = await fanOutQuery(manager, params, language, config.schema, scoring, searchOptionsFor(manager));
32999
+ fanOutResult = await fanOutQuery(
33000
+ manager,
33001
+ params,
33002
+ language,
33003
+ config.schema,
33004
+ scoring,
33005
+ searchOptionsFor(manager, context.patternWork)
33006
+ );
31899
33007
  }
31900
33008
  }
31901
33009
  const needsFullHits = params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
@@ -32015,7 +33123,8 @@ async function executeQuery(params, context) {
32015
33123
  coverage: coverageFor(manager, context.partitionIds)
32016
33124
  };
32017
33125
  }
32018
- async function executePreflight(params, context) {
33126
+ async function executePreflight(params, callerContext) {
33127
+ const context = withPatternWork(callerContext);
32019
33128
  const { manager, language, config, workerSearch, indexName } = context;
32020
33129
  const startTime = now();
32021
33130
  requireKnownMode(params);
@@ -32041,12 +33150,18 @@ async function executePreflight(params, context) {
32041
33150
  countExact = result.matchedExact !== false;
32042
33151
  } else if (countsWithoutScores(params)) {
32043
33152
  totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
32044
- searchOptions: searchOptionsFor(manager),
33153
+ searchOptions: searchOptionsFor(manager, context.patternWork),
32045
33154
  partitionIds: context.partitionIds
32046
33155
  });
32047
33156
  } else {
32048
33157
  const scoring = scoringConfigFor(params, context);
32049
- const workerResult = workerSearch ? await workerSearch(indexName, params, broadcastStatsForWorker(params, context, scoring), context.partitionIds) : null;
33158
+ const workerResult = workerSearch ? await workerSearch(
33159
+ indexName,
33160
+ params,
33161
+ broadcastStatsForWorker(params, context, scoring),
33162
+ context.partitionIds,
33163
+ context.patternWork
33164
+ ) : null;
32050
33165
  if (workerResult) {
32051
33166
  totalMatched = workerResult.totalMatched;
32052
33167
  } else {
@@ -32056,7 +33171,7 @@ async function executePreflight(params, context) {
32056
33171
  language,
32057
33172
  config.schema,
32058
33173
  scoring,
32059
- searchOptionsFor(manager)
33174
+ searchOptionsFor(manager, context.patternWork)
32060
33175
  );
32061
33176
  totalMatched = fanOutResult.totalMatched;
32062
33177
  }
@@ -32204,7 +33319,8 @@ async function runEngineQuery(core, indexName, params, options) {
32204
33319
  indexName,
32205
33320
  broadcastStats: broadcastStatsFor(core, options),
32206
33321
  partitionIds: options?.partitionIds,
32207
- cursorBinding: queryBindingOf(params)
33322
+ cursorBinding: queryBindingOf(params),
33323
+ patternWorkCap: core.patternWorkCap
32208
33324
  });
32209
33325
  if (core.pluginRegistry.hasHooks("afterSearch")) {
32210
33326
  const observed = { ...result, hits: [...result.hits] };
@@ -32245,7 +33361,8 @@ async function runEnginePreflight(core, indexName, params, options) {
32245
33361
  indexName,
32246
33362
  broadcastStats: broadcastStatsFor(core, options),
32247
33363
  partitionIds: options?.partitionIds,
32248
- cursorBinding: queryBindingOf(params)
33364
+ cursorBinding: queryBindingOf(params),
33365
+ patternWorkCap: core.patternWorkCap
32249
33366
  });
32250
33367
  if (core.analysisRebuild.isStale(indexName)) {
32251
33368
  result.analysisStale = true;
@@ -32282,7 +33399,8 @@ async function runEngineListDocuments(core, indexName, params, partitionIds) {
32282
33399
  manager: core.requireManager(indexName),
32283
33400
  schema: entry.config.schema,
32284
33401
  strict: entry.config.strict,
32285
- partitionIds
33402
+ partitionIds,
33403
+ patternWorkCap: core.patternWorkCap
32286
33404
  });
32287
33405
  } finally {
32288
33406
  release();
@@ -32316,15 +33434,18 @@ function createNarsilFromCore(core, config) {
32316
33434
  release();
32317
33435
  }
32318
33436
  }
33437
+ function withOpenIndexWrite(indexName, action) {
33438
+ return emitWriteEventAfter(core, indexName, withOpenIndex(indexName, action));
33439
+ }
32319
33440
  const narsil = {
32320
33441
  createIndex(name, indexConfig) {
32321
- return createEngineIndex(core, config, name, indexConfig);
33442
+ return emitWriteEventAfter(core, name, createEngineIndex(core, config, name, indexConfig));
32322
33443
  },
32323
33444
  registerEmbeddingAdapter(name, adapter) {
32324
33445
  registerEngineEmbeddingAdapter(core, name, adapter);
32325
33446
  },
32326
33447
  dropIndex(name) {
32327
- return dropEngineIndex(core, name);
33448
+ return emitWriteEventAfter(core, name, dropEngineIndex(core, name));
32328
33449
  },
32329
33450
  open(indexName) {
32330
33451
  guardShutdown();
@@ -32370,22 +33491,22 @@ function createNarsilFromCore(core, config) {
32370
33491
  return executor.getManager(indexName)?.getPartitionStats() ?? [];
32371
33492
  },
32372
33493
  insert(indexName, document2, docId, options) {
32373
- return withOpenIndex(indexName, () => insertDocument(mutationCtx, indexName, document2, docId, options));
33494
+ return withOpenIndexWrite(indexName, () => insertDocument(mutationCtx, indexName, document2, docId, options));
32374
33495
  },
32375
33496
  insertBatch(indexName, documents, options) {
32376
- return withOpenIndex(indexName, () => insertDocumentBatch(mutationCtx, indexName, documents, options));
33497
+ return withOpenIndexWrite(indexName, () => insertDocumentBatch(mutationCtx, indexName, documents, options));
32377
33498
  },
32378
33499
  remove(indexName, docId, options) {
32379
- return withOpenIndex(indexName, () => removeDocument(mutationCtx, indexName, docId, options));
33500
+ return withOpenIndexWrite(indexName, () => removeDocument(mutationCtx, indexName, docId, options));
32380
33501
  },
32381
33502
  removeBatch(indexName, docIds, options) {
32382
- return withOpenIndex(indexName, () => removeDocumentBatch(mutationCtx, indexName, docIds, options));
33503
+ return withOpenIndexWrite(indexName, () => removeDocumentBatch(mutationCtx, indexName, docIds, options));
32383
33504
  },
32384
33505
  update(indexName, docId, document2, options) {
32385
- return withOpenIndex(indexName, () => updateDocument(mutationCtx, indexName, docId, document2, options));
33506
+ return withOpenIndexWrite(indexName, () => updateDocument(mutationCtx, indexName, docId, document2, options));
32386
33507
  },
32387
33508
  updateBatch(indexName, updates, options) {
32388
- return withOpenIndex(indexName, () => updateDocumentBatch(mutationCtx, indexName, updates, options));
33509
+ return withOpenIndexWrite(indexName, () => updateDocumentBatch(mutationCtx, indexName, updates, options));
32389
33510
  },
32390
33511
  waitForWrites(indexName) {
32391
33512
  guardShutdown();
@@ -32433,7 +33554,7 @@ function createNarsilFromCore(core, config) {
32433
33554
  async rebuildAnalysis(indexName) {
32434
33555
  guardShutdown();
32435
33556
  requireIndex(indexName);
32436
- await withOpenIndex(indexName, () => core.analysisRebuild.rebuild(indexName));
33557
+ await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
32437
33558
  },
32438
33559
  async snapshot(indexName) {
32439
33560
  guardShutdown();
@@ -32441,11 +33562,11 @@ function createNarsilFromCore(core, config) {
32441
33562
  },
32442
33563
  async restore(indexName, data) {
32443
33564
  guardShutdown();
32444
- await restoreFromSnapshot(indexName, data, {
33565
+ const restored = restoreFromSnapshot(indexName, data, {
32445
33566
  executor,
32446
33567
  indexRegistry,
32447
33568
  getVectorFieldPaths,
32448
- dropIndex: narsil.dropIndex.bind(narsil),
33569
+ dropIndex: (name) => dropEngineIndex(core, name),
32449
33570
  announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
32450
33571
  requireManager,
32451
33572
  durability,
@@ -32456,6 +33577,7 @@ function createNarsilFromCore(core, config) {
32456
33577
  clearAnalysisStale: core.analysisRebuild.clearStale,
32457
33578
  indexState: core.indexState
32458
33579
  });
33580
+ await emitWriteEventAfter(core, indexName, restored);
32459
33581
  core.heapPressureNotifier.check(indexName);
32460
33582
  },
32461
33583
  async checkpoint(indexName) {
@@ -32468,7 +33590,7 @@ function createNarsilFromCore(core, config) {
32468
33590
  async clear(indexName) {
32469
33591
  guardShutdown();
32470
33592
  requireIndex(indexName);
32471
- await withOpenIndex(indexName, async () => {
33593
+ await withOpenIndexWrite(indexName, async () => {
32472
33594
  await executor.execute({ type: "clear", indexName, requestId: indexName });
32473
33595
  await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
32474
33596
  core.watermarkNotifier.forget(indexName);
@@ -32477,7 +33599,7 @@ function createNarsilFromCore(core, config) {
32477
33599
  async rebalance(indexName, targetPartitionCount) {
32478
33600
  guardShutdown();
32479
33601
  requireIndex(indexName);
32480
- return withOpenIndex(
33602
+ return withOpenIndexWrite(
32481
33603
  indexName,
32482
33604
  () => executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx)
32483
33605
  );
@@ -32543,14 +33665,14 @@ function createNarsilFromCore(core, config) {
32543
33665
  async compactVectors(indexName, fieldName) {
32544
33666
  guardShutdown();
32545
33667
  requireIndex(indexName);
32546
- await withOpenIndex(indexName, async () => {
33668
+ await withOpenIndexWrite(indexName, async () => {
32547
33669
  compactVectors(requireManager(indexName), indexName, fieldName);
32548
33670
  });
32549
33671
  },
32550
33672
  async optimizeVectors(indexName, fieldName) {
32551
33673
  guardShutdown();
32552
33674
  requireIndex(indexName);
32553
- await withOpenIndex(indexName, () => optimizeVectors(requireManager(indexName), indexName, fieldName));
33675
+ await withOpenIndexWrite(indexName, () => optimizeVectors(requireManager(indexName), indexName, fieldName));
32554
33676
  },
32555
33677
  vectorMaintenanceStatus(indexName) {
32556
33678
  guardShutdown();
@@ -32598,4 +33720,4 @@ async function createNarsil(config) {
32598
33720
  // src/index.ts
32599
33721
  var VERSION = "0.1.0";
32600
33722
 
32601
- export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNarsilError, isSimdAvailable, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache };
33723
+ export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNarsilError, isSimdAvailable, registerLanguage, registerPatternSearch, registerStopWords, registerTokenizer, resetNormalizationCache };