@delali/narsil 0.3.1-next.71 → 0.3.1-next.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-Dt-7B4Wz.d.ts → capabilities-D6gjjQyA.d.ts} +2 -2
- package/dist/{chunk-P4DSQ6M4.mjs → chunk-2X2ZQOAB.mjs} +69 -28
- package/dist/chunk-3HQPUCCT.mjs +785 -0
- package/dist/{chunk-IOR2K26O.mjs → chunk-7QMTA7AA.mjs} +146 -37
- package/dist/{chunk-FVLQ63B7.mjs → chunk-CVPA5Q7R.mjs} +24 -3
- package/dist/{chunk-2STVUTNA.mjs → chunk-EDMCZLSJ.mjs} +4 -3
- package/dist/{chunk-M76V3OGJ.mjs → chunk-HVJYTKK7.mjs} +940 -647
- package/dist/{chunk-HWQLTLNG.mjs → chunk-KZTI25D3.mjs} +1 -1
- package/dist/{chunk-YJDQ7NLA.mjs → chunk-LCCL6EEH.mjs} +3 -3
- package/dist/{chunk-OI42GWLR.mjs → chunk-LJYTW7GT.mjs} +57 -11
- package/dist/{chunk-653BNMAF.mjs → chunk-M6XF5HDE.mjs} +57 -17
- package/dist/{chunk-YNVIUGF5.mjs → chunk-P2ZZN32C.mjs} +1 -1
- package/dist/{chunk-LMO5TJUS.mjs → chunk-QR4OKRVY.mjs} +637 -196
- package/dist/{chunk-USKFU7J5.mjs → chunk-RV77FR2F.mjs} +1 -0
- package/dist/{chunk-GIVAGD3Q.mjs → chunk-XZ4PAREY.mjs} +1 -1
- package/dist/{chunk-3JGWULDF.mjs → chunk-Y52WZWAU.mjs} +5 -5
- package/dist/chunk-ZKRV5M2P.mjs +75 -0
- package/dist/{client-COek5GXK.d.ts → client-C2TiW29y.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +1 -0
- package/dist/{config-rBpbgTw8.d.ts → config-BLguB-om.d.ts} +2 -2
- package/dist/{config-Dfanyqqt.d.ts → config-C7Ip8Ife.d.ts} +17 -2
- package/dist/direct-executor-SVCHXR74.mjs +14 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +58 -35
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-igk0aMpQ.d.ts → engine-Dm4446j1.d.ts} +2 -2
- package/dist/index.browser.mjs +2183 -1087
- package/dist/index.d.ts +27 -8
- package/dist/index.mjs +14 -12
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-C7UMCMH3.d.ts → memory-CKHTU1CL.d.ts} +66 -25
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +7 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +17 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -12
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +1 -0
- package/dist/{schema-Bgzr4g7b.d.ts → schema-BBOCdhND.d.ts} +9 -7
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +13 -12
- package/dist/{tasks-BZt_zes6.d.ts → tasks-BECd0b__.d.ts} +1 -1
- package/dist/{thread-JEVAYETR.mjs → thread-IBSIZZ6W.mjs} +22 -13
- package/dist/{types-gOUqhs2F.d.ts → types-BiUmaiWT.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +4 -4
- package/package.json +15 -7
- package/dist/direct-executor-GYJEKNV5.mjs +0 -12
package/dist/index.browser.mjs
CHANGED
|
@@ -33,6 +33,7 @@ var ErrorCodes = {
|
|
|
33
33
|
SEARCH_INVALID_MODE: "SEARCH_INVALID_MODE",
|
|
34
34
|
SEARCH_INVALID_CURSOR: "SEARCH_INVALID_CURSOR",
|
|
35
35
|
SEARCH_RESULT_WINDOW_EXCEEDED: "SEARCH_RESULT_WINDOW_EXCEEDED",
|
|
36
|
+
SEARCH_WORK_CAP_EXCEEDED: "SEARCH_WORK_CAP_EXCEEDED",
|
|
36
37
|
LANGUAGE_NOT_SUPPORTED: "LANGUAGE_NOT_SUPPORTED",
|
|
37
38
|
ENVELOPE_VERSION_MISMATCH: "ENVELOPE_VERSION_MISMATCH",
|
|
38
39
|
ENVELOPE_INVALID_MAGIC: "ENVELOPE_INVALID_MAGIC",
|
|
@@ -209,6 +210,288 @@ function resolveIndexAnalysis(config) {
|
|
|
209
210
|
};
|
|
210
211
|
}
|
|
211
212
|
|
|
213
|
+
// src/core/pattern-index/platform.browser.ts
|
|
214
|
+
function platformPatternSearch() {
|
|
215
|
+
return null;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// src/schema/constants.ts
|
|
219
|
+
var MAX_NESTING_DEPTH = 4;
|
|
220
|
+
var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
221
|
+
var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
|
|
222
|
+
var MIN_PATTERN_VALUE_LIMIT = 1;
|
|
223
|
+
var MAX_PATTERN_VALUE_LIMIT = 65536;
|
|
224
|
+
|
|
225
|
+
// src/schema/validator/shared.ts
|
|
226
|
+
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
227
|
+
function isGeopointOrVectorType(fieldType) {
|
|
228
|
+
return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
|
|
229
|
+
}
|
|
230
|
+
var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
|
|
231
|
+
var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
|
|
232
|
+
var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
233
|
+
function isPlainObject(value) {
|
|
234
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// src/schema/validator/field-type.ts
|
|
238
|
+
var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
|
|
239
|
+
var OPTIONS_BY_BASE = {
|
|
240
|
+
string: FIELD_OPTION_ORDER,
|
|
241
|
+
"string[]": FIELD_OPTION_ORDER,
|
|
242
|
+
verbatim: ["sortable"],
|
|
243
|
+
"verbatim[]": ["sortable"],
|
|
244
|
+
number: [],
|
|
245
|
+
"number[]": [],
|
|
246
|
+
boolean: [],
|
|
247
|
+
"boolean[]": [],
|
|
248
|
+
enum: [],
|
|
249
|
+
"enum[]": [],
|
|
250
|
+
geopoint: [],
|
|
251
|
+
vector: []
|
|
252
|
+
};
|
|
253
|
+
var TYPE_SEPARATOR = ":";
|
|
254
|
+
function isNamedBase(name) {
|
|
255
|
+
return name !== "vector" && Object.hasOwn(OPTIONS_BY_BASE, name);
|
|
256
|
+
}
|
|
257
|
+
function describeAllowedOptions(baseName, allowed) {
|
|
258
|
+
if (allowed.length === 0) return `the "${baseName}" type has no options`;
|
|
259
|
+
const quoted = allowed.map((option) => `"${option}"`);
|
|
260
|
+
if (quoted.length === 1) return `the only option of the "${baseName}" type is ${quoted[0]}`;
|
|
261
|
+
return `the options of the "${baseName}" type are ${quoted.slice(0, -1).join(", ")}, and ${quoted[quoted.length - 1]}`;
|
|
262
|
+
}
|
|
263
|
+
function invalidType(path, type, message) {
|
|
264
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, message, { field: path, type });
|
|
265
|
+
}
|
|
266
|
+
function parseBase(baseName, path, type) {
|
|
267
|
+
if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
|
|
268
|
+
const vectorMatch = VECTOR_PATTERN.exec(baseName);
|
|
269
|
+
if (vectorMatch === null) {
|
|
270
|
+
invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
|
|
271
|
+
}
|
|
272
|
+
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
273
|
+
if (dimension <= 0) {
|
|
274
|
+
throw new NarsilError(
|
|
275
|
+
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
276
|
+
`Vector field "${path}" must have a positive dimension, got ${dimension}`,
|
|
277
|
+
{ field: path, dimension }
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
return { base: "vector", dimension };
|
|
281
|
+
}
|
|
282
|
+
function parseFieldType(type, path) {
|
|
283
|
+
const separatorIndex = type.indexOf(TYPE_SEPARATOR);
|
|
284
|
+
const baseName = separatorIndex === -1 ? type : type.slice(0, separatorIndex);
|
|
285
|
+
const { base, dimension } = parseBase(baseName, path, type);
|
|
286
|
+
const allowed = OPTIONS_BY_BASE[base];
|
|
287
|
+
const options = /* @__PURE__ */ new Set();
|
|
288
|
+
if (separatorIndex !== -1) {
|
|
289
|
+
for (const option of type.slice(separatorIndex + 1).split(TYPE_SEPARATOR)) {
|
|
290
|
+
if (!allowed.includes(option)) {
|
|
291
|
+
invalidType(
|
|
292
|
+
path,
|
|
293
|
+
type,
|
|
294
|
+
`Field "${path}" has type "${type}" with the option "${option}", but ${describeAllowedOptions(baseName, allowed)}`
|
|
295
|
+
);
|
|
296
|
+
}
|
|
297
|
+
if (options.has(option)) {
|
|
298
|
+
invalidType(path, type, `Field "${path}" has the "${option}" option twice in type "${type}"`);
|
|
299
|
+
}
|
|
300
|
+
options.add(option);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
const ordered = FIELD_OPTION_ORDER.filter((option) => options.has(option));
|
|
304
|
+
const verbatim = base === "verbatim" || base === "verbatim[]";
|
|
305
|
+
return {
|
|
306
|
+
type: [baseName, ...ordered].join(TYPE_SEPARATOR),
|
|
307
|
+
base,
|
|
308
|
+
sortable: options.has("sortable"),
|
|
309
|
+
pattern: verbatim || options.has("pattern"),
|
|
310
|
+
partial: options.has("partial"),
|
|
311
|
+
dimension
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
function parsedTypeOf(type) {
|
|
315
|
+
if (type === void 0) return void 0;
|
|
316
|
+
try {
|
|
317
|
+
return parseFieldType(type, "");
|
|
318
|
+
} catch {
|
|
319
|
+
return void 0;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
function withSortableOption(parsed) {
|
|
323
|
+
if (parsed.sortable) return parsed.type;
|
|
324
|
+
const separatorIndex = parsed.type.indexOf(TYPE_SEPARATOR);
|
|
325
|
+
const baseName = separatorIndex === -1 ? parsed.type : parsed.type.slice(0, separatorIndex);
|
|
326
|
+
return parseFieldType(`${baseName}${TYPE_SEPARATOR}sortable${parsed.type.slice(baseName.length)}`, "").type;
|
|
327
|
+
}
|
|
328
|
+
function isListBase(base) {
|
|
329
|
+
return base.endsWith("[]");
|
|
330
|
+
}
|
|
331
|
+
function isWordIndexedBase(base) {
|
|
332
|
+
return base === "string" || base === "string[]";
|
|
333
|
+
}
|
|
334
|
+
function isSingleStringFieldType(fieldType) {
|
|
335
|
+
return parsedTypeOf(fieldType)?.base === "string";
|
|
336
|
+
}
|
|
337
|
+
function isWordIndexedFieldType(fieldType) {
|
|
338
|
+
const parsed = parsedTypeOf(fieldType);
|
|
339
|
+
return parsed !== void 0 && isWordIndexedBase(parsed.base);
|
|
340
|
+
}
|
|
341
|
+
function isStringValuedBase(base) {
|
|
342
|
+
return base === "string" || base === "string[]" || base === "verbatim" || base === "verbatim[]";
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// src/schema/validator/schema.ts
|
|
346
|
+
function validateSchemaFields(schema, depth, prefix) {
|
|
347
|
+
if (depth > MAX_NESTING_DEPTH) {
|
|
348
|
+
throw new NarsilError(
|
|
349
|
+
ErrorCodes.SCHEMA_DEPTH_EXCEEDED,
|
|
350
|
+
`Schema nesting exceeds the maximum depth of ${MAX_NESTING_DEPTH} at "${prefix}"`,
|
|
351
|
+
{ path: prefix, maxDepth: MAX_NESTING_DEPTH }
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
let canonical = null;
|
|
355
|
+
const replace = (field, value) => {
|
|
356
|
+
if (canonical === null) canonical = { ...schema };
|
|
357
|
+
canonical[field] = value;
|
|
358
|
+
};
|
|
359
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
360
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
361
|
+
if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
|
|
362
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field name "${field}" is not allowed in a schema`, {
|
|
363
|
+
field,
|
|
364
|
+
path
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
if (!FIELD_NAME_PATTERN.test(field)) {
|
|
368
|
+
throw new NarsilError(
|
|
369
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
370
|
+
`Field name "${field}" contains characters that are not allowed; use letters, digits, and underscores only`,
|
|
371
|
+
{ field, path }
|
|
372
|
+
);
|
|
373
|
+
}
|
|
374
|
+
if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
|
|
375
|
+
throw new NarsilError(
|
|
376
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
377
|
+
`Field "${field}" is reserved and cannot be defined in a schema`,
|
|
378
|
+
{ field, path }
|
|
379
|
+
);
|
|
380
|
+
}
|
|
381
|
+
if (isPlainObject(type)) {
|
|
382
|
+
const nested = validateSchemaFields(type, depth + 1, path);
|
|
383
|
+
if (nested !== type) replace(field, nested);
|
|
384
|
+
continue;
|
|
385
|
+
}
|
|
386
|
+
if (typeof type !== "string") {
|
|
387
|
+
throw new NarsilError(
|
|
388
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
389
|
+
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
390
|
+
{
|
|
391
|
+
field: path,
|
|
392
|
+
type: String(type)
|
|
393
|
+
}
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
const parsed = parseFieldType(type, path);
|
|
397
|
+
if (parsed.type !== type) replace(field, parsed.type);
|
|
398
|
+
}
|
|
399
|
+
return canonical ?? schema;
|
|
400
|
+
}
|
|
401
|
+
function validateSchema(schema) {
|
|
402
|
+
if (!isPlainObject(schema)) {
|
|
403
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must be a plain object", {
|
|
404
|
+
received: typeof schema
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
if (Object.keys(schema).length === 0) {
|
|
408
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
|
|
409
|
+
}
|
|
410
|
+
return validateSchemaFields(schema, 1, "");
|
|
411
|
+
}
|
|
412
|
+
function requireValidFieldTypes(flatSchema) {
|
|
413
|
+
for (const [path, type] of Object.entries(flatSchema)) {
|
|
414
|
+
if (typeof type !== "string") {
|
|
415
|
+
throw new NarsilError(
|
|
416
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
417
|
+
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
418
|
+
{ field: path, type: String(type) }
|
|
419
|
+
);
|
|
420
|
+
}
|
|
421
|
+
parseFieldType(type, path);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
function flattenRecursive(schema, prefix, result) {
|
|
425
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
426
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
427
|
+
if (isPlainObject(type)) {
|
|
428
|
+
flattenRecursive(type, path, result);
|
|
429
|
+
} else {
|
|
430
|
+
result[path] = type;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
var flattenedSchemas = /* @__PURE__ */ new WeakMap();
|
|
435
|
+
function flattenSchema(schema) {
|
|
436
|
+
const cached = flattenedSchemas.get(schema);
|
|
437
|
+
if (cached !== void 0) return cached;
|
|
438
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
439
|
+
flattenRecursive(schema, "", result);
|
|
440
|
+
flattenedSchemas.set(schema, Object.freeze(result));
|
|
441
|
+
return result;
|
|
442
|
+
}
|
|
443
|
+
var parsedSchemas = /* @__PURE__ */ new WeakMap();
|
|
444
|
+
function schemaFieldsOf(schema) {
|
|
445
|
+
const cached = parsedSchemas.get(schema);
|
|
446
|
+
if (cached !== void 0) return cached;
|
|
447
|
+
const fields = [];
|
|
448
|
+
for (const [path, type] of Object.entries(flattenSchema(schema))) {
|
|
449
|
+
fields.push({ ...parseFieldType(type, path), path });
|
|
450
|
+
}
|
|
451
|
+
const frozen = Object.freeze(fields);
|
|
452
|
+
parsedSchemas.set(schema, frozen);
|
|
453
|
+
return frozen;
|
|
454
|
+
}
|
|
455
|
+
function extractVectorFieldsFromSchema(schema) {
|
|
456
|
+
const result = /* @__PURE__ */ new Map();
|
|
457
|
+
for (const field of schemaFieldsOf(schema)) {
|
|
458
|
+
if (field.base === "vector") result.set(field.path, field.dimension);
|
|
459
|
+
}
|
|
460
|
+
return result;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// src/core/pattern-index/registry.ts
|
|
464
|
+
var registered = null;
|
|
465
|
+
function isPatternSearchModule(value) {
|
|
466
|
+
return value !== null && typeof value === "object" && value.name === "pattern" && "createIndex" in value && typeof value.createIndex === "function" && "readArrays" in value && typeof value.readArrays === "function" && "mergeArrays" in value && typeof value.mergeArrays === "function" && "matchInto" in value && typeof value.matchInto === "function";
|
|
467
|
+
}
|
|
468
|
+
function registerPatternSearch(module) {
|
|
469
|
+
if (!isPatternSearchModule(module)) {
|
|
470
|
+
throw new NarsilError(
|
|
471
|
+
ErrorCodes.CONFIG_INVALID,
|
|
472
|
+
'registerPatternSearch takes the patternSearch object that "@delali/narsil/pattern" exports'
|
|
473
|
+
);
|
|
474
|
+
}
|
|
475
|
+
registered = module;
|
|
476
|
+
}
|
|
477
|
+
function availablePatternSearch() {
|
|
478
|
+
return registered ?? platformPatternSearch();
|
|
479
|
+
}
|
|
480
|
+
function requirePatternSearchFor(schema) {
|
|
481
|
+
for (const { path, pattern } of schemaFieldsOf(schema)) {
|
|
482
|
+
if (pattern) requirePatternSearch(path);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
function requirePatternSearch(fieldPath) {
|
|
486
|
+
const module = availablePatternSearch();
|
|
487
|
+
if (module !== null) return module;
|
|
488
|
+
throw new NarsilError(
|
|
489
|
+
ErrorCodes.CONFIG_INVALID,
|
|
490
|
+
`Field "${fieldPath}" is a pattern field, but this bundle contains no pattern search code. Add import { patternSearch } from '@delali/narsil/pattern', then call registerPatternSearch(patternSearch) before you create or load the index`,
|
|
491
|
+
{ field: fieldPath }
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
|
|
212
495
|
// src/core/tokenizer/constants.ts
|
|
213
496
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
214
497
|
var TOKEN_CACHE_SIZE_CEILING = 2e6;
|
|
@@ -1820,7 +2103,8 @@ var NARSIL_CONFIG_KEYS = {
|
|
|
1820
2103
|
embeddingAdapters: true,
|
|
1821
2104
|
durability: true,
|
|
1822
2105
|
analysis: true,
|
|
1823
|
-
lifecycle: true
|
|
2106
|
+
lifecycle: true,
|
|
2107
|
+
patternWorkCap: true
|
|
1824
2108
|
};
|
|
1825
2109
|
var DURABILITY_CONFIG_KEYS = {
|
|
1826
2110
|
tier: true,
|
|
@@ -1861,176 +2145,71 @@ function requireKnownKeys(section, value, known) {
|
|
|
1861
2145
|
);
|
|
1862
2146
|
}
|
|
1863
2147
|
}
|
|
1864
|
-
function requireNamedPlugins(plugins) {
|
|
1865
|
-
if (plugins === void 0) return;
|
|
1866
|
-
if (!Array.isArray(plugins)) {
|
|
1867
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The configuration sets "plugins" to something other than a list');
|
|
1868
|
-
}
|
|
1869
|
-
plugins.forEach((plugin, position) => {
|
|
1870
|
-
const name = plugin?.name;
|
|
1871
|
-
if (typeof name !== "string" || name.trim().length === 0) {
|
|
1872
|
-
throw new NarsilError(
|
|
1873
|
-
ErrorCodes.CONFIG_INVALID,
|
|
1874
|
-
`The plugin at position ${position} carries no name, and the engine names each plugin in its errors and event payloads`,
|
|
1875
|
-
{ position }
|
|
1876
|
-
);
|
|
1877
|
-
}
|
|
1878
|
-
});
|
|
1879
|
-
}
|
|
1880
|
-
function requireKnownConfig(config) {
|
|
1881
|
-
if (config === void 0) return;
|
|
1882
|
-
requireKnownKeys("", config, NARSIL_CONFIG_KEYS);
|
|
1883
|
-
requireKnownKeys("durability.", config.durability, DURABILITY_CONFIG_KEYS);
|
|
1884
|
-
requireKnownKeys("workers.", config.workers, WORKER_CONFIG_KEYS);
|
|
1885
|
-
requireKnownKeys("lifecycle.", config.lifecycle, LIFECYCLE_CONFIG_KEYS);
|
|
1886
|
-
requireKnownKeys("analysis.", config.analysis, ANALYSIS_CONFIG_KEYS);
|
|
1887
|
-
requireNamedPlugins(config.plugins);
|
|
1888
|
-
}
|
|
1889
|
-
|
|
1890
|
-
// src/core/id-generator.ts
|
|
1891
|
-
function generateId() {
|
|
1892
|
-
const bytes = new Uint8Array(16);
|
|
1893
|
-
crypto.getRandomValues(bytes);
|
|
1894
|
-
const now2 = Date.now();
|
|
1895
|
-
bytes[0] = Math.floor(now2 / 2 ** 40);
|
|
1896
|
-
bytes[1] = Math.floor(now2 / 2 ** 32) % 256;
|
|
1897
|
-
bytes[2] = Math.floor(now2 / 2 ** 24) % 256;
|
|
1898
|
-
bytes[3] = Math.floor(now2 / 2 ** 16) % 256;
|
|
1899
|
-
bytes[4] = Math.floor(now2 / 2 ** 8) % 256;
|
|
1900
|
-
bytes[5] = now2 % 256;
|
|
1901
|
-
bytes[6] = bytes[6] & 15 | 112;
|
|
1902
|
-
bytes[8] = bytes[8] & 63 | 128;
|
|
1903
|
-
const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
|
|
1904
|
-
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
1905
|
-
}
|
|
1906
|
-
|
|
1907
|
-
// src/schema/constants.ts
|
|
1908
|
-
var MAX_NESTING_DEPTH = 4;
|
|
1909
|
-
var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
1910
|
-
var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
|
|
1911
|
-
var MIN_PATTERN_VALUE_LIMIT = 1;
|
|
1912
|
-
var MAX_PATTERN_VALUE_LIMIT = 65536;
|
|
1913
|
-
|
|
1914
|
-
// src/schema/validator/shared.ts
|
|
1915
|
-
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
1916
|
-
function isGeopointOrVectorType(fieldType) {
|
|
1917
|
-
return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
|
|
1918
|
-
}
|
|
1919
|
-
var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
|
|
1920
|
-
var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
|
|
1921
|
-
var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
1922
|
-
function isPlainObject(value) {
|
|
1923
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1924
|
-
}
|
|
1925
|
-
|
|
1926
|
-
// src/schema/validator/field-type.ts
|
|
1927
|
-
var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
|
|
1928
|
-
var OPTIONS_BY_BASE = {
|
|
1929
|
-
string: FIELD_OPTION_ORDER,
|
|
1930
|
-
"string[]": FIELD_OPTION_ORDER,
|
|
1931
|
-
verbatim: ["sortable"],
|
|
1932
|
-
"verbatim[]": ["sortable"],
|
|
1933
|
-
number: [],
|
|
1934
|
-
"number[]": [],
|
|
1935
|
-
boolean: [],
|
|
1936
|
-
"boolean[]": [],
|
|
1937
|
-
enum: [],
|
|
1938
|
-
"enum[]": [],
|
|
1939
|
-
geopoint: [],
|
|
1940
|
-
vector: []
|
|
1941
|
-
};
|
|
1942
|
-
var TYPE_SEPARATOR = ":";
|
|
1943
|
-
function isNamedBase(name) {
|
|
1944
|
-
return name !== "vector" && Object.hasOwn(OPTIONS_BY_BASE, name);
|
|
1945
|
-
}
|
|
1946
|
-
function describeAllowedOptions(baseName, allowed) {
|
|
1947
|
-
if (allowed.length === 0) return `the "${baseName}" type has no options`;
|
|
1948
|
-
const quoted = allowed.map((option) => `"${option}"`);
|
|
1949
|
-
if (quoted.length === 1) return `the only option of the "${baseName}" type is ${quoted[0]}`;
|
|
1950
|
-
return `the options of the "${baseName}" type are ${quoted.slice(0, -1).join(", ")}, and ${quoted[quoted.length - 1]}`;
|
|
1951
|
-
}
|
|
1952
|
-
function invalidType(path, type, message) {
|
|
1953
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, message, { field: path, type });
|
|
1954
|
-
}
|
|
1955
|
-
function parseBase(baseName, path, type) {
|
|
1956
|
-
if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
|
|
1957
|
-
const vectorMatch = VECTOR_PATTERN.exec(baseName);
|
|
1958
|
-
if (vectorMatch === null) {
|
|
1959
|
-
invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
|
|
1960
|
-
}
|
|
1961
|
-
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
1962
|
-
if (dimension <= 0) {
|
|
1963
|
-
throw new NarsilError(
|
|
1964
|
-
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
1965
|
-
`Vector field "${path}" must have a positive dimension, got ${dimension}`,
|
|
1966
|
-
{ field: path, dimension }
|
|
1967
|
-
);
|
|
1968
|
-
}
|
|
1969
|
-
return { base: "vector", dimension };
|
|
1970
|
-
}
|
|
1971
|
-
function parseFieldType(type, path) {
|
|
1972
|
-
const separatorIndex = type.indexOf(TYPE_SEPARATOR);
|
|
1973
|
-
const baseName = separatorIndex === -1 ? type : type.slice(0, separatorIndex);
|
|
1974
|
-
const { base, dimension } = parseBase(baseName, path, type);
|
|
1975
|
-
const allowed = OPTIONS_BY_BASE[base];
|
|
1976
|
-
const options = /* @__PURE__ */ new Set();
|
|
1977
|
-
if (separatorIndex !== -1) {
|
|
1978
|
-
for (const option of type.slice(separatorIndex + 1).split(TYPE_SEPARATOR)) {
|
|
1979
|
-
if (!allowed.includes(option)) {
|
|
1980
|
-
invalidType(
|
|
1981
|
-
path,
|
|
1982
|
-
type,
|
|
1983
|
-
`Field "${path}" has type "${type}" with the option "${option}", but ${describeAllowedOptions(baseName, allowed)}`
|
|
1984
|
-
);
|
|
1985
|
-
}
|
|
1986
|
-
if (options.has(option)) {
|
|
1987
|
-
invalidType(path, type, `Field "${path}" has the "${option}" option twice in type "${type}"`);
|
|
1988
|
-
}
|
|
1989
|
-
options.add(option);
|
|
1990
|
-
}
|
|
1991
|
-
}
|
|
1992
|
-
const ordered = FIELD_OPTION_ORDER.filter((option) => options.has(option));
|
|
1993
|
-
const verbatim = base === "verbatim" || base === "verbatim[]";
|
|
1994
|
-
return {
|
|
1995
|
-
type: [baseName, ...ordered].join(TYPE_SEPARATOR),
|
|
1996
|
-
base,
|
|
1997
|
-
sortable: options.has("sortable"),
|
|
1998
|
-
pattern: verbatim || options.has("pattern"),
|
|
1999
|
-
partial: options.has("partial"),
|
|
2000
|
-
dimension
|
|
2001
|
-
};
|
|
2002
|
-
}
|
|
2003
|
-
function parsedTypeOf(type) {
|
|
2004
|
-
if (type === void 0) return void 0;
|
|
2005
|
-
try {
|
|
2006
|
-
return parseFieldType(type, "");
|
|
2007
|
-
} catch {
|
|
2008
|
-
return void 0;
|
|
2009
|
-
}
|
|
2010
|
-
}
|
|
2011
|
-
function withSortableOption(parsed) {
|
|
2012
|
-
if (parsed.sortable) return parsed.type;
|
|
2013
|
-
const separatorIndex = parsed.type.indexOf(TYPE_SEPARATOR);
|
|
2014
|
-
const baseName = separatorIndex === -1 ? parsed.type : parsed.type.slice(0, separatorIndex);
|
|
2015
|
-
return parseFieldType(`${baseName}${TYPE_SEPARATOR}sortable${parsed.type.slice(baseName.length)}`, "").type;
|
|
2016
|
-
}
|
|
2017
|
-
function isListBase(base) {
|
|
2018
|
-
return base.endsWith("[]");
|
|
2019
|
-
}
|
|
2020
|
-
function isWordIndexedBase(base) {
|
|
2021
|
-
return base === "string" || base === "string[]";
|
|
2148
|
+
function requireNamedPlugins(plugins) {
|
|
2149
|
+
if (plugins === void 0) return;
|
|
2150
|
+
if (!Array.isArray(plugins)) {
|
|
2151
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The configuration sets "plugins" to something other than a list');
|
|
2152
|
+
}
|
|
2153
|
+
plugins.forEach((plugin, position) => {
|
|
2154
|
+
const name = plugin?.name;
|
|
2155
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
2156
|
+
throw new NarsilError(
|
|
2157
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2158
|
+
`The plugin at position ${position} carries no name, and the engine names each plugin in its errors and event payloads`,
|
|
2159
|
+
{ position }
|
|
2160
|
+
);
|
|
2161
|
+
}
|
|
2162
|
+
});
|
|
2022
2163
|
}
|
|
2023
|
-
function
|
|
2024
|
-
|
|
2164
|
+
function requireKnownConfig(config) {
|
|
2165
|
+
if (config === void 0) return;
|
|
2166
|
+
requireKnownKeys("", config, NARSIL_CONFIG_KEYS);
|
|
2167
|
+
requireKnownKeys("durability.", config.durability, DURABILITY_CONFIG_KEYS);
|
|
2168
|
+
requireKnownKeys("workers.", config.workers, WORKER_CONFIG_KEYS);
|
|
2169
|
+
requireKnownKeys("lifecycle.", config.lifecycle, LIFECYCLE_CONFIG_KEYS);
|
|
2170
|
+
requireKnownKeys("analysis.", config.analysis, ANALYSIS_CONFIG_KEYS);
|
|
2171
|
+
requireNamedPlugins(config.plugins);
|
|
2172
|
+
requireValidPatternWorkCap(config.patternWorkCap);
|
|
2025
2173
|
}
|
|
2026
|
-
function
|
|
2027
|
-
|
|
2028
|
-
|
|
2174
|
+
function requireValidPatternWorkCap(cap) {
|
|
2175
|
+
if (cap === void 0) return;
|
|
2176
|
+
if (typeof cap === "number" && Number.isSafeInteger(cap) && cap >= 1) return;
|
|
2177
|
+
throw new NarsilError(
|
|
2178
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2179
|
+
`The configuration sets "patternWorkCap" to ${String(cap)}, while the cap must be a whole number from 1 to ${Number.MAX_SAFE_INTEGER}`,
|
|
2180
|
+
{ setting: "patternWorkCap" }
|
|
2181
|
+
);
|
|
2029
2182
|
}
|
|
2030
|
-
|
|
2031
|
-
|
|
2183
|
+
|
|
2184
|
+
// src/core/id-generator.ts
|
|
2185
|
+
function generateId() {
|
|
2186
|
+
const bytes = new Uint8Array(16);
|
|
2187
|
+
crypto.getRandomValues(bytes);
|
|
2188
|
+
const now2 = Date.now();
|
|
2189
|
+
bytes[0] = Math.floor(now2 / 2 ** 40);
|
|
2190
|
+
bytes[1] = Math.floor(now2 / 2 ** 32) % 256;
|
|
2191
|
+
bytes[2] = Math.floor(now2 / 2 ** 24) % 256;
|
|
2192
|
+
bytes[3] = Math.floor(now2 / 2 ** 16) % 256;
|
|
2193
|
+
bytes[4] = Math.floor(now2 / 2 ** 8) % 256;
|
|
2194
|
+
bytes[5] = now2 % 256;
|
|
2195
|
+
bytes[6] = bytes[6] & 15 | 112;
|
|
2196
|
+
bytes[8] = bytes[8] & 63 | 128;
|
|
2197
|
+
const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
|
|
2198
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
2032
2199
|
}
|
|
2033
2200
|
|
|
2201
|
+
// src/core/pattern-index/constants.ts
|
|
2202
|
+
var DEFAULT_PATTERN_WORK_CAP = 25e6;
|
|
2203
|
+
var MAX_PATTERN_TEXT_CODE_POINTS = 1024;
|
|
2204
|
+
var MAX_PATTERN_POSITIONS = 2048;
|
|
2205
|
+
var MATCHER_STATE_BUDGET_BYTES = 1e6;
|
|
2206
|
+
var MATCHER_CODE_POINT_BYTES = 8;
|
|
2207
|
+
var MATCHER_SIMULATION_BYTES_PER_POSITION = 16;
|
|
2208
|
+
var MATCHER_STATE_OVERHEAD_BYTES = 96;
|
|
2209
|
+
var PATTERN_WORK_FLUSH_UNITS = 65536;
|
|
2210
|
+
var PATTERN_RUN_CODE_POINTS = 3;
|
|
2211
|
+
var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "contains"];
|
|
2212
|
+
|
|
2034
2213
|
// src/core/ordering/code-points.ts
|
|
2035
2214
|
function exceedsCodePoints(value, maximum) {
|
|
2036
2215
|
if (value.length <= maximum) return false;
|
|
@@ -2048,6 +2227,31 @@ function exceedsCodePoints(value, maximum) {
|
|
|
2048
2227
|
}
|
|
2049
2228
|
return false;
|
|
2050
2229
|
}
|
|
2230
|
+
var SURROGATE_UNIT = /[\ud800-\udfff]/;
|
|
2231
|
+
function holdsUnpairedSurrogate(value) {
|
|
2232
|
+
if (!SURROGATE_UNIT.test(value)) return false;
|
|
2233
|
+
for (let i = 0; i < value.length; i++) {
|
|
2234
|
+
const unit = value.charCodeAt(i);
|
|
2235
|
+
if (unit < 55296 || unit >= 57344) continue;
|
|
2236
|
+
if (unit >= 56320) return true;
|
|
2237
|
+
const low = value.charCodeAt(i + 1);
|
|
2238
|
+
if (!(low >= 56320 && low < 57344)) return true;
|
|
2239
|
+
i++;
|
|
2240
|
+
}
|
|
2241
|
+
return false;
|
|
2242
|
+
}
|
|
2243
|
+
function codePointCount(value) {
|
|
2244
|
+
let counted = 0;
|
|
2245
|
+
for (let i = 0; i < value.length; i++) {
|
|
2246
|
+
const unit = value.charCodeAt(i);
|
|
2247
|
+
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
2248
|
+
const low = value.charCodeAt(i + 1);
|
|
2249
|
+
if (low >= 56320 && low < 57344) i++;
|
|
2250
|
+
}
|
|
2251
|
+
counted++;
|
|
2252
|
+
}
|
|
2253
|
+
return counted;
|
|
2254
|
+
}
|
|
2051
2255
|
function adjustUtf16Unit(unit) {
|
|
2052
2256
|
if (unit < 55296) return unit;
|
|
2053
2257
|
if (unit < 57344) return unit + 8192;
|
|
@@ -2061,6 +2265,29 @@ function compareCodePoints(a, b) {
|
|
|
2061
2265
|
return adjustUtf16Unit(a.charCodeAt(i)) - adjustUtf16Unit(b.charCodeAt(i));
|
|
2062
2266
|
}
|
|
2063
2267
|
|
|
2268
|
+
// src/geo/constants.ts
|
|
2269
|
+
var DEG_TO_RAD = Math.PI / 180;
|
|
2270
|
+
var MAX_LATITUDE_DEGREES = 90;
|
|
2271
|
+
var MAX_LONGITUDE_DEGREES = 180;
|
|
2272
|
+
var MIN_POLYGON_POINTS = 3;
|
|
2273
|
+
var ANTIMERIDIAN_SPAN_DEGREES = 180;
|
|
2274
|
+
var FULL_TURN_DEGREES = 360;
|
|
2275
|
+
var EARTH_RADIUS_METERS = 63710088e-1;
|
|
2276
|
+
var LOCATION_CODE_BITS_PER_AXIS = 26;
|
|
2277
|
+
var MAX_COVERING_CELLS = 16;
|
|
2278
|
+
var RADIUS_BOX_MARGIN_RATIO = 1.02;
|
|
2279
|
+
var RADIUS_BOX_MARGIN_RADIANS = 1e-9;
|
|
2280
|
+
var VINCENTY_MAX_ITERATIONS = 200;
|
|
2281
|
+
var VINCENTY_CONVERGENCE_THRESHOLD = 1e-12;
|
|
2282
|
+
|
|
2283
|
+
// src/geo/coordinates.ts
|
|
2284
|
+
function isLatitude(value) {
|
|
2285
|
+
return value >= -MAX_LATITUDE_DEGREES && value <= MAX_LATITUDE_DEGREES;
|
|
2286
|
+
}
|
|
2287
|
+
function isLongitude(value) {
|
|
2288
|
+
return value >= -MAX_LONGITUDE_DEGREES && value <= MAX_LONGITUDE_DEGREES;
|
|
2289
|
+
}
|
|
2290
|
+
|
|
2064
2291
|
// src/schema/validator/field-values.ts
|
|
2065
2292
|
function validateGeopoint(path, value) {
|
|
2066
2293
|
if (!isPlainObject(value)) {
|
|
@@ -2080,7 +2307,7 @@ function validateGeopoint(path, value) {
|
|
|
2080
2307
|
lon: Number.isNaN(value.lon) ? "NaN" : typeof value.lon
|
|
2081
2308
|
});
|
|
2082
2309
|
}
|
|
2083
|
-
if (value.lat
|
|
2310
|
+
if (!isLatitude(value.lat)) {
|
|
2084
2311
|
throw new NarsilError(
|
|
2085
2312
|
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
2086
2313
|
`Field "${path}" geopoint lat must be between -90 and 90`,
|
|
@@ -2090,7 +2317,7 @@ function validateGeopoint(path, value) {
|
|
|
2090
2317
|
}
|
|
2091
2318
|
);
|
|
2092
2319
|
}
|
|
2093
|
-
if (value.lon
|
|
2320
|
+
if (!isLongitude(value.lon)) {
|
|
2094
2321
|
throw new NarsilError(
|
|
2095
2322
|
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
2096
2323
|
`Field "${path}" geopoint lon must be between -180 and 180`,
|
|
@@ -2120,7 +2347,7 @@ function validateVector(path, value, expectedDimension) {
|
|
|
2120
2347
|
);
|
|
2121
2348
|
}
|
|
2122
2349
|
for (let i = 0; i < arr.length; i++) {
|
|
2123
|
-
if (typeof arr[i] !== "number" || Number.
|
|
2350
|
+
if (typeof arr[i] !== "number" || !Number.isFinite(arr[i])) {
|
|
2124
2351
|
throw new NarsilError(
|
|
2125
2352
|
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
2126
2353
|
`Field "${path}" vector element at index ${i} is not a valid number`,
|
|
@@ -2145,11 +2372,12 @@ function validateTypedArray(path, value, elementType) {
|
|
|
2145
2372
|
{ field: path, index: i, expected: elementType, received: typeof el }
|
|
2146
2373
|
);
|
|
2147
2374
|
}
|
|
2148
|
-
if (elementType === "number" && Number.
|
|
2149
|
-
throw new NarsilError(
|
|
2150
|
-
|
|
2151
|
-
index
|
|
2152
|
-
|
|
2375
|
+
if (elementType === "number" && !Number.isFinite(el)) {
|
|
2376
|
+
throw new NarsilError(
|
|
2377
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
2378
|
+
`Field "${path}" array element at index ${i} is ${String(el)}, while a number field holds finite numbers alone`,
|
|
2379
|
+
{ field: path, index: i }
|
|
2380
|
+
);
|
|
2153
2381
|
}
|
|
2154
2382
|
}
|
|
2155
2383
|
}
|
|
@@ -2180,11 +2408,11 @@ function validateFieldValue(path, value, type, patternValueLimit) {
|
|
|
2180
2408
|
if (type.pattern) validatePatternValue(path, value, patternValueLimit);
|
|
2181
2409
|
return;
|
|
2182
2410
|
case "number":
|
|
2183
|
-
if (typeof value !== "number" || Number.
|
|
2184
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a
|
|
2411
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
2412
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a finite number`, {
|
|
2185
2413
|
field: path,
|
|
2186
2414
|
expected: "number",
|
|
2187
|
-
received:
|
|
2415
|
+
received: typeof value === "number" ? String(value) : typeof value
|
|
2188
2416
|
});
|
|
2189
2417
|
}
|
|
2190
2418
|
return;
|
|
@@ -2242,6 +2470,7 @@ function validateFieldValue(path, value, type, patternValueLimit) {
|
|
|
2242
2470
|
|
|
2243
2471
|
// src/schema/validator/storable.ts
|
|
2244
2472
|
var RESERVED_OBJECT_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
2473
|
+
var UNPAIRED_SURROGATE_REASON = "holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8";
|
|
2245
2474
|
function reject(path, reason) {
|
|
2246
2475
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
2247
2476
|
field: path
|
|
@@ -2254,8 +2483,11 @@ function isPlainObject2(value) {
|
|
|
2254
2483
|
function walkStorable(value, path, depth, ancestors, skip) {
|
|
2255
2484
|
if (value === null || value === void 0) return;
|
|
2256
2485
|
const kind = typeof value;
|
|
2257
|
-
if (kind === "string"
|
|
2258
|
-
|
|
2486
|
+
if (kind === "string") {
|
|
2487
|
+
if (holdsUnpairedSurrogate(value)) reject(path, UNPAIRED_SURROGATE_REASON);
|
|
2488
|
+
return;
|
|
2489
|
+
}
|
|
2490
|
+
if (kind === "boolean" || kind === "number") return;
|
|
2259
2491
|
if (kind === "bigint" || kind === "symbol" || kind === "function") {
|
|
2260
2492
|
reject(path, `holds a ${kind}, which cannot be stored or recovered`);
|
|
2261
2493
|
}
|
|
@@ -2287,6 +2519,7 @@ function walkStorable(value, path, depth, ancestors, skip) {
|
|
|
2287
2519
|
if (RESERVED_OBJECT_KEYS.has(key)) {
|
|
2288
2520
|
reject(keyPath, "uses a reserved object key");
|
|
2289
2521
|
}
|
|
2522
|
+
if (holdsUnpairedSurrogate(key)) reject(keyPath, `has a name that ${UNPAIRED_SURROGATE_REASON}`);
|
|
2290
2523
|
if (skip.has(keyPath)) continue;
|
|
2291
2524
|
walkStorable(record[key], keyPath, depth + 1, ancestors, skip);
|
|
2292
2525
|
}
|
|
@@ -2353,9 +2586,20 @@ function validateDocument(document2, schema, patternValueLimit = DEFAULT_PATTERN
|
|
|
2353
2586
|
received: typeof document2
|
|
2354
2587
|
});
|
|
2355
2588
|
}
|
|
2589
|
+
requireIdFieldKey(document2);
|
|
2356
2590
|
assertStorableDocument(document2, vectorFieldPaths(schema));
|
|
2357
2591
|
validateDocumentFields(document2, schema, "", patternValueLimit);
|
|
2358
2592
|
}
|
|
2593
|
+
function requireIdFieldKey(document2) {
|
|
2594
|
+
if (!Object.hasOwn(document2, "id")) return;
|
|
2595
|
+
const id = document2.id;
|
|
2596
|
+
if (typeof id === "string" && id.length > 0) return;
|
|
2597
|
+
throw new NarsilError(
|
|
2598
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
2599
|
+
`Field "id" holds ${typeof id === "string" ? "an empty string" : id === null ? "null" : `a ${typeof id}`}, while the engine stores a document under its id field, so that field must hold a non-empty string`,
|
|
2600
|
+
{ field: "id", received: id === null ? "null" : typeof id }
|
|
2601
|
+
);
|
|
2602
|
+
}
|
|
2359
2603
|
function collectExtraFields(doc, schema, prefix, extras) {
|
|
2360
2604
|
for (const key of Object.keys(doc)) {
|
|
2361
2605
|
if (key === "id") continue;
|
|
@@ -2403,124 +2647,6 @@ function validateRequiredFields(document2, required) {
|
|
|
2403
2647
|
}
|
|
2404
2648
|
}
|
|
2405
2649
|
|
|
2406
|
-
// src/schema/validator/schema.ts
|
|
2407
|
-
function validateSchemaFields(schema, depth, prefix) {
|
|
2408
|
-
if (depth > MAX_NESTING_DEPTH) {
|
|
2409
|
-
throw new NarsilError(
|
|
2410
|
-
ErrorCodes.SCHEMA_DEPTH_EXCEEDED,
|
|
2411
|
-
`Schema nesting exceeds the maximum depth of ${MAX_NESTING_DEPTH} at "${prefix}"`,
|
|
2412
|
-
{ path: prefix, maxDepth: MAX_NESTING_DEPTH }
|
|
2413
|
-
);
|
|
2414
|
-
}
|
|
2415
|
-
let canonical = null;
|
|
2416
|
-
const replace = (field, value) => {
|
|
2417
|
-
if (canonical === null) canonical = { ...schema };
|
|
2418
|
-
canonical[field] = value;
|
|
2419
|
-
};
|
|
2420
|
-
for (const [field, type] of Object.entries(schema)) {
|
|
2421
|
-
const path = prefix ? `${prefix}.${field}` : field;
|
|
2422
|
-
if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
|
|
2423
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field name "${field}" is not allowed in a schema`, {
|
|
2424
|
-
field,
|
|
2425
|
-
path
|
|
2426
|
-
});
|
|
2427
|
-
}
|
|
2428
|
-
if (!FIELD_NAME_PATTERN.test(field)) {
|
|
2429
|
-
throw new NarsilError(
|
|
2430
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
2431
|
-
`Field name "${field}" contains characters that are not allowed; use letters, digits, and underscores only`,
|
|
2432
|
-
{ field, path }
|
|
2433
|
-
);
|
|
2434
|
-
}
|
|
2435
|
-
if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
|
|
2436
|
-
throw new NarsilError(
|
|
2437
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
2438
|
-
`Field "${field}" is reserved and cannot be defined in a schema`,
|
|
2439
|
-
{ field, path }
|
|
2440
|
-
);
|
|
2441
|
-
}
|
|
2442
|
-
if (isPlainObject(type)) {
|
|
2443
|
-
const nested = validateSchemaFields(type, depth + 1, path);
|
|
2444
|
-
if (nested !== type) replace(field, nested);
|
|
2445
|
-
continue;
|
|
2446
|
-
}
|
|
2447
|
-
if (typeof type !== "string") {
|
|
2448
|
-
throw new NarsilError(
|
|
2449
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
2450
|
-
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
2451
|
-
{
|
|
2452
|
-
field: path,
|
|
2453
|
-
type: String(type)
|
|
2454
|
-
}
|
|
2455
|
-
);
|
|
2456
|
-
}
|
|
2457
|
-
const parsed = parseFieldType(type, path);
|
|
2458
|
-
if (parsed.type !== type) replace(field, parsed.type);
|
|
2459
|
-
}
|
|
2460
|
-
return canonical ?? schema;
|
|
2461
|
-
}
|
|
2462
|
-
function validateSchema(schema) {
|
|
2463
|
-
if (!isPlainObject(schema)) {
|
|
2464
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must be a plain object", {
|
|
2465
|
-
received: typeof schema
|
|
2466
|
-
});
|
|
2467
|
-
}
|
|
2468
|
-
if (Object.keys(schema).length === 0) {
|
|
2469
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
|
|
2470
|
-
}
|
|
2471
|
-
return validateSchemaFields(schema, 1, "");
|
|
2472
|
-
}
|
|
2473
|
-
function requireValidFieldTypes(flatSchema) {
|
|
2474
|
-
for (const [path, type] of Object.entries(flatSchema)) {
|
|
2475
|
-
if (typeof type !== "string") {
|
|
2476
|
-
throw new NarsilError(
|
|
2477
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
2478
|
-
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
2479
|
-
{ field: path, type: String(type) }
|
|
2480
|
-
);
|
|
2481
|
-
}
|
|
2482
|
-
parseFieldType(type, path);
|
|
2483
|
-
}
|
|
2484
|
-
}
|
|
2485
|
-
function flattenRecursive(schema, prefix, result) {
|
|
2486
|
-
for (const [field, type] of Object.entries(schema)) {
|
|
2487
|
-
const path = prefix ? `${prefix}.${field}` : field;
|
|
2488
|
-
if (isPlainObject(type)) {
|
|
2489
|
-
flattenRecursive(type, path, result);
|
|
2490
|
-
} else {
|
|
2491
|
-
result[path] = type;
|
|
2492
|
-
}
|
|
2493
|
-
}
|
|
2494
|
-
}
|
|
2495
|
-
var flattenedSchemas = /* @__PURE__ */ new WeakMap();
|
|
2496
|
-
function flattenSchema(schema) {
|
|
2497
|
-
const cached = flattenedSchemas.get(schema);
|
|
2498
|
-
if (cached !== void 0) return cached;
|
|
2499
|
-
const result = /* @__PURE__ */ Object.create(null);
|
|
2500
|
-
flattenRecursive(schema, "", result);
|
|
2501
|
-
flattenedSchemas.set(schema, Object.freeze(result));
|
|
2502
|
-
return result;
|
|
2503
|
-
}
|
|
2504
|
-
var parsedSchemas = /* @__PURE__ */ new WeakMap();
|
|
2505
|
-
function schemaFieldsOf(schema) {
|
|
2506
|
-
const cached = parsedSchemas.get(schema);
|
|
2507
|
-
if (cached !== void 0) return cached;
|
|
2508
|
-
const fields = [];
|
|
2509
|
-
for (const [path, type] of Object.entries(flattenSchema(schema))) {
|
|
2510
|
-
fields.push({ ...parseFieldType(type, path), path });
|
|
2511
|
-
}
|
|
2512
|
-
const frozen = Object.freeze(fields);
|
|
2513
|
-
parsedSchemas.set(schema, frozen);
|
|
2514
|
-
return frozen;
|
|
2515
|
-
}
|
|
2516
|
-
function extractVectorFieldsFromSchema(schema) {
|
|
2517
|
-
const result = /* @__PURE__ */ new Map();
|
|
2518
|
-
for (const field of schemaFieldsOf(schema)) {
|
|
2519
|
-
if (field.base === "vector") result.set(field.path, field.dimension);
|
|
2520
|
-
}
|
|
2521
|
-
return result;
|
|
2522
|
-
}
|
|
2523
|
-
|
|
2524
2650
|
// src/vector/hnsw/constants.ts
|
|
2525
2651
|
var MAX_LAYER_CAP = 32;
|
|
2526
2652
|
var MAX_M = 512;
|
|
@@ -2868,6 +2994,44 @@ function sanitizeSurfaceForms(raw) {
|
|
|
2868
2994
|
}
|
|
2869
2995
|
return count > 0 ? result : void 0;
|
|
2870
2996
|
}
|
|
2997
|
+
function patternIndexesToWire(pattern) {
|
|
2998
|
+
if (pattern === void 0) return {};
|
|
2999
|
+
const wire = /* @__PURE__ */ Object.create(null);
|
|
3000
|
+
for (const [field, index] of Object.entries(pattern)) wire[field] = { doc_ids: index.docIds, runs: index.runs };
|
|
3001
|
+
return { pattern: wire };
|
|
3002
|
+
}
|
|
3003
|
+
function isWireRecord(value) {
|
|
3004
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
3005
|
+
}
|
|
3006
|
+
function corruptPatternEntry(field) {
|
|
3007
|
+
return new NarsilError(
|
|
3008
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3009
|
+
`The pattern entry of field "${field}" in this partition must hold a list of document IDs and a map of runs`,
|
|
3010
|
+
{ field }
|
|
3011
|
+
);
|
|
3012
|
+
}
|
|
3013
|
+
function patternIndexesFromWire(raw) {
|
|
3014
|
+
if (raw === void 0 || raw === null) return {};
|
|
3015
|
+
if (!isWireRecord(raw)) {
|
|
3016
|
+
throw new NarsilError(
|
|
3017
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3018
|
+
"The pattern entry of this partition must be a map of fields"
|
|
3019
|
+
);
|
|
3020
|
+
}
|
|
3021
|
+
const pattern = /* @__PURE__ */ Object.create(null);
|
|
3022
|
+
for (const [field, index] of Object.entries(raw)) {
|
|
3023
|
+
if (!isWireRecord(index) || !isWireRecord(index.runs) || !Array.isArray(index.doc_ids)) {
|
|
3024
|
+
throw corruptPatternEntry(field);
|
|
3025
|
+
}
|
|
3026
|
+
const docIds = [];
|
|
3027
|
+
for (const docId of index.doc_ids) {
|
|
3028
|
+
if (typeof docId !== "string") throw corruptPatternEntry(field);
|
|
3029
|
+
docIds.push(docId);
|
|
3030
|
+
}
|
|
3031
|
+
pattern[field] = { docIds, runs: index.runs };
|
|
3032
|
+
}
|
|
3033
|
+
return { pattern };
|
|
3034
|
+
}
|
|
2871
3035
|
function validateHnswMetric(value) {
|
|
2872
3036
|
if (value === void 0 || value === null) return void 0;
|
|
2873
3037
|
if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
|
|
@@ -2939,7 +3103,8 @@ function wireToPartition(raw) {
|
|
|
2939
3103
|
numeric,
|
|
2940
3104
|
boolean,
|
|
2941
3105
|
enum: raw.field_indexes?.enum ?? {},
|
|
2942
|
-
geopoint
|
|
3106
|
+
geopoint,
|
|
3107
|
+
...patternIndexesFromWire(raw.field_indexes?.pattern)
|
|
2943
3108
|
},
|
|
2944
3109
|
surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
|
|
2945
3110
|
vectorData,
|
|
@@ -3046,7 +3211,8 @@ function deserializePayloadV2(data) {
|
|
|
3046
3211
|
numeric,
|
|
3047
3212
|
boolean,
|
|
3048
3213
|
enum: raw.field_indexes?.enum ?? {},
|
|
3049
|
-
geopoint
|
|
3214
|
+
geopoint,
|
|
3215
|
+
...patternIndexesFromWire(raw.field_indexes?.pattern)
|
|
3050
3216
|
},
|
|
3051
3217
|
surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
|
|
3052
3218
|
vectorData,
|
|
@@ -3096,6 +3262,10 @@ function multiFoldTable() {
|
|
|
3096
3262
|
|
|
3097
3263
|
// src/core/ordering/fold-compare.ts
|
|
3098
3264
|
var EXHAUSTED = -1;
|
|
3265
|
+
function foldOf(codePoint, single, multi) {
|
|
3266
|
+
if (codePoint < 128) return codePoint >= 65 && codePoint <= 90 ? codePoint + 32 : codePoint;
|
|
3267
|
+
return single.get(codePoint) ?? multi.get(codePoint) ?? codePoint;
|
|
3268
|
+
}
|
|
3099
3269
|
function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
3100
3270
|
if (cursor.pending !== null) {
|
|
3101
3271
|
const cp2 = cursor.pending[cursor.pendingIndex];
|
|
@@ -3109,9 +3279,6 @@ function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
|
3109
3279
|
if (cursor.index >= text.length) return EXHAUSTED;
|
|
3110
3280
|
let cp = text.charCodeAt(cursor.index);
|
|
3111
3281
|
cursor.index++;
|
|
3112
|
-
if (cp < 128) {
|
|
3113
|
-
return cp >= 65 && cp <= 90 ? cp + 32 : cp;
|
|
3114
|
-
}
|
|
3115
3282
|
if (cp >= 55296 && cp < 56320 && cursor.index < text.length) {
|
|
3116
3283
|
const low = text.charCodeAt(cursor.index);
|
|
3117
3284
|
if (low >= 56320 && low < 57344) {
|
|
@@ -3119,15 +3286,13 @@ function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
|
3119
3286
|
cursor.index++;
|
|
3120
3287
|
}
|
|
3121
3288
|
}
|
|
3122
|
-
const
|
|
3123
|
-
if (
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
if (expansion.length > 1) {
|
|
3127
|
-
cursor.pending = expansion;
|
|
3289
|
+
const folded = foldOf(cp, single, multi);
|
|
3290
|
+
if (typeof folded === "number") return folded;
|
|
3291
|
+
if (folded.length > 1) {
|
|
3292
|
+
cursor.pending = folded;
|
|
3128
3293
|
cursor.pendingIndex = 1;
|
|
3129
3294
|
}
|
|
3130
|
-
return
|
|
3295
|
+
return folded[0];
|
|
3131
3296
|
}
|
|
3132
3297
|
function compareCaseFolded(a, b) {
|
|
3133
3298
|
const shorter = a.length < b.length ? a.length : b.length;
|
|
@@ -3154,6 +3319,28 @@ function compareSortStrings(a, b) {
|
|
|
3154
3319
|
if (folded !== 0) return folded;
|
|
3155
3320
|
return compareCodePoints(a, b);
|
|
3156
3321
|
}
|
|
3322
|
+
function appendFoldedCodePoint(codePoint, out) {
|
|
3323
|
+
const folded = foldOf(codePoint, singleFoldTable(), multiFoldTable());
|
|
3324
|
+
if (typeof folded === "number") {
|
|
3325
|
+
out.push(folded);
|
|
3326
|
+
return;
|
|
3327
|
+
}
|
|
3328
|
+
for (const foldedCodePoint of folded) out.push(foldedCodePoint);
|
|
3329
|
+
}
|
|
3330
|
+
function appendFoldedCodePoints(text, out) {
|
|
3331
|
+
const single = singleFoldTable();
|
|
3332
|
+
const multi = multiFoldTable();
|
|
3333
|
+
const cursor = { index: 0, pending: null, pendingIndex: 0 };
|
|
3334
|
+
for (let codePoint = nextFoldedCodePoint(text, cursor, single, multi); codePoint !== EXHAUSTED; ) {
|
|
3335
|
+
out.push(codePoint);
|
|
3336
|
+
codePoint = nextFoldedCodePoint(text, cursor, single, multi);
|
|
3337
|
+
}
|
|
3338
|
+
}
|
|
3339
|
+
function foldedCodePointCount(text) {
|
|
3340
|
+
const folded = [];
|
|
3341
|
+
appendFoldedCodePoints(text, folded);
|
|
3342
|
+
return folded.length;
|
|
3343
|
+
}
|
|
3157
3344
|
|
|
3158
3345
|
// src/core/ordering/constants.ts
|
|
3159
3346
|
var SORT_VALUE_MAX_CODE_POINTS = 512;
|
|
@@ -4281,17 +4468,18 @@ function bitsetAnd(a, b) {
|
|
|
4281
4468
|
}
|
|
4282
4469
|
return result;
|
|
4283
4470
|
}
|
|
4284
|
-
function
|
|
4285
|
-
const
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4289
|
-
|
|
4290
|
-
|
|
4291
|
-
for (let i =
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4471
|
+
function bitsetAndInto(target, other) {
|
|
4472
|
+
const shared2 = Math.min(target.length, other.length);
|
|
4473
|
+
for (let i = 0; i < shared2; i++) target[i] &= other[i];
|
|
4474
|
+
target.fill(0, shared2);
|
|
4475
|
+
}
|
|
4476
|
+
function bitsetOrInto(target, other) {
|
|
4477
|
+
const shared2 = Math.min(target.length, other.length);
|
|
4478
|
+
for (let i = 0; i < shared2; i++) target[i] |= other[i];
|
|
4479
|
+
}
|
|
4480
|
+
function bitsetAndNotInto(target, removed) {
|
|
4481
|
+
const shared2 = Math.min(target.length, removed.length);
|
|
4482
|
+
for (let i = 0; i < shared2; i++) target[i] &= ~removed[i];
|
|
4295
4483
|
}
|
|
4296
4484
|
function bitsetNot(bs, capacity) {
|
|
4297
4485
|
const result = new Uint32Array(bs.length);
|
|
@@ -4385,13 +4573,6 @@ function getFieldValueByInternalId(docStore, internalId, fieldPath) {
|
|
|
4385
4573
|
if (externalId === void 0) return void 0;
|
|
4386
4574
|
return getFieldValueForDoc(docStore, externalId, fieldPath);
|
|
4387
4575
|
}
|
|
4388
|
-
function getAllInternalDocIds(docStore) {
|
|
4389
|
-
const ids = /* @__PURE__ */ new Set();
|
|
4390
|
-
for (const internalId of docStore.allInternalIds()) {
|
|
4391
|
-
ids.add(internalId);
|
|
4392
|
-
}
|
|
4393
|
-
return ids;
|
|
4394
|
-
}
|
|
4395
4576
|
|
|
4396
4577
|
// src/core/partition/facets.ts
|
|
4397
4578
|
var UNDECLARED_FIELD_TYPE = "";
|
|
@@ -4524,19 +4705,116 @@ function computeFacets(state, matched, config, schema) {
|
|
|
4524
4705
|
}
|
|
4525
4706
|
result[fieldPath] = { values, count: entries.length, errorBound };
|
|
4526
4707
|
}
|
|
4527
|
-
return result;
|
|
4708
|
+
return result;
|
|
4709
|
+
}
|
|
4710
|
+
|
|
4711
|
+
// src/core/pattern-index/host.ts
|
|
4712
|
+
var PATTERN_HOST = { appendFoldedCodePoint };
|
|
4713
|
+
function patternIndexOf(indexes, fieldPath) {
|
|
4714
|
+
let index = indexes.get(fieldPath);
|
|
4715
|
+
if (index === void 0) {
|
|
4716
|
+
index = requirePatternSearch(fieldPath).createIndex(PATTERN_HOST);
|
|
4717
|
+
indexes.set(fieldPath, index);
|
|
4718
|
+
}
|
|
4719
|
+
return index;
|
|
4528
4720
|
}
|
|
4529
4721
|
|
|
4530
|
-
// src/geo/
|
|
4531
|
-
var
|
|
4532
|
-
|
|
4533
|
-
|
|
4534
|
-
|
|
4535
|
-
|
|
4536
|
-
|
|
4722
|
+
// src/geo/cells.ts
|
|
4723
|
+
var AXIS_CELLS = 2 ** LOCATION_CODE_BITS_PER_AXIS;
|
|
4724
|
+
function axisCell(value, lowest, span) {
|
|
4725
|
+
const cell = Math.floor((value - lowest) / span * AXIS_CELLS);
|
|
4726
|
+
if (!(cell > 0)) return 0;
|
|
4727
|
+
return cell >= AXIS_CELLS ? AXIS_CELLS - 1 : cell;
|
|
4728
|
+
}
|
|
4729
|
+
function latitudeCell(lat) {
|
|
4730
|
+
return axisCell(lat, -MAX_LATITUDE_DEGREES, 2 * MAX_LATITUDE_DEGREES);
|
|
4731
|
+
}
|
|
4732
|
+
function longitudeCell(lon) {
|
|
4733
|
+
return axisCell(lon, -MAX_LONGITUDE_DEGREES, FULL_TURN_DEGREES);
|
|
4734
|
+
}
|
|
4735
|
+
function interleave(row, column, bits) {
|
|
4736
|
+
let code = 0;
|
|
4737
|
+
for (let bit = bits - 1; bit >= 0; bit--) code = code * 4 + (row >>> bit & 1) * 2 + (column >>> bit & 1);
|
|
4738
|
+
return code;
|
|
4739
|
+
}
|
|
4740
|
+
function locationCode(lat, lon) {
|
|
4741
|
+
return interleave(latitudeCell(lat), longitudeCell(lon), LOCATION_CODE_BITS_PER_AXIS);
|
|
4742
|
+
}
|
|
4743
|
+
function appendBoxRanges(box, out) {
|
|
4744
|
+
const south = latitudeCell(box.south);
|
|
4745
|
+
const north = latitudeCell(box.north);
|
|
4746
|
+
const west = longitudeCell(box.west);
|
|
4747
|
+
const east = longitudeCell(box.east);
|
|
4748
|
+
let level2 = LOCATION_CODE_BITS_PER_AXIS;
|
|
4749
|
+
while (level2 > 0) {
|
|
4750
|
+
const shift2 = LOCATION_CODE_BITS_PER_AXIS - level2;
|
|
4751
|
+
const rows = (north >>> shift2) - (south >>> shift2) + 1;
|
|
4752
|
+
const columns = (east >>> shift2) - (west >>> shift2) + 1;
|
|
4753
|
+
if (rows * columns <= MAX_COVERING_CELLS) break;
|
|
4754
|
+
level2--;
|
|
4755
|
+
}
|
|
4756
|
+
const shift = LOCATION_CODE_BITS_PER_AXIS - level2;
|
|
4757
|
+
const codesPerCell = 4 ** shift;
|
|
4758
|
+
for (let row = south >>> shift; row <= north >>> shift; row++) {
|
|
4759
|
+
for (let column = west >>> shift; column <= east >>> shift; column++) {
|
|
4760
|
+
const from = interleave(row, column, level2) * codesPerCell;
|
|
4761
|
+
out.push({ from, to: from + codesPerCell });
|
|
4762
|
+
}
|
|
4763
|
+
}
|
|
4764
|
+
}
|
|
4765
|
+
function coveringRanges(boxes) {
|
|
4766
|
+
const ranges = [];
|
|
4767
|
+
for (const box of boxes) appendBoxRanges(box, ranges);
|
|
4768
|
+
ranges.sort((left, right) => left.from - right.from);
|
|
4769
|
+
const merged = [];
|
|
4770
|
+
for (const range of ranges) {
|
|
4771
|
+
const last = merged[merged.length - 1];
|
|
4772
|
+
if (last !== void 0 && range.from <= last.to) {
|
|
4773
|
+
if (range.to > last.to) last.to = range.to;
|
|
4774
|
+
} else {
|
|
4775
|
+
merged.push({ from: range.from, to: range.to });
|
|
4776
|
+
}
|
|
4777
|
+
}
|
|
4778
|
+
return merged;
|
|
4779
|
+
}
|
|
4780
|
+
function longitudeBoxes(south, north, west, east) {
|
|
4781
|
+
if (east - west >= FULL_TURN_DEGREES) {
|
|
4782
|
+
return [{ south, north, west: -MAX_LONGITUDE_DEGREES, east: MAX_LONGITUDE_DEGREES }];
|
|
4783
|
+
}
|
|
4784
|
+
if (west < -MAX_LONGITUDE_DEGREES) {
|
|
4785
|
+
return [
|
|
4786
|
+
{ south, north, west: west + FULL_TURN_DEGREES, east: MAX_LONGITUDE_DEGREES },
|
|
4787
|
+
{ south, north, west: -MAX_LONGITUDE_DEGREES, east }
|
|
4788
|
+
];
|
|
4789
|
+
}
|
|
4790
|
+
if (east > MAX_LONGITUDE_DEGREES) {
|
|
4791
|
+
return [
|
|
4792
|
+
{ south, north, west, east: MAX_LONGITUDE_DEGREES },
|
|
4793
|
+
{ south, north, west: -MAX_LONGITUDE_DEGREES, east: east - FULL_TURN_DEGREES }
|
|
4794
|
+
];
|
|
4795
|
+
}
|
|
4796
|
+
return [{ south, north, west, east }];
|
|
4797
|
+
}
|
|
4798
|
+
function radiusBoxes(lat, lon, distanceMeters) {
|
|
4799
|
+
const angle = distanceMeters / EARTH_RADIUS_METERS * RADIUS_BOX_MARGIN_RATIO + RADIUS_BOX_MARGIN_RADIANS;
|
|
4800
|
+
const angleDegrees = angle / DEG_TO_RAD;
|
|
4801
|
+
const south = lat - angleDegrees;
|
|
4802
|
+
const north = lat + angleDegrees;
|
|
4803
|
+
if (angle >= Math.PI || south <= -MAX_LATITUDE_DEGREES || north >= MAX_LATITUDE_DEGREES) {
|
|
4804
|
+
return [
|
|
4805
|
+
{
|
|
4806
|
+
south: Math.max(south, -MAX_LATITUDE_DEGREES),
|
|
4807
|
+
north: Math.min(north, MAX_LATITUDE_DEGREES),
|
|
4808
|
+
west: -MAX_LONGITUDE_DEGREES,
|
|
4809
|
+
east: MAX_LONGITUDE_DEGREES
|
|
4810
|
+
}
|
|
4811
|
+
];
|
|
4812
|
+
}
|
|
4813
|
+
const spread = Math.asin(Math.sin(angle) / Math.cos(lat * DEG_TO_RAD)) / DEG_TO_RAD;
|
|
4814
|
+
return longitudeBoxes(south, north, lon - spread, lon + spread);
|
|
4815
|
+
}
|
|
4537
4816
|
|
|
4538
4817
|
// src/geo/haversine.ts
|
|
4539
|
-
var EARTH_RADIUS_METERS = 63710088e-1;
|
|
4540
4818
|
function haversineDistance(lat1, lon1, lat2, lon2) {
|
|
4541
4819
|
const phi1 = lat1 * DEG_TO_RAD;
|
|
4542
4820
|
const phi2 = lat2 * DEG_TO_RAD;
|
|
@@ -4549,6 +4827,53 @@ function haversineDistance(lat1, lon1, lat2, lon2) {
|
|
|
4549
4827
|
return EARTH_RADIUS_METERS * c;
|
|
4550
4828
|
}
|
|
4551
4829
|
|
|
4830
|
+
// src/geo/edge-tree.ts
|
|
4831
|
+
function buildLongitudeEdgeTree(lons) {
|
|
4832
|
+
const n = lons.length;
|
|
4833
|
+
const edges = [];
|
|
4834
|
+
for (let i = 0; i < n; i++) {
|
|
4835
|
+
if (lons[i] !== lons[i === 0 ? n - 1 : i - 1]) edges.push(i);
|
|
4836
|
+
}
|
|
4837
|
+
const lowOf = (i) => Math.min(lons[i], lons[i === 0 ? n - 1 : i - 1]);
|
|
4838
|
+
edges.sort((left, right) => lowOf(left) - lowOf(right));
|
|
4839
|
+
const count = edges.length;
|
|
4840
|
+
const tree = {
|
|
4841
|
+
from: new Int32Array(count),
|
|
4842
|
+
to: new Int32Array(count),
|
|
4843
|
+
low: new Float64Array(count),
|
|
4844
|
+
high: new Float64Array(count),
|
|
4845
|
+
subtreeHigh: new Float64Array(count)
|
|
4846
|
+
};
|
|
4847
|
+
edges.forEach((i, at) => {
|
|
4848
|
+
const j = i === 0 ? n - 1 : i - 1;
|
|
4849
|
+
tree.from[at] = i;
|
|
4850
|
+
tree.to[at] = j;
|
|
4851
|
+
tree.low[at] = Math.min(lons[i], lons[j]);
|
|
4852
|
+
tree.high[at] = Math.max(lons[i], lons[j]);
|
|
4853
|
+
});
|
|
4854
|
+
const fillSubtreeHigh = (start, end) => {
|
|
4855
|
+
if (start >= end) return Number.NEGATIVE_INFINITY;
|
|
4856
|
+
const mid = start + end >>> 1;
|
|
4857
|
+
const highest = Math.max(tree.high[mid], fillSubtreeHigh(start, mid), fillSubtreeHigh(mid + 1, end));
|
|
4858
|
+
tree.subtreeHigh[mid] = highest;
|
|
4859
|
+
return highest;
|
|
4860
|
+
};
|
|
4861
|
+
fillSubtreeHigh(0, count);
|
|
4862
|
+
return tree;
|
|
4863
|
+
}
|
|
4864
|
+
function forEachEdgeSpanning(tree, lon, visit) {
|
|
4865
|
+
const walk = (start, end) => {
|
|
4866
|
+
if (start >= end) return;
|
|
4867
|
+
const mid = start + end >>> 1;
|
|
4868
|
+
if (tree.subtreeHigh[mid] <= lon) return;
|
|
4869
|
+
walk(start, mid);
|
|
4870
|
+
if (tree.low[mid] > lon) return;
|
|
4871
|
+
if (lon < tree.high[mid]) visit(tree.from[mid], tree.to[mid]);
|
|
4872
|
+
walk(mid + 1, end);
|
|
4873
|
+
};
|
|
4874
|
+
walk(0, tree.from.length);
|
|
4875
|
+
}
|
|
4876
|
+
|
|
4552
4877
|
// src/geo/polygon.ts
|
|
4553
4878
|
function requirePolygonRing(points) {
|
|
4554
4879
|
if (!Array.isArray(points) || points.length < MIN_POLYGON_POINTS) {
|
|
@@ -4560,11 +4885,18 @@ function requirePolygonRing(points) {
|
|
|
4560
4885
|
}
|
|
4561
4886
|
points.forEach((point, position) => {
|
|
4562
4887
|
const { lat, lon } = point ?? {};
|
|
4563
|
-
if (typeof lat
|
|
4888
|
+
if (typeof lat !== "number" || !Number.isFinite(lat) || typeof lon !== "number" || !Number.isFinite(lon)) {
|
|
4889
|
+
throw new NarsilError(
|
|
4890
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4891
|
+
`A polygon point carries a finite "lat" and "lon", and the point at position ${position} does not`,
|
|
4892
|
+
{ position }
|
|
4893
|
+
);
|
|
4894
|
+
}
|
|
4895
|
+
if (isLatitude(lat) && isLongitude(lon)) return;
|
|
4564
4896
|
throw new NarsilError(
|
|
4565
4897
|
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4566
|
-
`A polygon point
|
|
4567
|
-
{ position }
|
|
4898
|
+
`A polygon point must hold a "lat" from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES} and a "lon" from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, while the point at position ${position} holds ${lat}, ${lon}`,
|
|
4899
|
+
{ position, lat, lon }
|
|
4568
4900
|
);
|
|
4569
4901
|
});
|
|
4570
4902
|
}
|
|
@@ -4597,20 +4929,35 @@ function polygonContainment(polygon) {
|
|
|
4597
4929
|
lats[i] = polygon[i].lat;
|
|
4598
4930
|
lons[i] = unwrap ? eastward(polygon[i].lon) : polygon[i].lon;
|
|
4599
4931
|
}
|
|
4932
|
+
const edges = buildLongitudeEdgeTree(lons);
|
|
4600
4933
|
return (lat, rawLon) => {
|
|
4601
4934
|
const lon = unwrap ? eastward(rawLon) : rawLon;
|
|
4602
4935
|
let inside = false;
|
|
4603
|
-
|
|
4604
|
-
const
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
const intersectLat = lats[i] + (lats[j] - lats[i]) / (lonJ - lonI) * (lon - lonI);
|
|
4608
|
-
if (lat < intersectLat) inside = !inside;
|
|
4609
|
-
}
|
|
4610
|
-
}
|
|
4936
|
+
forEachEdgeSpanning(edges, lon, (i, j) => {
|
|
4937
|
+
const intersectLat = lats[i] + (lats[j] - lats[i]) / (lons[j] - lons[i]) * (lon - lons[i]);
|
|
4938
|
+
if (lat < intersectLat) inside = !inside;
|
|
4939
|
+
});
|
|
4611
4940
|
return inside;
|
|
4612
4941
|
};
|
|
4613
4942
|
}
|
|
4943
|
+
function polygonBoxes(polygon) {
|
|
4944
|
+
if (polygon.length < MIN_POLYGON_POINTS) return [];
|
|
4945
|
+
const unwrap = crossesAntimeridian(polygon);
|
|
4946
|
+
let south = Number.POSITIVE_INFINITY;
|
|
4947
|
+
let north = Number.NEGATIVE_INFINITY;
|
|
4948
|
+
let west = Number.POSITIVE_INFINITY;
|
|
4949
|
+
let east = Number.NEGATIVE_INFINITY;
|
|
4950
|
+
for (const point of polygon) {
|
|
4951
|
+
const lon = unwrap ? eastward(point.lon) : point.lon;
|
|
4952
|
+
south = Math.min(south, point.lat);
|
|
4953
|
+
north = Math.max(north, point.lat);
|
|
4954
|
+
west = Math.min(west, lon);
|
|
4955
|
+
east = Math.max(east, lon);
|
|
4956
|
+
}
|
|
4957
|
+
if (west >= MAX_LONGITUDE_DEGREES)
|
|
4958
|
+
return longitudeBoxes(south, north, west - FULL_TURN_DEGREES, east - FULL_TURN_DEGREES);
|
|
4959
|
+
return longitudeBoxes(south, north, west, east);
|
|
4960
|
+
}
|
|
4614
4961
|
|
|
4615
4962
|
// src/geo/vincenty.ts
|
|
4616
4963
|
var WGS84_A = 6378137;
|
|
@@ -4662,61 +5009,94 @@ function vincentyDistance(lat1, lon1, lat2, lon2) {
|
|
|
4662
5009
|
}
|
|
4663
5010
|
|
|
4664
5011
|
// src/geo/geo-index.ts
|
|
5012
|
+
function firstAtOrAfter(entries, code) {
|
|
5013
|
+
let lo = 0;
|
|
5014
|
+
let hi = entries.length;
|
|
5015
|
+
while (lo < hi) {
|
|
5016
|
+
const mid = lo + hi >>> 1;
|
|
5017
|
+
if (entries[mid].code < code) lo = mid + 1;
|
|
5018
|
+
else hi = mid;
|
|
5019
|
+
}
|
|
5020
|
+
return lo;
|
|
5021
|
+
}
|
|
4665
5022
|
function createGeoIndex() {
|
|
4666
5023
|
const entries = [];
|
|
4667
|
-
|
|
4668
|
-
|
|
4669
|
-
|
|
4670
|
-
|
|
5024
|
+
const byDocument = /* @__PURE__ */ new Map();
|
|
5025
|
+
let sorted = true;
|
|
5026
|
+
function ensureSorted() {
|
|
5027
|
+
if (sorted) return;
|
|
5028
|
+
entries.sort((left, right) => left.code - right.code || left.docId - right.docId);
|
|
5029
|
+
sorted = true;
|
|
5030
|
+
}
|
|
5031
|
+
function removeEntry(internalId) {
|
|
5032
|
+
const entry = byDocument.get(internalId);
|
|
5033
|
+
if (entry === void 0) return;
|
|
5034
|
+
byDocument.delete(internalId);
|
|
5035
|
+
ensureSorted();
|
|
5036
|
+
for (let at = firstAtOrAfter(entries, entry.code); at < entries.length; at++) {
|
|
5037
|
+
if (entries[at] !== entry) continue;
|
|
5038
|
+
entries.splice(at, 1);
|
|
5039
|
+
return;
|
|
5040
|
+
}
|
|
5041
|
+
}
|
|
5042
|
+
function matching(boxes, passes, inside) {
|
|
5043
|
+
ensureSorted();
|
|
5044
|
+
const within = /* @__PURE__ */ new Set();
|
|
5045
|
+
for (const range of coveringRanges(boxes)) {
|
|
5046
|
+
for (let at = firstAtOrAfter(entries, range.from); at < entries.length && entries[at].code < range.to; at++) {
|
|
5047
|
+
if (passes(entries[at])) within.add(entries[at].docId);
|
|
5048
|
+
}
|
|
5049
|
+
}
|
|
5050
|
+
if (inside) return within;
|
|
5051
|
+
const outside = /* @__PURE__ */ new Set();
|
|
5052
|
+
for (const entry of entries) {
|
|
5053
|
+
if (!within.has(entry.docId)) outside.add(entry.docId);
|
|
5054
|
+
}
|
|
5055
|
+
return outside;
|
|
5056
|
+
}
|
|
5057
|
+
const index = {
|
|
4671
5058
|
insert(internalId, lat, lon) {
|
|
4672
|
-
|
|
5059
|
+
removeEntry(internalId);
|
|
5060
|
+
const entry = { lat, lon, docId: internalId, code: locationCode(lat, lon) };
|
|
5061
|
+
entries.push(entry);
|
|
5062
|
+
byDocument.set(internalId, entry);
|
|
5063
|
+
sorted = false;
|
|
4673
5064
|
},
|
|
4674
5065
|
remove(internalId) {
|
|
4675
|
-
|
|
4676
|
-
if (entries[i].docId === internalId) {
|
|
4677
|
-
entries.splice(i, 1);
|
|
4678
|
-
return;
|
|
4679
|
-
}
|
|
4680
|
-
}
|
|
5066
|
+
removeEntry(internalId);
|
|
4681
5067
|
},
|
|
4682
5068
|
radiusQuery(lat, lon, distanceMeters, inside, highPrecision) {
|
|
4683
|
-
const result = /* @__PURE__ */ new Set();
|
|
4684
5069
|
const distanceFn = highPrecision ? vincentyDistance : haversineDistance;
|
|
4685
|
-
|
|
4686
|
-
|
|
4687
|
-
|
|
4688
|
-
|
|
4689
|
-
|
|
4690
|
-
result.add(entry.docId);
|
|
4691
|
-
}
|
|
4692
|
-
}
|
|
4693
|
-
return result;
|
|
5070
|
+
return matching(
|
|
5071
|
+
radiusBoxes(lat, lon, distanceMeters),
|
|
5072
|
+
(entry) => distanceFn(lat, lon, entry.lat, entry.lon) <= distanceMeters,
|
|
5073
|
+
inside
|
|
5074
|
+
);
|
|
4694
5075
|
},
|
|
4695
5076
|
polygonQuery(points, inside) {
|
|
4696
|
-
const result = /* @__PURE__ */ new Set();
|
|
4697
5077
|
const contains = polygonContainment(points);
|
|
4698
|
-
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
}
|
|
4705
|
-
return result;
|
|
5078
|
+
return matching(polygonBoxes(points), (entry) => contains(entry.lat, entry.lon), inside);
|
|
5079
|
+
},
|
|
5080
|
+
documentsBitset(capacity) {
|
|
5081
|
+
const bits = createBitSet(capacity);
|
|
5082
|
+
for (const entry of entries) bitsetSet(bits, entry.docId);
|
|
5083
|
+
return bits;
|
|
4706
5084
|
},
|
|
4707
5085
|
clear() {
|
|
4708
5086
|
entries.length = 0;
|
|
5087
|
+
byDocument.clear();
|
|
5088
|
+
sorted = true;
|
|
4709
5089
|
},
|
|
4710
5090
|
serialize() {
|
|
4711
|
-
|
|
5091
|
+
ensureSorted();
|
|
5092
|
+
return entries.map((entry) => ({ lat: entry.lat, lon: entry.lon, docId: entry.docId }));
|
|
4712
5093
|
},
|
|
4713
5094
|
deserialize(data) {
|
|
4714
|
-
|
|
4715
|
-
for (const entry of data)
|
|
4716
|
-
entries.push({ lat: entry.lat, lon: entry.lon, docId: entry.docId });
|
|
4717
|
-
}
|
|
5095
|
+
index.clear();
|
|
5096
|
+
for (const entry of data) index.insert(entry.docId, entry.lat, entry.lon);
|
|
4718
5097
|
}
|
|
4719
5098
|
};
|
|
5099
|
+
return index;
|
|
4720
5100
|
}
|
|
4721
5101
|
|
|
4722
5102
|
// src/core/field-index/boolean.ts
|
|
@@ -4790,9 +5170,90 @@ function createBooleanIndex() {
|
|
|
4790
5170
|
};
|
|
4791
5171
|
}
|
|
4792
5172
|
|
|
5173
|
+
// src/core/field-index/affixes.ts
|
|
5174
|
+
function compareUnits(left, right) {
|
|
5175
|
+
if (left < right) return -1;
|
|
5176
|
+
return left > right ? 1 : 0;
|
|
5177
|
+
}
|
|
5178
|
+
function reversedUnits(value) {
|
|
5179
|
+
let reversed = "";
|
|
5180
|
+
for (let at = value.length - 1; at >= 0; at--) reversed += value[at];
|
|
5181
|
+
return reversed;
|
|
5182
|
+
}
|
|
5183
|
+
function firstAtOrAfter2(sorted, key) {
|
|
5184
|
+
let lo = 0;
|
|
5185
|
+
let hi = sorted.length;
|
|
5186
|
+
while (lo < hi) {
|
|
5187
|
+
const mid = lo + hi >>> 1;
|
|
5188
|
+
if (sorted[mid] < key) lo = mid + 1;
|
|
5189
|
+
else hi = mid;
|
|
5190
|
+
}
|
|
5191
|
+
return lo;
|
|
5192
|
+
}
|
|
5193
|
+
function settledColumn(column, holds) {
|
|
5194
|
+
const { keys, values } = column;
|
|
5195
|
+
const order = [];
|
|
5196
|
+
for (let at = 0; at < keys.length; at++) if (holds(values[at])) order.push(at);
|
|
5197
|
+
order.sort((left, right) => compareUnits(keys[left], keys[right]) || compareUnits(values[left], values[right]));
|
|
5198
|
+
const settled2 = { keys: [], values: [] };
|
|
5199
|
+
for (const at of order) {
|
|
5200
|
+
const last = settled2.values.length - 1;
|
|
5201
|
+
if (last >= 0 && settled2.keys[last] === keys[at] && settled2.values[last] === values[at]) continue;
|
|
5202
|
+
settled2.keys.push(keys[at]);
|
|
5203
|
+
settled2.values.push(values[at]);
|
|
5204
|
+
}
|
|
5205
|
+
return settled2;
|
|
5206
|
+
}
|
|
5207
|
+
var ValueAffixes = class {
|
|
5208
|
+
constructor(holds, values) {
|
|
5209
|
+
this.holds = holds;
|
|
5210
|
+
for (const value of values) this.add(value);
|
|
5211
|
+
}
|
|
5212
|
+
forward = { keys: [], values: [] };
|
|
5213
|
+
backward = { keys: [], values: [] };
|
|
5214
|
+
settled = true;
|
|
5215
|
+
add(value) {
|
|
5216
|
+
this.forward.keys.push(value);
|
|
5217
|
+
this.forward.values.push(value);
|
|
5218
|
+
this.backward.keys.push(reversedUnits(value));
|
|
5219
|
+
this.backward.values.push(value);
|
|
5220
|
+
this.settled = false;
|
|
5221
|
+
}
|
|
5222
|
+
forget() {
|
|
5223
|
+
this.settled = false;
|
|
5224
|
+
}
|
|
5225
|
+
withPrefix(prefix) {
|
|
5226
|
+
this.settle();
|
|
5227
|
+
return this.matching(this.forward, prefix);
|
|
5228
|
+
}
|
|
5229
|
+
withSuffix(suffix) {
|
|
5230
|
+
this.settle();
|
|
5231
|
+
return this.matching(this.backward, reversedUnits(suffix));
|
|
5232
|
+
}
|
|
5233
|
+
settle() {
|
|
5234
|
+
if (this.settled) return;
|
|
5235
|
+
this.forward = settledColumn(this.forward, this.holds);
|
|
5236
|
+
this.backward = settledColumn(this.backward, this.holds);
|
|
5237
|
+
this.settled = true;
|
|
5238
|
+
}
|
|
5239
|
+
matching(column, start) {
|
|
5240
|
+
const found = [];
|
|
5241
|
+
for (let at = firstAtOrAfter2(column.keys, start); at < column.keys.length; at++) {
|
|
5242
|
+
if (!column.keys[at].startsWith(start)) break;
|
|
5243
|
+
found.push(column.values[at]);
|
|
5244
|
+
}
|
|
5245
|
+
return found;
|
|
5246
|
+
}
|
|
5247
|
+
};
|
|
5248
|
+
|
|
4793
5249
|
// src/core/field-index/enum.ts
|
|
4794
5250
|
function createEnumIndex() {
|
|
4795
5251
|
const valueMap = /* @__PURE__ */ new Map();
|
|
5252
|
+
let affixes = null;
|
|
5253
|
+
function affixesOf() {
|
|
5254
|
+
affixes ??= new ValueAffixes((value) => valueMap.has(value), valueMap.keys());
|
|
5255
|
+
return affixes;
|
|
5256
|
+
}
|
|
4796
5257
|
function allDocs() {
|
|
4797
5258
|
const result = /* @__PURE__ */ new Set();
|
|
4798
5259
|
for (const docSet of valueMap.values()) {
|
|
@@ -4806,6 +5267,7 @@ function createEnumIndex() {
|
|
|
4806
5267
|
if (!docSet) {
|
|
4807
5268
|
docSet = /* @__PURE__ */ new Set();
|
|
4808
5269
|
valueMap.set(value, docSet);
|
|
5270
|
+
affixes?.add(value);
|
|
4809
5271
|
}
|
|
4810
5272
|
docSet.add(internalId);
|
|
4811
5273
|
},
|
|
@@ -4813,7 +5275,15 @@ function createEnumIndex() {
|
|
|
4813
5275
|
const docSet = valueMap.get(value);
|
|
4814
5276
|
if (!docSet) return;
|
|
4815
5277
|
docSet.delete(internalId);
|
|
4816
|
-
if (docSet.size
|
|
5278
|
+
if (docSet.size > 0) return;
|
|
5279
|
+
valueMap.delete(value);
|
|
5280
|
+
affixes?.forget();
|
|
5281
|
+
},
|
|
5282
|
+
queryPrefixBitset(prefix, capacity) {
|
|
5283
|
+
return this.queryInBitset(affixesOf().withPrefix(prefix), capacity);
|
|
5284
|
+
},
|
|
5285
|
+
querySuffixBitset(suffix, capacity) {
|
|
5286
|
+
return this.queryInBitset(affixesOf().withSuffix(suffix), capacity);
|
|
4817
5287
|
},
|
|
4818
5288
|
queryEq(value) {
|
|
4819
5289
|
const docSet = valueMap.get(value);
|
|
@@ -4889,6 +5359,7 @@ function createEnumIndex() {
|
|
|
4889
5359
|
},
|
|
4890
5360
|
clear() {
|
|
4891
5361
|
valueMap.clear();
|
|
5362
|
+
affixes = null;
|
|
4892
5363
|
},
|
|
4893
5364
|
serialize() {
|
|
4894
5365
|
const result = /* @__PURE__ */ Object.create(null);
|
|
@@ -4899,6 +5370,7 @@ function createEnumIndex() {
|
|
|
4899
5370
|
},
|
|
4900
5371
|
deserialize(data) {
|
|
4901
5372
|
valueMap.clear();
|
|
5373
|
+
affixes = null;
|
|
4902
5374
|
for (const value of Object.keys(data)) {
|
|
4903
5375
|
valueMap.set(value, new Set(data[value]));
|
|
4904
5376
|
}
|
|
@@ -5080,6 +5552,44 @@ function createNumericIndex() {
|
|
|
5080
5552
|
};
|
|
5081
5553
|
}
|
|
5082
5554
|
|
|
5555
|
+
// src/core/partition/value-sizes.ts
|
|
5556
|
+
function keepsValueSize(base) {
|
|
5557
|
+
return base === "string" || base === "verbatim" || base.endsWith("[]");
|
|
5558
|
+
}
|
|
5559
|
+
function valueSizeOf(value) {
|
|
5560
|
+
if (Array.isArray(value)) return value.length;
|
|
5561
|
+
return value === "" ? 0 : 1;
|
|
5562
|
+
}
|
|
5563
|
+
function sizeIndexOf(indexes, fieldPath) {
|
|
5564
|
+
let index = indexes.get(fieldPath);
|
|
5565
|
+
if (index === void 0) {
|
|
5566
|
+
index = createNumericIndex();
|
|
5567
|
+
indexes.set(fieldPath, index);
|
|
5568
|
+
}
|
|
5569
|
+
return index;
|
|
5570
|
+
}
|
|
5571
|
+
function recordValueSize(indexes, internalId, fieldPath, base, value) {
|
|
5572
|
+
if (value === void 0 || value === null || !keepsValueSize(base)) return;
|
|
5573
|
+
sizeIndexOf(indexes, fieldPath).insert(internalId, valueSizeOf(value));
|
|
5574
|
+
}
|
|
5575
|
+
function forgetValueSize(indexes, internalId, fieldPath, base, value) {
|
|
5576
|
+
if (value === void 0 || value === null || !keepsValueSize(base)) return;
|
|
5577
|
+
indexes.get(fieldPath)?.remove(internalId, valueSizeOf(value));
|
|
5578
|
+
}
|
|
5579
|
+
function rebuildSizeIndexes(state, schema) {
|
|
5580
|
+
state.sizeIndexes.clear();
|
|
5581
|
+
const fields = schemaFieldsOf(schema).filter((field) => keepsValueSize(field.base));
|
|
5582
|
+
if (fields.length === 0) return;
|
|
5583
|
+
for (const [docId, stored] of state.docStore.all()) {
|
|
5584
|
+
const internalId = state.docStore.getInternalId(docId);
|
|
5585
|
+
if (internalId === void 0) continue;
|
|
5586
|
+
for (const { path, base } of fields) {
|
|
5587
|
+
const value = getNestedValue(stored.fields, path);
|
|
5588
|
+
recordValueSize(state.sizeIndexes, internalId, path, base, value);
|
|
5589
|
+
}
|
|
5590
|
+
}
|
|
5591
|
+
}
|
|
5592
|
+
|
|
5083
5593
|
// src/core/partition/indexing.ts
|
|
5084
5594
|
function ensureFieldIndex(state, fieldPath, base) {
|
|
5085
5595
|
if ((base === "number" || base === "number[]") && !state.numericIndexes.has(fieldPath)) {
|
|
@@ -5218,10 +5728,12 @@ function indexDocument(state, docId, document2, fields, language, options) {
|
|
|
5218
5728
|
const fieldLengths = {};
|
|
5219
5729
|
const tokensByField = {};
|
|
5220
5730
|
const internalId = resolveInternalId(state, docId);
|
|
5221
|
-
for (const { path: fieldPath, base } of fields) {
|
|
5731
|
+
for (const { path: fieldPath, base, pattern } of fields) {
|
|
5222
5732
|
const value = getNestedValue(document2, fieldPath);
|
|
5223
5733
|
if (value === void 0 || value === null) continue;
|
|
5734
|
+
recordValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
5224
5735
|
ensureFieldIndex(state, fieldPath, base);
|
|
5736
|
+
if (pattern) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
|
|
5225
5737
|
if (base === "string") {
|
|
5226
5738
|
indexStringField(state, internalId, fieldPath, value, language, options, fieldLengths, tokensByField);
|
|
5227
5739
|
} else if (base === "number") {
|
|
@@ -5265,9 +5777,11 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
5265
5777
|
const fields = storedDoc.fields;
|
|
5266
5778
|
const internalId = resolveInternalId(state, docId);
|
|
5267
5779
|
const opts = tokenizeOptions(options);
|
|
5268
|
-
for (const { path: fieldPath, base } of schemaFields) {
|
|
5780
|
+
for (const { path: fieldPath, base, pattern } of schemaFields) {
|
|
5269
5781
|
const value = getNestedValue(fields, fieldPath);
|
|
5270
5782
|
if (value === void 0 || value === null) continue;
|
|
5783
|
+
forgetValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
5784
|
+
if (pattern) state.patternIndexes.get(fieldPath)?.remove(internalId, value);
|
|
5271
5785
|
if (base === "string") {
|
|
5272
5786
|
const result = tokenize(value, language, opts);
|
|
5273
5787
|
fieldLengths[fieldPath] = result.tokens.length;
|
|
@@ -5339,16 +5853,24 @@ function resolveInternalId2(state, docId) {
|
|
|
5339
5853
|
}
|
|
5340
5854
|
function updateFieldIndexOnly(state, docId, oldFields, newDoc, fields) {
|
|
5341
5855
|
const internalId = resolveInternalId2(state, docId);
|
|
5342
|
-
for (const { path: fieldPath, base: fieldType } of fields) {
|
|
5856
|
+
for (const { path: fieldPath, base: fieldType, pattern } of fields) {
|
|
5343
5857
|
if (isWordIndexedBase(fieldType)) continue;
|
|
5344
5858
|
const oldVal = getNestedValue(oldFields, fieldPath);
|
|
5345
5859
|
const newVal = getNestedValue(newDoc, fieldPath);
|
|
5346
5860
|
if (oldVal === newVal) continue;
|
|
5347
5861
|
if (oldVal !== void 0 && oldVal !== null && newVal !== void 0 && newVal !== null && typeof oldVal === typeof newVal && JSON.stringify(oldVal) === JSON.stringify(newVal))
|
|
5348
5862
|
continue;
|
|
5863
|
+
forgetValueSize(state.sizeIndexes, internalId, fieldPath, fieldType, oldVal);
|
|
5864
|
+
recordValueSize(state.sizeIndexes, internalId, fieldPath, fieldType, newVal);
|
|
5349
5865
|
if (newVal !== void 0 && newVal !== null) {
|
|
5350
5866
|
ensureFieldIndex(state, fieldPath, fieldType);
|
|
5351
5867
|
}
|
|
5868
|
+
if (pattern) {
|
|
5869
|
+
if (oldVal !== void 0 && oldVal !== null) state.patternIndexes.get(fieldPath)?.remove(internalId, oldVal);
|
|
5870
|
+
if (newVal !== void 0 && newVal !== null) {
|
|
5871
|
+
patternIndexOf(state.patternIndexes, fieldPath).add(internalId, newVal);
|
|
5872
|
+
}
|
|
5873
|
+
}
|
|
5352
5874
|
if (fieldType === "number") {
|
|
5353
5875
|
const numIdx = state.numericIndexes.get(fieldPath);
|
|
5354
5876
|
if (numIdx) {
|
|
@@ -5410,27 +5932,224 @@ function updateFieldIndexOnly(state, docId, oldFields, newDoc, fields) {
|
|
|
5410
5932
|
}
|
|
5411
5933
|
}
|
|
5412
5934
|
|
|
5413
|
-
// src/filters/
|
|
5414
|
-
|
|
5415
|
-
|
|
5416
|
-
|
|
5417
|
-
|
|
5418
|
-
|
|
5419
|
-
|
|
5935
|
+
// src/filters/constants.ts
|
|
5936
|
+
var MAX_FILTER_DEPTH = 30;
|
|
5937
|
+
var MAX_FILTER_TESTS = 1024;
|
|
5938
|
+
var MAX_FILTER_ARRAY_SIZE = 65536;
|
|
5939
|
+
var GEO_UNIT_METRES = { km: 1e3, mi: 1609.344, m: 1 };
|
|
5940
|
+
|
|
5941
|
+
// src/filters/pattern-tests.ts
|
|
5942
|
+
function excluding(context, fieldPath, excluded) {
|
|
5943
|
+
const universe = bitsetAnd(context.presentBitset(fieldPath), context.allDocIdsBitset);
|
|
5944
|
+
return bitsetAnd(universe, bitsetNot(excluded, context.capacity));
|
|
5945
|
+
}
|
|
5946
|
+
function textsOf(operand) {
|
|
5947
|
+
const texts = /* @__PURE__ */ new Set();
|
|
5948
|
+
if (!Array.isArray(operand)) return texts;
|
|
5949
|
+
for (const text of operand) if (typeof text === "string") texts.add(text);
|
|
5950
|
+
return texts;
|
|
5951
|
+
}
|
|
5952
|
+
function evaluatePatternTests(fieldPath, filter, context, bitsets) {
|
|
5953
|
+
const caseFold = filter.caseFold === true;
|
|
5954
|
+
const matching = (operator, texts) => {
|
|
5955
|
+
const into = createBitSet(context.capacity);
|
|
5956
|
+
for (const text of texts) context.matchPattern(fieldPath, operator, text, caseFold, into);
|
|
5957
|
+
return into;
|
|
5958
|
+
};
|
|
5959
|
+
if (typeof filter.eq === "string") bitsets.push(matching("eq", [filter.eq]));
|
|
5960
|
+
if (typeof filter.ne === "string") bitsets.push(excluding(context, fieldPath, matching("eq", [filter.ne])));
|
|
5961
|
+
if (filter.in !== void 0) bitsets.push(matching("eq", textsOf(filter.in)));
|
|
5962
|
+
if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq", textsOf(filter.nin))));
|
|
5963
|
+
if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
|
|
5964
|
+
if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
|
|
5965
|
+
if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
|
|
5966
|
+
}
|
|
5967
|
+
|
|
5968
|
+
// src/filters/field-tests.ts
|
|
5969
|
+
var COMPARISON_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
|
|
5970
|
+
var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
|
|
5971
|
+
var FieldBits = class {
|
|
5972
|
+
constructor(context, fieldPath) {
|
|
5973
|
+
this.context = context;
|
|
5974
|
+
this.fieldPath = fieldPath;
|
|
5420
5975
|
}
|
|
5421
|
-
|
|
5976
|
+
none() {
|
|
5977
|
+
return createBitSet(this.context.capacity);
|
|
5978
|
+
}
|
|
5979
|
+
present() {
|
|
5980
|
+
return this.context.presentBitset(this.fieldPath);
|
|
5981
|
+
}
|
|
5982
|
+
holdingSomething() {
|
|
5983
|
+
const empty = this.context.sizeIndex(this.fieldPath)?.queryEqBitset(0, this.context.capacity);
|
|
5984
|
+
return empty === void 0 ? this.present() : this.without(this.present(), empty);
|
|
5985
|
+
}
|
|
5986
|
+
without(from, removed) {
|
|
5987
|
+
const kept = from.slice();
|
|
5988
|
+
bitsetAndNotInto(kept, removed);
|
|
5989
|
+
return kept;
|
|
5990
|
+
}
|
|
5991
|
+
complement(bits) {
|
|
5992
|
+
return this.without(this.context.allDocIdsBitset, bits);
|
|
5993
|
+
}
|
|
5994
|
+
};
|
|
5995
|
+
function comparisonBits(index, operator, operand, bits, capacity) {
|
|
5996
|
+
if (operator === "between") {
|
|
5997
|
+
const [low, high] = operand;
|
|
5998
|
+
return index?.queryBetweenBitset(low, high, capacity) ?? bits.none();
|
|
5999
|
+
}
|
|
6000
|
+
const value = operand;
|
|
6001
|
+
if (operator === "ne") return bits.without(bits.present(), index?.queryEqBitset(value, capacity) ?? bits.none());
|
|
6002
|
+
if (index === void 0) return bits.none();
|
|
6003
|
+
if (operator === "eq") return index.queryEqBitset(value, capacity);
|
|
6004
|
+
if (operator === "gt") return index.queryGtBitset(value, capacity);
|
|
6005
|
+
if (operator === "gte") return index.queryGteBitset(value, capacity);
|
|
6006
|
+
if (operator === "lt") return index.queryLtBitset(value, capacity);
|
|
6007
|
+
return index.queryLteBitset(value, capacity);
|
|
6008
|
+
}
|
|
6009
|
+
function elementBits(context, fieldPath, parsed, value) {
|
|
6010
|
+
const { capacity } = context;
|
|
6011
|
+
if (parsed.pattern) {
|
|
6012
|
+
const matches = createBitSet(capacity);
|
|
6013
|
+
context.matchPattern(fieldPath, "eq", value, false, matches);
|
|
6014
|
+
return matches;
|
|
6015
|
+
}
|
|
6016
|
+
if (typeof value === "number")
|
|
6017
|
+
return context.numericIndex(fieldPath)?.queryEqBitset(value, capacity) ?? createBitSet(capacity);
|
|
6018
|
+
if (typeof value === "boolean")
|
|
6019
|
+
return context.booleanIndex(fieldPath)?.queryEqBitset(value, capacity) ?? createBitSet(capacity);
|
|
6020
|
+
return context.enumIndex(fieldPath)?.queryEqBitset(value, capacity) ?? createBitSet(capacity);
|
|
5422
6021
|
}
|
|
5423
|
-
function
|
|
5424
|
-
|
|
5425
|
-
|
|
5426
|
-
|
|
5427
|
-
|
|
5428
|
-
|
|
6022
|
+
function anyElementBits(context, fieldPath, parsed, values) {
|
|
6023
|
+
const { capacity } = context;
|
|
6024
|
+
const union = createBitSet(capacity);
|
|
6025
|
+
if (parsed.pattern) {
|
|
6026
|
+
for (const value of values) context.matchPattern(fieldPath, "eq", value, false, union);
|
|
6027
|
+
return union;
|
|
5429
6028
|
}
|
|
5430
|
-
|
|
6029
|
+
if (parsed.base === "enum[]") {
|
|
6030
|
+
return context.enumIndex(fieldPath)?.queryInBitset(values, capacity) ?? union;
|
|
6031
|
+
}
|
|
6032
|
+
const numbers = context.numericIndex(fieldPath);
|
|
6033
|
+
const booleans = context.booleanIndex(fieldPath);
|
|
6034
|
+
for (const value of values) {
|
|
6035
|
+
if (typeof value === "number") for (const docId of numbers?.queryEq(value) ?? []) bitsetSet(union, docId);
|
|
6036
|
+
if (typeof value === "boolean") for (const docId of booleans?.queryEq(value) ?? []) bitsetSet(union, docId);
|
|
6037
|
+
}
|
|
6038
|
+
return union;
|
|
6039
|
+
}
|
|
6040
|
+
function distinctScalars(values) {
|
|
6041
|
+
return [...new Set(values)];
|
|
6042
|
+
}
|
|
6043
|
+
function narrowByElements(fieldPath, f, parsed, context, bits, narrow) {
|
|
6044
|
+
if (f.matchesAny !== void 0) narrow(anyElementBits(context, fieldPath, parsed, distinctScalars(f.matchesAny)));
|
|
6045
|
+
if (f.containsAll !== void 0) {
|
|
6046
|
+
const all = bits.present().slice();
|
|
6047
|
+
for (const value of distinctScalars(f.containsAll)) {
|
|
6048
|
+
if (bitsetIsEmpty(all)) break;
|
|
6049
|
+
bitsetAndInto(all, elementBits(context, fieldPath, parsed, value));
|
|
6050
|
+
}
|
|
6051
|
+
narrow(all);
|
|
6052
|
+
}
|
|
6053
|
+
if (f.size !== void 0) {
|
|
6054
|
+
const sizes = context.sizeIndex(fieldPath);
|
|
6055
|
+
const comparison = f.size;
|
|
6056
|
+
for (const operator of COMPARISON_OPERATORS) {
|
|
6057
|
+
if (comparison[operator] === void 0) continue;
|
|
6058
|
+
narrow(comparisonBits(sizes, operator, comparison[operator], bits, context.capacity));
|
|
6059
|
+
}
|
|
6060
|
+
}
|
|
6061
|
+
}
|
|
6062
|
+
function narrowByValue(fieldPath, f, parsed, context, bits, narrow) {
|
|
6063
|
+
const { capacity } = context;
|
|
6064
|
+
const base = parsed.base;
|
|
6065
|
+
if (base === "number" || base === "number[]") {
|
|
6066
|
+
const index = context.numericIndex(fieldPath);
|
|
6067
|
+
for (const operator of COMPARISON_OPERATORS) {
|
|
6068
|
+
const operand = f[operator];
|
|
6069
|
+
if (operand === void 0 || typeof operand === "string") continue;
|
|
6070
|
+
if (operator === "between" && typeof operand[0] === "string") continue;
|
|
6071
|
+
narrow(comparisonBits(index, operator, operand, bits, capacity));
|
|
6072
|
+
}
|
|
6073
|
+
return;
|
|
6074
|
+
}
|
|
6075
|
+
if (base === "boolean" || base === "boolean[]") {
|
|
6076
|
+
const index = context.booleanIndex(fieldPath);
|
|
6077
|
+
const equal = (value) => index?.queryEqBitset(value, capacity) ?? bits.none();
|
|
6078
|
+
if (typeof f.eq === "boolean") narrow(equal(f.eq));
|
|
6079
|
+
if (typeof f.ne === "boolean") narrow(bits.without(bits.present(), equal(f.ne)));
|
|
6080
|
+
return;
|
|
6081
|
+
}
|
|
6082
|
+
if (base === "enum" || base === "enum[]") {
|
|
6083
|
+
const index = context.enumIndex(fieldPath);
|
|
6084
|
+
const equal = (value) => index?.queryEqBitset(value, capacity) ?? bits.none();
|
|
6085
|
+
const anyOf = (values) => index?.queryInBitset(values, capacity) ?? bits.none();
|
|
6086
|
+
if (typeof f.eq === "string") narrow(equal(f.eq));
|
|
6087
|
+
if (typeof f.ne === "string") narrow(bits.without(bits.present(), equal(f.ne)));
|
|
6088
|
+
if (Array.isArray(f.in)) narrow(anyOf(f.in));
|
|
6089
|
+
if (Array.isArray(f.nin)) narrow(bits.without(bits.present(), anyOf(f.nin)));
|
|
6090
|
+
if (typeof f.startsWith === "string") narrow(index?.queryPrefixBitset(f.startsWith, capacity) ?? bits.none());
|
|
6091
|
+
if (typeof f.endsWith === "string") narrow(index?.querySuffixBitset(f.endsWith, capacity) ?? bits.none());
|
|
6092
|
+
}
|
|
6093
|
+
}
|
|
6094
|
+
function narrowByLocation(fieldPath, f, context, narrow) {
|
|
6095
|
+
const index = context.geoIndex(fieldPath);
|
|
6096
|
+
if (f.radius !== void 0) {
|
|
6097
|
+
const circle = f.radius;
|
|
6098
|
+
const found = index?.radiusQuery(
|
|
6099
|
+
circle.lat,
|
|
6100
|
+
circle.lon,
|
|
6101
|
+
circle.distance * GEO_UNIT_METRES[circle.unit],
|
|
6102
|
+
circle.inside ?? true,
|
|
6103
|
+
circle.highPrecision ?? false
|
|
6104
|
+
);
|
|
6105
|
+
narrow(bitsetFromSet(found ?? /* @__PURE__ */ new Set(), context.capacity));
|
|
6106
|
+
}
|
|
6107
|
+
if (f.polygon !== void 0) {
|
|
6108
|
+
const polygon = f.polygon;
|
|
6109
|
+
narrow(bitsetFromSet(index?.polygonQuery(polygon.points, polygon.inside ?? true) ?? /* @__PURE__ */ new Set(), context.capacity));
|
|
6110
|
+
}
|
|
6111
|
+
}
|
|
6112
|
+
function narrowByPresence(f, bits, narrow) {
|
|
6113
|
+
if (typeof f.exists === "boolean") narrow(f.exists ? bits.present() : bits.complement(bits.present()));
|
|
6114
|
+
if (typeof f.notExists === "boolean") narrow(f.notExists ? bits.complement(bits.present()) : bits.present());
|
|
6115
|
+
if (typeof f.isNotEmpty === "boolean") {
|
|
6116
|
+
const holding = bits.holdingSomething();
|
|
6117
|
+
narrow(f.isNotEmpty ? holding : bits.complement(holding));
|
|
6118
|
+
}
|
|
6119
|
+
if (typeof f.isEmpty === "boolean") {
|
|
6120
|
+
const holding = bits.holdingSomething();
|
|
6121
|
+
narrow(f.isEmpty ? bits.complement(holding) : holding);
|
|
6122
|
+
}
|
|
6123
|
+
}
|
|
6124
|
+
function textBoundsOf(operator, bound) {
|
|
6125
|
+
if (operator === "gt" || operator === "gte") {
|
|
6126
|
+
return { lower: bound, upper: null, lowerInclusive: operator === "gte", upperInclusive: false };
|
|
6127
|
+
}
|
|
6128
|
+
return { lower: null, upper: bound, upperInclusive: operator === "lte", lowerInclusive: false };
|
|
5431
6129
|
}
|
|
5432
|
-
function
|
|
5433
|
-
|
|
6130
|
+
function narrowByTextRange(fieldPath, f, context, narrow) {
|
|
6131
|
+
for (const operator of TEXT_RANGE_OPERATORS) {
|
|
6132
|
+
const bound = f[operator];
|
|
6133
|
+
if (typeof bound === "string") narrow(context.textRangeBitset(fieldPath, textBoundsOf(operator, bound)));
|
|
6134
|
+
}
|
|
6135
|
+
if (!Array.isArray(f.between)) return;
|
|
6136
|
+
const [low, high] = f.between;
|
|
6137
|
+
if (typeof low !== "string" || typeof high !== "string") return;
|
|
6138
|
+
narrow(context.textRangeBitset(fieldPath, { lower: low, lowerInclusive: true, upper: high, upperInclusive: true }));
|
|
6139
|
+
}
|
|
6140
|
+
function narrowByFieldTests(fieldPath, f, parsed, context, narrow) {
|
|
6141
|
+
const bits = new FieldBits(context, fieldPath);
|
|
6142
|
+
narrowByPresence(f, bits, narrow);
|
|
6143
|
+
narrowByLocation(fieldPath, f, context, narrow);
|
|
6144
|
+
narrowByTextRange(fieldPath, f, context, narrow);
|
|
6145
|
+
if (parsed.pattern) {
|
|
6146
|
+
const patternBits = [];
|
|
6147
|
+
evaluatePatternTests(fieldPath, f, context, patternBits);
|
|
6148
|
+
for (const matched of patternBits) narrow(matched);
|
|
6149
|
+
} else {
|
|
6150
|
+
narrowByValue(fieldPath, f, parsed, context, bits, narrow);
|
|
6151
|
+
}
|
|
6152
|
+
narrowByElements(fieldPath, f, parsed, context, bits, narrow);
|
|
5434
6153
|
}
|
|
5435
6154
|
|
|
5436
6155
|
// src/filters/keys.ts
|
|
@@ -5447,6 +6166,8 @@ var FIELD_FILTER_OPERATORS = [
|
|
|
5447
6166
|
"nin",
|
|
5448
6167
|
"startsWith",
|
|
5449
6168
|
"endsWith",
|
|
6169
|
+
"contains",
|
|
6170
|
+
"caseFold",
|
|
5450
6171
|
"containsAll",
|
|
5451
6172
|
"matchesAny",
|
|
5452
6173
|
"size",
|
|
@@ -5458,14 +6179,7 @@ var FIELD_FILTER_OPERATORS = [
|
|
|
5458
6179
|
"polygon"
|
|
5459
6180
|
];
|
|
5460
6181
|
|
|
5461
|
-
// src/filters/
|
|
5462
|
-
var MAX_FILTER_DEPTH = 30;
|
|
5463
|
-
|
|
5464
|
-
// src/filters/operands.ts
|
|
5465
|
-
var RANGE_OPERATORS = ["gt", "lt", "gte", "lte"];
|
|
5466
|
-
var TEXT_OPERATORS = ["startsWith", "endsWith"];
|
|
5467
|
-
var EQUALITY_OPERATORS = ["eq", "ne"];
|
|
5468
|
-
var LIST_OPERATORS = ["in", "nin"];
|
|
6182
|
+
// src/filters/operands/checks.ts
|
|
5469
6183
|
function operandKindOf(type) {
|
|
5470
6184
|
if (type === void 0) return void 0;
|
|
5471
6185
|
if (type === "number" || type === "number[]") return "number";
|
|
@@ -5483,6 +6197,9 @@ function kindOfValue(value) {
|
|
|
5483
6197
|
if (typeof value === "string") return "text";
|
|
5484
6198
|
return void 0;
|
|
5485
6199
|
}
|
|
6200
|
+
function isObjectRecord(value) {
|
|
6201
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
6202
|
+
}
|
|
5486
6203
|
function invalid(message, fieldPath, operator) {
|
|
5487
6204
|
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message, { fieldPath, operator });
|
|
5488
6205
|
}
|
|
@@ -5495,40 +6212,247 @@ function requireFieldKind(fieldPath, operator, type, accepted) {
|
|
|
5495
6212
|
operator
|
|
5496
6213
|
);
|
|
5497
6214
|
}
|
|
5498
|
-
function requireTextRangeField(fieldPath, operator, type) {
|
|
5499
|
-
const parsed = parsedTypeOf(type);
|
|
5500
|
-
if (parsed?.sortable && (parsed.base === "string" || parsed.base === "verbatim")) return;
|
|
5501
|
-
const declared = type === void 0 ? `the schema has no field "${fieldPath}"` : `field "${fieldPath}" has type "${type}"`;
|
|
6215
|
+
function requireTextRangeField(fieldPath, operator, type) {
|
|
6216
|
+
const parsed = parsedTypeOf(type);
|
|
6217
|
+
if (parsed?.sortable && (parsed.base === "string" || parsed.base === "verbatim")) return;
|
|
6218
|
+
const declared = type === void 0 ? `the schema has no field "${fieldPath}"` : `field "${fieldPath}" has type "${type}"`;
|
|
6219
|
+
invalid(
|
|
6220
|
+
`The engine applies "${operator}" with a text bound to a "string" or "verbatim" field whose type includes "sortable", but ${declared}`,
|
|
6221
|
+
fieldPath,
|
|
6222
|
+
operator
|
|
6223
|
+
);
|
|
6224
|
+
}
|
|
6225
|
+
function requireListLength(fieldPath, operator, values) {
|
|
6226
|
+
if (values.length <= MAX_FILTER_ARRAY_SIZE) return;
|
|
6227
|
+
invalid(
|
|
6228
|
+
`"${operator}" on field "${fieldPath}" holds ${values.length} values, while a filter list holds at most ${MAX_FILTER_ARRAY_SIZE}`,
|
|
6229
|
+
fieldPath,
|
|
6230
|
+
operator
|
|
6231
|
+
);
|
|
6232
|
+
}
|
|
6233
|
+
|
|
6234
|
+
// src/filters/operands/geo.ts
|
|
6235
|
+
function requireFlag(fieldPath, operator, flag, value) {
|
|
6236
|
+
if (value === void 0 || typeof value === "boolean") return;
|
|
6237
|
+
invalid(`"${operator}.${flag}" on field "${fieldPath}" must be true or false`, fieldPath, operator);
|
|
6238
|
+
}
|
|
6239
|
+
function requireRadius(fieldPath, radius) {
|
|
6240
|
+
const circle = radius;
|
|
6241
|
+
if (circle === null || typeof circle !== "object") {
|
|
6242
|
+
invalid(`"radius" on field "${fieldPath}" must be an object`, fieldPath, "radius");
|
|
6243
|
+
}
|
|
6244
|
+
for (const key of ["lat", "lon", "distance"]) {
|
|
6245
|
+
if (!isFiniteNumber(circle[key])) {
|
|
6246
|
+
invalid(`"radius.${key}" on field "${fieldPath}" must be a finite number`, fieldPath, "radius");
|
|
6247
|
+
}
|
|
6248
|
+
}
|
|
6249
|
+
if (typeof circle.unit !== "string" || !Object.hasOwn(GEO_UNIT_METRES, circle.unit)) {
|
|
6250
|
+
invalid(
|
|
6251
|
+
`"radius.unit" on field "${fieldPath}" must be one of ${Object.keys(GEO_UNIT_METRES).join(", ")}, and this filter sets ${String(circle.unit)}`,
|
|
6252
|
+
fieldPath,
|
|
6253
|
+
"radius"
|
|
6254
|
+
);
|
|
6255
|
+
}
|
|
6256
|
+
requireFlag(fieldPath, "radius", "inside", circle.inside);
|
|
6257
|
+
requireFlag(fieldPath, "radius", "highPrecision", circle.highPrecision);
|
|
6258
|
+
if (circle.distance < 0) {
|
|
6259
|
+
invalid(
|
|
6260
|
+
`"radius.distance" on field "${fieldPath}" must be zero or more, and this filter sets ${String(circle.distance)}`,
|
|
6261
|
+
fieldPath,
|
|
6262
|
+
"radius"
|
|
6263
|
+
);
|
|
6264
|
+
}
|
|
6265
|
+
if (!isLatitude(circle.lat)) {
|
|
6266
|
+
invalid(
|
|
6267
|
+
`"radius.lat" on field "${fieldPath}" must be from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES}, and this filter sets ${String(circle.lat)}`,
|
|
6268
|
+
fieldPath,
|
|
6269
|
+
"radius"
|
|
6270
|
+
);
|
|
6271
|
+
}
|
|
6272
|
+
if (!isLongitude(circle.lon)) {
|
|
6273
|
+
invalid(
|
|
6274
|
+
`"radius.lon" on field "${fieldPath}" must be from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, and this filter sets ${String(circle.lon)}`,
|
|
6275
|
+
fieldPath,
|
|
6276
|
+
"radius"
|
|
6277
|
+
);
|
|
6278
|
+
}
|
|
6279
|
+
}
|
|
6280
|
+
function requireGeoOperands(fieldPath, filter) {
|
|
6281
|
+
if (filter.radius !== void 0) requireRadius(fieldPath, filter.radius);
|
|
6282
|
+
if (filter.polygon === void 0) return;
|
|
6283
|
+
if (!isObjectRecord(filter.polygon)) {
|
|
6284
|
+
invalid(`"polygon" on field "${fieldPath}" must be an object`, fieldPath, "polygon");
|
|
6285
|
+
}
|
|
6286
|
+
requireFlag(fieldPath, "polygon", "inside", filter.polygon.inside);
|
|
6287
|
+
const points = filter.polygon.points;
|
|
6288
|
+
if (Array.isArray(points)) requireListLength(fieldPath, "polygon", points);
|
|
6289
|
+
requirePolygonRing(points);
|
|
6290
|
+
}
|
|
6291
|
+
|
|
6292
|
+
// src/filters/operands/lists.ts
|
|
6293
|
+
var WHOLE_VALUE_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "matchesAny", "containsAll"];
|
|
6294
|
+
var ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
|
|
6295
|
+
var PRESENCE_OPERATORS = ["exists", "notExists", "isEmpty", "isNotEmpty"];
|
|
6296
|
+
var SIZE_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
|
|
6297
|
+
var SIZE_OPERATOR_SET = new Set(SIZE_OPERATORS);
|
|
6298
|
+
var ELEMENT_KIND_BY_BASE = {
|
|
6299
|
+
"number[]": "number",
|
|
6300
|
+
"boolean[]": "boolean",
|
|
6301
|
+
"enum[]": "text",
|
|
6302
|
+
"string[]": "text",
|
|
6303
|
+
"verbatim[]": "text"
|
|
6304
|
+
};
|
|
6305
|
+
function requireWholeValueIndex(fieldPath, filter, type) {
|
|
6306
|
+
const parsed = parsedTypeOf(type);
|
|
6307
|
+
if (parsed === void 0 || parsed.pattern || parsed.base !== "string" && parsed.base !== "string[]") return;
|
|
6308
|
+
for (const operator of WHOLE_VALUE_OPERATORS) {
|
|
6309
|
+
if (filter[operator] === void 0) continue;
|
|
6310
|
+
invalid(
|
|
6311
|
+
`"${operator}" compares whole values of field "${fieldPath}", which the engine finds only through a pattern index. A "${parsed.base}" field keeps a pattern index where its type includes "pattern", so declare the field as "${parsed.base}:pattern" in a new index to filter it this way`,
|
|
6312
|
+
fieldPath,
|
|
6313
|
+
operator
|
|
6314
|
+
);
|
|
6315
|
+
}
|
|
6316
|
+
}
|
|
6317
|
+
function requirePresenceOperands(fieldPath, filter) {
|
|
6318
|
+
for (const operator of PRESENCE_OPERATORS) {
|
|
6319
|
+
const operand = filter[operator];
|
|
6320
|
+
if (operand !== void 0 && typeof operand !== "boolean") {
|
|
6321
|
+
invalid(`"${operator}" on field "${fieldPath}" must be true or false`, fieldPath, operator);
|
|
6322
|
+
}
|
|
6323
|
+
}
|
|
6324
|
+
}
|
|
6325
|
+
function listElementKind(fieldPath, operator, type) {
|
|
6326
|
+
const base = parsedTypeOf(type)?.base;
|
|
6327
|
+
const kind = base === void 0 ? void 0 : ELEMENT_KIND_BY_BASE[base];
|
|
6328
|
+
if (kind !== void 0) return kind;
|
|
6329
|
+
return invalid(
|
|
6330
|
+
`"${operator}" tests the elements of a list field, while the schema declares field "${fieldPath}" as "${type}"`,
|
|
6331
|
+
fieldPath,
|
|
6332
|
+
operator
|
|
6333
|
+
);
|
|
6334
|
+
}
|
|
6335
|
+
function requireElementOperands(fieldPath, filter, type) {
|
|
6336
|
+
for (const operator of ELEMENT_OPERATORS) {
|
|
6337
|
+
const values = filter[operator];
|
|
6338
|
+
if (values === void 0) continue;
|
|
6339
|
+
if (!Array.isArray(values)) invalid(`"${operator}" on field "${fieldPath}" must be a list`, fieldPath, operator);
|
|
6340
|
+
requireListLength(fieldPath, operator, values);
|
|
6341
|
+
const kind = listElementKind(fieldPath, operator, type);
|
|
6342
|
+
if (values.some((value) => kindOfValue(value) !== kind)) {
|
|
6343
|
+
invalid(
|
|
6344
|
+
`Every value of "${operator}" on field "${fieldPath}" must be a ${kind === "text" ? "string" : kind === "number" ? "finite number" : "boolean"}, because the field holds a list of them`,
|
|
6345
|
+
fieldPath,
|
|
6346
|
+
operator
|
|
6347
|
+
);
|
|
6348
|
+
}
|
|
6349
|
+
}
|
|
6350
|
+
}
|
|
6351
|
+
function requireSizeOperands(fieldPath, filter, type) {
|
|
6352
|
+
const size = filter.size;
|
|
6353
|
+
if (size === void 0) return;
|
|
6354
|
+
if (!isObjectRecord(size)) invalid(`"size" on field "${fieldPath}" must be a comparison object`, fieldPath, "size");
|
|
6355
|
+
listElementKind(fieldPath, "size", type);
|
|
6356
|
+
for (const [operator, operand] of Object.entries(size)) {
|
|
6357
|
+
if (!SIZE_OPERATOR_SET.has(operator)) {
|
|
6358
|
+
invalid(
|
|
6359
|
+
`"size" on field "${fieldPath}" takes ${SIZE_OPERATORS.join(", ")}, while "${operator}" is none of them`,
|
|
6360
|
+
fieldPath,
|
|
6361
|
+
"size"
|
|
6362
|
+
);
|
|
6363
|
+
}
|
|
6364
|
+
const bounds = operator === "between" ? operand : [operand];
|
|
6365
|
+
if (!Array.isArray(bounds) || bounds.length !== (operator === "between" ? 2 : 1) || !bounds.every(isFiniteNumber)) {
|
|
6366
|
+
invalid(
|
|
6367
|
+
`"size.${operator}" on field "${fieldPath}" must be ${operator === "between" ? "two finite numbers, the lower bound first" : "a finite number"}`,
|
|
6368
|
+
fieldPath,
|
|
6369
|
+
"size"
|
|
6370
|
+
);
|
|
6371
|
+
}
|
|
6372
|
+
}
|
|
6373
|
+
}
|
|
6374
|
+
|
|
6375
|
+
// src/filters/operands/pattern.ts
|
|
6376
|
+
function requirePatternField(fieldPath, operator, type) {
|
|
6377
|
+
if (parsedTypeOf(type)?.pattern === true) return;
|
|
6378
|
+
const declared = type === void 0 ? `The schema declares no field "${fieldPath}"` : `Field "${fieldPath}" has type "${type}"`;
|
|
5502
6379
|
invalid(
|
|
5503
|
-
`
|
|
6380
|
+
`"${operator}" applies only to a pattern field, because only a pattern field has a pattern index. A pattern field is a "verbatim" field, or a text field whose type includes "pattern". ${declared}`,
|
|
5504
6381
|
fieldPath,
|
|
5505
6382
|
operator
|
|
5506
6383
|
);
|
|
5507
6384
|
}
|
|
5508
|
-
function
|
|
5509
|
-
|
|
5510
|
-
|
|
5511
|
-
|
|
5512
|
-
|
|
5513
|
-
|
|
5514
|
-
|
|
5515
|
-
invalid(`"radius.${key}" on field "${fieldPath}" must be a finite number`, fieldPath, "radius");
|
|
5516
|
-
}
|
|
6385
|
+
function requirePatternText(fieldPath, operator, text, caseFold) {
|
|
6386
|
+
if (exceedsCodePoints(text, MAX_PATTERN_TEXT_CODE_POINTS)) {
|
|
6387
|
+
invalid(
|
|
6388
|
+
`The text of "${operator}" on pattern field "${fieldPath}" holds more than ${MAX_PATTERN_TEXT_CODE_POINTS} code points`,
|
|
6389
|
+
fieldPath,
|
|
6390
|
+
operator
|
|
6391
|
+
);
|
|
5517
6392
|
}
|
|
5518
|
-
if (
|
|
6393
|
+
if (caseFold && foldedCodePointCount(text) > MAX_PATTERN_POSITIONS) {
|
|
5519
6394
|
invalid(
|
|
5520
|
-
`"
|
|
6395
|
+
`The text of "${operator}" on pattern field "${fieldPath}" has a case fold of more than ${MAX_PATTERN_POSITIONS} code points, while a pattern holds at most ${MAX_PATTERN_POSITIONS} positions`,
|
|
5521
6396
|
fieldPath,
|
|
5522
|
-
|
|
6397
|
+
operator
|
|
5523
6398
|
);
|
|
5524
6399
|
}
|
|
5525
6400
|
}
|
|
6401
|
+
var ELEMENT_OPERATORS2 = ["matchesAny", "containsAll"];
|
|
6402
|
+
function requirePatternOperands(fieldPath, filter, type) {
|
|
6403
|
+
if (filter.contains !== void 0) {
|
|
6404
|
+
if (typeof filter.contains !== "string") {
|
|
6405
|
+
invalid(`"contains" on field "${fieldPath}" must be a string`, fieldPath, "contains");
|
|
6406
|
+
}
|
|
6407
|
+
requirePatternField(fieldPath, "contains", type);
|
|
6408
|
+
}
|
|
6409
|
+
if (filter.caseFold !== void 0) {
|
|
6410
|
+
if (typeof filter.caseFold !== "boolean") {
|
|
6411
|
+
invalid(`"caseFold" on field "${fieldPath}" must be true or false`, fieldPath, "caseFold");
|
|
6412
|
+
}
|
|
6413
|
+
requirePatternField(fieldPath, "caseFold", type);
|
|
6414
|
+
}
|
|
6415
|
+
if (parsedTypeOf(type)?.pattern !== true) return;
|
|
6416
|
+
const caseFold = filter.caseFold === true;
|
|
6417
|
+
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
6418
|
+
const operand = filter[operator];
|
|
6419
|
+
if (typeof operand === "string") requirePatternText(fieldPath, operator, operand, caseFold);
|
|
6420
|
+
if (!Array.isArray(operand)) continue;
|
|
6421
|
+
for (const text of operand) {
|
|
6422
|
+
if (typeof text === "string") requirePatternText(fieldPath, operator, text, caseFold);
|
|
6423
|
+
}
|
|
6424
|
+
}
|
|
6425
|
+
for (const operator of ELEMENT_OPERATORS2) {
|
|
6426
|
+
const operand = filter[operator];
|
|
6427
|
+
if (!Array.isArray(operand)) continue;
|
|
6428
|
+
for (const text of operand) {
|
|
6429
|
+
if (typeof text === "string") requirePatternText(fieldPath, operator, text, false);
|
|
6430
|
+
}
|
|
6431
|
+
}
|
|
6432
|
+
}
|
|
6433
|
+
function requireWellFormedText(fieldPath, filter) {
|
|
6434
|
+
for (const [operator, operand] of Object.entries(filter)) {
|
|
6435
|
+
const texts = Array.isArray(operand) ? operand : [operand];
|
|
6436
|
+
for (const text of texts) {
|
|
6437
|
+
if (typeof text !== "string" || !holdsUnpairedSurrogate(text)) continue;
|
|
6438
|
+
invalid(
|
|
6439
|
+
`The text of "${operator}" on field "${fieldPath}" holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so no stored value can hold it`,
|
|
6440
|
+
fieldPath,
|
|
6441
|
+
operator
|
|
6442
|
+
);
|
|
6443
|
+
}
|
|
6444
|
+
}
|
|
6445
|
+
}
|
|
6446
|
+
|
|
6447
|
+
// src/filters/operands/index.ts
|
|
6448
|
+
var RANGE_OPERATORS = ["gt", "lt", "gte", "lte"];
|
|
6449
|
+
var TEXT_OPERATORS = ["startsWith", "endsWith"];
|
|
6450
|
+
var EQUALITY_OPERATORS = ["eq", "ne"];
|
|
6451
|
+
var LIST_OPERATORS = ["in", "nin"];
|
|
6452
|
+
var GEO_OPERATORS = ["radius", "polygon"];
|
|
5526
6453
|
function invalidShape(message) {
|
|
5527
6454
|
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message);
|
|
5528
6455
|
}
|
|
5529
|
-
function isObjectRecord(value) {
|
|
5530
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
5531
|
-
}
|
|
5532
6456
|
function undeclaredFieldOnStrictIndex(field, option) {
|
|
5533
6457
|
return new NarsilError(
|
|
5534
6458
|
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
@@ -5536,34 +6460,14 @@ function undeclaredFieldOnStrictIndex(field, option) {
|
|
|
5536
6460
|
{ field, option }
|
|
5537
6461
|
);
|
|
5538
6462
|
}
|
|
5539
|
-
function
|
|
5540
|
-
|
|
5541
|
-
|
|
5542
|
-
|
|
5543
|
-
|
|
5544
|
-
|
|
5545
|
-
for (const [fieldPath, filter] of Object.entries(fields)) {
|
|
5546
|
-
if (rules.declaredOnly && !Object.hasOwn(rules.fieldTypes, fieldPath)) {
|
|
5547
|
-
throw undeclaredFieldOnStrictIndex(fieldPath, "filter");
|
|
5548
|
-
}
|
|
5549
|
-
if (isObjectRecord(filter)) requireValidOperands(fieldPath, filter, rules.fieldTypes[fieldPath]);
|
|
5550
|
-
}
|
|
5551
|
-
}
|
|
5552
|
-
for (const [clause, list] of [
|
|
5553
|
-
["and", and],
|
|
5554
|
-
["or", or]
|
|
5555
|
-
]) {
|
|
5556
|
-
if (list === void 0) continue;
|
|
5557
|
-
if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
|
|
5558
|
-
for (const nested of list) requireValidExpression(nested, rules, depth + 1);
|
|
5559
|
-
}
|
|
5560
|
-
if (not !== void 0) requireValidExpression(not, rules, depth + 1);
|
|
5561
|
-
}
|
|
5562
|
-
function requireValidFilter(expression, fieldTypes, declaredOnly = false) {
|
|
5563
|
-
if (expression === void 0 || expression === null) return;
|
|
5564
|
-
requireValidExpression(expression, { fieldTypes, declaredOnly }, 0);
|
|
6463
|
+
function undeclaredFilterField(field) {
|
|
6464
|
+
return new NarsilError(
|
|
6465
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
6466
|
+
`The schema declares no field "${field}", while the engine answers a filter only from the indexes that it keeps for declared fields. Declare the field in the schema of a new index to filter by it`,
|
|
6467
|
+
{ field, option: "filter" }
|
|
6468
|
+
);
|
|
5565
6469
|
}
|
|
5566
|
-
function
|
|
6470
|
+
function requireRangeOperands(fieldPath, filter, type) {
|
|
5567
6471
|
for (const operator of RANGE_OPERATORS) {
|
|
5568
6472
|
const bound = filter[operator];
|
|
5569
6473
|
if (bound === void 0) continue;
|
|
@@ -5576,22 +6480,23 @@ function requireValidOperands(fieldPath, filter, type) {
|
|
|
5576
6480
|
}
|
|
5577
6481
|
requireFieldKind(fieldPath, operator, type, ["number"]);
|
|
5578
6482
|
}
|
|
5579
|
-
if (filter.between
|
|
5580
|
-
|
|
5581
|
-
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
5585
|
-
|
|
5586
|
-
|
|
5587
|
-
|
|
5588
|
-
|
|
5589
|
-
|
|
5590
|
-
|
|
5591
|
-
|
|
5592
|
-
requireFieldKind(fieldPath, "between", type, ["number"]);
|
|
5593
|
-
}
|
|
6483
|
+
if (filter.between === void 0) return;
|
|
6484
|
+
const bounds = filter.between;
|
|
6485
|
+
const isPair = Array.isArray(bounds) && bounds.length === 2;
|
|
6486
|
+
if (isPair && typeof bounds[0] === "string" && typeof bounds[1] === "string") {
|
|
6487
|
+
requireTextRangeField(fieldPath, "between", type);
|
|
6488
|
+
return;
|
|
6489
|
+
}
|
|
6490
|
+
if (!isPair || !isFiniteNumber(bounds[0]) || !isFiniteNumber(bounds[1])) {
|
|
6491
|
+
invalid(
|
|
6492
|
+
`"between" on field "${fieldPath}" must be two finite numbers or two strings, the lower bound first`,
|
|
6493
|
+
fieldPath,
|
|
6494
|
+
"between"
|
|
6495
|
+
);
|
|
5594
6496
|
}
|
|
6497
|
+
requireFieldKind(fieldPath, "between", type, ["number"]);
|
|
6498
|
+
}
|
|
6499
|
+
function requireValueOperands(fieldPath, filter, type) {
|
|
5595
6500
|
for (const operator of TEXT_OPERATORS) {
|
|
5596
6501
|
if (filter[operator] === void 0) continue;
|
|
5597
6502
|
if (typeof filter[operator] !== "string") {
|
|
@@ -5617,247 +6522,74 @@ function requireValidOperands(fieldPath, filter, type) {
|
|
|
5617
6522
|
if (!Array.isArray(values) || values.some((value) => typeof value !== "string")) {
|
|
5618
6523
|
invalid(`"${operator}" on field "${fieldPath}" must be a list of strings`, fieldPath, operator);
|
|
5619
6524
|
}
|
|
6525
|
+
requireListLength(fieldPath, operator, values);
|
|
5620
6526
|
requireFieldKind(fieldPath, operator, type, ["text"]);
|
|
5621
6527
|
}
|
|
5622
|
-
|
|
5623
|
-
|
|
5624
|
-
if (size === null || typeof size !== "object") {
|
|
5625
|
-
invalid(`"size" on field "${fieldPath}" must be a comparison object`, fieldPath, "size");
|
|
5626
|
-
}
|
|
5627
|
-
requireValidOperands(`${fieldPath}.size`, size, "number");
|
|
5628
|
-
}
|
|
5629
|
-
if (filter.radius !== void 0) requireRadius(fieldPath, filter.radius);
|
|
5630
|
-
if (filter.polygon !== void 0) {
|
|
5631
|
-
if (!isObjectRecord(filter.polygon)) {
|
|
5632
|
-
invalid(`"polygon" on field "${fieldPath}" must be an object`, fieldPath, "polygon");
|
|
5633
|
-
}
|
|
5634
|
-
requirePolygonRing(filter.polygon.points);
|
|
5635
|
-
}
|
|
5636
|
-
}
|
|
5637
|
-
|
|
5638
|
-
// src/filters/operators/shared.ts
|
|
5639
|
-
var METERS_PER_KM = 1e3;
|
|
5640
|
-
var METERS_PER_MI = 1609.344;
|
|
5641
|
-
function convertToMeters(distance, unit) {
|
|
5642
|
-
if (unit === "km") return distance * METERS_PER_KM;
|
|
5643
|
-
if (unit === "mi") return distance * METERS_PER_MI;
|
|
5644
|
-
if (unit === "m") return distance;
|
|
5645
|
-
throw new NarsilError(
|
|
5646
|
-
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
5647
|
-
`The engine measures a radius in "km", "mi", or "m", and "${String(unit)}" is none of them`,
|
|
5648
|
-
{ unit: String(unit) }
|
|
5649
|
-
);
|
|
5650
|
-
}
|
|
5651
|
-
function matchesNumericComparison(value, filter) {
|
|
5652
|
-
if (filter.eq !== void 0 && value !== filter.eq) return false;
|
|
5653
|
-
if (filter.ne !== void 0 && value === filter.ne) return false;
|
|
5654
|
-
if (filter.gt !== void 0 && !(value > filter.gt)) return false;
|
|
5655
|
-
if (filter.lt !== void 0 && !(value < filter.lt)) return false;
|
|
5656
|
-
if (filter.gte !== void 0 && !(value >= filter.gte)) return false;
|
|
5657
|
-
if (filter.lte !== void 0 && !(value <= filter.lte)) return false;
|
|
5658
|
-
if (filter.between !== void 0 && !(value >= filter.between[0] && value <= filter.between[1])) return false;
|
|
5659
|
-
return true;
|
|
5660
|
-
}
|
|
5661
|
-
function holdsValue(stored, value) {
|
|
5662
|
-
return Array.isArray(stored) ? stored.includes(value) : stored === value;
|
|
5663
|
-
}
|
|
5664
|
-
function lacksValue(stored, value) {
|
|
5665
|
-
if (stored === void 0 || stored === null) return false;
|
|
5666
|
-
return Array.isArray(stored) ? !stored.includes(value) : stored !== value;
|
|
5667
|
-
}
|
|
5668
|
-
function isStringIn(values) {
|
|
5669
|
-
return (element) => typeof element === "string" && values.has(element);
|
|
5670
|
-
}
|
|
5671
|
-
function holdsStringIn(stored, values) {
|
|
5672
|
-
return Array.isArray(stored) ? stored.some(isStringIn(values)) : isStringIn(values)(stored);
|
|
5673
|
-
}
|
|
5674
|
-
function holdsNoStringIn(stored, values) {
|
|
5675
|
-
if (Array.isArray(stored)) return !stored.some(isStringIn(values));
|
|
5676
|
-
return typeof stored === "string" && !values.has(stored);
|
|
5677
|
-
}
|
|
5678
|
-
function holdsStringWhere(stored, test) {
|
|
5679
|
-
const matches = (element) => typeof element === "string" && test(element);
|
|
5680
|
-
return Array.isArray(stored) ? stored.some(matches) : matches(stored);
|
|
5681
|
-
}
|
|
5682
|
-
|
|
5683
|
-
// src/filters/operators/bitset.ts
|
|
5684
|
-
function resolveBitset(bs) {
|
|
5685
|
-
return typeof bs === "function" ? bs() : bs;
|
|
5686
|
-
}
|
|
5687
|
-
function scanToBitset(docIds, capacity, getValue, predicate) {
|
|
5688
|
-
const resolved = resolveBitset(docIds);
|
|
5689
|
-
const result = createBitSet(capacity);
|
|
5690
|
-
for (let wi = 0; wi < resolved.length; wi++) {
|
|
5691
|
-
let word = resolved[wi];
|
|
5692
|
-
if (word === 0) continue;
|
|
5693
|
-
const base = wi << 5;
|
|
5694
|
-
while (word !== 0) {
|
|
5695
|
-
const tz = Math.clz32(word & -word) ^ 31;
|
|
5696
|
-
const id = base + tz;
|
|
5697
|
-
if (predicate(getValue(id))) {
|
|
5698
|
-
bitsetSet(result, id);
|
|
5699
|
-
}
|
|
5700
|
-
word &= word - 1;
|
|
5701
|
-
}
|
|
5702
|
-
}
|
|
5703
|
-
return result;
|
|
5704
|
-
}
|
|
5705
|
-
function applyEqBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5706
|
-
if (fieldIndex?.type === "numeric" && typeof value === "number") {
|
|
5707
|
-
return fieldIndex.index.eqBitset(value, capacity);
|
|
5708
|
-
}
|
|
5709
|
-
if (fieldIndex?.type === "boolean" && typeof value === "boolean") {
|
|
5710
|
-
return value ? fieldIndex.index.getTrueBitset(capacity) : fieldIndex.index.getFalseBitset(capacity);
|
|
5711
|
-
}
|
|
5712
|
-
if (fieldIndex?.type === "enum" && typeof value === "string") {
|
|
5713
|
-
return fieldIndex.index.getDocIdsBitset(value, capacity);
|
|
5714
|
-
}
|
|
5715
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsValue(v, value));
|
|
5716
|
-
}
|
|
5717
|
-
function applyNeBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5718
|
-
if (fieldIndex?.type === "numeric" && typeof value === "number") {
|
|
5719
|
-
const all = fieldIndex.index.allDocIdsBitset(capacity);
|
|
5720
|
-
const eq = fieldIndex.index.eqBitset(value, capacity);
|
|
5721
|
-
return bitsetAnd(all, bitsetNot(eq, capacity));
|
|
5722
|
-
}
|
|
5723
|
-
if (fieldIndex?.type === "boolean" && typeof value === "boolean") {
|
|
5724
|
-
return value ? fieldIndex.index.getFalseBitset(capacity) : fieldIndex.index.getTrueBitset(capacity);
|
|
5725
|
-
}
|
|
5726
|
-
if (fieldIndex?.type === "enum" && typeof value === "string") {
|
|
5727
|
-
const all = fieldIndex.index.allDocIdsBitset(capacity);
|
|
5728
|
-
const eq = fieldIndex.index.getDocIdsBitset(value, capacity);
|
|
5729
|
-
return bitsetAnd(all, bitsetNot(eq, capacity));
|
|
5730
|
-
}
|
|
5731
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => lacksValue(v, value));
|
|
5732
|
-
}
|
|
5733
|
-
function applyGtBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5734
|
-
if (fieldIndex?.type === "numeric") {
|
|
5735
|
-
return fieldIndex.index.gtBitset(value, capacity);
|
|
5736
|
-
}
|
|
5737
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v > value);
|
|
5738
|
-
}
|
|
5739
|
-
function applyLtBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5740
|
-
if (fieldIndex?.type === "numeric") {
|
|
5741
|
-
return fieldIndex.index.ltBitset(value, capacity);
|
|
5742
|
-
}
|
|
5743
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v < value);
|
|
5744
|
-
}
|
|
5745
|
-
function applyGteBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5746
|
-
if (fieldIndex?.type === "numeric") {
|
|
5747
|
-
return fieldIndex.index.gteBitset(value, capacity);
|
|
5748
|
-
}
|
|
5749
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v >= value);
|
|
5750
|
-
}
|
|
5751
|
-
function applyLteBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5752
|
-
if (fieldIndex?.type === "numeric") {
|
|
5753
|
-
return fieldIndex.index.lteBitset(value, capacity);
|
|
5754
|
-
}
|
|
5755
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v <= value);
|
|
5756
|
-
}
|
|
5757
|
-
function applyBetweenBitset(range, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5758
|
-
if (fieldIndex?.type === "numeric") {
|
|
5759
|
-
return fieldIndex.index.betweenBitset(range[0], range[1], capacity);
|
|
6528
|
+
for (const operator of GEO_OPERATORS) {
|
|
6529
|
+
if (filter[operator] !== void 0) requireFieldKind(fieldPath, operator, type, ["geopoint"]);
|
|
5760
6530
|
}
|
|
5761
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v >= range[0] && v <= range[1]);
|
|
5762
6531
|
}
|
|
5763
|
-
function
|
|
5764
|
-
|
|
5765
|
-
|
|
6532
|
+
function requireValidOperands(fieldPath, filter, type) {
|
|
6533
|
+
requireWellFormedText(fieldPath, filter);
|
|
6534
|
+
requireRangeOperands(fieldPath, filter, type);
|
|
6535
|
+
requireValueOperands(fieldPath, filter, type);
|
|
6536
|
+
requireWholeValueIndex(fieldPath, filter, type);
|
|
6537
|
+
requirePresenceOperands(fieldPath, filter);
|
|
6538
|
+
requireElementOperands(fieldPath, filter, type);
|
|
6539
|
+
requireSizeOperands(fieldPath, filter, type);
|
|
6540
|
+
requirePatternOperands(fieldPath, filter, type);
|
|
6541
|
+
requireGeoOperands(fieldPath, filter);
|
|
6542
|
+
}
|
|
6543
|
+
function testsIn(filter) {
|
|
6544
|
+
let tests = 0;
|
|
6545
|
+
for (const [operator, operand] of Object.entries(filter)) {
|
|
6546
|
+
if (operator === "caseFold") continue;
|
|
6547
|
+
tests += operator === "size" && isObjectRecord(operand) ? Math.max(1, Object.keys(operand).length) : 1;
|
|
6548
|
+
}
|
|
6549
|
+
return Math.max(1, tests);
|
|
6550
|
+
}
|
|
6551
|
+
function countTests(walk, tests) {
|
|
6552
|
+
walk.tests += tests;
|
|
6553
|
+
if (walk.tests > MAX_FILTER_TESTS) {
|
|
6554
|
+
invalidShape(`A filter holds at most ${MAX_FILTER_TESTS} tests across all of its fields and nested expressions`);
|
|
6555
|
+
}
|
|
6556
|
+
}
|
|
6557
|
+
function holdsNestedExpressions(expression) {
|
|
6558
|
+
const { and, or, not } = expression;
|
|
6559
|
+
return Array.isArray(and) && and.length > 0 || Array.isArray(or) && or.length > 0 || not !== void 0;
|
|
6560
|
+
}
|
|
6561
|
+
function requireValidExpression(expression, walk, depth) {
|
|
6562
|
+
if (depth > MAX_FILTER_DEPTH) invalidShape(`A filter expression nests at most ${MAX_FILTER_DEPTH} levels deep`);
|
|
6563
|
+
if (!isObjectRecord(expression)) invalidShape("Each filter expression must be an object");
|
|
6564
|
+
const { fields, and, or, not } = expression;
|
|
6565
|
+
if (fields !== void 0 && !isObjectRecord(fields)) {
|
|
6566
|
+
invalidShape('"fields" in a filter expression must be an object keyed by field path');
|
|
6567
|
+
}
|
|
6568
|
+
const fieldFilters = Object.entries(fields ?? {});
|
|
6569
|
+
if (fieldFilters.length === 0 && !holdsNestedExpressions(expression)) countTests(walk, 1);
|
|
6570
|
+
for (const [fieldPath, filter] of fieldFilters) {
|
|
6571
|
+
const type = walk.fieldTypes[fieldPath];
|
|
6572
|
+
if (!Object.hasOwn(walk.fieldTypes, fieldPath) || type === void 0) throw undeclaredFilterField(fieldPath);
|
|
6573
|
+
if (!isObjectRecord(filter)) invalidShape(`The filter on field "${fieldPath}" must be an object of tests`);
|
|
6574
|
+
countTests(walk, testsIn(filter));
|
|
6575
|
+
requireValidOperands(fieldPath, filter, type);
|
|
5766
6576
|
}
|
|
5767
|
-
const
|
|
5768
|
-
|
|
5769
|
-
|
|
5770
|
-
|
|
5771
|
-
|
|
5772
|
-
|
|
5773
|
-
const
|
|
5774
|
-
return bitsetAnd(all, bitsetNot(matched, capacity));
|
|
6577
|
+
for (const [clause, list] of [
|
|
6578
|
+
["and", and],
|
|
6579
|
+
["or", or]
|
|
6580
|
+
]) {
|
|
6581
|
+
if (list === void 0) continue;
|
|
6582
|
+
if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
|
|
6583
|
+
for (const nested of list) requireValidExpression(nested, walk, depth + 1);
|
|
5775
6584
|
}
|
|
5776
|
-
|
|
5777
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsNoStringIn(v, valSet));
|
|
5778
|
-
}
|
|
5779
|
-
function applyStartsWithBitset(prefix, allDocsBitset, capacity, getValue) {
|
|
5780
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsStringWhere(v, (text) => text.startsWith(prefix)));
|
|
5781
|
-
}
|
|
5782
|
-
function applyEndsWithBitset(suffix, allDocsBitset, capacity, getValue) {
|
|
5783
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsStringWhere(v, (text) => text.endsWith(suffix)));
|
|
5784
|
-
}
|
|
5785
|
-
function applyContainsAllBitset(values, allDocsBitset, capacity, getValue) {
|
|
5786
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
|
|
5787
|
-
if (!Array.isArray(v)) return false;
|
|
5788
|
-
const arr = v;
|
|
5789
|
-
for (const val of values) {
|
|
5790
|
-
if (!arr.includes(val)) return false;
|
|
5791
|
-
}
|
|
5792
|
-
return true;
|
|
5793
|
-
});
|
|
5794
|
-
}
|
|
5795
|
-
function applyMatchesAnyBitset(values, allDocsBitset, capacity, getValue) {
|
|
5796
|
-
const valSet = new Set(values);
|
|
5797
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
|
|
5798
|
-
if (!Array.isArray(v)) return false;
|
|
5799
|
-
for (const item of v) {
|
|
5800
|
-
if (valSet.has(item)) return true;
|
|
5801
|
-
}
|
|
5802
|
-
return false;
|
|
5803
|
-
});
|
|
5804
|
-
}
|
|
5805
|
-
function applySizeBitset(sizeFilter, allDocsBitset, capacity, getValue) {
|
|
5806
|
-
return scanToBitset(
|
|
5807
|
-
allDocsBitset,
|
|
5808
|
-
capacity,
|
|
5809
|
-
getValue,
|
|
5810
|
-
(v) => Array.isArray(v) && matchesNumericComparison(v.length, sizeFilter)
|
|
5811
|
-
);
|
|
5812
|
-
}
|
|
5813
|
-
function applyExistsBitset(allDocsBitset, capacity, getValue) {
|
|
5814
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => v !== void 0 && v !== null);
|
|
5815
|
-
}
|
|
5816
|
-
function applyNotExistsBitset(allDocsBitset, capacity, getValue) {
|
|
5817
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => v === void 0 || v === null);
|
|
5818
|
-
}
|
|
5819
|
-
function applyIsEmptyBitset(allDocsBitset, capacity, getValue) {
|
|
5820
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
|
|
5821
|
-
if (v === void 0 || v === null) return true;
|
|
5822
|
-
if (typeof v === "string" && v === "") return true;
|
|
5823
|
-
if (Array.isArray(v) && v.length === 0) return true;
|
|
5824
|
-
return false;
|
|
5825
|
-
});
|
|
5826
|
-
}
|
|
5827
|
-
function applyIsNotEmptyBitset(allDocsBitset, capacity, getValue) {
|
|
5828
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
|
|
5829
|
-
if (v === void 0 || v === null) return false;
|
|
5830
|
-
if (typeof v === "string" && v === "") return false;
|
|
5831
|
-
if (Array.isArray(v) && v.length === 0) return false;
|
|
5832
|
-
return true;
|
|
5833
|
-
});
|
|
5834
|
-
}
|
|
5835
|
-
function applyGeoRadiusBitset(filter, geoIndex, capacity) {
|
|
5836
|
-
const distanceMeters = convertToMeters(filter.distance, filter.unit);
|
|
5837
|
-
const setResult = geoIndex.radiusQuery(
|
|
5838
|
-
filter.lat,
|
|
5839
|
-
filter.lon,
|
|
5840
|
-
distanceMeters,
|
|
5841
|
-
filter.inside ?? true,
|
|
5842
|
-
filter.highPrecision ?? false
|
|
5843
|
-
);
|
|
5844
|
-
return bitsetFromSet(setResult, capacity);
|
|
6585
|
+
if (not !== void 0) requireValidExpression(not, walk, depth + 1);
|
|
5845
6586
|
}
|
|
5846
|
-
function
|
|
5847
|
-
|
|
5848
|
-
|
|
5849
|
-
return bitsetFromSet(setResult, capacity);
|
|
6587
|
+
function requireValidFilter(expression, fieldTypes) {
|
|
6588
|
+
if (expression === void 0 || expression === null) return;
|
|
6589
|
+
requireValidExpression(expression, { fieldTypes, tests: 0 }, 0);
|
|
5850
6590
|
}
|
|
5851
6591
|
|
|
5852
6592
|
// src/filters/evaluator.ts
|
|
5853
|
-
var UNBOUNDED = { lower: null, upper: null };
|
|
5854
|
-
function textBoundsOf(operator, bound) {
|
|
5855
|
-
if (operator === "gt" || operator === "gte") {
|
|
5856
|
-
return { ...UNBOUNDED, lower: bound, lowerInclusive: operator === "gte", upperInclusive: false };
|
|
5857
|
-
}
|
|
5858
|
-
return { ...UNBOUNDED, upper: bound, upperInclusive: operator === "lte", lowerInclusive: false };
|
|
5859
|
-
}
|
|
5860
|
-
var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
|
|
5861
6593
|
var EXPRESSION_KEYS = new Set(FILTER_EXPRESSION_KEYS);
|
|
5862
6594
|
var OPERATOR_KEYS = new Set(FIELD_FILTER_OPERATORS);
|
|
5863
6595
|
function requireKnownKeys2(value, known, describe) {
|
|
@@ -5867,160 +6599,66 @@ function requireKnownKeys2(value, known, describe) {
|
|
|
5867
6599
|
}
|
|
5868
6600
|
}
|
|
5869
6601
|
}
|
|
6602
|
+
var Narrowing = class {
|
|
6603
|
+
bits = null;
|
|
6604
|
+
narrow = (matched) => {
|
|
6605
|
+
if (this.bits === null) this.bits = matched.slice();
|
|
6606
|
+
else bitsetAndInto(this.bits, matched);
|
|
6607
|
+
};
|
|
6608
|
+
value(universe) {
|
|
6609
|
+
return this.bits ?? universe().slice();
|
|
6610
|
+
}
|
|
6611
|
+
};
|
|
6612
|
+
function narrowByField(fieldPath, filter, context, narrow) {
|
|
6613
|
+
requireKnownKeys2(
|
|
6614
|
+
filter,
|
|
6615
|
+
OPERATOR_KEYS,
|
|
6616
|
+
(key) => `Field "${fieldPath}" is tested with "${key}", which is no filter operator`
|
|
6617
|
+
);
|
|
6618
|
+
const type = context.fieldTypes[fieldPath];
|
|
6619
|
+
const parsed = parsedTypeOf(type);
|
|
6620
|
+
if (type === void 0 || parsed === void 0) throw undeclaredFilterField(fieldPath);
|
|
6621
|
+
const f = filter;
|
|
6622
|
+
requireValidOperands(fieldPath, f, type);
|
|
6623
|
+
narrowByFieldTests(fieldPath, f, parsed, context, narrow.narrow);
|
|
6624
|
+
}
|
|
5870
6625
|
function evaluateFilters(expression, context) {
|
|
5871
6626
|
requireKnownKeys2(
|
|
5872
6627
|
expression,
|
|
5873
6628
|
EXPRESSION_KEYS,
|
|
5874
6629
|
(key) => `A filter expression takes "fields", "and", "or", and "not", and "${key}" is none of them. A per-field test such as { ${key}: { eq: ... } } belongs under "fields"`
|
|
5875
6630
|
);
|
|
5876
|
-
const
|
|
6631
|
+
const narrowing = new Narrowing();
|
|
5877
6632
|
if (expression.fields) {
|
|
5878
6633
|
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
5879
|
-
|
|
6634
|
+
narrowByField(fieldPath, filter, context, narrowing);
|
|
5880
6635
|
}
|
|
5881
6636
|
}
|
|
5882
6637
|
if (expression.and?.length) {
|
|
5883
|
-
const
|
|
5884
|
-
bitsets.push(applyAndBitset(andBitsets));
|
|
6638
|
+
for (const nested of expression.and) narrowing.narrow(evaluateFilters(nested, context));
|
|
5885
6639
|
}
|
|
5886
6640
|
if (expression.or?.length) {
|
|
5887
|
-
const
|
|
5888
|
-
|
|
6641
|
+
const union = createBitSet(context.capacity);
|
|
6642
|
+
for (const nested of expression.or) bitsetOrInto(union, evaluateFilters(nested, context));
|
|
6643
|
+
narrowing.narrow(union);
|
|
5889
6644
|
}
|
|
5890
6645
|
if (expression.not) {
|
|
5891
6646
|
const excluded = evaluateFilters(expression.not, context);
|
|
5892
|
-
const
|
|
5893
|
-
|
|
5894
|
-
|
|
5895
|
-
if (bitsets.length === 0) return context.allDocIdsBitset;
|
|
5896
|
-
if (bitsets.length === 1) return bitsets[0];
|
|
5897
|
-
return applyAndBitset(bitsets);
|
|
5898
|
-
}
|
|
5899
|
-
function evaluateFieldFilter(fieldPath, filter, context) {
|
|
5900
|
-
requireKnownKeys2(
|
|
5901
|
-
filter,
|
|
5902
|
-
OPERATOR_KEYS,
|
|
5903
|
-
(key) => `Field "${fieldPath}" is tested with "${key}", which is no filter operator`
|
|
5904
|
-
);
|
|
5905
|
-
const f = filter;
|
|
5906
|
-
requireValidOperands(fieldPath, f, context.fieldTypes[fieldPath]);
|
|
5907
|
-
const fieldIndex = context.fieldIndexes[fieldPath];
|
|
5908
|
-
const getValue = (internalId) => context.getFieldValue(internalId, fieldPath);
|
|
5909
|
-
const bitsets = [];
|
|
5910
|
-
const { capacity } = context;
|
|
5911
|
-
let allDocsBitset = null;
|
|
5912
|
-
const getAllDocsBitset = () => {
|
|
5913
|
-
if (allDocsBitset === null) {
|
|
5914
|
-
allDocsBitset = context.allDocIdsBitset;
|
|
5915
|
-
}
|
|
5916
|
-
return allDocsBitset;
|
|
5917
|
-
};
|
|
5918
|
-
if ("radius" in f && f.radius) {
|
|
5919
|
-
const geoIndex = resolveGeoIndex(fieldPath, fieldIndex);
|
|
5920
|
-
bitsets.push(applyGeoRadiusBitset(filter.radius, geoIndex, capacity));
|
|
6647
|
+
const kept = context.allDocIdsBitset.slice();
|
|
6648
|
+
bitsetAndNotInto(kept, excluded);
|
|
6649
|
+
narrowing.narrow(kept);
|
|
5921
6650
|
}
|
|
5922
|
-
|
|
5923
|
-
const geoIndex = resolveGeoIndex(fieldPath, fieldIndex);
|
|
5924
|
-
bitsets.push(applyGeoPolygonBitset(filter.polygon, geoIndex, capacity));
|
|
5925
|
-
}
|
|
5926
|
-
if ("eq" in f && f.eq !== void 0) {
|
|
5927
|
-
bitsets.push(applyEqBitset(f.eq, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5928
|
-
}
|
|
5929
|
-
if ("ne" in f && f.ne !== void 0) {
|
|
5930
|
-
bitsets.push(applyNeBitset(f.ne, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5931
|
-
}
|
|
5932
|
-
for (const operator of TEXT_RANGE_OPERATORS) {
|
|
5933
|
-
const bound = f[operator];
|
|
5934
|
-
if (typeof bound === "string") bitsets.push(context.textRangeBitset(fieldPath, textBoundsOf(operator, bound)));
|
|
5935
|
-
}
|
|
5936
|
-
if (typeof f.gt === "number") {
|
|
5937
|
-
bitsets.push(applyGtBitset(f.gt, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5938
|
-
}
|
|
5939
|
-
if (typeof f.lt === "number") {
|
|
5940
|
-
bitsets.push(applyLtBitset(f.lt, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5941
|
-
}
|
|
5942
|
-
if (typeof f.gte === "number") {
|
|
5943
|
-
bitsets.push(applyGteBitset(f.gte, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5944
|
-
}
|
|
5945
|
-
if (typeof f.lte === "number") {
|
|
5946
|
-
bitsets.push(applyLteBitset(f.lte, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5947
|
-
}
|
|
5948
|
-
if (Array.isArray(f.between)) {
|
|
5949
|
-
const [low, high] = f.between;
|
|
5950
|
-
if (typeof low === "string" && typeof high === "string") {
|
|
5951
|
-
bitsets.push(
|
|
5952
|
-
context.textRangeBitset(fieldPath, { lower: low, lowerInclusive: true, upper: high, upperInclusive: true })
|
|
5953
|
-
);
|
|
5954
|
-
} else {
|
|
5955
|
-
bitsets.push(
|
|
5956
|
-
applyBetweenBitset([low, high], getAllDocsBitset, capacity, getValue, fieldIndex)
|
|
5957
|
-
);
|
|
5958
|
-
}
|
|
5959
|
-
}
|
|
5960
|
-
if ("in" in f && f.in !== void 0) {
|
|
5961
|
-
bitsets.push(applyInBitset(f.in, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5962
|
-
}
|
|
5963
|
-
if ("nin" in f && f.nin !== void 0) {
|
|
5964
|
-
bitsets.push(applyNinBitset(f.nin, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5965
|
-
}
|
|
5966
|
-
if ("startsWith" in f && f.startsWith !== void 0) {
|
|
5967
|
-
bitsets.push(applyStartsWithBitset(f.startsWith, getAllDocsBitset, capacity, getValue));
|
|
5968
|
-
}
|
|
5969
|
-
if ("endsWith" in f && f.endsWith !== void 0) {
|
|
5970
|
-
bitsets.push(applyEndsWithBitset(f.endsWith, getAllDocsBitset, capacity, getValue));
|
|
5971
|
-
}
|
|
5972
|
-
if ("containsAll" in f && f.containsAll !== void 0) {
|
|
5973
|
-
bitsets.push(
|
|
5974
|
-
applyContainsAllBitset(f.containsAll, getAllDocsBitset, capacity, getValue)
|
|
5975
|
-
);
|
|
5976
|
-
}
|
|
5977
|
-
if ("matchesAny" in f && f.matchesAny !== void 0) {
|
|
5978
|
-
bitsets.push(
|
|
5979
|
-
applyMatchesAnyBitset(f.matchesAny, getAllDocsBitset, capacity, getValue)
|
|
5980
|
-
);
|
|
5981
|
-
}
|
|
5982
|
-
if ("size" in f && f.size !== void 0) {
|
|
5983
|
-
bitsets.push(applySizeBitset(f.size, getAllDocsBitset, capacity, getValue));
|
|
5984
|
-
}
|
|
5985
|
-
if ("exists" in f && f.exists !== void 0) {
|
|
5986
|
-
bitsets.push(
|
|
5987
|
-
f.exists ? applyExistsBitset(getAllDocsBitset, capacity, getValue) : applyNotExistsBitset(getAllDocsBitset, capacity, getValue)
|
|
5988
|
-
);
|
|
5989
|
-
}
|
|
5990
|
-
if ("notExists" in f && f.notExists !== void 0) {
|
|
5991
|
-
bitsets.push(
|
|
5992
|
-
f.notExists ? applyNotExistsBitset(getAllDocsBitset, capacity, getValue) : applyExistsBitset(getAllDocsBitset, capacity, getValue)
|
|
5993
|
-
);
|
|
5994
|
-
}
|
|
5995
|
-
if ("isEmpty" in f && f.isEmpty !== void 0) {
|
|
5996
|
-
bitsets.push(
|
|
5997
|
-
f.isEmpty ? applyIsEmptyBitset(getAllDocsBitset, capacity, getValue) : applyIsNotEmptyBitset(getAllDocsBitset, capacity, getValue)
|
|
5998
|
-
);
|
|
5999
|
-
}
|
|
6000
|
-
if ("isNotEmpty" in f && f.isNotEmpty !== void 0) {
|
|
6001
|
-
bitsets.push(
|
|
6002
|
-
f.isNotEmpty ? applyIsNotEmptyBitset(getAllDocsBitset, capacity, getValue) : applyIsEmptyBitset(getAllDocsBitset, capacity, getValue)
|
|
6003
|
-
);
|
|
6004
|
-
}
|
|
6005
|
-
if (bitsets.length === 0) return context.allDocIdsBitset;
|
|
6006
|
-
if (bitsets.length === 1) return bitsets[0];
|
|
6007
|
-
return applyAndBitset(bitsets);
|
|
6008
|
-
}
|
|
6009
|
-
function resolveGeoIndex(fieldPath, fieldIndex) {
|
|
6010
|
-
if (fieldIndex?.type !== "geopoint") {
|
|
6011
|
-
throw new NarsilError(
|
|
6012
|
-
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
6013
|
-
`Field "${fieldPath}" requires a geopoint index for geo filters`,
|
|
6014
|
-
{ fieldPath }
|
|
6015
|
-
);
|
|
6016
|
-
}
|
|
6017
|
-
return fieldIndex.index;
|
|
6651
|
+
return narrowing.value(() => context.allDocIdsBitset);
|
|
6018
6652
|
}
|
|
6019
6653
|
|
|
6020
6654
|
// src/core/partition/constants.ts
|
|
6021
6655
|
var TARGET_BLOCK_ENTRIES = 128;
|
|
6022
6656
|
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
6023
6657
|
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
6658
|
+
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
6659
|
+
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
6660
|
+
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
6661
|
+
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
6024
6662
|
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
6025
6663
|
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
6026
6664
|
|
|
@@ -6637,99 +7275,72 @@ function sortColumnBytes(state) {
|
|
|
6637
7275
|
}
|
|
6638
7276
|
|
|
6639
7277
|
// src/core/partition/filters.ts
|
|
6640
|
-
|
|
6641
|
-
|
|
6642
|
-
|
|
6643
|
-
|
|
6644
|
-
|
|
7278
|
+
function presentBitsetOf(state, fieldPath, type, capacity) {
|
|
7279
|
+
const base = parsedTypeOf(type)?.base;
|
|
7280
|
+
let present;
|
|
7281
|
+
if (base === "number") present = state.numericIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
7282
|
+
else if (base === "boolean") present = state.booleanIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
7283
|
+
else if (base === "enum") present = state.enumIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
7284
|
+
else if (base === "geopoint") present = state.geoIndexes.get(fieldPath)?.documentsBitset(capacity);
|
|
7285
|
+
else if (base !== void 0 && keepsValueSize(base)) {
|
|
7286
|
+
present = state.sizeIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
7287
|
+
}
|
|
7288
|
+
return present ?? createBitSet(capacity);
|
|
7289
|
+
}
|
|
7290
|
+
function liveDocumentsBitset(state, capacity) {
|
|
7291
|
+
const bits = createBitSet(capacity);
|
|
7292
|
+
for (const internalId of state.docStore.allInternalIds()) bitsetSet(bits, internalId);
|
|
7293
|
+
return bits;
|
|
7294
|
+
}
|
|
7295
|
+
function buildFilterContext(state, schema, meter) {
|
|
6645
7296
|
const flat = getFlatSchema(state, schema);
|
|
6646
|
-
const fieldIndexes = {};
|
|
6647
7297
|
const capacity = state.docStore.internalIdCapacity();
|
|
6648
|
-
|
|
6649
|
-
|
|
6650
|
-
|
|
6651
|
-
if (numIdx) {
|
|
6652
|
-
fieldIndexes[fieldPath] = {
|
|
6653
|
-
type: "numeric",
|
|
6654
|
-
index: {
|
|
6655
|
-
eq: (v) => numIdx.queryEq(v),
|
|
6656
|
-
gt: (v) => numIdx.queryGt(v),
|
|
6657
|
-
gte: (v) => numIdx.queryGte(v),
|
|
6658
|
-
lt: (v) => numIdx.queryLt(v),
|
|
6659
|
-
lte: (v) => numIdx.queryLte(v),
|
|
6660
|
-
between: (min, max) => numIdx.queryBetween(min, max),
|
|
6661
|
-
allDocIds: () => numIdx.getAllDocIds(),
|
|
6662
|
-
eqBitset: (v, cap) => numIdx.queryEqBitset(v, cap),
|
|
6663
|
-
gtBitset: (v, cap) => numIdx.queryGtBitset(v, cap),
|
|
6664
|
-
gteBitset: (v, cap) => numIdx.queryGteBitset(v, cap),
|
|
6665
|
-
ltBitset: (v, cap) => numIdx.queryLtBitset(v, cap),
|
|
6666
|
-
lteBitset: (v, cap) => numIdx.queryLteBitset(v, cap),
|
|
6667
|
-
betweenBitset: (min, max, cap) => numIdx.queryBetweenBitset(min, max, cap),
|
|
6668
|
-
allDocIdsBitset: (cap) => numIdx.getAllDocIdsBitset(cap)
|
|
6669
|
-
}
|
|
6670
|
-
};
|
|
6671
|
-
}
|
|
6672
|
-
} else if (fieldType === "boolean" || fieldType === "boolean[]") {
|
|
6673
|
-
const boolIdx = state.booleanIndexes.get(fieldPath);
|
|
6674
|
-
if (boolIdx) {
|
|
6675
|
-
fieldIndexes[fieldPath] = {
|
|
6676
|
-
type: "boolean",
|
|
6677
|
-
index: {
|
|
6678
|
-
getTrue: () => boolIdx.queryEq(true),
|
|
6679
|
-
getFalse: () => boolIdx.queryEq(false),
|
|
6680
|
-
allDocIds: () => boolIdx.getAllDocIds(),
|
|
6681
|
-
getTrueBitset: (cap) => boolIdx.queryEqBitset(true, cap),
|
|
6682
|
-
getFalseBitset: (cap) => boolIdx.queryEqBitset(false, cap),
|
|
6683
|
-
allDocIdsBitset: (cap) => boolIdx.getAllDocIdsBitset(cap)
|
|
6684
|
-
}
|
|
6685
|
-
};
|
|
6686
|
-
}
|
|
6687
|
-
} else if (fieldType === "enum" || fieldType === "enum[]") {
|
|
6688
|
-
const enumIdx = state.enumIndexes.get(fieldPath);
|
|
6689
|
-
if (enumIdx) {
|
|
6690
|
-
fieldIndexes[fieldPath] = {
|
|
6691
|
-
type: "enum",
|
|
6692
|
-
index: {
|
|
6693
|
-
getDocIds: (v) => enumIdx.queryEq(v),
|
|
6694
|
-
allDocIds: () => enumIdx.getAllDocIds(),
|
|
6695
|
-
getDocIdsBitset: (v, cap) => enumIdx.queryEqBitset(v, cap),
|
|
6696
|
-
getDocIdsInBitset: (values, cap) => enumIdx.queryInBitset(values, cap),
|
|
6697
|
-
allDocIdsBitset: (cap) => enumIdx.getAllDocIdsBitset(cap)
|
|
6698
|
-
}
|
|
6699
|
-
};
|
|
6700
|
-
}
|
|
6701
|
-
} else if (fieldType === "geopoint") {
|
|
6702
|
-
const geoIdx = state.geoIndexes.get(fieldPath);
|
|
6703
|
-
fieldIndexes[fieldPath] = {
|
|
6704
|
-
type: "geopoint",
|
|
6705
|
-
index: geoIdx === void 0 ? EMPTY_GEO_INDEX : geoIdx
|
|
6706
|
-
};
|
|
6707
|
-
}
|
|
7298
|
+
const patternFields = /* @__PURE__ */ new Set();
|
|
7299
|
+
for (const { path: fieldPath, pattern } of schemaFieldsOf(schema)) {
|
|
7300
|
+
if (pattern) patternFields.add(fieldPath);
|
|
6708
7301
|
}
|
|
6709
|
-
|
|
7302
|
+
const getFieldValue = (internalId, fieldPath) => getFieldValueByInternalId(state.docStore, internalId, fieldPath);
|
|
6710
7303
|
let cachedAllDocIdsBitset = null;
|
|
7304
|
+
const cachedPresentBitsets = /* @__PURE__ */ new Map();
|
|
6711
7305
|
return {
|
|
6712
7306
|
fieldTypes: flat,
|
|
6713
|
-
|
|
6714
|
-
|
|
7307
|
+
numericIndex: (fieldPath) => state.numericIndexes.get(fieldPath),
|
|
7308
|
+
booleanIndex: (fieldPath) => state.booleanIndexes.get(fieldPath),
|
|
7309
|
+
enumIndex: (fieldPath) => state.enumIndexes.get(fieldPath),
|
|
7310
|
+
geoIndex: (fieldPath) => state.geoIndexes.get(fieldPath),
|
|
7311
|
+
sizeIndex: (fieldPath) => state.sizeIndexes.get(fieldPath),
|
|
6715
7312
|
textRangeBitset: (fieldPath, bounds) => textRangeBitsetOf(state, fieldPath, flat[fieldPath], bounds, capacity),
|
|
6716
|
-
|
|
6717
|
-
|
|
6718
|
-
|
|
6719
|
-
|
|
6720
|
-
|
|
7313
|
+
isPatternField: (fieldPath) => patternFields.has(fieldPath),
|
|
7314
|
+
matchPattern: (fieldPath, operator, text, caseFold, into) => {
|
|
7315
|
+
const index = state.patternIndexes.get(fieldPath);
|
|
7316
|
+
if (index === void 0) return;
|
|
7317
|
+
requirePatternSearch(fieldPath).matchInto({
|
|
7318
|
+
operator,
|
|
7319
|
+
text,
|
|
7320
|
+
caseFold,
|
|
7321
|
+
index,
|
|
7322
|
+
matches: into,
|
|
7323
|
+
meter,
|
|
7324
|
+
host: PATTERN_HOST,
|
|
7325
|
+
valueOf: (internalId) => getFieldValue(internalId, fieldPath)
|
|
7326
|
+
});
|
|
7327
|
+
},
|
|
7328
|
+
presentBitset: (fieldPath) => {
|
|
7329
|
+
const cached = cachedPresentBitsets.get(fieldPath);
|
|
7330
|
+
if (cached !== void 0) return cached;
|
|
7331
|
+
const present = presentBitsetOf(state, fieldPath, flat[fieldPath], capacity);
|
|
7332
|
+
cachedPresentBitsets.set(fieldPath, present);
|
|
7333
|
+
return present;
|
|
6721
7334
|
},
|
|
6722
7335
|
capacity,
|
|
6723
7336
|
get allDocIdsBitset() {
|
|
6724
|
-
|
|
6725
|
-
cachedAllDocIdsBitset = bitsetFromSet(this.allDocIds, capacity);
|
|
6726
|
-
}
|
|
7337
|
+
cachedAllDocIdsBitset ??= liveDocumentsBitset(state, capacity);
|
|
6727
7338
|
return cachedAllDocIdsBitset;
|
|
6728
7339
|
}
|
|
6729
7340
|
};
|
|
6730
7341
|
}
|
|
6731
|
-
function applyPartitionFilters(state, filters, schema) {
|
|
6732
|
-
const context = buildFilterContext(state, schema);
|
|
7342
|
+
function applyPartitionFilters(state, filters, schema, meter) {
|
|
7343
|
+
const context = buildFilterContext(state, schema, meter);
|
|
6733
7344
|
const resultBitset = evaluateFilters(filters, context);
|
|
6734
7345
|
const resolver = state.docStore.resolver();
|
|
6735
7346
|
const externalResult = /* @__PURE__ */ new Set();
|
|
@@ -6749,12 +7360,12 @@ function applyPartitionFilters(state, filters, schema) {
|
|
|
6749
7360
|
}
|
|
6750
7361
|
return externalResult;
|
|
6751
7362
|
}
|
|
6752
|
-
function applyPartitionFiltersBitset(state, filters, schema) {
|
|
6753
|
-
const context = buildFilterContext(state, schema);
|
|
7363
|
+
function applyPartitionFiltersBitset(state, filters, schema, meter) {
|
|
7364
|
+
const context = buildFilterContext(state, schema, meter);
|
|
6754
7365
|
return evaluateFilters(filters, context);
|
|
6755
7366
|
}
|
|
6756
|
-
function partitionFilterMatches(state, filters, schema) {
|
|
6757
|
-
const accepted = applyPartitionFiltersBitset(state, filters, schema);
|
|
7367
|
+
function partitionFilterMatches(state, filters, schema, meter) {
|
|
7368
|
+
const accepted = applyPartitionFiltersBitset(state, filters, schema, meter);
|
|
6758
7369
|
const docStore = state.docStore;
|
|
6759
7370
|
const resolver = docStore.resolver();
|
|
6760
7371
|
let count = 0;
|
|
@@ -6911,6 +7522,8 @@ function estimatePartitionBytes(state) {
|
|
|
6911
7522
|
bytes += docCount * state.booleanIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
6912
7523
|
bytes += docCount * state.enumIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
6913
7524
|
bytes += docCount * state.geoIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
7525
|
+
bytes += docCount * state.sizeIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
7526
|
+
for (const index of state.patternIndexes.values()) bytes += index.bytes();
|
|
6914
7527
|
return bytes + sortColumnBytes(state);
|
|
6915
7528
|
}
|
|
6916
7529
|
|
|
@@ -6970,12 +7583,12 @@ function mergeSurfaceForms(target, source) {
|
|
|
6970
7583
|
target.surfaceRegistry.add(surface, value[1], value[0]);
|
|
6971
7584
|
}
|
|
6972
7585
|
}
|
|
6973
|
-
function
|
|
6974
|
-
for (const [fieldPath, index] of source
|
|
6975
|
-
let targetIndex = target.
|
|
7586
|
+
function mergeNumericIndexes(target, source, idMapping) {
|
|
7587
|
+
for (const [fieldPath, index] of source) {
|
|
7588
|
+
let targetIndex = target.get(fieldPath);
|
|
6976
7589
|
if (targetIndex === void 0) {
|
|
6977
7590
|
targetIndex = createNumericIndex();
|
|
6978
|
-
target.
|
|
7591
|
+
target.set(fieldPath, targetIndex);
|
|
6979
7592
|
}
|
|
6980
7593
|
for (const entry of index.serialize()) {
|
|
6981
7594
|
const targetInternalId = idMapping.get(entry.docId);
|
|
@@ -6983,6 +7596,10 @@ function mergeFieldIndexes(target, source, idMapping) {
|
|
|
6983
7596
|
targetIndex.insert(targetInternalId, entry.value);
|
|
6984
7597
|
}
|
|
6985
7598
|
}
|
|
7599
|
+
}
|
|
7600
|
+
function mergeFieldIndexes(target, source, idMapping) {
|
|
7601
|
+
mergeNumericIndexes(target.numericIndexes, source.numericIndexes, idMapping);
|
|
7602
|
+
mergeNumericIndexes(target.sizeIndexes, source.sizeIndexes, idMapping);
|
|
6986
7603
|
for (const [fieldPath, index] of source.booleanIndexes) {
|
|
6987
7604
|
let targetIndex = target.booleanIndexes.get(fieldPath);
|
|
6988
7605
|
if (targetIndex === void 0) {
|
|
@@ -7024,6 +7641,12 @@ function mergeFieldIndexes(target, source, idMapping) {
|
|
|
7024
7641
|
targetIndex.insert(targetInternalId, entry.lat, entry.lon);
|
|
7025
7642
|
}
|
|
7026
7643
|
}
|
|
7644
|
+
for (const [fieldPath, index] of source.patternIndexes) {
|
|
7645
|
+
patternIndexOf(target.patternIndexes, fieldPath).addArrays(
|
|
7646
|
+
index.toArrays(fieldPath, null),
|
|
7647
|
+
(sourceInternalId) => idMapping.get(sourceInternalId) ?? -1
|
|
7648
|
+
);
|
|
7649
|
+
}
|
|
7027
7650
|
}
|
|
7028
7651
|
function mergeSegmentState(target, source) {
|
|
7029
7652
|
if (source.stats.totalDocuments === 0 && source.docStore.count() === 0) return;
|
|
@@ -8044,19 +8667,23 @@ function searchFulltext(state, params) {
|
|
|
8044
8667
|
}
|
|
8045
8668
|
|
|
8046
8669
|
// src/core/partition/segment-field-indexes.ts
|
|
8047
|
-
function
|
|
8048
|
-
const
|
|
8049
|
-
const
|
|
8050
|
-
|
|
8051
|
-
const entries = index.serialize().filter((entry) => survives(entry.docId));
|
|
8670
|
+
function encodeNumericIndexes(indexes, remap) {
|
|
8671
|
+
const encoded = [];
|
|
8672
|
+
for (const [fieldPath, index] of indexes) {
|
|
8673
|
+
const entries = index.serialize().filter((entry) => remap[entry.docId] >= 0);
|
|
8052
8674
|
const docIds = new Uint32Array(entries.length);
|
|
8053
8675
|
const values = new Float64Array(entries.length);
|
|
8054
8676
|
for (let i = 0; i < entries.length; i++) {
|
|
8055
8677
|
docIds[i] = remap[entries[i].docId];
|
|
8056
8678
|
values[i] = entries[i].value;
|
|
8057
8679
|
}
|
|
8058
|
-
|
|
8680
|
+
encoded.push({ fieldPath, docIds, values });
|
|
8059
8681
|
}
|
|
8682
|
+
return encoded;
|
|
8683
|
+
}
|
|
8684
|
+
function encodeFieldIndexes(state, remap) {
|
|
8685
|
+
const survives = (internalId) => remap[internalId] >= 0;
|
|
8686
|
+
const numeric = encodeNumericIndexes(state.numericIndexes, remap);
|
|
8060
8687
|
const booleans = [];
|
|
8061
8688
|
for (const [fieldPath, index] of state.booleanIndexes) {
|
|
8062
8689
|
const { trueDocs, falseDocs } = index.serialize();
|
|
@@ -8097,7 +8724,9 @@ function encodeFieldIndexes(state, remap) {
|
|
|
8097
8724
|
}
|
|
8098
8725
|
geo.push({ fieldPath, docIds, latitudes, longitudes });
|
|
8099
8726
|
}
|
|
8100
|
-
|
|
8727
|
+
const pattern = [];
|
|
8728
|
+
for (const [fieldPath, index] of state.patternIndexes) pattern.push(index.toArrays(fieldPath, remap));
|
|
8729
|
+
return { numeric, boolean: booleans, enums, geo, pattern, sizes: encodeNumericIndexes(state.sizeIndexes, remap) };
|
|
8101
8730
|
}
|
|
8102
8731
|
|
|
8103
8732
|
// src/core/partition/segment-payload.ts
|
|
@@ -8254,15 +8883,19 @@ function mergePayloadPostings(target, payload, ordinalBase) {
|
|
|
8254
8883
|
}
|
|
8255
8884
|
}
|
|
8256
8885
|
}
|
|
8257
|
-
function
|
|
8258
|
-
for (const entry of
|
|
8259
|
-
let index = target.
|
|
8886
|
+
function mergePayloadNumericIndexes(target, entries, ordinalBase) {
|
|
8887
|
+
for (const entry of entries) {
|
|
8888
|
+
let index = target.get(entry.fieldPath);
|
|
8260
8889
|
if (index === void 0) {
|
|
8261
8890
|
index = createNumericIndex();
|
|
8262
|
-
target.
|
|
8891
|
+
target.set(entry.fieldPath, index);
|
|
8263
8892
|
}
|
|
8264
8893
|
for (let i = 0; i < entry.docIds.length; i++) index.insert(ordinalBase + entry.docIds[i], entry.values[i]);
|
|
8265
8894
|
}
|
|
8895
|
+
}
|
|
8896
|
+
function mergePayloadFieldIndexes(target, payload, ordinalBase) {
|
|
8897
|
+
mergePayloadNumericIndexes(target.numericIndexes, payload.numeric, ordinalBase);
|
|
8898
|
+
mergePayloadNumericIndexes(target.sizeIndexes, payload.sizes, ordinalBase);
|
|
8266
8899
|
for (const entry of payload.boolean) {
|
|
8267
8900
|
let index = target.booleanIndexes.get(entry.fieldPath);
|
|
8268
8901
|
if (index === void 0) {
|
|
@@ -8293,6 +8926,9 @@ function mergePayloadFieldIndexes(target, payload, ordinalBase) {
|
|
|
8293
8926
|
index.insert(ordinalBase + entry.docIds[i], entry.latitudes[i], entry.longitudes[i]);
|
|
8294
8927
|
}
|
|
8295
8928
|
}
|
|
8929
|
+
for (const entry of payload.pattern) {
|
|
8930
|
+
patternIndexOf(target.patternIndexes, entry.fieldPath).addArrays(entry, (ordinal) => ordinalBase + ordinal);
|
|
8931
|
+
}
|
|
8296
8932
|
}
|
|
8297
8933
|
function mergeSegmentPayload(target, payload, documents) {
|
|
8298
8934
|
if (payload.documentCount === 0) return;
|
|
@@ -8337,6 +8973,123 @@ function mergeSegmentPayload(target, payload, documents) {
|
|
|
8337
8973
|
target.scoreBuffer = null;
|
|
8338
8974
|
}
|
|
8339
8975
|
|
|
8976
|
+
// src/core/partition/pattern-snapshot.ts
|
|
8977
|
+
function loadFailed(fieldPath, problem) {
|
|
8978
|
+
return new NarsilError(
|
|
8979
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
8980
|
+
`The pattern index of field "${fieldPath}" in this partition is corrupt: ${problem}`,
|
|
8981
|
+
{ field: fieldPath }
|
|
8982
|
+
);
|
|
8983
|
+
}
|
|
8984
|
+
function serializePatternIndex(state, fieldPath, index) {
|
|
8985
|
+
const resolver = state.docStore.resolver();
|
|
8986
|
+
const arrays = index.toArrays(fieldPath, null);
|
|
8987
|
+
const positionOf = /* @__PURE__ */ new Map();
|
|
8988
|
+
const docIds = [];
|
|
8989
|
+
for (const ordinal of arrays.docIds) {
|
|
8990
|
+
const externalId = resolver.toExternal(ordinal);
|
|
8991
|
+
if (externalId === void 0) continue;
|
|
8992
|
+
positionOf.set(ordinal, docIds.length);
|
|
8993
|
+
docIds.push(externalId);
|
|
8994
|
+
}
|
|
8995
|
+
const runs = /* @__PURE__ */ Object.create(null);
|
|
8996
|
+
for (let run = 0; run + 1 < arrays.offsets.length; run++) {
|
|
8997
|
+
const positions = [];
|
|
8998
|
+
for (let at = arrays.offsets[run]; at < arrays.offsets[run + 1]; at++) {
|
|
8999
|
+
const position = positionOf.get(arrays.postings[at]);
|
|
9000
|
+
if (position !== void 0) positions.push(position);
|
|
9001
|
+
}
|
|
9002
|
+
if (positions.length === 0) continue;
|
|
9003
|
+
const base = run * PATTERN_RUN_CODE_POINTS;
|
|
9004
|
+
runs[String.fromCodePoint(arrays.runs[base], arrays.runs[base + 1], arrays.runs[base + 2])] = positions;
|
|
9005
|
+
}
|
|
9006
|
+
return { docIds, runs };
|
|
9007
|
+
}
|
|
9008
|
+
function serializePatternIndexes(state, schema) {
|
|
9009
|
+
const patternFields = schemaFieldsOf(schema).filter((field) => field.pattern);
|
|
9010
|
+
if (patternFields.length === 0) return void 0;
|
|
9011
|
+
const serialized = /* @__PURE__ */ Object.create(null);
|
|
9012
|
+
for (const { path: fieldPath } of patternFields) {
|
|
9013
|
+
const index = state.patternIndexes.get(fieldPath);
|
|
9014
|
+
serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } : serializePatternIndex(state, fieldPath, index);
|
|
9015
|
+
}
|
|
9016
|
+
return serialized;
|
|
9017
|
+
}
|
|
9018
|
+
function appendRunCodePoints(fieldPath, key, out) {
|
|
9019
|
+
const start = out.length;
|
|
9020
|
+
for (const character of key) out.push(character.codePointAt(0) ?? 0);
|
|
9021
|
+
const count = out.length - start;
|
|
9022
|
+
if (count !== PATTERN_RUN_CODE_POINTS) {
|
|
9023
|
+
throw loadFailed(
|
|
9024
|
+
fieldPath,
|
|
9025
|
+
`the key "${key}" holds ${count} code points, while each key must hold ${PATTERN_RUN_CODE_POINTS}`
|
|
9026
|
+
);
|
|
9027
|
+
}
|
|
9028
|
+
}
|
|
9029
|
+
function requirePositions(fieldPath, key, list, documentCount) {
|
|
9030
|
+
if (!Array.isArray(list)) throw loadFailed(fieldPath, `the run "${key}" must map to a list of positions`);
|
|
9031
|
+
let previous = -1;
|
|
9032
|
+
for (const position of list) {
|
|
9033
|
+
if (!Number.isInteger(position) || position < 0 || position >= documentCount) {
|
|
9034
|
+
throw loadFailed(fieldPath, `the run "${key}" names position ${String(position)} outside its document list`);
|
|
9035
|
+
}
|
|
9036
|
+
if (position <= previous) {
|
|
9037
|
+
throw loadFailed(fieldPath, `the positions of the run "${key}" are out of ascending order`);
|
|
9038
|
+
}
|
|
9039
|
+
previous = position;
|
|
9040
|
+
}
|
|
9041
|
+
return list;
|
|
9042
|
+
}
|
|
9043
|
+
function requireDistinctDocuments(fieldPath, docIds) {
|
|
9044
|
+
const seen = /* @__PURE__ */ new Set();
|
|
9045
|
+
for (const docId of docIds) {
|
|
9046
|
+
if (seen.has(docId)) throw loadFailed(fieldPath, `the document list names "${docId}" twice`);
|
|
9047
|
+
seen.add(docId);
|
|
9048
|
+
}
|
|
9049
|
+
}
|
|
9050
|
+
function decodePatternEntry(fieldPath, entry) {
|
|
9051
|
+
requireDistinctDocuments(fieldPath, entry.docIds);
|
|
9052
|
+
const runs = [];
|
|
9053
|
+
const offsets = [0];
|
|
9054
|
+
const postings = [];
|
|
9055
|
+
for (const [key, list] of Object.entries(entry.runs)) {
|
|
9056
|
+
appendRunCodePoints(fieldPath, key, runs);
|
|
9057
|
+
for (const position of requirePositions(fieldPath, key, list, entry.docIds.length)) postings.push(position);
|
|
9058
|
+
offsets.push(postings.length);
|
|
9059
|
+
}
|
|
9060
|
+
const positions = new Uint32Array(entry.docIds.length);
|
|
9061
|
+
for (let position = 0; position < positions.length; position++) positions[position] = position;
|
|
9062
|
+
return {
|
|
9063
|
+
fieldPath,
|
|
9064
|
+
docIds: positions,
|
|
9065
|
+
runs: Uint32Array.from(runs),
|
|
9066
|
+
offsets: Uint32Array.from(offsets),
|
|
9067
|
+
postings: Uint32Array.from(postings)
|
|
9068
|
+
};
|
|
9069
|
+
}
|
|
9070
|
+
function rebuildPatternIndex(state, fieldPath) {
|
|
9071
|
+
for (const [docId, stored] of state.docStore.all()) {
|
|
9072
|
+
const value = getNestedValue(stored.fields, fieldPath);
|
|
9073
|
+
if (value === void 0 || value === null) continue;
|
|
9074
|
+
const internalId = state.docStore.getInternalId(docId);
|
|
9075
|
+
if (internalId !== void 0) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
|
|
9076
|
+
}
|
|
9077
|
+
}
|
|
9078
|
+
function loadPatternIndexes(state, serialized, schema) {
|
|
9079
|
+
const resolver = state.docStore.resolver();
|
|
9080
|
+
for (const { path: fieldPath, pattern } of schemaFieldsOf(schema)) {
|
|
9081
|
+
if (!pattern) continue;
|
|
9082
|
+
const entry = serialized !== void 0 && Object.hasOwn(serialized, fieldPath) ? serialized[fieldPath] : void 0;
|
|
9083
|
+
if (entry === void 0) {
|
|
9084
|
+
rebuildPatternIndex(state, fieldPath);
|
|
9085
|
+
continue;
|
|
9086
|
+
}
|
|
9087
|
+
const decoded = decodePatternEntry(fieldPath, entry);
|
|
9088
|
+
const ordinals = entry.docIds.map((docId) => resolver.toInternal(docId) ?? -1);
|
|
9089
|
+
patternIndexOf(state.patternIndexes, fieldPath).addArrays(decoded, (position) => ordinals[position]);
|
|
9090
|
+
}
|
|
9091
|
+
}
|
|
9092
|
+
|
|
8340
9093
|
// src/core/partition/serialization.ts
|
|
8341
9094
|
function serializePartition(state, partitionId, indexName, totalPartitions, language, schema) {
|
|
8342
9095
|
const flatSchema = getFlatSchema(state, schema);
|
|
@@ -8409,6 +9162,7 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
|
|
|
8409
9162
|
}));
|
|
8410
9163
|
}
|
|
8411
9164
|
const serializedStats = state.stats.serialize();
|
|
9165
|
+
const serializedPattern = serializePatternIndexes(state, schema);
|
|
8412
9166
|
return {
|
|
8413
9167
|
indexName,
|
|
8414
9168
|
partitionId,
|
|
@@ -8423,7 +9177,8 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
|
|
|
8423
9177
|
numeric: serializedNumeric,
|
|
8424
9178
|
boolean: serializedBoolean,
|
|
8425
9179
|
enum: serializedEnum,
|
|
8426
|
-
geopoint: serializedGeo
|
|
9180
|
+
geopoint: serializedGeo,
|
|
9181
|
+
...serializedPattern === void 0 ? {} : { pattern: serializedPattern }
|
|
8427
9182
|
},
|
|
8428
9183
|
surfaceForms: state.surfaceRegistry.serialize(),
|
|
8429
9184
|
statistics: serializedStats
|
|
@@ -8501,6 +9256,8 @@ function deserializePartition(state, data, clearFn, schema) {
|
|
|
8501
9256
|
throw new Error(`Schema mismatch: serialized data contains unknown field "${field}" not present in schema`);
|
|
8502
9257
|
}
|
|
8503
9258
|
}
|
|
9259
|
+
loadPatternIndexes(state, data.fieldIndexes.pattern, schema);
|
|
9260
|
+
rebuildSizeIndexes(state, schema);
|
|
8504
9261
|
}
|
|
8505
9262
|
|
|
8506
9263
|
// src/core/partition/suggestions.ts
|
|
@@ -8663,7 +9420,8 @@ function serializePartitionToWirePayloadV2(state, partitionId, indexName, totalP
|
|
|
8663
9420
|
numeric: wireNumeric,
|
|
8664
9421
|
boolean: wireBoolean,
|
|
8665
9422
|
enum: wireEnum,
|
|
8666
|
-
geopoint: wireGeo
|
|
9423
|
+
geopoint: wireGeo,
|
|
9424
|
+
...patternIndexesToWire(serializePatternIndexes(state, schema))
|
|
8667
9425
|
},
|
|
8668
9426
|
surface_forms: state.surfaceRegistry.serialize(),
|
|
8669
9427
|
statistics: {
|
|
@@ -8698,6 +9456,8 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8698
9456
|
booleanIndexes: /* @__PURE__ */ new Map(),
|
|
8699
9457
|
enumIndexes: /* @__PURE__ */ new Map(),
|
|
8700
9458
|
geoIndexes: /* @__PURE__ */ new Map(),
|
|
9459
|
+
patternIndexes: /* @__PURE__ */ new Map(),
|
|
9460
|
+
sizeIndexes: /* @__PURE__ */ new Map(),
|
|
8701
9461
|
fieldNameTable,
|
|
8702
9462
|
flatSchemaCache: null,
|
|
8703
9463
|
lastSchemaRef: null,
|
|
@@ -8717,6 +9477,8 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8717
9477
|
state.booleanIndexes.clear();
|
|
8718
9478
|
state.enumIndexes.clear();
|
|
8719
9479
|
state.geoIndexes.clear();
|
|
9480
|
+
state.patternIndexes.clear();
|
|
9481
|
+
state.sizeIndexes.clear();
|
|
8720
9482
|
state.stats.deserialize({ totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} });
|
|
8721
9483
|
state.flatSchemaCache = null;
|
|
8722
9484
|
state.lastSchemaRef = null;
|
|
@@ -8892,14 +9654,14 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8892
9654
|
sortValues(docId, fields, fieldTypes, modes) {
|
|
8893
9655
|
return sortValuesOf(state, docId, fields, fieldTypes, modes);
|
|
8894
9656
|
},
|
|
8895
|
-
applyFilters(filters, schema) {
|
|
8896
|
-
return applyPartitionFilters(state, filters, schema);
|
|
9657
|
+
applyFilters(filters, schema, meter) {
|
|
9658
|
+
return applyPartitionFilters(state, filters, schema, meter);
|
|
8897
9659
|
},
|
|
8898
|
-
applyFiltersBitset(filters, schema) {
|
|
8899
|
-
return applyPartitionFiltersBitset(state, filters, schema);
|
|
9660
|
+
applyFiltersBitset(filters, schema, meter) {
|
|
9661
|
+
return applyPartitionFiltersBitset(state, filters, schema, meter);
|
|
8900
9662
|
},
|
|
8901
|
-
filterMatches(filters, schema) {
|
|
8902
|
-
return partitionFilterMatches(state, filters, schema);
|
|
9663
|
+
filterMatches(filters, schema, meter) {
|
|
9664
|
+
return partitionFilterMatches(state, filters, schema, meter);
|
|
8903
9665
|
},
|
|
8904
9666
|
computeFacets(matched, config, schema) {
|
|
8905
9667
|
return computeFacets(state, matched, config, schema);
|
|
@@ -9281,6 +10043,100 @@ function readHeapStatistics() {
|
|
|
9281
10043
|
return null;
|
|
9282
10044
|
}
|
|
9283
10045
|
|
|
10046
|
+
// src/core/pattern-index/work-meter.ts
|
|
10047
|
+
var COUNTER_BYTES = 8;
|
|
10048
|
+
function capExceeded(cap) {
|
|
10049
|
+
return new NarsilError(
|
|
10050
|
+
ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
|
|
10051
|
+
`The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
|
|
10052
|
+
{ cap }
|
|
10053
|
+
);
|
|
10054
|
+
}
|
|
10055
|
+
function localMeter(cap, spent, matcherStateBytes, flushUnits) {
|
|
10056
|
+
let total = spent;
|
|
10057
|
+
let sharedCount = null;
|
|
10058
|
+
const spentSoFar = () => sharedCount === null ? total : Number(Atomics.load(sharedCount, 0));
|
|
10059
|
+
const meter = {
|
|
10060
|
+
cap,
|
|
10061
|
+
matcherStateBytes,
|
|
10062
|
+
flushUnits,
|
|
10063
|
+
add(units) {
|
|
10064
|
+
if (units <= 0) return;
|
|
10065
|
+
if (sharedCount === null) total += units;
|
|
10066
|
+
else total = Number(Atomics.add(sharedCount, 0, BigInt(units))) + units;
|
|
10067
|
+
if (total > cap) throw capExceeded(cap);
|
|
10068
|
+
},
|
|
10069
|
+
remaining() {
|
|
10070
|
+
return cap - spentSoFar();
|
|
10071
|
+
},
|
|
10072
|
+
fork(workerCount) {
|
|
10073
|
+
const copies = Math.max(1, workerCount);
|
|
10074
|
+
const seed = spentSoFar();
|
|
10075
|
+
let counter = null;
|
|
10076
|
+
if (typeof SharedArrayBuffer === "function") {
|
|
10077
|
+
counter = new SharedArrayBuffer(COUNTER_BYTES);
|
|
10078
|
+
new BigInt64Array(counter)[0] = BigInt(seed);
|
|
10079
|
+
}
|
|
10080
|
+
const shared2 = {
|
|
10081
|
+
cap,
|
|
10082
|
+
spent: seed,
|
|
10083
|
+
counter,
|
|
10084
|
+
matcherStateBytes: Math.floor(matcherStateBytes / copies),
|
|
10085
|
+
flushUnits: Math.max(1, Math.floor(flushUnits / copies))
|
|
10086
|
+
};
|
|
10087
|
+
return {
|
|
10088
|
+
shared: shared2,
|
|
10089
|
+
settle(reportedUnits) {
|
|
10090
|
+
if (counter === null) {
|
|
10091
|
+
meter.add(reportedUnits);
|
|
10092
|
+
return;
|
|
10093
|
+
}
|
|
10094
|
+
const unitsSinceFork = spentSoFar() - seed;
|
|
10095
|
+
const view = new BigInt64Array(counter);
|
|
10096
|
+
if (unitsSinceFork > 0) Atomics.add(view, 0, BigInt(unitsSinceFork));
|
|
10097
|
+
sharedCount = view;
|
|
10098
|
+
total = spentSoFar();
|
|
10099
|
+
if (total > cap) throw capExceeded(cap);
|
|
10100
|
+
}
|
|
10101
|
+
};
|
|
10102
|
+
}
|
|
10103
|
+
};
|
|
10104
|
+
return meter;
|
|
10105
|
+
}
|
|
10106
|
+
function createPatternWorkMeter(cap = DEFAULT_PATTERN_WORK_CAP) {
|
|
10107
|
+
return localMeter(cap, 0, MATCHER_STATE_BUDGET_BYTES, PATTERN_WORK_FLUSH_UNITS);
|
|
10108
|
+
}
|
|
10109
|
+
function sharedMeter(work, counter) {
|
|
10110
|
+
const view = new BigInt64Array(counter);
|
|
10111
|
+
return {
|
|
10112
|
+
matcherStateBytes: work.matcherStateBytes,
|
|
10113
|
+
flushUnits: work.flushUnits,
|
|
10114
|
+
add(units) {
|
|
10115
|
+
if (units <= 0) return;
|
|
10116
|
+
const total = Number(Atomics.add(view, 0, BigInt(units))) + units;
|
|
10117
|
+
if (total > work.cap) throw capExceeded(work.cap);
|
|
10118
|
+
},
|
|
10119
|
+
remaining() {
|
|
10120
|
+
return work.cap - Number(Atomics.load(view, 0));
|
|
10121
|
+
}
|
|
10122
|
+
};
|
|
10123
|
+
}
|
|
10124
|
+
function joinPatternWorkMeter(work) {
|
|
10125
|
+
const meter = work.counter === null ? localMeter(work.cap, work.spent, work.matcherStateBytes, work.flushUnits) : sharedMeter(work, work.counter);
|
|
10126
|
+
let spent = 0;
|
|
10127
|
+
return {
|
|
10128
|
+
matcherStateBytes: meter.matcherStateBytes,
|
|
10129
|
+
flushUnits: meter.flushUnits,
|
|
10130
|
+
add(units) {
|
|
10131
|
+
if (units <= 0) return;
|
|
10132
|
+
spent += units;
|
|
10133
|
+
meter.add(units);
|
|
10134
|
+
},
|
|
10135
|
+
remaining: () => meter.remaining(),
|
|
10136
|
+
spent: () => spent
|
|
10137
|
+
};
|
|
10138
|
+
}
|
|
10139
|
+
|
|
9284
10140
|
// src/partitioning/distributed-scoring.ts
|
|
9285
10141
|
function analysedQueryTokens(term, language, analysis) {
|
|
9286
10142
|
const { tokens } = tokenize(term, language, {
|
|
@@ -9938,7 +10794,11 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
9938
10794
|
}
|
|
9939
10795
|
let filterBitset;
|
|
9940
10796
|
if (params.filters) {
|
|
9941
|
-
filterBitset = partition.applyFiltersBitset(
|
|
10797
|
+
filterBitset = partition.applyFiltersBitset(
|
|
10798
|
+
params.filters,
|
|
10799
|
+
schema,
|
|
10800
|
+
options?.patternWork ?? createPatternWorkMeter()
|
|
10801
|
+
);
|
|
9942
10802
|
if (bitsetIsEmpty(filterBitset)) return null;
|
|
9943
10803
|
}
|
|
9944
10804
|
return { queryTokens, prefixExpansion, filterBitset };
|
|
@@ -10181,7 +11041,8 @@ function resolveEffectiveMode(config) {
|
|
|
10181
11041
|
function buildSearchOptions(base, globalStats) {
|
|
10182
11042
|
return {
|
|
10183
11043
|
...base,
|
|
10184
|
-
globalStats: globalStats ?? base?.globalStats
|
|
11044
|
+
globalStats: globalStats ?? base?.globalStats,
|
|
11045
|
+
patternWork: base?.patternWork ?? createPatternWorkMeter()
|
|
10185
11046
|
};
|
|
10186
11047
|
}
|
|
10187
11048
|
function dispatchWithDispatcher(partitions, params, language, schema, options, dispatcher) {
|
|
@@ -10216,12 +11077,13 @@ function frozenSegmentBytes(segment) {
|
|
|
10216
11077
|
bytes += arrays.documentTable.blob.byteLength + arrays.documentTable.offsets.byteLength;
|
|
10217
11078
|
}
|
|
10218
11079
|
for (const column of arrays.fieldLengthColumns) bytes += column.byteLength;
|
|
10219
|
-
for (const entry of arrays.numeric) bytes += entry.docIds.byteLength + entry.values.byteLength;
|
|
11080
|
+
for (const entry of [...arrays.numeric, ...arrays.sizes]) bytes += entry.docIds.byteLength + entry.values.byteLength;
|
|
10220
11081
|
for (const entry of arrays.boolean) bytes += entry.trueDocs.byteLength + entry.falseDocs.byteLength;
|
|
10221
11082
|
for (const entry of arrays.enums) bytes += entry.offsets.byteLength + entry.docIds.byteLength;
|
|
10222
11083
|
for (const entry of arrays.geo) {
|
|
10223
11084
|
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
10224
11085
|
}
|
|
11086
|
+
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
10225
11087
|
return bytes;
|
|
10226
11088
|
}
|
|
10227
11089
|
|
|
@@ -10299,24 +11161,24 @@ function placeSubBitset(composite, layout, subIndex, subBits) {
|
|
|
10299
11161
|
if (words === 0) return;
|
|
10300
11162
|
composite.set(subBits.length > words ? subBits.subarray(0, words) : subBits, layout.bases[subIndex] >> 5);
|
|
10301
11163
|
}
|
|
10302
|
-
function compositeFiltersBitset(subs, layout, filters, schema) {
|
|
11164
|
+
function compositeFiltersBitset(subs, layout, filters, schema, meter) {
|
|
10303
11165
|
const composite = new Uint32Array(Math.ceil(layout.totalCapacity / WORD_BITS));
|
|
10304
11166
|
for (let i = 0; i < subs.length; i++) {
|
|
10305
|
-
placeSubBitset(composite, layout, i, applyPartitionFiltersBitset(subs[i], filters, schema));
|
|
11167
|
+
placeSubBitset(composite, layout, i, applyPartitionFiltersBitset(subs[i], filters, schema, meter));
|
|
10306
11168
|
}
|
|
10307
11169
|
return composite;
|
|
10308
11170
|
}
|
|
10309
|
-
function compositeFilters(subs, filters, schema) {
|
|
11171
|
+
function compositeFilters(subs, filters, schema, meter) {
|
|
10310
11172
|
const combined = /* @__PURE__ */ new Set();
|
|
10311
11173
|
for (const sub of subs) {
|
|
10312
|
-
for (const docId of applyPartitionFilters(sub, filters, schema)) {
|
|
11174
|
+
for (const docId of applyPartitionFilters(sub, filters, schema, meter)) {
|
|
10313
11175
|
combined.add(docId);
|
|
10314
11176
|
}
|
|
10315
11177
|
}
|
|
10316
11178
|
return combined;
|
|
10317
11179
|
}
|
|
10318
|
-
function compositeFilterMatches(subs, layout, filters, schema) {
|
|
10319
|
-
const bits = compositeFiltersBitset(subs, layout, filters, schema);
|
|
11180
|
+
function compositeFilterMatches(subs, layout, filters, schema, meter) {
|
|
11181
|
+
const bits = compositeFiltersBitset(subs, layout, filters, schema, meter);
|
|
10320
11182
|
let count = 0;
|
|
10321
11183
|
for (let i = 0; i < subs.length; i++) {
|
|
10322
11184
|
const resolver = subs[i].docStore.resolver();
|
|
@@ -10777,14 +11639,14 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10777
11639
|
searchFulltextMatches(params) {
|
|
10778
11640
|
return compositeSearchMatches(subs(), layout(), params);
|
|
10779
11641
|
},
|
|
10780
|
-
applyFilters(filters, schema) {
|
|
10781
|
-
return compositeFilters(subs(), filters, schema);
|
|
11642
|
+
applyFilters(filters, schema, meter) {
|
|
11643
|
+
return compositeFilters(subs(), filters, schema, meter);
|
|
10782
11644
|
},
|
|
10783
|
-
applyFiltersBitset(filters, schema) {
|
|
10784
|
-
return compositeFiltersBitset(subs(), layout(), filters, schema);
|
|
11645
|
+
applyFiltersBitset(filters, schema, meter) {
|
|
11646
|
+
return compositeFiltersBitset(subs(), layout(), filters, schema, meter);
|
|
10785
11647
|
},
|
|
10786
|
-
filterMatches(filters, schema) {
|
|
10787
|
-
return compositeFilterMatches(subs(), layout(), filters, schema);
|
|
11648
|
+
filterMatches(filters, schema, meter) {
|
|
11649
|
+
return compositeFilterMatches(subs(), layout(), filters, schema, meter);
|
|
10788
11650
|
},
|
|
10789
11651
|
computeFacets(matched, config, schema) {
|
|
10790
11652
|
const everyValue = everyValueFacetConfig(config);
|
|
@@ -10954,6 +11816,7 @@ function setNestedValue(obj, path, value) {
|
|
|
10954
11816
|
|
|
10955
11817
|
// src/partitioning/manager/index.ts
|
|
10956
11818
|
function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
|
|
11819
|
+
requirePatternSearchFor(config.schema);
|
|
10957
11820
|
const count = initialPartitionCount ?? 1;
|
|
10958
11821
|
const analysis = resolveIndexAnalysis(config);
|
|
10959
11822
|
const trackPositions = config.trackPositions ?? true;
|
|
@@ -11265,8 +12128,12 @@ function countsWithoutScores(params) {
|
|
|
11265
12128
|
}
|
|
11266
12129
|
function fanOutMatchCount(manager, params, language, schema, options) {
|
|
11267
12130
|
let count = 0;
|
|
12131
|
+
const searchOptions = {
|
|
12132
|
+
...options?.searchOptions,
|
|
12133
|
+
patternWork: options?.searchOptions?.patternWork ?? createPatternWorkMeter()
|
|
12134
|
+
};
|
|
11268
12135
|
for (const partition of partitionsIn(manager, options?.partitionIds)) {
|
|
11269
|
-
const matches = fulltextMatches(partition, params, language, schema,
|
|
12136
|
+
const matches = fulltextMatches(partition, params, language, schema, searchOptions);
|
|
11270
12137
|
if (matches !== null) count += matches.count;
|
|
11271
12138
|
}
|
|
11272
12139
|
return count;
|
|
@@ -17951,7 +18818,22 @@ function createFrozenEnumReader(entry) {
|
|
|
17951
18818
|
function rangeOf(value) {
|
|
17952
18819
|
return rangeByValue.get(value) ?? { from: 0, to: 0 };
|
|
17953
18820
|
}
|
|
18821
|
+
let affixes = null;
|
|
18822
|
+
function affixesOf() {
|
|
18823
|
+
affixes ??= new ValueAffixes(() => true, values);
|
|
18824
|
+
return affixes;
|
|
18825
|
+
}
|
|
18826
|
+
function bitsetOfValues(matching, capacity) {
|
|
18827
|
+
const bits = createBitSet(capacity);
|
|
18828
|
+
for (const value of matching) {
|
|
18829
|
+
const { from, to } = rangeOf(value);
|
|
18830
|
+
for (let i = from; i < to; i++) bitsetSet(bits, docIds[i]);
|
|
18831
|
+
}
|
|
18832
|
+
return bits;
|
|
18833
|
+
}
|
|
17954
18834
|
return {
|
|
18835
|
+
queryPrefixBitset: (prefix, capacity) => bitsetOfValues(affixesOf().withPrefix(prefix), capacity),
|
|
18836
|
+
querySuffixBitset: (suffix, capacity) => bitsetOfValues(affixesOf().withSuffix(suffix), capacity),
|
|
17955
18837
|
serialize: () => {
|
|
17956
18838
|
const byValue = /* @__PURE__ */ Object.create(null);
|
|
17957
18839
|
for (let i = 0; i < values.length; i++) {
|
|
@@ -18396,6 +19278,11 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
|
18396
19278
|
docIds: sharedUint32(entry.docIds),
|
|
18397
19279
|
values: sharedFloat64(entry.values)
|
|
18398
19280
|
})),
|
|
19281
|
+
sizes: payload.sizes.map((entry) => ({
|
|
19282
|
+
fieldPath: entry.fieldPath,
|
|
19283
|
+
docIds: sharedUint32(entry.docIds),
|
|
19284
|
+
values: sharedFloat64(entry.values)
|
|
19285
|
+
})),
|
|
18399
19286
|
boolean: payload.boolean.map((entry) => ({
|
|
18400
19287
|
fieldPath: entry.fieldPath,
|
|
18401
19288
|
trueDocs: sharedUint32(entry.trueDocs),
|
|
@@ -18413,6 +19300,13 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
|
18413
19300
|
latitudes: sharedFloat64(entry.latitudes),
|
|
18414
19301
|
longitudes: sharedFloat64(entry.longitudes)
|
|
18415
19302
|
})),
|
|
19303
|
+
pattern: payload.pattern.map((entry) => ({
|
|
19304
|
+
fieldPath: entry.fieldPath,
|
|
19305
|
+
docIds: sharedUint32(entry.docIds),
|
|
19306
|
+
runs: sharedUint32(entry.runs),
|
|
19307
|
+
offsets: sharedUint32(entry.offsets),
|
|
19308
|
+
postings: sharedUint32(entry.postings)
|
|
19309
|
+
})),
|
|
18416
19310
|
surfaceTable: sharedSurfaceTable(payload)
|
|
18417
19311
|
};
|
|
18418
19312
|
}
|
|
@@ -18459,10 +19353,16 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
18459
19353
|
const docStore = createFrozenDocTable(source.idTable, lengthColumns, documentSource, tombstones);
|
|
18460
19354
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
18461
19355
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
19356
|
+
const sizeIndexes = /* @__PURE__ */ new Map();
|
|
19357
|
+
for (const entry of source.sizes) sizeIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
18462
19358
|
const booleanIndexes = /* @__PURE__ */ new Map();
|
|
18463
19359
|
for (const entry of source.boolean) booleanIndexes.set(entry.fieldPath, createFrozenBooleanReader(entry));
|
|
18464
19360
|
const enumIndexes = /* @__PURE__ */ new Map();
|
|
18465
19361
|
for (const entry of source.enums) enumIndexes.set(entry.fieldPath, createFrozenEnumReader(entry));
|
|
19362
|
+
const patternIndexes = /* @__PURE__ */ new Map();
|
|
19363
|
+
for (const entry of source.pattern) {
|
|
19364
|
+
patternIndexes.set(entry.fieldPath, requirePatternSearch(entry.fieldPath).readArrays(entry));
|
|
19365
|
+
}
|
|
18466
19366
|
const fieldNameTable = {
|
|
18467
19367
|
names: [...source.fieldNames],
|
|
18468
19368
|
indexMap: new Map(source.fieldNames.map((name, index) => [name, index]))
|
|
@@ -18480,6 +19380,8 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
18480
19380
|
booleanIndexes,
|
|
18481
19381
|
enumIndexes,
|
|
18482
19382
|
geoIndexes: buildGeoReaders(source.geo),
|
|
19383
|
+
patternIndexes,
|
|
19384
|
+
sizeIndexes,
|
|
18483
19385
|
fieldNameTable,
|
|
18484
19386
|
trackPositions: source.positionOffsets !== null,
|
|
18485
19387
|
flatSchemaCache: null,
|
|
@@ -18607,12 +19509,8 @@ function remapDocIds(inputs, pick) {
|
|
|
18607
19509
|
}
|
|
18608
19510
|
return out;
|
|
18609
19511
|
}
|
|
18610
|
-
function mergeNumeric(inputs) {
|
|
18611
|
-
const byFieldPath = rowsByFieldPath(
|
|
18612
|
-
inputs,
|
|
18613
|
-
(input) => input.segment.arrays.numeric,
|
|
18614
|
-
(entry, at) => [entry.values[at]]
|
|
18615
|
-
);
|
|
19512
|
+
function mergeNumeric(inputs, pick) {
|
|
19513
|
+
const byFieldPath = rowsByFieldPath(inputs, pick, (entry, at) => [entry.values[at]]);
|
|
18616
19514
|
const merged = [];
|
|
18617
19515
|
for (const [fieldPath, rows] of byFieldPath) {
|
|
18618
19516
|
rows.sort((a, b) => a.values[0] - b.values[0] || a.docId - b.docId);
|
|
@@ -18685,12 +19583,32 @@ function mergeEnums(inputs) {
|
|
|
18685
19583
|
}
|
|
18686
19584
|
return merged;
|
|
18687
19585
|
}
|
|
19586
|
+
function mergePattern(inputs) {
|
|
19587
|
+
const byFieldPath = /* @__PURE__ */ new Map();
|
|
19588
|
+
for (const input of inputs) {
|
|
19589
|
+
for (const arrays of input.segment.arrays.pattern) {
|
|
19590
|
+
let entries = byFieldPath.get(arrays.fieldPath);
|
|
19591
|
+
if (entries === void 0) {
|
|
19592
|
+
entries = [];
|
|
19593
|
+
byFieldPath.set(arrays.fieldPath, entries);
|
|
19594
|
+
}
|
|
19595
|
+
entries.push({ arrays, remap: input.remap });
|
|
19596
|
+
}
|
|
19597
|
+
}
|
|
19598
|
+
const merged = [];
|
|
19599
|
+
for (const [fieldPath, entries] of byFieldPath) {
|
|
19600
|
+
merged.push(requirePatternSearch(fieldPath).mergeArrays(fieldPath, entries));
|
|
19601
|
+
}
|
|
19602
|
+
return merged;
|
|
19603
|
+
}
|
|
18688
19604
|
function mergeFieldIndexes2(inputs) {
|
|
18689
19605
|
return {
|
|
18690
|
-
numeric: mergeNumeric(inputs),
|
|
19606
|
+
numeric: mergeNumeric(inputs, (input) => input.segment.arrays.numeric),
|
|
18691
19607
|
boolean: mergeBoolean(inputs),
|
|
18692
19608
|
enums: mergeEnums(inputs),
|
|
18693
|
-
geo: mergeGeo(inputs)
|
|
19609
|
+
geo: mergeGeo(inputs),
|
|
19610
|
+
pattern: mergePattern(inputs),
|
|
19611
|
+
sizes: mergeNumeric(inputs, (input) => input.segment.arrays.sizes)
|
|
18694
19612
|
};
|
|
18695
19613
|
}
|
|
18696
19614
|
|
|
@@ -19032,54 +19950,7 @@ var GrowableUint32 = class {
|
|
|
19032
19950
|
}
|
|
19033
19951
|
};
|
|
19034
19952
|
|
|
19035
|
-
// src/core/partition/segment-
|
|
19036
|
-
var TOKEN_TABLE_START = 1 << 12;
|
|
19037
|
-
var POSTING_LOG_START = 1 << 14;
|
|
19038
|
-
var POSITION_POOL_START = 1 << 16;
|
|
19039
|
-
var FIELD_SCRATCH_START = 1 << 10;
|
|
19040
|
-
function indexOf(indexes, fieldPath, create) {
|
|
19041
|
-
let index = indexes.get(fieldPath);
|
|
19042
|
-
if (index === void 0) {
|
|
19043
|
-
index = create();
|
|
19044
|
-
indexes.set(fieldPath, index);
|
|
19045
|
-
}
|
|
19046
|
-
return index;
|
|
19047
|
-
}
|
|
19048
|
-
function indexFilterValue(indexes, ordinal, fieldPath, fieldType, value) {
|
|
19049
|
-
if (fieldType === "number" || fieldType === "number[]") {
|
|
19050
|
-
const index = indexOf(indexes.numericIndexes, fieldPath, createNumericIndex);
|
|
19051
|
-
for (const entry of fieldType === "number" ? [value] : value) index.insert(ordinal, entry);
|
|
19052
|
-
} else if (fieldType === "boolean" || fieldType === "boolean[]") {
|
|
19053
|
-
const index = indexOf(indexes.booleanIndexes, fieldPath, createBooleanIndex);
|
|
19054
|
-
for (const entry of fieldType === "boolean" ? [value] : value)
|
|
19055
|
-
index.insert(ordinal, entry);
|
|
19056
|
-
} else if (fieldType === "enum" || fieldType === "enum[]") {
|
|
19057
|
-
const index = indexOf(indexes.enumIndexes, fieldPath, createEnumIndex);
|
|
19058
|
-
for (const entry of fieldType === "enum" ? [value] : value) index.insert(ordinal, entry);
|
|
19059
|
-
} else if (fieldType === "geopoint") {
|
|
19060
|
-
const point = value;
|
|
19061
|
-
indexOf(indexes.geoIndexes, fieldPath, createGeoIndex).insert(ordinal, point.lat, point.lon);
|
|
19062
|
-
}
|
|
19063
|
-
}
|
|
19064
|
-
function refuseRepeatedDocIds(documents) {
|
|
19065
|
-
const seen = /* @__PURE__ */ new Set();
|
|
19066
|
-
for (const { docId } of documents) {
|
|
19067
|
-
if (seen.has(docId)) {
|
|
19068
|
-
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists in partition 0`, {
|
|
19069
|
-
docId,
|
|
19070
|
-
partitionId: 0
|
|
19071
|
-
});
|
|
19072
|
-
}
|
|
19073
|
-
seen.add(docId);
|
|
19074
|
-
}
|
|
19075
|
-
}
|
|
19076
|
-
function validateAsAnInsertDoes(documents, schema, options) {
|
|
19077
|
-
if (options?.validate === false) return;
|
|
19078
|
-
for (const { document: document2 } of documents) {
|
|
19079
|
-
validateDocument(document2, schema, options?.patternValueLimit);
|
|
19080
|
-
if (options?.strict) validateDocumentStrict(document2, schema);
|
|
19081
|
-
}
|
|
19082
|
-
}
|
|
19953
|
+
// src/core/partition/segment-postings.ts
|
|
19083
19954
|
var TextPostings = class {
|
|
19084
19955
|
constructor(trackPositions) {
|
|
19085
19956
|
this.trackPositions = trackPositions;
|
|
@@ -19088,22 +19959,22 @@ var TextPostings = class {
|
|
|
19088
19959
|
fieldNames = [];
|
|
19089
19960
|
tokenIds = /* @__PURE__ */ new Map();
|
|
19090
19961
|
fieldNameIndexes = /* @__PURE__ */ new Map();
|
|
19091
|
-
postingsPerToken = new GrowableUint32(
|
|
19092
|
-
docFrequencyPerToken = new GrowableUint32(
|
|
19093
|
-
lastDocPerToken = new GrowableUint32(
|
|
19094
|
-
fieldStampPerToken = new GrowableUint32(
|
|
19095
|
-
fieldSlotPerToken = new GrowableUint32(
|
|
19096
|
-
logToken = new GrowableUint32(
|
|
19097
|
-
logDoc = new GrowableUint32(
|
|
19098
|
-
logFrequency = new GrowableUint32(
|
|
19099
|
-
logField = new GrowableUint32(
|
|
19100
|
-
logPositionStart = new GrowableUint32(
|
|
19101
|
-
positionPool = new GrowableUint32(
|
|
19102
|
-
slotToken = new GrowableUint32(
|
|
19103
|
-
slotCount = new GrowableUint32(
|
|
19104
|
-
slotCursor = new GrowableUint32(
|
|
19105
|
-
eventSlot = new GrowableUint32(
|
|
19106
|
-
eventPosition = new GrowableUint32(
|
|
19962
|
+
postingsPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19963
|
+
docFrequencyPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19964
|
+
lastDocPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19965
|
+
fieldStampPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19966
|
+
fieldSlotPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19967
|
+
logToken = new GrowableUint32(SEGMENT_POSTING_LOG_START);
|
|
19968
|
+
logDoc = new GrowableUint32(SEGMENT_POSTING_LOG_START);
|
|
19969
|
+
logFrequency = new GrowableUint32(SEGMENT_POSTING_LOG_START);
|
|
19970
|
+
logField = new GrowableUint32(SEGMENT_POSTING_LOG_START);
|
|
19971
|
+
logPositionStart = new GrowableUint32(SEGMENT_POSTING_LOG_START);
|
|
19972
|
+
positionPool = new GrowableUint32(SEGMENT_POSITION_POOL_START);
|
|
19973
|
+
slotToken = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
|
|
19974
|
+
slotCount = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
|
|
19975
|
+
slotCursor = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
|
|
19976
|
+
eventSlot = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
|
|
19977
|
+
eventPosition = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
|
|
19107
19978
|
fieldStamp = 0;
|
|
19108
19979
|
beginField() {
|
|
19109
19980
|
this.fieldStamp++;
|
|
@@ -19231,11 +20102,56 @@ var TextPostings = class {
|
|
|
19231
20102
|
};
|
|
19232
20103
|
}
|
|
19233
20104
|
};
|
|
20105
|
+
|
|
20106
|
+
// src/core/partition/segment-builder.ts
|
|
20107
|
+
function indexOf(indexes, fieldPath, create) {
|
|
20108
|
+
let index = indexes.get(fieldPath);
|
|
20109
|
+
if (index === void 0) {
|
|
20110
|
+
index = create();
|
|
20111
|
+
indexes.set(fieldPath, index);
|
|
20112
|
+
}
|
|
20113
|
+
return index;
|
|
20114
|
+
}
|
|
20115
|
+
function indexFilterValue(indexes, ordinal, fieldPath, fieldType, value) {
|
|
20116
|
+
if (fieldType === "number" || fieldType === "number[]") {
|
|
20117
|
+
const index = indexOf(indexes.numericIndexes, fieldPath, createNumericIndex);
|
|
20118
|
+
for (const entry of fieldType === "number" ? [value] : value) index.insert(ordinal, entry);
|
|
20119
|
+
} else if (fieldType === "boolean" || fieldType === "boolean[]") {
|
|
20120
|
+
const index = indexOf(indexes.booleanIndexes, fieldPath, createBooleanIndex);
|
|
20121
|
+
for (const entry of fieldType === "boolean" ? [value] : value)
|
|
20122
|
+
index.insert(ordinal, entry);
|
|
20123
|
+
} else if (fieldType === "enum" || fieldType === "enum[]") {
|
|
20124
|
+
const index = indexOf(indexes.enumIndexes, fieldPath, createEnumIndex);
|
|
20125
|
+
for (const entry of fieldType === "enum" ? [value] : value) index.insert(ordinal, entry);
|
|
20126
|
+
} else if (fieldType === "geopoint") {
|
|
20127
|
+
const point = value;
|
|
20128
|
+
indexOf(indexes.geoIndexes, fieldPath, createGeoIndex).insert(ordinal, point.lat, point.lon);
|
|
20129
|
+
}
|
|
20130
|
+
}
|
|
20131
|
+
function refuseRepeatedDocIds(documents) {
|
|
20132
|
+
const seen = /* @__PURE__ */ new Set();
|
|
20133
|
+
for (const { docId } of documents) {
|
|
20134
|
+
if (seen.has(docId)) {
|
|
20135
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists in partition 0`, {
|
|
20136
|
+
docId,
|
|
20137
|
+
partitionId: 0
|
|
20138
|
+
});
|
|
20139
|
+
}
|
|
20140
|
+
seen.add(docId);
|
|
20141
|
+
}
|
|
20142
|
+
}
|
|
20143
|
+
function validateAsAnInsertDoes(documents, schema, options) {
|
|
20144
|
+
if (options?.validate === false) return;
|
|
20145
|
+
for (const { document: document2 } of documents) {
|
|
20146
|
+
validateDocument(document2, schema, options?.patternValueLimit);
|
|
20147
|
+
if (options?.strict) validateDocumentStrict(document2, schema);
|
|
20148
|
+
}
|
|
20149
|
+
}
|
|
19234
20150
|
var SurfaceCounts = class {
|
|
19235
20151
|
ids = /* @__PURE__ */ new Map();
|
|
19236
20152
|
surfaces = [];
|
|
19237
20153
|
tokens = [];
|
|
19238
|
-
counts = new GrowableUint32(
|
|
20154
|
+
counts = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19239
20155
|
count(surface, token) {
|
|
19240
20156
|
if (surface === void 0 || surface.length === 0) return;
|
|
19241
20157
|
const known = this.ids.get(surface);
|
|
@@ -19295,7 +20211,9 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
19295
20211
|
numericIndexes: /* @__PURE__ */ new Map(),
|
|
19296
20212
|
booleanIndexes: /* @__PURE__ */ new Map(),
|
|
19297
20213
|
enumIndexes: /* @__PURE__ */ new Map(),
|
|
19298
|
-
geoIndexes: /* @__PURE__ */ new Map()
|
|
20214
|
+
geoIndexes: /* @__PURE__ */ new Map(),
|
|
20215
|
+
patternIndexes: /* @__PURE__ */ new Map(),
|
|
20216
|
+
sizeIndexes: /* @__PURE__ */ new Map()
|
|
19299
20217
|
};
|
|
19300
20218
|
let positionOffset = 0;
|
|
19301
20219
|
const addToken = (token, position, surface) => {
|
|
@@ -19304,9 +20222,11 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
19304
20222
|
};
|
|
19305
20223
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
19306
20224
|
const document2 = documents[ordinal].document;
|
|
19307
|
-
for (const { path: fieldPath, base: fieldType } of fields) {
|
|
20225
|
+
for (const { path: fieldPath, base: fieldType, pattern } of fields) {
|
|
19308
20226
|
const value = getNestedValue(document2, fieldPath);
|
|
19309
20227
|
if (value === void 0 || value === null) continue;
|
|
20228
|
+
recordValueSize(filters.sizeIndexes, ordinal, fieldPath, fieldType, value);
|
|
20229
|
+
if (pattern) patternIndexOf(filters.patternIndexes, fieldPath).add(ordinal, value);
|
|
19310
20230
|
if (!isWordIndexedBase(fieldType)) {
|
|
19311
20231
|
indexFilterValue(filters, ordinal, fieldPath, fieldType, value);
|
|
19312
20232
|
continue;
|
|
@@ -19831,6 +20751,7 @@ function createDirectExecutor(options) {
|
|
|
19831
20751
|
}
|
|
19832
20752
|
case "query": {
|
|
19833
20753
|
const entry = requireIndex(action.indexName);
|
|
20754
|
+
const patternWork = joinPatternWorkMeter(action.patternWork);
|
|
19834
20755
|
const result = await fanOutQuery(
|
|
19835
20756
|
entry.manager,
|
|
19836
20757
|
action.params,
|
|
@@ -19841,15 +20762,17 @@ function createDirectExecutor(options) {
|
|
|
19841
20762
|
partitionIds: action.partitionIds,
|
|
19842
20763
|
...action.globalStats !== void 0 ? { globalStats: sanitizeGlobalStats(action.globalStats) } : {}
|
|
19843
20764
|
},
|
|
19844
|
-
entry.searchOptions
|
|
20765
|
+
{ ...entry.searchOptions, patternWork }
|
|
19845
20766
|
);
|
|
20767
|
+
result.patternWorkSpent = patternWork.spent();
|
|
19846
20768
|
return result;
|
|
19847
20769
|
}
|
|
19848
20770
|
case "preflight": {
|
|
19849
20771
|
const entry = requireIndex(action.indexName);
|
|
20772
|
+
const searchOptions = { ...entry.searchOptions, patternWork: joinPatternWorkMeter(action.patternWork) };
|
|
19850
20773
|
if (countsWithoutScores(action.params)) {
|
|
19851
20774
|
const count = fanOutMatchCount(entry.manager, action.params, entry.language, entry.config.schema, {
|
|
19852
|
-
searchOptions
|
|
20775
|
+
searchOptions
|
|
19853
20776
|
});
|
|
19854
20777
|
return { count };
|
|
19855
20778
|
}
|
|
@@ -19859,7 +20782,7 @@ function createDirectExecutor(options) {
|
|
|
19859
20782
|
entry.language,
|
|
19860
20783
|
entry.config.schema,
|
|
19861
20784
|
{ scoringMode: entry.config.defaultScoring ?? "local" },
|
|
19862
|
-
|
|
20785
|
+
searchOptions
|
|
19863
20786
|
);
|
|
19864
20787
|
return { count: result.totalMatched };
|
|
19865
20788
|
}
|
|
@@ -25245,6 +26168,13 @@ function validateDocId(docId) {
|
|
|
25245
26168
|
if (docId.includes("\0")) {
|
|
25246
26169
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
|
|
25247
26170
|
}
|
|
26171
|
+
if (holdsUnpairedSurrogate(docId)) {
|
|
26172
|
+
throw new NarsilError(
|
|
26173
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
26174
|
+
"Document ID holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8",
|
|
26175
|
+
{ docId }
|
|
26176
|
+
);
|
|
26177
|
+
}
|
|
25248
26178
|
}
|
|
25249
26179
|
function validatePartitionConfig(partitions) {
|
|
25250
26180
|
const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
|
|
@@ -27707,6 +28637,64 @@ function stopIdleSweep(state) {
|
|
|
27707
28637
|
state.idleSweep = null;
|
|
27708
28638
|
}
|
|
27709
28639
|
|
|
28640
|
+
// src/core/pattern-index/matcher-budget.ts
|
|
28641
|
+
var POSITIONS_PER_WORD = 32;
|
|
28642
|
+
function positionWords(positions) {
|
|
28643
|
+
return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
|
|
28644
|
+
}
|
|
28645
|
+
function automatonBytes(positions) {
|
|
28646
|
+
return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES;
|
|
28647
|
+
}
|
|
28648
|
+
function smallestMatcherBytes(positions) {
|
|
28649
|
+
return automatonBytes(positions) + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
|
|
28650
|
+
}
|
|
28651
|
+
function widestPatternSplit(positions) {
|
|
28652
|
+
return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / smallestMatcherBytes(positions)));
|
|
28653
|
+
}
|
|
28654
|
+
|
|
28655
|
+
// src/core/pattern-index/query-positions.ts
|
|
28656
|
+
function isRecord5(value) {
|
|
28657
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
28658
|
+
}
|
|
28659
|
+
function textPositions(text, caseFold) {
|
|
28660
|
+
if (typeof text !== "string") return 0;
|
|
28661
|
+
return caseFold ? foldedCodePointCount(text) : codePointCount(text);
|
|
28662
|
+
}
|
|
28663
|
+
function larger(left, right) {
|
|
28664
|
+
if (left === null) return right;
|
|
28665
|
+
if (right === null) return left;
|
|
28666
|
+
return Math.max(left, right);
|
|
28667
|
+
}
|
|
28668
|
+
function largestTestOf(filter) {
|
|
28669
|
+
const caseFold = filter.caseFold === true;
|
|
28670
|
+
let largest = null;
|
|
28671
|
+
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
28672
|
+
const operand = filter[operator];
|
|
28673
|
+
if (operand === void 0) continue;
|
|
28674
|
+
const texts = Array.isArray(operand) ? operand : [operand];
|
|
28675
|
+
for (const text of texts) largest = larger(largest, textPositions(text, caseFold));
|
|
28676
|
+
}
|
|
28677
|
+
return largest;
|
|
28678
|
+
}
|
|
28679
|
+
function largestIn(expression, patternFields) {
|
|
28680
|
+
let largest = null;
|
|
28681
|
+
if (isRecord5(expression.fields)) {
|
|
28682
|
+
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
28683
|
+
if (patternFields.has(fieldPath) && isRecord5(filter)) largest = larger(largest, largestTestOf(filter));
|
|
28684
|
+
}
|
|
28685
|
+
}
|
|
28686
|
+
const nested = [...expression.and ?? [], ...expression.or ?? []];
|
|
28687
|
+
if (expression.not !== void 0) nested.push(expression.not);
|
|
28688
|
+
for (const child of nested) largest = larger(largest, largestIn(child, patternFields));
|
|
28689
|
+
return largest;
|
|
28690
|
+
}
|
|
28691
|
+
function largestPatternTest(expression, schema) {
|
|
28692
|
+
if (expression === void 0) return null;
|
|
28693
|
+
const patternFields = /* @__PURE__ */ new Set();
|
|
28694
|
+
for (const { path, pattern } of schemaFieldsOf(schema)) if (pattern) patternFields.add(path);
|
|
28695
|
+
return patternFields.size === 0 ? null : largestIn(expression, patternFields);
|
|
28696
|
+
}
|
|
28697
|
+
|
|
27710
28698
|
// src/engine/orchestration/search.ts
|
|
27711
28699
|
function partitionsPerLease(scope, leaseCount) {
|
|
27712
28700
|
const assignments = Array.from({ length: leaseCount }, () => []);
|
|
@@ -27718,15 +28706,18 @@ function partitionsPerLease(scope, leaseCount) {
|
|
|
27718
28706
|
function mergeWorkerResults(results, facetConfig) {
|
|
27719
28707
|
const merged = kWayMerge(results.map((result) => result.scored));
|
|
27720
28708
|
let totalMatched = 0;
|
|
28709
|
+
let patternWorkSpent = 0;
|
|
27721
28710
|
const workerFacets = [];
|
|
27722
28711
|
for (const result of results) {
|
|
27723
28712
|
totalMatched += result.totalMatched;
|
|
28713
|
+
patternWorkSpent += result.patternWorkSpent ?? 0;
|
|
27724
28714
|
if (result.facets !== void 0) workerFacets.push(result.facets);
|
|
27725
28715
|
}
|
|
27726
28716
|
return {
|
|
27727
28717
|
scored: merged,
|
|
27728
28718
|
totalMatched,
|
|
27729
|
-
facets: facetConfig !== void 0 && workerFacets.length > 0 ? mergeFacets(workerFacets, facetConfig) : void 0
|
|
28719
|
+
facets: facetConfig !== void 0 && workerFacets.length > 0 ? mergeFacets(workerFacets, facetConfig) : void 0,
|
|
28720
|
+
patternWorkSpent
|
|
27730
28721
|
};
|
|
27731
28722
|
}
|
|
27732
28723
|
function scoresPerPartition(state, indexName, params, globalStats) {
|
|
@@ -27734,26 +28725,45 @@ function scoresPerPartition(state, indexName, params, globalStats) {
|
|
|
27734
28725
|
if (mode === "local") return true;
|
|
27735
28726
|
return mode === "broadcast" && globalStats !== void 0;
|
|
27736
28727
|
}
|
|
27737
|
-
function queryAction(indexName, params, globalStats, partitionIds) {
|
|
28728
|
+
function queryAction(indexName, params, globalStats, partitionIds, patternWork) {
|
|
27738
28729
|
return {
|
|
27739
28730
|
type: "query",
|
|
27740
28731
|
indexName,
|
|
27741
28732
|
params,
|
|
27742
28733
|
requestId: createRequestId(),
|
|
28734
|
+
patternWork,
|
|
27743
28735
|
...partitionIds !== void 0 ? { partitionIds } : {},
|
|
27744
28736
|
...globalStats !== void 0 ? { globalStats } : {}
|
|
27745
28737
|
};
|
|
27746
28738
|
}
|
|
27747
|
-
async function runSplit(leases, scope, indexName, params, globalStats) {
|
|
28739
|
+
async function runSplit(leases, scope, indexName, params, globalStats, patternWork) {
|
|
27748
28740
|
const assignments = partitionsPerLease(scope, leases.length);
|
|
27749
28741
|
const workerParams = params.facets !== void 0 ? { ...params, facets: oversampledFacetConfig(params.facets) } : params;
|
|
27750
|
-
const
|
|
28742
|
+
const outcomes = await Promise.allSettled(
|
|
27751
28743
|
assignments.map(
|
|
27752
|
-
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds)).finally(() => leases[at].release())
|
|
28744
|
+
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds, patternWork)).finally(() => leases[at].release())
|
|
27753
28745
|
)
|
|
27754
28746
|
);
|
|
28747
|
+
const results = [];
|
|
28748
|
+
const failures = [];
|
|
28749
|
+
for (const outcome of outcomes) {
|
|
28750
|
+
if (outcome.status === "fulfilled") results.push(outcome.value);
|
|
28751
|
+
else failures.push(outcome.reason);
|
|
28752
|
+
}
|
|
28753
|
+
if (failures.length > 0) throw failures.find(isWorkCapError) ?? failures[0];
|
|
27755
28754
|
return mergeWorkerResults(results, params.facets);
|
|
27756
28755
|
}
|
|
28756
|
+
function isWorkCapError(err) {
|
|
28757
|
+
return err instanceof NarsilError && err.code === ErrorCodes.SEARCH_WORK_CAP_EXCEEDED;
|
|
28758
|
+
}
|
|
28759
|
+
function patternSplitWidth(state, indexName, params) {
|
|
28760
|
+
const schema = state.indexRegistry.get(indexName)?.config.schema;
|
|
28761
|
+
if (schema === void 0) return Number.POSITIVE_INFINITY;
|
|
28762
|
+
const largest = largestPatternTest(params.filters, schema);
|
|
28763
|
+
if (largest === null) return Number.POSITIVE_INFINITY;
|
|
28764
|
+
if (typeof SharedArrayBuffer !== "function") return 1;
|
|
28765
|
+
return widestPatternSplit(largest);
|
|
28766
|
+
}
|
|
27757
28767
|
function takeMainCopyTurn(state) {
|
|
27758
28768
|
if (state.mainCopyQueries === "none" || state.mainCopyTurnTaken) return false;
|
|
27759
28769
|
state.mainCopyTurnTaken = true;
|
|
@@ -27767,7 +28777,7 @@ function answersFasterOnMainCopy(state, pool, manager) {
|
|
|
27767
28777
|
if (manager.countDocuments() > MAIN_COPY_LONE_QUERY_DOCUMENTS) return false;
|
|
27768
28778
|
return takeMainCopyTurn(state);
|
|
27769
28779
|
}
|
|
27770
|
-
async function searchViaWorker(state, indexName, params, globalStats, partitionIds) {
|
|
28780
|
+
async function searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork) {
|
|
27771
28781
|
const pool = state.workerPool;
|
|
27772
28782
|
if (!pool) return null;
|
|
27773
28783
|
if (!state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName)) return null;
|
|
@@ -27775,19 +28785,30 @@ async function searchViaWorker(state, indexName, params, globalStats, partitionI
|
|
|
27775
28785
|
if (!manager) return null;
|
|
27776
28786
|
if (answersFasterOnMainCopy(state, pool, manager)) return null;
|
|
27777
28787
|
const scope = partitionIds ?? Array.from({ length: manager.partitionCount }, (_, partitionId) => partitionId);
|
|
27778
|
-
const idle = pool.leaseIdle(scope.length);
|
|
28788
|
+
const idle = pool.leaseIdle(Math.min(scope.length, patternSplitWidth(state, indexName, params)));
|
|
27779
28789
|
const leases = [];
|
|
28790
|
+
let fork = null;
|
|
27780
28791
|
try {
|
|
27781
28792
|
if (idle.length >= 2 && scoresPerPartition(state, indexName, params, globalStats)) {
|
|
27782
28793
|
leases.push(...idle);
|
|
27783
|
-
|
|
28794
|
+
fork = patternWork.fork(leases.length);
|
|
28795
|
+
const merged = await runSplit(leases, scope, indexName, params, globalStats, fork.shared);
|
|
28796
|
+
fork.settle(merged.patternWorkSpent ?? 0);
|
|
28797
|
+
return merged;
|
|
27784
28798
|
}
|
|
27785
28799
|
for (const lease2 of idle.slice(1)) lease2.release();
|
|
27786
28800
|
const lease = idle[0] ?? (takeMainCopyTurn(state) ? null : pool.leaseLeastBusy());
|
|
27787
28801
|
if (lease === null) return null;
|
|
27788
28802
|
leases.push(lease);
|
|
27789
|
-
|
|
28803
|
+
fork = patternWork.fork(1);
|
|
28804
|
+
const result = await lease.executor.execute(
|
|
28805
|
+
queryAction(indexName, params, globalStats, partitionIds, fork.shared)
|
|
28806
|
+
);
|
|
28807
|
+
fork.settle(result.patternWorkSpent ?? 0);
|
|
28808
|
+
return result;
|
|
27790
28809
|
} catch (err) {
|
|
28810
|
+
if (isWorkCapError(err)) throw err;
|
|
28811
|
+
fork?.settle(0);
|
|
27791
28812
|
console.warn("Worker search failed, falling back to local:", err);
|
|
27792
28813
|
return null;
|
|
27793
28814
|
} finally {
|
|
@@ -28016,7 +29037,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
28016
29037
|
segmentBuildConcurrency: (indexName) => segmentBuildConcurrency(state, indexName),
|
|
28017
29038
|
holdUnbroadcastSegments: (indexName, segmentIds) => holdUnbroadcastSegments(state, indexName, segmentIds),
|
|
28018
29039
|
releaseUnbroadcastSegments: (indexName, segmentIds) => releaseUnbroadcastSegments(state, indexName, segmentIds),
|
|
28019
|
-
searchViaWorker: (indexName, params, globalStats, partitionIds) => searchViaWorker(state, indexName, params, globalStats, partitionIds),
|
|
29040
|
+
searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork),
|
|
28020
29041
|
hasWorkerPool: () => state.workerPool !== null,
|
|
28021
29042
|
mainCopyQueries: () => state.mainCopyQueries ?? "lone",
|
|
28022
29043
|
shareMainThread: () => {
|
|
@@ -28305,7 +29326,8 @@ function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
28305
29326
|
analysisRebuild,
|
|
28306
29327
|
indexState,
|
|
28307
29328
|
mutationCtx,
|
|
28308
|
-
rebalanceCtx
|
|
29329
|
+
rebalanceCtx,
|
|
29330
|
+
patternWorkCap: config?.patternWorkCap ?? DEFAULT_PATTERN_WORK_CAP
|
|
28309
29331
|
};
|
|
28310
29332
|
}
|
|
28311
29333
|
|
|
@@ -28404,6 +29426,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
28404
29426
|
}
|
|
28405
29427
|
const schema = validateSchema(requestedConfig.schema);
|
|
28406
29428
|
const indexConfig = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
|
|
29429
|
+
requirePatternSearchFor(schema);
|
|
28407
29430
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
28408
29431
|
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
28409
29432
|
validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
|
|
@@ -28434,10 +29457,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
28434
29457
|
const fields = indexConfig.embedding.fields;
|
|
28435
29458
|
if (typeof configuredAdapter === "string") {
|
|
28436
29459
|
embeddingAdapterName = configuredAdapter;
|
|
28437
|
-
const
|
|
28438
|
-
if (
|
|
29460
|
+
const registered2 = core.embeddingAdapters.get(configuredAdapter);
|
|
29461
|
+
if (registered2 !== void 0) {
|
|
28439
29462
|
resolvedEmbeddingAdapter = validateEmbeddingConfig(
|
|
28440
|
-
{ fields, adapter:
|
|
29463
|
+
{ fields, adapter: registered2 },
|
|
28441
29464
|
indexConfig.schema,
|
|
28442
29465
|
config?.embedding
|
|
28443
29466
|
);
|
|
@@ -28791,9 +29814,30 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
28791
29814
|
|
|
28792
29815
|
// src/engine/mutations/insert-admission.ts
|
|
28793
29816
|
function providedDocId(document2) {
|
|
29817
|
+
if (document2 === null || typeof document2 !== "object") return void 0;
|
|
28794
29818
|
const id = document2.id;
|
|
28795
29819
|
return typeof id === "string" && id.length > 0 ? id : void 0;
|
|
28796
29820
|
}
|
|
29821
|
+
function requireDocumentObject(document2) {
|
|
29822
|
+
if (document2 !== null && typeof document2 === "object" && !Array.isArray(document2)) return;
|
|
29823
|
+
const received = Array.isArray(document2) ? "a list" : document2 === null ? "null" : `a ${typeof document2}`;
|
|
29824
|
+
throw new NarsilError(
|
|
29825
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
29826
|
+
`Each document must be an object, while this entry is ${received}`,
|
|
29827
|
+
{
|
|
29828
|
+
received
|
|
29829
|
+
}
|
|
29830
|
+
);
|
|
29831
|
+
}
|
|
29832
|
+
function requireMatchingDocId(docId, document2) {
|
|
29833
|
+
const embeddedDocId = providedDocId(document2);
|
|
29834
|
+
if (embeddedDocId === void 0 || embeddedDocId === docId) return;
|
|
29835
|
+
throw new NarsilError(
|
|
29836
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
29837
|
+
`The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
|
|
29838
|
+
{ docId, documentId: embeddedDocId }
|
|
29839
|
+
);
|
|
29840
|
+
}
|
|
28797
29841
|
function asBatchInsertError(err) {
|
|
28798
29842
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
28799
29843
|
}
|
|
@@ -28904,15 +29948,8 @@ async function embedChunkDocuments(entry, documents, chunkStart, chunkEnd, signa
|
|
|
28904
29948
|
async function insertDocument(ctx, indexName, document2, docId, options) {
|
|
28905
29949
|
ctx.guardShutdown();
|
|
28906
29950
|
const entry = ctx.requireIndex(indexName);
|
|
28907
|
-
|
|
28908
|
-
|
|
28909
|
-
throw new NarsilError(
|
|
28910
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
28911
|
-
`The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
|
|
28912
|
-
{ docId, documentId: embeddedDocId }
|
|
28913
|
-
);
|
|
28914
|
-
}
|
|
28915
|
-
const resolvedDocId = docId ?? embeddedDocId ?? ctx.idGenerator();
|
|
29951
|
+
if (docId !== void 0) requireMatchingDocId(docId, document2);
|
|
29952
|
+
const resolvedDocId = docId ?? providedDocId(document2) ?? ctx.idGenerator();
|
|
28916
29953
|
validateDocId(resolvedDocId);
|
|
28917
29954
|
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document: document2 });
|
|
28918
29955
|
if (entry.config.required && entry.config.required.length > 0) {
|
|
@@ -29149,6 +30186,7 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
|
|
|
29149
30186
|
if (chunkFailedIndexes.has(i)) continue;
|
|
29150
30187
|
const docId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
29151
30188
|
try {
|
|
30189
|
+
requireDocumentObject(documents[i]);
|
|
29152
30190
|
validateDocId(docId);
|
|
29153
30191
|
if (hasBeforeHook) {
|
|
29154
30192
|
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents[i] });
|
|
@@ -29647,6 +30685,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
29647
30685
|
if (chunkFailedIndexes.has(i)) continue;
|
|
29648
30686
|
const batchDocId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
29649
30687
|
try {
|
|
30688
|
+
requireDocumentObject(documents[i]);
|
|
29650
30689
|
validateDocId(batchDocId);
|
|
29651
30690
|
if (hasBeforeHook) {
|
|
29652
30691
|
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents[i] });
|
|
@@ -29869,6 +30908,7 @@ async function settleUpdate(ctx, indexName, prepared) {
|
|
|
29869
30908
|
async function prepareUpdate(ctx, indexName, entry, docId, document2) {
|
|
29870
30909
|
ctx.guardShutdown();
|
|
29871
30910
|
validateDocId(docId);
|
|
30911
|
+
requireMatchingDocId(docId, document2);
|
|
29872
30912
|
const manager = ctx.requireManager(indexName);
|
|
29873
30913
|
if (ctx.pluginRegistry.hasHooks("beforeUpdate")) {
|
|
29874
30914
|
await ctx.pluginRegistry.runHook("beforeUpdate", {
|
|
@@ -29978,6 +31018,22 @@ async function updateDocument(ctx, indexName, docId, document2, options) {
|
|
|
29978
31018
|
ctx.checkHeapPressure(indexName);
|
|
29979
31019
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
29980
31020
|
}
|
|
31021
|
+
function reportedDocIdOf(update) {
|
|
31022
|
+
const docId = update !== null && typeof update === "object" ? update.docId : void 0;
|
|
31023
|
+
return typeof docId === "string" ? docId : "";
|
|
31024
|
+
}
|
|
31025
|
+
function requireUpdateEntry(update) {
|
|
31026
|
+
if (update !== null && typeof update === "object" && !Array.isArray(update)) {
|
|
31027
|
+
const { docId, document: document2 } = update;
|
|
31028
|
+
if (typeof docId === "string" && document2 !== null && typeof document2 === "object" && !Array.isArray(document2)) {
|
|
31029
|
+
return { document: document2 };
|
|
31030
|
+
}
|
|
31031
|
+
}
|
|
31032
|
+
throw new NarsilError(
|
|
31033
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
31034
|
+
"Each entry of a batch update must be an object whose docId is a string and whose document is an object"
|
|
31035
|
+
);
|
|
31036
|
+
}
|
|
29981
31037
|
async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
29982
31038
|
ctx.guardShutdown();
|
|
29983
31039
|
const entry = ctx.requireIndex(indexName);
|
|
@@ -29990,10 +31046,13 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
29990
31046
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
29991
31047
|
const prepared = [];
|
|
29992
31048
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
31049
|
+
const update = updates[i];
|
|
31050
|
+
const docId = reportedDocIdOf(update);
|
|
29993
31051
|
try {
|
|
29994
|
-
|
|
31052
|
+
const { document: document2 } = requireUpdateEntry(update);
|
|
31053
|
+
prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document2));
|
|
29995
31054
|
} catch (err) {
|
|
29996
|
-
failed.push({ docId
|
|
31055
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
29997
31056
|
}
|
|
29998
31057
|
}
|
|
29999
31058
|
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
@@ -30978,6 +32037,9 @@ function requireValidTypoTolerance(params) {
|
|
|
30978
32037
|
}
|
|
30979
32038
|
|
|
30980
32039
|
// src/engine/query/shared.ts
|
|
32040
|
+
function withPatternWork(context) {
|
|
32041
|
+
return { ...context, patternWork: createPatternWorkMeter(context.patternWorkCap) };
|
|
32042
|
+
}
|
|
30981
32043
|
function partitionsFor(manager, partitionIds) {
|
|
30982
32044
|
return partitionsIn(manager, partitionIds);
|
|
30983
32045
|
}
|
|
@@ -31024,18 +32086,19 @@ function broadcastStatsForWorker(params, context, scoring) {
|
|
|
31024
32086
|
}
|
|
31025
32087
|
return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, context.manager.analysis);
|
|
31026
32088
|
}
|
|
31027
|
-
function searchOptionsFor(manager) {
|
|
32089
|
+
function searchOptionsFor(manager, patternWork) {
|
|
31028
32090
|
return {
|
|
31029
32091
|
bm25Params: manager.config.bm25,
|
|
31030
32092
|
stopWords: manager.analysis.stopWords,
|
|
31031
|
-
customTokenizer: manager.analysis.customTokenizer
|
|
32093
|
+
customTokenizer: manager.analysis.customTokenizer,
|
|
32094
|
+
patternWork
|
|
31032
32095
|
};
|
|
31033
32096
|
}
|
|
31034
|
-
function collectFilterDocIds(manager, params, schema, partitionIds) {
|
|
32097
|
+
function collectFilterDocIds(manager, params, schema, partitionIds, patternWork) {
|
|
31035
32098
|
const filterDocIds = /* @__PURE__ */ new Set();
|
|
31036
32099
|
if (!params.filters) return filterDocIds;
|
|
31037
32100
|
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
31038
|
-
const partitionFiltered = partition.applyFilters(params.filters, schema);
|
|
32101
|
+
const partitionFiltered = partition.applyFilters(params.filters, schema, patternWork);
|
|
31039
32102
|
for (const docId of partitionFiltered) {
|
|
31040
32103
|
filterDocIds.add(docId);
|
|
31041
32104
|
}
|
|
@@ -31070,11 +32133,11 @@ function requireValidQueryOptions(params, config) {
|
|
|
31070
32133
|
const strict = config.strict === true;
|
|
31071
32134
|
requireValidTypoTolerance(params);
|
|
31072
32135
|
requireSortableFields(params.sort, config.schema, strict);
|
|
31073
|
-
requireValidQueryFilter(params.filters, config.schema
|
|
32136
|
+
requireValidQueryFilter(params.filters, config.schema);
|
|
31074
32137
|
requireUsableQueryFields(params, config.schema, strict);
|
|
31075
32138
|
}
|
|
31076
|
-
function requireValidQueryFilter(filters, schema
|
|
31077
|
-
if (filters !== void 0) requireValidFilter(filters, flattenSchema(schema)
|
|
32139
|
+
function requireValidQueryFilter(filters, schema) {
|
|
32140
|
+
if (filters !== void 0) requireValidFilter(filters, flattenSchema(schema));
|
|
31078
32141
|
}
|
|
31079
32142
|
function requireVectorSearchable(params, context, needsVector) {
|
|
31080
32143
|
if (!needsVector) return;
|
|
@@ -31110,8 +32173,9 @@ function firstIdAfter(sorted, anchor) {
|
|
|
31110
32173
|
}
|
|
31111
32174
|
return low;
|
|
31112
32175
|
}
|
|
31113
|
-
function collectFilterMatches(partitions, filters, schema) {
|
|
31114
|
-
const
|
|
32176
|
+
function collectFilterMatches(partitions, filters, schema, patternWorkCap) {
|
|
32177
|
+
const meter = createPatternWorkMeter(patternWorkCap);
|
|
32178
|
+
const perPartition = partitions.map((partition) => partition.filterMatches(filters, schema, meter));
|
|
31115
32179
|
let total = 0;
|
|
31116
32180
|
for (const matches of perPartition) total += matches.count;
|
|
31117
32181
|
return { matchesFor: (partitionIndex) => perPartition[partitionIndex] ?? null, total };
|
|
@@ -31189,7 +32253,7 @@ function executeListDocuments(params, context) {
|
|
|
31189
32253
|
const limit = clampListLimit(params.limit);
|
|
31190
32254
|
const signature = sortSignatureOf(params.sort);
|
|
31191
32255
|
requireSortableFields(params.sort, schema, context.strict === true);
|
|
31192
|
-
requireValidQueryFilter(params.filters, schema
|
|
32256
|
+
requireValidQueryFilter(params.filters, schema);
|
|
31193
32257
|
const binding = listBindingOf(params.filters);
|
|
31194
32258
|
let cursor = null;
|
|
31195
32259
|
if (params.cursor !== void 0) {
|
|
@@ -31197,7 +32261,7 @@ function executeListDocuments(params, context) {
|
|
|
31197
32261
|
requireMatchingCursor(cursor, params.cursor, signature, false, binding);
|
|
31198
32262
|
}
|
|
31199
32263
|
const partitions = context.partitionIds === void 0 ? manager.getAllPartitions() : context.partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
|
|
31200
|
-
const filtered = params.filters === void 0 ? null : collectFilterMatches(partitions, params.filters, schema);
|
|
32264
|
+
const filtered = params.filters === void 0 ? null : collectFilterMatches(partitions, params.filters, schema, context.patternWorkCap);
|
|
31201
32265
|
const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
|
|
31202
32266
|
const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
|
|
31203
32267
|
const projection = resolveProjection(params.document);
|
|
@@ -31564,7 +32628,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
31564
32628
|
const flatSchema = flattenSchema(schema);
|
|
31565
32629
|
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
31566
32630
|
const modes = sortModesOf(normalized);
|
|
31567
|
-
const options = searchOptionsFor(manager);
|
|
32631
|
+
const options = searchOptionsFor(manager, context.patternWork);
|
|
31568
32632
|
const partitions = partitionsFor(manager, context.partitionIds);
|
|
31569
32633
|
const partitionLimit = offset + limit + 1;
|
|
31570
32634
|
const merged = [];
|
|
@@ -31765,7 +32829,7 @@ async function executeVectorSearch(params, context, limit, offset, cursorDepth =
|
|
|
31765
32829
|
let filterDocIds;
|
|
31766
32830
|
let filterPartitions;
|
|
31767
32831
|
if (params.filters) {
|
|
31768
|
-
filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds);
|
|
32832
|
+
filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds, context.patternWork);
|
|
31769
32833
|
if (filterDocIds.size === 0) {
|
|
31770
32834
|
return { scored: [], totalMatched: 0 };
|
|
31771
32835
|
}
|
|
@@ -31793,11 +32857,19 @@ async function textLeg(textOnlyParams, context) {
|
|
|
31793
32857
|
indexName,
|
|
31794
32858
|
textOnlyParams,
|
|
31795
32859
|
broadcastStatsForWorker(textOnlyParams, context, scoring),
|
|
31796
|
-
context.partitionIds
|
|
32860
|
+
context.partitionIds,
|
|
32861
|
+
context.patternWork
|
|
31797
32862
|
);
|
|
31798
32863
|
if (viaWorker) return viaWorker;
|
|
31799
32864
|
}
|
|
31800
|
-
return fanOutQuery(
|
|
32865
|
+
return fanOutQuery(
|
|
32866
|
+
manager,
|
|
32867
|
+
textOnlyParams,
|
|
32868
|
+
language,
|
|
32869
|
+
config.schema,
|
|
32870
|
+
scoring,
|
|
32871
|
+
searchOptionsFor(manager, context.patternWork)
|
|
32872
|
+
);
|
|
31801
32873
|
}
|
|
31802
32874
|
async function vectorLeg(vectorConfig, queryVector, context, filterDocIds, k) {
|
|
31803
32875
|
const { manager } = context;
|
|
@@ -31823,7 +32895,7 @@ async function executeHybridSearch(params, context, limit, offset, cursorDepth =
|
|
|
31823
32895
|
const fusion = resolveHybridFusion(params.hybrid);
|
|
31824
32896
|
let filterDocIds;
|
|
31825
32897
|
if (params.filters) {
|
|
31826
|
-
filterDocIds = collectFilterDocIds(manager, params, config.schema, context.partitionIds);
|
|
32898
|
+
filterDocIds = collectFilterDocIds(manager, params, config.schema, context.partitionIds, context.patternWork);
|
|
31827
32899
|
if (filterDocIds.size === 0) {
|
|
31828
32900
|
return { scored: [], totalMatched: 0 };
|
|
31829
32901
|
}
|
|
@@ -31867,7 +32939,8 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
31867
32939
|
requireMatchingCursor(decoded, cursor, signature, true, binding);
|
|
31868
32940
|
return decoded.depth ?? 0;
|
|
31869
32941
|
}
|
|
31870
|
-
async function executeQuery(params,
|
|
32942
|
+
async function executeQuery(params, callerContext) {
|
|
32943
|
+
const context = withPatternWork(callerContext);
|
|
31871
32944
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
31872
32945
|
const startTime = now();
|
|
31873
32946
|
const limit = clampLimit(params.limit);
|
|
@@ -31913,11 +32986,24 @@ async function executeQuery(params, context) {
|
|
|
31913
32986
|
fanOutResult = await executeHybridSearch(params, context, limit, offset, depth);
|
|
31914
32987
|
} else {
|
|
31915
32988
|
const scoring = scoringConfigFor(params, context);
|
|
31916
|
-
const workerResult = workerSearch ? await workerSearch(
|
|
32989
|
+
const workerResult = workerSearch ? await workerSearch(
|
|
32990
|
+
indexName,
|
|
32991
|
+
params,
|
|
32992
|
+
broadcastStatsForWorker(params, context, scoring),
|
|
32993
|
+
context.partitionIds,
|
|
32994
|
+
context.patternWork
|
|
32995
|
+
) : null;
|
|
31917
32996
|
if (workerResult) {
|
|
31918
32997
|
fanOutResult = workerResult;
|
|
31919
32998
|
} else {
|
|
31920
|
-
fanOutResult = await fanOutQuery(
|
|
32999
|
+
fanOutResult = await fanOutQuery(
|
|
33000
|
+
manager,
|
|
33001
|
+
params,
|
|
33002
|
+
language,
|
|
33003
|
+
config.schema,
|
|
33004
|
+
scoring,
|
|
33005
|
+
searchOptionsFor(manager, context.patternWork)
|
|
33006
|
+
);
|
|
31921
33007
|
}
|
|
31922
33008
|
}
|
|
31923
33009
|
const needsFullHits = params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
@@ -32037,7 +33123,8 @@ async function executeQuery(params, context) {
|
|
|
32037
33123
|
coverage: coverageFor(manager, context.partitionIds)
|
|
32038
33124
|
};
|
|
32039
33125
|
}
|
|
32040
|
-
async function executePreflight(params,
|
|
33126
|
+
async function executePreflight(params, callerContext) {
|
|
33127
|
+
const context = withPatternWork(callerContext);
|
|
32041
33128
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
32042
33129
|
const startTime = now();
|
|
32043
33130
|
requireKnownMode(params);
|
|
@@ -32063,12 +33150,18 @@ async function executePreflight(params, context) {
|
|
|
32063
33150
|
countExact = result.matchedExact !== false;
|
|
32064
33151
|
} else if (countsWithoutScores(params)) {
|
|
32065
33152
|
totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
|
|
32066
|
-
searchOptions: searchOptionsFor(manager),
|
|
33153
|
+
searchOptions: searchOptionsFor(manager, context.patternWork),
|
|
32067
33154
|
partitionIds: context.partitionIds
|
|
32068
33155
|
});
|
|
32069
33156
|
} else {
|
|
32070
33157
|
const scoring = scoringConfigFor(params, context);
|
|
32071
|
-
const workerResult = workerSearch ? await workerSearch(
|
|
33158
|
+
const workerResult = workerSearch ? await workerSearch(
|
|
33159
|
+
indexName,
|
|
33160
|
+
params,
|
|
33161
|
+
broadcastStatsForWorker(params, context, scoring),
|
|
33162
|
+
context.partitionIds,
|
|
33163
|
+
context.patternWork
|
|
33164
|
+
) : null;
|
|
32072
33165
|
if (workerResult) {
|
|
32073
33166
|
totalMatched = workerResult.totalMatched;
|
|
32074
33167
|
} else {
|
|
@@ -32078,7 +33171,7 @@ async function executePreflight(params, context) {
|
|
|
32078
33171
|
language,
|
|
32079
33172
|
config.schema,
|
|
32080
33173
|
scoring,
|
|
32081
|
-
searchOptionsFor(manager)
|
|
33174
|
+
searchOptionsFor(manager, context.patternWork)
|
|
32082
33175
|
);
|
|
32083
33176
|
totalMatched = fanOutResult.totalMatched;
|
|
32084
33177
|
}
|
|
@@ -32226,7 +33319,8 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
32226
33319
|
indexName,
|
|
32227
33320
|
broadcastStats: broadcastStatsFor(core, options),
|
|
32228
33321
|
partitionIds: options?.partitionIds,
|
|
32229
|
-
cursorBinding: queryBindingOf(params)
|
|
33322
|
+
cursorBinding: queryBindingOf(params),
|
|
33323
|
+
patternWorkCap: core.patternWorkCap
|
|
32230
33324
|
});
|
|
32231
33325
|
if (core.pluginRegistry.hasHooks("afterSearch")) {
|
|
32232
33326
|
const observed = { ...result, hits: [...result.hits] };
|
|
@@ -32267,7 +33361,8 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
32267
33361
|
indexName,
|
|
32268
33362
|
broadcastStats: broadcastStatsFor(core, options),
|
|
32269
33363
|
partitionIds: options?.partitionIds,
|
|
32270
|
-
cursorBinding: queryBindingOf(params)
|
|
33364
|
+
cursorBinding: queryBindingOf(params),
|
|
33365
|
+
patternWorkCap: core.patternWorkCap
|
|
32271
33366
|
});
|
|
32272
33367
|
if (core.analysisRebuild.isStale(indexName)) {
|
|
32273
33368
|
result.analysisStale = true;
|
|
@@ -32304,7 +33399,8 @@ async function runEngineListDocuments(core, indexName, params, partitionIds) {
|
|
|
32304
33399
|
manager: core.requireManager(indexName),
|
|
32305
33400
|
schema: entry.config.schema,
|
|
32306
33401
|
strict: entry.config.strict,
|
|
32307
|
-
partitionIds
|
|
33402
|
+
partitionIds,
|
|
33403
|
+
patternWorkCap: core.patternWorkCap
|
|
32308
33404
|
});
|
|
32309
33405
|
} finally {
|
|
32310
33406
|
release();
|
|
@@ -32624,4 +33720,4 @@ async function createNarsil(config) {
|
|
|
32624
33720
|
// src/index.ts
|
|
32625
33721
|
var VERSION = "0.1.0";
|
|
32626
33722
|
|
|
32627
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNarsilError, isSimdAvailable, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache };
|
|
33723
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNarsilError, isSimdAvailable, registerLanguage, registerPatternSearch, registerStopWords, registerTokenizer, resetNormalizationCache };
|