@delali/narsil 0.3.1-next.70 → 0.3.1-next.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-Dt-7B4Wz.d.ts → capabilities-D6gjjQyA.d.ts} +2 -2
- package/dist/{chunk-P4DSQ6M4.mjs → chunk-2X2ZQOAB.mjs} +69 -28
- package/dist/chunk-3HQPUCCT.mjs +785 -0
- package/dist/{chunk-VNMTB3KC.mjs → chunk-7QMTA7AA.mjs} +194 -59
- package/dist/{chunk-FVLQ63B7.mjs → chunk-CVPA5Q7R.mjs} +24 -3
- package/dist/{chunk-2STVUTNA.mjs → chunk-EDMCZLSJ.mjs} +4 -3
- package/dist/{chunk-M76V3OGJ.mjs → chunk-HVJYTKK7.mjs} +940 -647
- package/dist/{chunk-HWQLTLNG.mjs → chunk-KZTI25D3.mjs} +1 -1
- package/dist/{chunk-YJDQ7NLA.mjs → chunk-LCCL6EEH.mjs} +3 -3
- package/dist/{chunk-OI42GWLR.mjs → chunk-LJYTW7GT.mjs} +57 -11
- package/dist/{chunk-653BNMAF.mjs → chunk-M6XF5HDE.mjs} +57 -17
- package/dist/{chunk-YNVIUGF5.mjs → chunk-P2ZZN32C.mjs} +1 -1
- package/dist/{chunk-LMO5TJUS.mjs → chunk-QR4OKRVY.mjs} +637 -196
- package/dist/{chunk-USKFU7J5.mjs → chunk-RV77FR2F.mjs} +1 -0
- package/dist/{chunk-GIVAGD3Q.mjs → chunk-XZ4PAREY.mjs} +1 -1
- package/dist/{chunk-3JGWULDF.mjs → chunk-Y52WZWAU.mjs} +5 -5
- package/dist/chunk-ZKRV5M2P.mjs +75 -0
- package/dist/{client-COek5GXK.d.ts → client-C2TiW29y.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +1 -0
- package/dist/{config-rBpbgTw8.d.ts → config-BLguB-om.d.ts} +2 -2
- package/dist/{config-Dfanyqqt.d.ts → config-C7Ip8Ife.d.ts} +17 -2
- package/dist/direct-executor-SVCHXR74.mjs +14 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +58 -35
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-CXKfmK7I.d.ts → engine-Dm4446j1.d.ts} +104 -62
- package/dist/index.browser.mjs +2222 -1100
- package/dist/index.d.ts +27 -8
- package/dist/index.mjs +14 -12
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-C7UMCMH3.d.ts → memory-CKHTU1CL.d.ts} +66 -25
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +7 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +17 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -12
- package/dist/react.d.ts +277 -182
- package/dist/react.mjs +190 -45
- package/dist/{schema-Bgzr4g7b.d.ts → schema-BBOCdhND.d.ts} +9 -7
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +13 -12
- package/dist/{tasks-BZt_zes6.d.ts → tasks-BECd0b__.d.ts} +1 -1
- package/dist/{thread-JEVAYETR.mjs → thread-IBSIZZ6W.mjs} +22 -13
- package/dist/{types-gOUqhs2F.d.ts → types-BiUmaiWT.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +4 -4
- package/package.json +15 -7
- package/dist/direct-executor-GYJEKNV5.mjs +0 -12
package/dist/index.browser.mjs
CHANGED
|
@@ -33,6 +33,7 @@ var ErrorCodes = {
|
|
|
33
33
|
SEARCH_INVALID_MODE: "SEARCH_INVALID_MODE",
|
|
34
34
|
SEARCH_INVALID_CURSOR: "SEARCH_INVALID_CURSOR",
|
|
35
35
|
SEARCH_RESULT_WINDOW_EXCEEDED: "SEARCH_RESULT_WINDOW_EXCEEDED",
|
|
36
|
+
SEARCH_WORK_CAP_EXCEEDED: "SEARCH_WORK_CAP_EXCEEDED",
|
|
36
37
|
LANGUAGE_NOT_SUPPORTED: "LANGUAGE_NOT_SUPPORTED",
|
|
37
38
|
ENVELOPE_VERSION_MISMATCH: "ENVELOPE_VERSION_MISMATCH",
|
|
38
39
|
ENVELOPE_INVALID_MAGIC: "ENVELOPE_INVALID_MAGIC",
|
|
@@ -209,6 +210,288 @@ function resolveIndexAnalysis(config) {
|
|
|
209
210
|
};
|
|
210
211
|
}
|
|
211
212
|
|
|
213
|
+
// src/core/pattern-index/platform.browser.ts
|
|
214
|
+
function platformPatternSearch() {
|
|
215
|
+
return null;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// src/schema/constants.ts
|
|
219
|
+
var MAX_NESTING_DEPTH = 4;
|
|
220
|
+
var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
221
|
+
var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
|
|
222
|
+
var MIN_PATTERN_VALUE_LIMIT = 1;
|
|
223
|
+
var MAX_PATTERN_VALUE_LIMIT = 65536;
|
|
224
|
+
|
|
225
|
+
// src/schema/validator/shared.ts
|
|
226
|
+
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
227
|
+
function isGeopointOrVectorType(fieldType) {
|
|
228
|
+
return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
|
|
229
|
+
}
|
|
230
|
+
var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
|
|
231
|
+
var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
|
|
232
|
+
var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
233
|
+
function isPlainObject(value) {
|
|
234
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// src/schema/validator/field-type.ts
|
|
238
|
+
var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
|
|
239
|
+
var OPTIONS_BY_BASE = {
|
|
240
|
+
string: FIELD_OPTION_ORDER,
|
|
241
|
+
"string[]": FIELD_OPTION_ORDER,
|
|
242
|
+
verbatim: ["sortable"],
|
|
243
|
+
"verbatim[]": ["sortable"],
|
|
244
|
+
number: [],
|
|
245
|
+
"number[]": [],
|
|
246
|
+
boolean: [],
|
|
247
|
+
"boolean[]": [],
|
|
248
|
+
enum: [],
|
|
249
|
+
"enum[]": [],
|
|
250
|
+
geopoint: [],
|
|
251
|
+
vector: []
|
|
252
|
+
};
|
|
253
|
+
var TYPE_SEPARATOR = ":";
|
|
254
|
+
function isNamedBase(name) {
|
|
255
|
+
return name !== "vector" && Object.hasOwn(OPTIONS_BY_BASE, name);
|
|
256
|
+
}
|
|
257
|
+
function describeAllowedOptions(baseName, allowed) {
|
|
258
|
+
if (allowed.length === 0) return `the "${baseName}" type has no options`;
|
|
259
|
+
const quoted = allowed.map((option) => `"${option}"`);
|
|
260
|
+
if (quoted.length === 1) return `the only option of the "${baseName}" type is ${quoted[0]}`;
|
|
261
|
+
return `the options of the "${baseName}" type are ${quoted.slice(0, -1).join(", ")}, and ${quoted[quoted.length - 1]}`;
|
|
262
|
+
}
|
|
263
|
+
function invalidType(path, type, message) {
|
|
264
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, message, { field: path, type });
|
|
265
|
+
}
|
|
266
|
+
function parseBase(baseName, path, type) {
|
|
267
|
+
if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
|
|
268
|
+
const vectorMatch = VECTOR_PATTERN.exec(baseName);
|
|
269
|
+
if (vectorMatch === null) {
|
|
270
|
+
invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
|
|
271
|
+
}
|
|
272
|
+
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
273
|
+
if (dimension <= 0) {
|
|
274
|
+
throw new NarsilError(
|
|
275
|
+
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
276
|
+
`Vector field "${path}" must have a positive dimension, got ${dimension}`,
|
|
277
|
+
{ field: path, dimension }
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
return { base: "vector", dimension };
|
|
281
|
+
}
|
|
282
|
+
function parseFieldType(type, path) {
|
|
283
|
+
const separatorIndex = type.indexOf(TYPE_SEPARATOR);
|
|
284
|
+
const baseName = separatorIndex === -1 ? type : type.slice(0, separatorIndex);
|
|
285
|
+
const { base, dimension } = parseBase(baseName, path, type);
|
|
286
|
+
const allowed = OPTIONS_BY_BASE[base];
|
|
287
|
+
const options = /* @__PURE__ */ new Set();
|
|
288
|
+
if (separatorIndex !== -1) {
|
|
289
|
+
for (const option of type.slice(separatorIndex + 1).split(TYPE_SEPARATOR)) {
|
|
290
|
+
if (!allowed.includes(option)) {
|
|
291
|
+
invalidType(
|
|
292
|
+
path,
|
|
293
|
+
type,
|
|
294
|
+
`Field "${path}" has type "${type}" with the option "${option}", but ${describeAllowedOptions(baseName, allowed)}`
|
|
295
|
+
);
|
|
296
|
+
}
|
|
297
|
+
if (options.has(option)) {
|
|
298
|
+
invalidType(path, type, `Field "${path}" has the "${option}" option twice in type "${type}"`);
|
|
299
|
+
}
|
|
300
|
+
options.add(option);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
const ordered = FIELD_OPTION_ORDER.filter((option) => options.has(option));
|
|
304
|
+
const verbatim = base === "verbatim" || base === "verbatim[]";
|
|
305
|
+
return {
|
|
306
|
+
type: [baseName, ...ordered].join(TYPE_SEPARATOR),
|
|
307
|
+
base,
|
|
308
|
+
sortable: options.has("sortable"),
|
|
309
|
+
pattern: verbatim || options.has("pattern"),
|
|
310
|
+
partial: options.has("partial"),
|
|
311
|
+
dimension
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
function parsedTypeOf(type) {
|
|
315
|
+
if (type === void 0) return void 0;
|
|
316
|
+
try {
|
|
317
|
+
return parseFieldType(type, "");
|
|
318
|
+
} catch {
|
|
319
|
+
return void 0;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
function withSortableOption(parsed) {
|
|
323
|
+
if (parsed.sortable) return parsed.type;
|
|
324
|
+
const separatorIndex = parsed.type.indexOf(TYPE_SEPARATOR);
|
|
325
|
+
const baseName = separatorIndex === -1 ? parsed.type : parsed.type.slice(0, separatorIndex);
|
|
326
|
+
return parseFieldType(`${baseName}${TYPE_SEPARATOR}sortable${parsed.type.slice(baseName.length)}`, "").type;
|
|
327
|
+
}
|
|
328
|
+
function isListBase(base) {
|
|
329
|
+
return base.endsWith("[]");
|
|
330
|
+
}
|
|
331
|
+
function isWordIndexedBase(base) {
|
|
332
|
+
return base === "string" || base === "string[]";
|
|
333
|
+
}
|
|
334
|
+
function isSingleStringFieldType(fieldType) {
|
|
335
|
+
return parsedTypeOf(fieldType)?.base === "string";
|
|
336
|
+
}
|
|
337
|
+
function isWordIndexedFieldType(fieldType) {
|
|
338
|
+
const parsed = parsedTypeOf(fieldType);
|
|
339
|
+
return parsed !== void 0 && isWordIndexedBase(parsed.base);
|
|
340
|
+
}
|
|
341
|
+
function isStringValuedBase(base) {
|
|
342
|
+
return base === "string" || base === "string[]" || base === "verbatim" || base === "verbatim[]";
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// src/schema/validator/schema.ts
|
|
346
|
+
function validateSchemaFields(schema, depth, prefix) {
|
|
347
|
+
if (depth > MAX_NESTING_DEPTH) {
|
|
348
|
+
throw new NarsilError(
|
|
349
|
+
ErrorCodes.SCHEMA_DEPTH_EXCEEDED,
|
|
350
|
+
`Schema nesting exceeds the maximum depth of ${MAX_NESTING_DEPTH} at "${prefix}"`,
|
|
351
|
+
{ path: prefix, maxDepth: MAX_NESTING_DEPTH }
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
let canonical = null;
|
|
355
|
+
const replace = (field, value) => {
|
|
356
|
+
if (canonical === null) canonical = { ...schema };
|
|
357
|
+
canonical[field] = value;
|
|
358
|
+
};
|
|
359
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
360
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
361
|
+
if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
|
|
362
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field name "${field}" is not allowed in a schema`, {
|
|
363
|
+
field,
|
|
364
|
+
path
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
if (!FIELD_NAME_PATTERN.test(field)) {
|
|
368
|
+
throw new NarsilError(
|
|
369
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
370
|
+
`Field name "${field}" contains characters that are not allowed; use letters, digits, and underscores only`,
|
|
371
|
+
{ field, path }
|
|
372
|
+
);
|
|
373
|
+
}
|
|
374
|
+
if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
|
|
375
|
+
throw new NarsilError(
|
|
376
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
377
|
+
`Field "${field}" is reserved and cannot be defined in a schema`,
|
|
378
|
+
{ field, path }
|
|
379
|
+
);
|
|
380
|
+
}
|
|
381
|
+
if (isPlainObject(type)) {
|
|
382
|
+
const nested = validateSchemaFields(type, depth + 1, path);
|
|
383
|
+
if (nested !== type) replace(field, nested);
|
|
384
|
+
continue;
|
|
385
|
+
}
|
|
386
|
+
if (typeof type !== "string") {
|
|
387
|
+
throw new NarsilError(
|
|
388
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
389
|
+
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
390
|
+
{
|
|
391
|
+
field: path,
|
|
392
|
+
type: String(type)
|
|
393
|
+
}
|
|
394
|
+
);
|
|
395
|
+
}
|
|
396
|
+
const parsed = parseFieldType(type, path);
|
|
397
|
+
if (parsed.type !== type) replace(field, parsed.type);
|
|
398
|
+
}
|
|
399
|
+
return canonical ?? schema;
|
|
400
|
+
}
|
|
401
|
+
function validateSchema(schema) {
|
|
402
|
+
if (!isPlainObject(schema)) {
|
|
403
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must be a plain object", {
|
|
404
|
+
received: typeof schema
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
if (Object.keys(schema).length === 0) {
|
|
408
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
|
|
409
|
+
}
|
|
410
|
+
return validateSchemaFields(schema, 1, "");
|
|
411
|
+
}
|
|
412
|
+
function requireValidFieldTypes(flatSchema) {
|
|
413
|
+
for (const [path, type] of Object.entries(flatSchema)) {
|
|
414
|
+
if (typeof type !== "string") {
|
|
415
|
+
throw new NarsilError(
|
|
416
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
417
|
+
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
418
|
+
{ field: path, type: String(type) }
|
|
419
|
+
);
|
|
420
|
+
}
|
|
421
|
+
parseFieldType(type, path);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
function flattenRecursive(schema, prefix, result) {
|
|
425
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
426
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
427
|
+
if (isPlainObject(type)) {
|
|
428
|
+
flattenRecursive(type, path, result);
|
|
429
|
+
} else {
|
|
430
|
+
result[path] = type;
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
var flattenedSchemas = /* @__PURE__ */ new WeakMap();
|
|
435
|
+
function flattenSchema(schema) {
|
|
436
|
+
const cached = flattenedSchemas.get(schema);
|
|
437
|
+
if (cached !== void 0) return cached;
|
|
438
|
+
const result = /* @__PURE__ */ Object.create(null);
|
|
439
|
+
flattenRecursive(schema, "", result);
|
|
440
|
+
flattenedSchemas.set(schema, Object.freeze(result));
|
|
441
|
+
return result;
|
|
442
|
+
}
|
|
443
|
+
var parsedSchemas = /* @__PURE__ */ new WeakMap();
|
|
444
|
+
function schemaFieldsOf(schema) {
|
|
445
|
+
const cached = parsedSchemas.get(schema);
|
|
446
|
+
if (cached !== void 0) return cached;
|
|
447
|
+
const fields = [];
|
|
448
|
+
for (const [path, type] of Object.entries(flattenSchema(schema))) {
|
|
449
|
+
fields.push({ ...parseFieldType(type, path), path });
|
|
450
|
+
}
|
|
451
|
+
const frozen = Object.freeze(fields);
|
|
452
|
+
parsedSchemas.set(schema, frozen);
|
|
453
|
+
return frozen;
|
|
454
|
+
}
|
|
455
|
+
function extractVectorFieldsFromSchema(schema) {
|
|
456
|
+
const result = /* @__PURE__ */ new Map();
|
|
457
|
+
for (const field of schemaFieldsOf(schema)) {
|
|
458
|
+
if (field.base === "vector") result.set(field.path, field.dimension);
|
|
459
|
+
}
|
|
460
|
+
return result;
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
// src/core/pattern-index/registry.ts
|
|
464
|
+
var registered = null;
|
|
465
|
+
function isPatternSearchModule(value) {
|
|
466
|
+
return value !== null && typeof value === "object" && value.name === "pattern" && "createIndex" in value && typeof value.createIndex === "function" && "readArrays" in value && typeof value.readArrays === "function" && "mergeArrays" in value && typeof value.mergeArrays === "function" && "matchInto" in value && typeof value.matchInto === "function";
|
|
467
|
+
}
|
|
468
|
+
function registerPatternSearch(module) {
|
|
469
|
+
if (!isPatternSearchModule(module)) {
|
|
470
|
+
throw new NarsilError(
|
|
471
|
+
ErrorCodes.CONFIG_INVALID,
|
|
472
|
+
'registerPatternSearch takes the patternSearch object that "@delali/narsil/pattern" exports'
|
|
473
|
+
);
|
|
474
|
+
}
|
|
475
|
+
registered = module;
|
|
476
|
+
}
|
|
477
|
+
function availablePatternSearch() {
|
|
478
|
+
return registered ?? platformPatternSearch();
|
|
479
|
+
}
|
|
480
|
+
function requirePatternSearchFor(schema) {
|
|
481
|
+
for (const { path, pattern } of schemaFieldsOf(schema)) {
|
|
482
|
+
if (pattern) requirePatternSearch(path);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
function requirePatternSearch(fieldPath) {
|
|
486
|
+
const module = availablePatternSearch();
|
|
487
|
+
if (module !== null) return module;
|
|
488
|
+
throw new NarsilError(
|
|
489
|
+
ErrorCodes.CONFIG_INVALID,
|
|
490
|
+
`Field "${fieldPath}" is a pattern field, but this bundle contains no pattern search code. Add import { patternSearch } from '@delali/narsil/pattern', then call registerPatternSearch(patternSearch) before you create or load the index`,
|
|
491
|
+
{ field: fieldPath }
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
|
|
212
495
|
// src/core/tokenizer/constants.ts
|
|
213
496
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
214
497
|
var TOKEN_CACHE_SIZE_CEILING = 2e6;
|
|
@@ -1820,7 +2103,8 @@ var NARSIL_CONFIG_KEYS = {
|
|
|
1820
2103
|
embeddingAdapters: true,
|
|
1821
2104
|
durability: true,
|
|
1822
2105
|
analysis: true,
|
|
1823
|
-
lifecycle: true
|
|
2106
|
+
lifecycle: true,
|
|
2107
|
+
patternWorkCap: true
|
|
1824
2108
|
};
|
|
1825
2109
|
var DURABILITY_CONFIG_KEYS = {
|
|
1826
2110
|
tier: true,
|
|
@@ -1867,170 +2151,65 @@ function requireNamedPlugins(plugins) {
|
|
|
1867
2151
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, 'The configuration sets "plugins" to something other than a list');
|
|
1868
2152
|
}
|
|
1869
2153
|
plugins.forEach((plugin, position) => {
|
|
1870
|
-
const name = plugin?.name;
|
|
1871
|
-
if (typeof name !== "string" || name.trim().length === 0) {
|
|
1872
|
-
throw new NarsilError(
|
|
1873
|
-
ErrorCodes.CONFIG_INVALID,
|
|
1874
|
-
`The plugin at position ${position} carries no name, and the engine names each plugin in its errors and event payloads`,
|
|
1875
|
-
{ position }
|
|
1876
|
-
);
|
|
1877
|
-
}
|
|
1878
|
-
});
|
|
1879
|
-
}
|
|
1880
|
-
function requireKnownConfig(config) {
|
|
1881
|
-
if (config === void 0) return;
|
|
1882
|
-
requireKnownKeys("", config, NARSIL_CONFIG_KEYS);
|
|
1883
|
-
requireKnownKeys("durability.", config.durability, DURABILITY_CONFIG_KEYS);
|
|
1884
|
-
requireKnownKeys("workers.", config.workers, WORKER_CONFIG_KEYS);
|
|
1885
|
-
requireKnownKeys("lifecycle.", config.lifecycle, LIFECYCLE_CONFIG_KEYS);
|
|
1886
|
-
requireKnownKeys("analysis.", config.analysis, ANALYSIS_CONFIG_KEYS);
|
|
1887
|
-
requireNamedPlugins(config.plugins);
|
|
1888
|
-
}
|
|
1889
|
-
|
|
1890
|
-
// src/core/id-generator.ts
|
|
1891
|
-
function generateId() {
|
|
1892
|
-
const bytes = new Uint8Array(16);
|
|
1893
|
-
crypto.getRandomValues(bytes);
|
|
1894
|
-
const now2 = Date.now();
|
|
1895
|
-
bytes[0] = Math.floor(now2 / 2 ** 40);
|
|
1896
|
-
bytes[1] = Math.floor(now2 / 2 ** 32) % 256;
|
|
1897
|
-
bytes[2] = Math.floor(now2 / 2 ** 24) % 256;
|
|
1898
|
-
bytes[3] = Math.floor(now2 / 2 ** 16) % 256;
|
|
1899
|
-
bytes[4] = Math.floor(now2 / 2 ** 8) % 256;
|
|
1900
|
-
bytes[5] = now2 % 256;
|
|
1901
|
-
bytes[6] = bytes[6] & 15 | 112;
|
|
1902
|
-
bytes[8] = bytes[8] & 63 | 128;
|
|
1903
|
-
const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
|
|
1904
|
-
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
1905
|
-
}
|
|
1906
|
-
|
|
1907
|
-
// src/schema/constants.ts
|
|
1908
|
-
var MAX_NESTING_DEPTH = 4;
|
|
1909
|
-
var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
1910
|
-
var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
|
|
1911
|
-
var MIN_PATTERN_VALUE_LIMIT = 1;
|
|
1912
|
-
var MAX_PATTERN_VALUE_LIMIT = 65536;
|
|
1913
|
-
|
|
1914
|
-
// src/schema/validator/shared.ts
|
|
1915
|
-
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
1916
|
-
function isGeopointOrVectorType(fieldType) {
|
|
1917
|
-
return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
|
|
1918
|
-
}
|
|
1919
|
-
var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
|
|
1920
|
-
var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
|
|
1921
|
-
var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
1922
|
-
function isPlainObject(value) {
|
|
1923
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1924
|
-
}
|
|
1925
|
-
|
|
1926
|
-
// src/schema/validator/field-type.ts
|
|
1927
|
-
var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
|
|
1928
|
-
var OPTIONS_BY_BASE = {
|
|
1929
|
-
string: FIELD_OPTION_ORDER,
|
|
1930
|
-
"string[]": FIELD_OPTION_ORDER,
|
|
1931
|
-
verbatim: ["sortable"],
|
|
1932
|
-
"verbatim[]": ["sortable"],
|
|
1933
|
-
number: [],
|
|
1934
|
-
"number[]": [],
|
|
1935
|
-
boolean: [],
|
|
1936
|
-
"boolean[]": [],
|
|
1937
|
-
enum: [],
|
|
1938
|
-
"enum[]": [],
|
|
1939
|
-
geopoint: [],
|
|
1940
|
-
vector: []
|
|
1941
|
-
};
|
|
1942
|
-
var TYPE_SEPARATOR = ":";
|
|
1943
|
-
function isNamedBase(name) {
|
|
1944
|
-
return name !== "vector" && Object.hasOwn(OPTIONS_BY_BASE, name);
|
|
1945
|
-
}
|
|
1946
|
-
function describeAllowedOptions(baseName, allowed) {
|
|
1947
|
-
if (allowed.length === 0) return `the "${baseName}" type has no options`;
|
|
1948
|
-
const quoted = allowed.map((option) => `"${option}"`);
|
|
1949
|
-
if (quoted.length === 1) return `the only option of the "${baseName}" type is ${quoted[0]}`;
|
|
1950
|
-
return `the options of the "${baseName}" type are ${quoted.slice(0, -1).join(", ")}, and ${quoted[quoted.length - 1]}`;
|
|
1951
|
-
}
|
|
1952
|
-
function invalidType(path, type, message) {
|
|
1953
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, message, { field: path, type });
|
|
1954
|
-
}
|
|
1955
|
-
function parseBase(baseName, path, type) {
|
|
1956
|
-
if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
|
|
1957
|
-
const vectorMatch = VECTOR_PATTERN.exec(baseName);
|
|
1958
|
-
if (vectorMatch === null) {
|
|
1959
|
-
invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
|
|
1960
|
-
}
|
|
1961
|
-
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
1962
|
-
if (dimension <= 0) {
|
|
1963
|
-
throw new NarsilError(
|
|
1964
|
-
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
1965
|
-
`Vector field "${path}" must have a positive dimension, got ${dimension}`,
|
|
1966
|
-
{ field: path, dimension }
|
|
1967
|
-
);
|
|
1968
|
-
}
|
|
1969
|
-
return { base: "vector", dimension };
|
|
1970
|
-
}
|
|
1971
|
-
function parseFieldType(type, path) {
|
|
1972
|
-
const separatorIndex = type.indexOf(TYPE_SEPARATOR);
|
|
1973
|
-
const baseName = separatorIndex === -1 ? type : type.slice(0, separatorIndex);
|
|
1974
|
-
const { base, dimension } = parseBase(baseName, path, type);
|
|
1975
|
-
const allowed = OPTIONS_BY_BASE[base];
|
|
1976
|
-
const options = /* @__PURE__ */ new Set();
|
|
1977
|
-
if (separatorIndex !== -1) {
|
|
1978
|
-
for (const option of type.slice(separatorIndex + 1).split(TYPE_SEPARATOR)) {
|
|
1979
|
-
if (!allowed.includes(option)) {
|
|
1980
|
-
invalidType(
|
|
1981
|
-
path,
|
|
1982
|
-
type,
|
|
1983
|
-
`Field "${path}" has type "${type}" with the option "${option}", but ${describeAllowedOptions(baseName, allowed)}`
|
|
1984
|
-
);
|
|
1985
|
-
}
|
|
1986
|
-
if (options.has(option)) {
|
|
1987
|
-
invalidType(path, type, `Field "${path}" has the "${option}" option twice in type "${type}"`);
|
|
1988
|
-
}
|
|
1989
|
-
options.add(option);
|
|
1990
|
-
}
|
|
1991
|
-
}
|
|
1992
|
-
const ordered = FIELD_OPTION_ORDER.filter((option) => options.has(option));
|
|
1993
|
-
const verbatim = base === "verbatim" || base === "verbatim[]";
|
|
1994
|
-
return {
|
|
1995
|
-
type: [baseName, ...ordered].join(TYPE_SEPARATOR),
|
|
1996
|
-
base,
|
|
1997
|
-
sortable: options.has("sortable"),
|
|
1998
|
-
pattern: verbatim || options.has("pattern"),
|
|
1999
|
-
partial: options.has("partial"),
|
|
2000
|
-
dimension
|
|
2001
|
-
};
|
|
2002
|
-
}
|
|
2003
|
-
function parsedTypeOf(type) {
|
|
2004
|
-
if (type === void 0) return void 0;
|
|
2005
|
-
try {
|
|
2006
|
-
return parseFieldType(type, "");
|
|
2007
|
-
} catch {
|
|
2008
|
-
return void 0;
|
|
2009
|
-
}
|
|
2010
|
-
}
|
|
2011
|
-
function withSortableOption(parsed) {
|
|
2012
|
-
if (parsed.sortable) return parsed.type;
|
|
2013
|
-
const separatorIndex = parsed.type.indexOf(TYPE_SEPARATOR);
|
|
2014
|
-
const baseName = separatorIndex === -1 ? parsed.type : parsed.type.slice(0, separatorIndex);
|
|
2015
|
-
return parseFieldType(`${baseName}${TYPE_SEPARATOR}sortable${parsed.type.slice(baseName.length)}`, "").type;
|
|
2016
|
-
}
|
|
2017
|
-
function isListBase(base) {
|
|
2018
|
-
return base.endsWith("[]");
|
|
2019
|
-
}
|
|
2020
|
-
function isWordIndexedBase(base) {
|
|
2021
|
-
return base === "string" || base === "string[]";
|
|
2154
|
+
const name = plugin?.name;
|
|
2155
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
2156
|
+
throw new NarsilError(
|
|
2157
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2158
|
+
`The plugin at position ${position} carries no name, and the engine names each plugin in its errors and event payloads`,
|
|
2159
|
+
{ position }
|
|
2160
|
+
);
|
|
2161
|
+
}
|
|
2162
|
+
});
|
|
2022
2163
|
}
|
|
2023
|
-
function
|
|
2024
|
-
|
|
2164
|
+
function requireKnownConfig(config) {
|
|
2165
|
+
if (config === void 0) return;
|
|
2166
|
+
requireKnownKeys("", config, NARSIL_CONFIG_KEYS);
|
|
2167
|
+
requireKnownKeys("durability.", config.durability, DURABILITY_CONFIG_KEYS);
|
|
2168
|
+
requireKnownKeys("workers.", config.workers, WORKER_CONFIG_KEYS);
|
|
2169
|
+
requireKnownKeys("lifecycle.", config.lifecycle, LIFECYCLE_CONFIG_KEYS);
|
|
2170
|
+
requireKnownKeys("analysis.", config.analysis, ANALYSIS_CONFIG_KEYS);
|
|
2171
|
+
requireNamedPlugins(config.plugins);
|
|
2172
|
+
requireValidPatternWorkCap(config.patternWorkCap);
|
|
2025
2173
|
}
|
|
2026
|
-
function
|
|
2027
|
-
|
|
2028
|
-
|
|
2174
|
+
function requireValidPatternWorkCap(cap) {
|
|
2175
|
+
if (cap === void 0) return;
|
|
2176
|
+
if (typeof cap === "number" && Number.isSafeInteger(cap) && cap >= 1) return;
|
|
2177
|
+
throw new NarsilError(
|
|
2178
|
+
ErrorCodes.CONFIG_INVALID,
|
|
2179
|
+
`The configuration sets "patternWorkCap" to ${String(cap)}, while the cap must be a whole number from 1 to ${Number.MAX_SAFE_INTEGER}`,
|
|
2180
|
+
{ setting: "patternWorkCap" }
|
|
2181
|
+
);
|
|
2029
2182
|
}
|
|
2030
|
-
|
|
2031
|
-
|
|
2183
|
+
|
|
2184
|
+
// src/core/id-generator.ts
|
|
2185
|
+
function generateId() {
|
|
2186
|
+
const bytes = new Uint8Array(16);
|
|
2187
|
+
crypto.getRandomValues(bytes);
|
|
2188
|
+
const now2 = Date.now();
|
|
2189
|
+
bytes[0] = Math.floor(now2 / 2 ** 40);
|
|
2190
|
+
bytes[1] = Math.floor(now2 / 2 ** 32) % 256;
|
|
2191
|
+
bytes[2] = Math.floor(now2 / 2 ** 24) % 256;
|
|
2192
|
+
bytes[3] = Math.floor(now2 / 2 ** 16) % 256;
|
|
2193
|
+
bytes[4] = Math.floor(now2 / 2 ** 8) % 256;
|
|
2194
|
+
bytes[5] = now2 % 256;
|
|
2195
|
+
bytes[6] = bytes[6] & 15 | 112;
|
|
2196
|
+
bytes[8] = bytes[8] & 63 | 128;
|
|
2197
|
+
const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
|
|
2198
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
2032
2199
|
}
|
|
2033
2200
|
|
|
2201
|
+
// src/core/pattern-index/constants.ts
|
|
2202
|
+
var DEFAULT_PATTERN_WORK_CAP = 25e6;
|
|
2203
|
+
var MAX_PATTERN_TEXT_CODE_POINTS = 1024;
|
|
2204
|
+
var MAX_PATTERN_POSITIONS = 2048;
|
|
2205
|
+
var MATCHER_STATE_BUDGET_BYTES = 1e6;
|
|
2206
|
+
var MATCHER_CODE_POINT_BYTES = 8;
|
|
2207
|
+
var MATCHER_SIMULATION_BYTES_PER_POSITION = 16;
|
|
2208
|
+
var MATCHER_STATE_OVERHEAD_BYTES = 96;
|
|
2209
|
+
var PATTERN_WORK_FLUSH_UNITS = 65536;
|
|
2210
|
+
var PATTERN_RUN_CODE_POINTS = 3;
|
|
2211
|
+
var PATTERN_TEST_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "contains"];
|
|
2212
|
+
|
|
2034
2213
|
// src/core/ordering/code-points.ts
|
|
2035
2214
|
function exceedsCodePoints(value, maximum) {
|
|
2036
2215
|
if (value.length <= maximum) return false;
|
|
@@ -2048,6 +2227,31 @@ function exceedsCodePoints(value, maximum) {
|
|
|
2048
2227
|
}
|
|
2049
2228
|
return false;
|
|
2050
2229
|
}
|
|
2230
|
+
var SURROGATE_UNIT = /[\ud800-\udfff]/;
|
|
2231
|
+
function holdsUnpairedSurrogate(value) {
|
|
2232
|
+
if (!SURROGATE_UNIT.test(value)) return false;
|
|
2233
|
+
for (let i = 0; i < value.length; i++) {
|
|
2234
|
+
const unit = value.charCodeAt(i);
|
|
2235
|
+
if (unit < 55296 || unit >= 57344) continue;
|
|
2236
|
+
if (unit >= 56320) return true;
|
|
2237
|
+
const low = value.charCodeAt(i + 1);
|
|
2238
|
+
if (!(low >= 56320 && low < 57344)) return true;
|
|
2239
|
+
i++;
|
|
2240
|
+
}
|
|
2241
|
+
return false;
|
|
2242
|
+
}
|
|
2243
|
+
function codePointCount(value) {
|
|
2244
|
+
let counted = 0;
|
|
2245
|
+
for (let i = 0; i < value.length; i++) {
|
|
2246
|
+
const unit = value.charCodeAt(i);
|
|
2247
|
+
if (unit >= 55296 && unit < 56320 && i + 1 < value.length) {
|
|
2248
|
+
const low = value.charCodeAt(i + 1);
|
|
2249
|
+
if (low >= 56320 && low < 57344) i++;
|
|
2250
|
+
}
|
|
2251
|
+
counted++;
|
|
2252
|
+
}
|
|
2253
|
+
return counted;
|
|
2254
|
+
}
|
|
2051
2255
|
function adjustUtf16Unit(unit) {
|
|
2052
2256
|
if (unit < 55296) return unit;
|
|
2053
2257
|
if (unit < 57344) return unit + 8192;
|
|
@@ -2061,6 +2265,29 @@ function compareCodePoints(a, b) {
|
|
|
2061
2265
|
return adjustUtf16Unit(a.charCodeAt(i)) - adjustUtf16Unit(b.charCodeAt(i));
|
|
2062
2266
|
}
|
|
2063
2267
|
|
|
2268
|
+
// src/geo/constants.ts
|
|
2269
|
+
var DEG_TO_RAD = Math.PI / 180;
|
|
2270
|
+
var MAX_LATITUDE_DEGREES = 90;
|
|
2271
|
+
var MAX_LONGITUDE_DEGREES = 180;
|
|
2272
|
+
var MIN_POLYGON_POINTS = 3;
|
|
2273
|
+
var ANTIMERIDIAN_SPAN_DEGREES = 180;
|
|
2274
|
+
var FULL_TURN_DEGREES = 360;
|
|
2275
|
+
var EARTH_RADIUS_METERS = 63710088e-1;
|
|
2276
|
+
var LOCATION_CODE_BITS_PER_AXIS = 26;
|
|
2277
|
+
var MAX_COVERING_CELLS = 16;
|
|
2278
|
+
var RADIUS_BOX_MARGIN_RATIO = 1.02;
|
|
2279
|
+
var RADIUS_BOX_MARGIN_RADIANS = 1e-9;
|
|
2280
|
+
var VINCENTY_MAX_ITERATIONS = 200;
|
|
2281
|
+
var VINCENTY_CONVERGENCE_THRESHOLD = 1e-12;
|
|
2282
|
+
|
|
2283
|
+
// src/geo/coordinates.ts
|
|
2284
|
+
function isLatitude(value) {
|
|
2285
|
+
return value >= -MAX_LATITUDE_DEGREES && value <= MAX_LATITUDE_DEGREES;
|
|
2286
|
+
}
|
|
2287
|
+
function isLongitude(value) {
|
|
2288
|
+
return value >= -MAX_LONGITUDE_DEGREES && value <= MAX_LONGITUDE_DEGREES;
|
|
2289
|
+
}
|
|
2290
|
+
|
|
2064
2291
|
// src/schema/validator/field-values.ts
|
|
2065
2292
|
function validateGeopoint(path, value) {
|
|
2066
2293
|
if (!isPlainObject(value)) {
|
|
@@ -2080,7 +2307,7 @@ function validateGeopoint(path, value) {
|
|
|
2080
2307
|
lon: Number.isNaN(value.lon) ? "NaN" : typeof value.lon
|
|
2081
2308
|
});
|
|
2082
2309
|
}
|
|
2083
|
-
if (value.lat
|
|
2310
|
+
if (!isLatitude(value.lat)) {
|
|
2084
2311
|
throw new NarsilError(
|
|
2085
2312
|
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
2086
2313
|
`Field "${path}" geopoint lat must be between -90 and 90`,
|
|
@@ -2090,7 +2317,7 @@ function validateGeopoint(path, value) {
|
|
|
2090
2317
|
}
|
|
2091
2318
|
);
|
|
2092
2319
|
}
|
|
2093
|
-
if (value.lon
|
|
2320
|
+
if (!isLongitude(value.lon)) {
|
|
2094
2321
|
throw new NarsilError(
|
|
2095
2322
|
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
2096
2323
|
`Field "${path}" geopoint lon must be between -180 and 180`,
|
|
@@ -2120,7 +2347,7 @@ function validateVector(path, value, expectedDimension) {
|
|
|
2120
2347
|
);
|
|
2121
2348
|
}
|
|
2122
2349
|
for (let i = 0; i < arr.length; i++) {
|
|
2123
|
-
if (typeof arr[i] !== "number" || Number.
|
|
2350
|
+
if (typeof arr[i] !== "number" || !Number.isFinite(arr[i])) {
|
|
2124
2351
|
throw new NarsilError(
|
|
2125
2352
|
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
2126
2353
|
`Field "${path}" vector element at index ${i} is not a valid number`,
|
|
@@ -2145,11 +2372,12 @@ function validateTypedArray(path, value, elementType) {
|
|
|
2145
2372
|
{ field: path, index: i, expected: elementType, received: typeof el }
|
|
2146
2373
|
);
|
|
2147
2374
|
}
|
|
2148
|
-
if (elementType === "number" && Number.
|
|
2149
|
-
throw new NarsilError(
|
|
2150
|
-
|
|
2151
|
-
index
|
|
2152
|
-
|
|
2375
|
+
if (elementType === "number" && !Number.isFinite(el)) {
|
|
2376
|
+
throw new NarsilError(
|
|
2377
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
2378
|
+
`Field "${path}" array element at index ${i} is ${String(el)}, while a number field holds finite numbers alone`,
|
|
2379
|
+
{ field: path, index: i }
|
|
2380
|
+
);
|
|
2153
2381
|
}
|
|
2154
2382
|
}
|
|
2155
2383
|
}
|
|
@@ -2180,11 +2408,11 @@ function validateFieldValue(path, value, type, patternValueLimit) {
|
|
|
2180
2408
|
if (type.pattern) validatePatternValue(path, value, patternValueLimit);
|
|
2181
2409
|
return;
|
|
2182
2410
|
case "number":
|
|
2183
|
-
if (typeof value !== "number" || Number.
|
|
2184
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a
|
|
2411
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
2412
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a finite number`, {
|
|
2185
2413
|
field: path,
|
|
2186
2414
|
expected: "number",
|
|
2187
|
-
received:
|
|
2415
|
+
received: typeof value === "number" ? String(value) : typeof value
|
|
2188
2416
|
});
|
|
2189
2417
|
}
|
|
2190
2418
|
return;
|
|
@@ -2242,6 +2470,7 @@ function validateFieldValue(path, value, type, patternValueLimit) {
|
|
|
2242
2470
|
|
|
2243
2471
|
// src/schema/validator/storable.ts
|
|
2244
2472
|
var RESERVED_OBJECT_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
2473
|
+
var UNPAIRED_SURROGATE_REASON = "holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8";
|
|
2245
2474
|
function reject(path, reason) {
|
|
2246
2475
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
2247
2476
|
field: path
|
|
@@ -2254,8 +2483,11 @@ function isPlainObject2(value) {
|
|
|
2254
2483
|
function walkStorable(value, path, depth, ancestors, skip) {
|
|
2255
2484
|
if (value === null || value === void 0) return;
|
|
2256
2485
|
const kind = typeof value;
|
|
2257
|
-
if (kind === "string"
|
|
2258
|
-
|
|
2486
|
+
if (kind === "string") {
|
|
2487
|
+
if (holdsUnpairedSurrogate(value)) reject(path, UNPAIRED_SURROGATE_REASON);
|
|
2488
|
+
return;
|
|
2489
|
+
}
|
|
2490
|
+
if (kind === "boolean" || kind === "number") return;
|
|
2259
2491
|
if (kind === "bigint" || kind === "symbol" || kind === "function") {
|
|
2260
2492
|
reject(path, `holds a ${kind}, which cannot be stored or recovered`);
|
|
2261
2493
|
}
|
|
@@ -2287,6 +2519,7 @@ function walkStorable(value, path, depth, ancestors, skip) {
|
|
|
2287
2519
|
if (RESERVED_OBJECT_KEYS.has(key)) {
|
|
2288
2520
|
reject(keyPath, "uses a reserved object key");
|
|
2289
2521
|
}
|
|
2522
|
+
if (holdsUnpairedSurrogate(key)) reject(keyPath, `has a name that ${UNPAIRED_SURROGATE_REASON}`);
|
|
2290
2523
|
if (skip.has(keyPath)) continue;
|
|
2291
2524
|
walkStorable(record[key], keyPath, depth + 1, ancestors, skip);
|
|
2292
2525
|
}
|
|
@@ -2353,9 +2586,20 @@ function validateDocument(document2, schema, patternValueLimit = DEFAULT_PATTERN
|
|
|
2353
2586
|
received: typeof document2
|
|
2354
2587
|
});
|
|
2355
2588
|
}
|
|
2589
|
+
requireIdFieldKey(document2);
|
|
2356
2590
|
assertStorableDocument(document2, vectorFieldPaths(schema));
|
|
2357
2591
|
validateDocumentFields(document2, schema, "", patternValueLimit);
|
|
2358
2592
|
}
|
|
2593
|
+
function requireIdFieldKey(document2) {
|
|
2594
|
+
if (!Object.hasOwn(document2, "id")) return;
|
|
2595
|
+
const id = document2.id;
|
|
2596
|
+
if (typeof id === "string" && id.length > 0) return;
|
|
2597
|
+
throw new NarsilError(
|
|
2598
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
2599
|
+
`Field "id" holds ${typeof id === "string" ? "an empty string" : id === null ? "null" : `a ${typeof id}`}, while the engine stores a document under its id field, so that field must hold a non-empty string`,
|
|
2600
|
+
{ field: "id", received: id === null ? "null" : typeof id }
|
|
2601
|
+
);
|
|
2602
|
+
}
|
|
2359
2603
|
function collectExtraFields(doc, schema, prefix, extras) {
|
|
2360
2604
|
for (const key of Object.keys(doc)) {
|
|
2361
2605
|
if (key === "id") continue;
|
|
@@ -2403,124 +2647,6 @@ function validateRequiredFields(document2, required) {
|
|
|
2403
2647
|
}
|
|
2404
2648
|
}
|
|
2405
2649
|
|
|
2406
|
-
// src/schema/validator/schema.ts
|
|
2407
|
-
function validateSchemaFields(schema, depth, prefix) {
|
|
2408
|
-
if (depth > MAX_NESTING_DEPTH) {
|
|
2409
|
-
throw new NarsilError(
|
|
2410
|
-
ErrorCodes.SCHEMA_DEPTH_EXCEEDED,
|
|
2411
|
-
`Schema nesting exceeds the maximum depth of ${MAX_NESTING_DEPTH} at "${prefix}"`,
|
|
2412
|
-
{ path: prefix, maxDepth: MAX_NESTING_DEPTH }
|
|
2413
|
-
);
|
|
2414
|
-
}
|
|
2415
|
-
let canonical = null;
|
|
2416
|
-
const replace = (field, value) => {
|
|
2417
|
-
if (canonical === null) canonical = { ...schema };
|
|
2418
|
-
canonical[field] = value;
|
|
2419
|
-
};
|
|
2420
|
-
for (const [field, type] of Object.entries(schema)) {
|
|
2421
|
-
const path = prefix ? `${prefix}.${field}` : field;
|
|
2422
|
-
if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
|
|
2423
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field name "${field}" is not allowed in a schema`, {
|
|
2424
|
-
field,
|
|
2425
|
-
path
|
|
2426
|
-
});
|
|
2427
|
-
}
|
|
2428
|
-
if (!FIELD_NAME_PATTERN.test(field)) {
|
|
2429
|
-
throw new NarsilError(
|
|
2430
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
2431
|
-
`Field name "${field}" contains characters that are not allowed; use letters, digits, and underscores only`,
|
|
2432
|
-
{ field, path }
|
|
2433
|
-
);
|
|
2434
|
-
}
|
|
2435
|
-
if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
|
|
2436
|
-
throw new NarsilError(
|
|
2437
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
2438
|
-
`Field "${field}" is reserved and cannot be defined in a schema`,
|
|
2439
|
-
{ field, path }
|
|
2440
|
-
);
|
|
2441
|
-
}
|
|
2442
|
-
if (isPlainObject(type)) {
|
|
2443
|
-
const nested = validateSchemaFields(type, depth + 1, path);
|
|
2444
|
-
if (nested !== type) replace(field, nested);
|
|
2445
|
-
continue;
|
|
2446
|
-
}
|
|
2447
|
-
if (typeof type !== "string") {
|
|
2448
|
-
throw new NarsilError(
|
|
2449
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
2450
|
-
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
2451
|
-
{
|
|
2452
|
-
field: path,
|
|
2453
|
-
type: String(type)
|
|
2454
|
-
}
|
|
2455
|
-
);
|
|
2456
|
-
}
|
|
2457
|
-
const parsed = parseFieldType(type, path);
|
|
2458
|
-
if (parsed.type !== type) replace(field, parsed.type);
|
|
2459
|
-
}
|
|
2460
|
-
return canonical ?? schema;
|
|
2461
|
-
}
|
|
2462
|
-
function validateSchema(schema) {
|
|
2463
|
-
if (!isPlainObject(schema)) {
|
|
2464
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must be a plain object", {
|
|
2465
|
-
received: typeof schema
|
|
2466
|
-
});
|
|
2467
|
-
}
|
|
2468
|
-
if (Object.keys(schema).length === 0) {
|
|
2469
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
|
|
2470
|
-
}
|
|
2471
|
-
return validateSchemaFields(schema, 1, "");
|
|
2472
|
-
}
|
|
2473
|
-
function requireValidFieldTypes(flatSchema) {
|
|
2474
|
-
for (const [path, type] of Object.entries(flatSchema)) {
|
|
2475
|
-
if (typeof type !== "string") {
|
|
2476
|
-
throw new NarsilError(
|
|
2477
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
2478
|
-
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
2479
|
-
{ field: path, type: String(type) }
|
|
2480
|
-
);
|
|
2481
|
-
}
|
|
2482
|
-
parseFieldType(type, path);
|
|
2483
|
-
}
|
|
2484
|
-
}
|
|
2485
|
-
function flattenRecursive(schema, prefix, result) {
|
|
2486
|
-
for (const [field, type] of Object.entries(schema)) {
|
|
2487
|
-
const path = prefix ? `${prefix}.${field}` : field;
|
|
2488
|
-
if (isPlainObject(type)) {
|
|
2489
|
-
flattenRecursive(type, path, result);
|
|
2490
|
-
} else {
|
|
2491
|
-
result[path] = type;
|
|
2492
|
-
}
|
|
2493
|
-
}
|
|
2494
|
-
}
|
|
2495
|
-
var flattenedSchemas = /* @__PURE__ */ new WeakMap();
|
|
2496
|
-
function flattenSchema(schema) {
|
|
2497
|
-
const cached = flattenedSchemas.get(schema);
|
|
2498
|
-
if (cached !== void 0) return cached;
|
|
2499
|
-
const result = /* @__PURE__ */ Object.create(null);
|
|
2500
|
-
flattenRecursive(schema, "", result);
|
|
2501
|
-
flattenedSchemas.set(schema, Object.freeze(result));
|
|
2502
|
-
return result;
|
|
2503
|
-
}
|
|
2504
|
-
var parsedSchemas = /* @__PURE__ */ new WeakMap();
|
|
2505
|
-
function schemaFieldsOf(schema) {
|
|
2506
|
-
const cached = parsedSchemas.get(schema);
|
|
2507
|
-
if (cached !== void 0) return cached;
|
|
2508
|
-
const fields = [];
|
|
2509
|
-
for (const [path, type] of Object.entries(flattenSchema(schema))) {
|
|
2510
|
-
fields.push({ ...parseFieldType(type, path), path });
|
|
2511
|
-
}
|
|
2512
|
-
const frozen = Object.freeze(fields);
|
|
2513
|
-
parsedSchemas.set(schema, frozen);
|
|
2514
|
-
return frozen;
|
|
2515
|
-
}
|
|
2516
|
-
function extractVectorFieldsFromSchema(schema) {
|
|
2517
|
-
const result = /* @__PURE__ */ new Map();
|
|
2518
|
-
for (const field of schemaFieldsOf(schema)) {
|
|
2519
|
-
if (field.base === "vector") result.set(field.path, field.dimension);
|
|
2520
|
-
}
|
|
2521
|
-
return result;
|
|
2522
|
-
}
|
|
2523
|
-
|
|
2524
2650
|
// src/vector/hnsw/constants.ts
|
|
2525
2651
|
var MAX_LAYER_CAP = 32;
|
|
2526
2652
|
var MAX_M = 512;
|
|
@@ -2868,6 +2994,44 @@ function sanitizeSurfaceForms(raw) {
|
|
|
2868
2994
|
}
|
|
2869
2995
|
return count > 0 ? result : void 0;
|
|
2870
2996
|
}
|
|
2997
|
+
function patternIndexesToWire(pattern) {
|
|
2998
|
+
if (pattern === void 0) return {};
|
|
2999
|
+
const wire = /* @__PURE__ */ Object.create(null);
|
|
3000
|
+
for (const [field, index] of Object.entries(pattern)) wire[field] = { doc_ids: index.docIds, runs: index.runs };
|
|
3001
|
+
return { pattern: wire };
|
|
3002
|
+
}
|
|
3003
|
+
function isWireRecord(value) {
|
|
3004
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
3005
|
+
}
|
|
3006
|
+
function corruptPatternEntry(field) {
|
|
3007
|
+
return new NarsilError(
|
|
3008
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3009
|
+
`The pattern entry of field "${field}" in this partition must hold a list of document IDs and a map of runs`,
|
|
3010
|
+
{ field }
|
|
3011
|
+
);
|
|
3012
|
+
}
|
|
3013
|
+
function patternIndexesFromWire(raw) {
|
|
3014
|
+
if (raw === void 0 || raw === null) return {};
|
|
3015
|
+
if (!isWireRecord(raw)) {
|
|
3016
|
+
throw new NarsilError(
|
|
3017
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
3018
|
+
"The pattern entry of this partition must be a map of fields"
|
|
3019
|
+
);
|
|
3020
|
+
}
|
|
3021
|
+
const pattern = /* @__PURE__ */ Object.create(null);
|
|
3022
|
+
for (const [field, index] of Object.entries(raw)) {
|
|
3023
|
+
if (!isWireRecord(index) || !isWireRecord(index.runs) || !Array.isArray(index.doc_ids)) {
|
|
3024
|
+
throw corruptPatternEntry(field);
|
|
3025
|
+
}
|
|
3026
|
+
const docIds = [];
|
|
3027
|
+
for (const docId of index.doc_ids) {
|
|
3028
|
+
if (typeof docId !== "string") throw corruptPatternEntry(field);
|
|
3029
|
+
docIds.push(docId);
|
|
3030
|
+
}
|
|
3031
|
+
pattern[field] = { docIds, runs: index.runs };
|
|
3032
|
+
}
|
|
3033
|
+
return { pattern };
|
|
3034
|
+
}
|
|
2871
3035
|
function validateHnswMetric(value) {
|
|
2872
3036
|
if (value === void 0 || value === null) return void 0;
|
|
2873
3037
|
if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
|
|
@@ -2939,7 +3103,8 @@ function wireToPartition(raw) {
|
|
|
2939
3103
|
numeric,
|
|
2940
3104
|
boolean,
|
|
2941
3105
|
enum: raw.field_indexes?.enum ?? {},
|
|
2942
|
-
geopoint
|
|
3106
|
+
geopoint,
|
|
3107
|
+
...patternIndexesFromWire(raw.field_indexes?.pattern)
|
|
2943
3108
|
},
|
|
2944
3109
|
surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
|
|
2945
3110
|
vectorData,
|
|
@@ -3046,7 +3211,8 @@ function deserializePayloadV2(data) {
|
|
|
3046
3211
|
numeric,
|
|
3047
3212
|
boolean,
|
|
3048
3213
|
enum: raw.field_indexes?.enum ?? {},
|
|
3049
|
-
geopoint
|
|
3214
|
+
geopoint,
|
|
3215
|
+
...patternIndexesFromWire(raw.field_indexes?.pattern)
|
|
3050
3216
|
},
|
|
3051
3217
|
surfaceForms: sanitizeSurfaceForms(raw.surface_forms),
|
|
3052
3218
|
vectorData,
|
|
@@ -3096,6 +3262,10 @@ function multiFoldTable() {
|
|
|
3096
3262
|
|
|
3097
3263
|
// src/core/ordering/fold-compare.ts
|
|
3098
3264
|
var EXHAUSTED = -1;
|
|
3265
|
+
function foldOf(codePoint, single, multi) {
|
|
3266
|
+
if (codePoint < 128) return codePoint >= 65 && codePoint <= 90 ? codePoint + 32 : codePoint;
|
|
3267
|
+
return single.get(codePoint) ?? multi.get(codePoint) ?? codePoint;
|
|
3268
|
+
}
|
|
3099
3269
|
function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
3100
3270
|
if (cursor.pending !== null) {
|
|
3101
3271
|
const cp2 = cursor.pending[cursor.pendingIndex];
|
|
@@ -3109,9 +3279,6 @@ function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
|
3109
3279
|
if (cursor.index >= text.length) return EXHAUSTED;
|
|
3110
3280
|
let cp = text.charCodeAt(cursor.index);
|
|
3111
3281
|
cursor.index++;
|
|
3112
|
-
if (cp < 128) {
|
|
3113
|
-
return cp >= 65 && cp <= 90 ? cp + 32 : cp;
|
|
3114
|
-
}
|
|
3115
3282
|
if (cp >= 55296 && cp < 56320 && cursor.index < text.length) {
|
|
3116
3283
|
const low = text.charCodeAt(cursor.index);
|
|
3117
3284
|
if (low >= 56320 && low < 57344) {
|
|
@@ -3119,15 +3286,13 @@ function nextFoldedCodePoint(text, cursor, single, multi) {
|
|
|
3119
3286
|
cursor.index++;
|
|
3120
3287
|
}
|
|
3121
3288
|
}
|
|
3122
|
-
const
|
|
3123
|
-
if (
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
if (expansion.length > 1) {
|
|
3127
|
-
cursor.pending = expansion;
|
|
3289
|
+
const folded = foldOf(cp, single, multi);
|
|
3290
|
+
if (typeof folded === "number") return folded;
|
|
3291
|
+
if (folded.length > 1) {
|
|
3292
|
+
cursor.pending = folded;
|
|
3128
3293
|
cursor.pendingIndex = 1;
|
|
3129
3294
|
}
|
|
3130
|
-
return
|
|
3295
|
+
return folded[0];
|
|
3131
3296
|
}
|
|
3132
3297
|
function compareCaseFolded(a, b) {
|
|
3133
3298
|
const shorter = a.length < b.length ? a.length : b.length;
|
|
@@ -3154,6 +3319,28 @@ function compareSortStrings(a, b) {
|
|
|
3154
3319
|
if (folded !== 0) return folded;
|
|
3155
3320
|
return compareCodePoints(a, b);
|
|
3156
3321
|
}
|
|
3322
|
+
function appendFoldedCodePoint(codePoint, out) {
|
|
3323
|
+
const folded = foldOf(codePoint, singleFoldTable(), multiFoldTable());
|
|
3324
|
+
if (typeof folded === "number") {
|
|
3325
|
+
out.push(folded);
|
|
3326
|
+
return;
|
|
3327
|
+
}
|
|
3328
|
+
for (const foldedCodePoint of folded) out.push(foldedCodePoint);
|
|
3329
|
+
}
|
|
3330
|
+
function appendFoldedCodePoints(text, out) {
|
|
3331
|
+
const single = singleFoldTable();
|
|
3332
|
+
const multi = multiFoldTable();
|
|
3333
|
+
const cursor = { index: 0, pending: null, pendingIndex: 0 };
|
|
3334
|
+
for (let codePoint = nextFoldedCodePoint(text, cursor, single, multi); codePoint !== EXHAUSTED; ) {
|
|
3335
|
+
out.push(codePoint);
|
|
3336
|
+
codePoint = nextFoldedCodePoint(text, cursor, single, multi);
|
|
3337
|
+
}
|
|
3338
|
+
}
|
|
3339
|
+
function foldedCodePointCount(text) {
|
|
3340
|
+
const folded = [];
|
|
3341
|
+
appendFoldedCodePoints(text, folded);
|
|
3342
|
+
return folded.length;
|
|
3343
|
+
}
|
|
3157
3344
|
|
|
3158
3345
|
// src/core/ordering/constants.ts
|
|
3159
3346
|
var SORT_VALUE_MAX_CODE_POINTS = 512;
|
|
@@ -4281,17 +4468,18 @@ function bitsetAnd(a, b) {
|
|
|
4281
4468
|
}
|
|
4282
4469
|
return result;
|
|
4283
4470
|
}
|
|
4284
|
-
function
|
|
4285
|
-
const
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4289
|
-
|
|
4290
|
-
|
|
4291
|
-
for (let i =
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4471
|
+
function bitsetAndInto(target, other) {
|
|
4472
|
+
const shared2 = Math.min(target.length, other.length);
|
|
4473
|
+
for (let i = 0; i < shared2; i++) target[i] &= other[i];
|
|
4474
|
+
target.fill(0, shared2);
|
|
4475
|
+
}
|
|
4476
|
+
function bitsetOrInto(target, other) {
|
|
4477
|
+
const shared2 = Math.min(target.length, other.length);
|
|
4478
|
+
for (let i = 0; i < shared2; i++) target[i] |= other[i];
|
|
4479
|
+
}
|
|
4480
|
+
function bitsetAndNotInto(target, removed) {
|
|
4481
|
+
const shared2 = Math.min(target.length, removed.length);
|
|
4482
|
+
for (let i = 0; i < shared2; i++) target[i] &= ~removed[i];
|
|
4295
4483
|
}
|
|
4296
4484
|
function bitsetNot(bs, capacity) {
|
|
4297
4485
|
const result = new Uint32Array(bs.length);
|
|
@@ -4385,13 +4573,6 @@ function getFieldValueByInternalId(docStore, internalId, fieldPath) {
|
|
|
4385
4573
|
if (externalId === void 0) return void 0;
|
|
4386
4574
|
return getFieldValueForDoc(docStore, externalId, fieldPath);
|
|
4387
4575
|
}
|
|
4388
|
-
function getAllInternalDocIds(docStore) {
|
|
4389
|
-
const ids = /* @__PURE__ */ new Set();
|
|
4390
|
-
for (const internalId of docStore.allInternalIds()) {
|
|
4391
|
-
ids.add(internalId);
|
|
4392
|
-
}
|
|
4393
|
-
return ids;
|
|
4394
|
-
}
|
|
4395
4576
|
|
|
4396
4577
|
// src/core/partition/facets.ts
|
|
4397
4578
|
var UNDECLARED_FIELD_TYPE = "";
|
|
@@ -4527,16 +4708,113 @@ function computeFacets(state, matched, config, schema) {
|
|
|
4527
4708
|
return result;
|
|
4528
4709
|
}
|
|
4529
4710
|
|
|
4530
|
-
// src/
|
|
4531
|
-
var
|
|
4532
|
-
|
|
4533
|
-
|
|
4534
|
-
|
|
4535
|
-
|
|
4536
|
-
|
|
4711
|
+
// src/core/pattern-index/host.ts
|
|
4712
|
+
var PATTERN_HOST = { appendFoldedCodePoint };
|
|
4713
|
+
function patternIndexOf(indexes, fieldPath) {
|
|
4714
|
+
let index = indexes.get(fieldPath);
|
|
4715
|
+
if (index === void 0) {
|
|
4716
|
+
index = requirePatternSearch(fieldPath).createIndex(PATTERN_HOST);
|
|
4717
|
+
indexes.set(fieldPath, index);
|
|
4718
|
+
}
|
|
4719
|
+
return index;
|
|
4720
|
+
}
|
|
4721
|
+
|
|
4722
|
+
// src/geo/cells.ts
|
|
4723
|
+
var AXIS_CELLS = 2 ** LOCATION_CODE_BITS_PER_AXIS;
|
|
4724
|
+
function axisCell(value, lowest, span) {
|
|
4725
|
+
const cell = Math.floor((value - lowest) / span * AXIS_CELLS);
|
|
4726
|
+
if (!(cell > 0)) return 0;
|
|
4727
|
+
return cell >= AXIS_CELLS ? AXIS_CELLS - 1 : cell;
|
|
4728
|
+
}
|
|
4729
|
+
function latitudeCell(lat) {
|
|
4730
|
+
return axisCell(lat, -MAX_LATITUDE_DEGREES, 2 * MAX_LATITUDE_DEGREES);
|
|
4731
|
+
}
|
|
4732
|
+
function longitudeCell(lon) {
|
|
4733
|
+
return axisCell(lon, -MAX_LONGITUDE_DEGREES, FULL_TURN_DEGREES);
|
|
4734
|
+
}
|
|
4735
|
+
function interleave(row, column, bits) {
|
|
4736
|
+
let code = 0;
|
|
4737
|
+
for (let bit = bits - 1; bit >= 0; bit--) code = code * 4 + (row >>> bit & 1) * 2 + (column >>> bit & 1);
|
|
4738
|
+
return code;
|
|
4739
|
+
}
|
|
4740
|
+
function locationCode(lat, lon) {
|
|
4741
|
+
return interleave(latitudeCell(lat), longitudeCell(lon), LOCATION_CODE_BITS_PER_AXIS);
|
|
4742
|
+
}
|
|
4743
|
+
function appendBoxRanges(box, out) {
|
|
4744
|
+
const south = latitudeCell(box.south);
|
|
4745
|
+
const north = latitudeCell(box.north);
|
|
4746
|
+
const west = longitudeCell(box.west);
|
|
4747
|
+
const east = longitudeCell(box.east);
|
|
4748
|
+
let level2 = LOCATION_CODE_BITS_PER_AXIS;
|
|
4749
|
+
while (level2 > 0) {
|
|
4750
|
+
const shift2 = LOCATION_CODE_BITS_PER_AXIS - level2;
|
|
4751
|
+
const rows = (north >>> shift2) - (south >>> shift2) + 1;
|
|
4752
|
+
const columns = (east >>> shift2) - (west >>> shift2) + 1;
|
|
4753
|
+
if (rows * columns <= MAX_COVERING_CELLS) break;
|
|
4754
|
+
level2--;
|
|
4755
|
+
}
|
|
4756
|
+
const shift = LOCATION_CODE_BITS_PER_AXIS - level2;
|
|
4757
|
+
const codesPerCell = 4 ** shift;
|
|
4758
|
+
for (let row = south >>> shift; row <= north >>> shift; row++) {
|
|
4759
|
+
for (let column = west >>> shift; column <= east >>> shift; column++) {
|
|
4760
|
+
const from = interleave(row, column, level2) * codesPerCell;
|
|
4761
|
+
out.push({ from, to: from + codesPerCell });
|
|
4762
|
+
}
|
|
4763
|
+
}
|
|
4764
|
+
}
|
|
4765
|
+
function coveringRanges(boxes) {
|
|
4766
|
+
const ranges = [];
|
|
4767
|
+
for (const box of boxes) appendBoxRanges(box, ranges);
|
|
4768
|
+
ranges.sort((left, right) => left.from - right.from);
|
|
4769
|
+
const merged = [];
|
|
4770
|
+
for (const range of ranges) {
|
|
4771
|
+
const last = merged[merged.length - 1];
|
|
4772
|
+
if (last !== void 0 && range.from <= last.to) {
|
|
4773
|
+
if (range.to > last.to) last.to = range.to;
|
|
4774
|
+
} else {
|
|
4775
|
+
merged.push({ from: range.from, to: range.to });
|
|
4776
|
+
}
|
|
4777
|
+
}
|
|
4778
|
+
return merged;
|
|
4779
|
+
}
|
|
4780
|
+
function longitudeBoxes(south, north, west, east) {
|
|
4781
|
+
if (east - west >= FULL_TURN_DEGREES) {
|
|
4782
|
+
return [{ south, north, west: -MAX_LONGITUDE_DEGREES, east: MAX_LONGITUDE_DEGREES }];
|
|
4783
|
+
}
|
|
4784
|
+
if (west < -MAX_LONGITUDE_DEGREES) {
|
|
4785
|
+
return [
|
|
4786
|
+
{ south, north, west: west + FULL_TURN_DEGREES, east: MAX_LONGITUDE_DEGREES },
|
|
4787
|
+
{ south, north, west: -MAX_LONGITUDE_DEGREES, east }
|
|
4788
|
+
];
|
|
4789
|
+
}
|
|
4790
|
+
if (east > MAX_LONGITUDE_DEGREES) {
|
|
4791
|
+
return [
|
|
4792
|
+
{ south, north, west, east: MAX_LONGITUDE_DEGREES },
|
|
4793
|
+
{ south, north, west: -MAX_LONGITUDE_DEGREES, east: east - FULL_TURN_DEGREES }
|
|
4794
|
+
];
|
|
4795
|
+
}
|
|
4796
|
+
return [{ south, north, west, east }];
|
|
4797
|
+
}
|
|
4798
|
+
function radiusBoxes(lat, lon, distanceMeters) {
|
|
4799
|
+
const angle = distanceMeters / EARTH_RADIUS_METERS * RADIUS_BOX_MARGIN_RATIO + RADIUS_BOX_MARGIN_RADIANS;
|
|
4800
|
+
const angleDegrees = angle / DEG_TO_RAD;
|
|
4801
|
+
const south = lat - angleDegrees;
|
|
4802
|
+
const north = lat + angleDegrees;
|
|
4803
|
+
if (angle >= Math.PI || south <= -MAX_LATITUDE_DEGREES || north >= MAX_LATITUDE_DEGREES) {
|
|
4804
|
+
return [
|
|
4805
|
+
{
|
|
4806
|
+
south: Math.max(south, -MAX_LATITUDE_DEGREES),
|
|
4807
|
+
north: Math.min(north, MAX_LATITUDE_DEGREES),
|
|
4808
|
+
west: -MAX_LONGITUDE_DEGREES,
|
|
4809
|
+
east: MAX_LONGITUDE_DEGREES
|
|
4810
|
+
}
|
|
4811
|
+
];
|
|
4812
|
+
}
|
|
4813
|
+
const spread = Math.asin(Math.sin(angle) / Math.cos(lat * DEG_TO_RAD)) / DEG_TO_RAD;
|
|
4814
|
+
return longitudeBoxes(south, north, lon - spread, lon + spread);
|
|
4815
|
+
}
|
|
4537
4816
|
|
|
4538
4817
|
// src/geo/haversine.ts
|
|
4539
|
-
var EARTH_RADIUS_METERS = 63710088e-1;
|
|
4540
4818
|
function haversineDistance(lat1, lon1, lat2, lon2) {
|
|
4541
4819
|
const phi1 = lat1 * DEG_TO_RAD;
|
|
4542
4820
|
const phi2 = lat2 * DEG_TO_RAD;
|
|
@@ -4549,6 +4827,53 @@ function haversineDistance(lat1, lon1, lat2, lon2) {
|
|
|
4549
4827
|
return EARTH_RADIUS_METERS * c;
|
|
4550
4828
|
}
|
|
4551
4829
|
|
|
4830
|
+
// src/geo/edge-tree.ts
|
|
4831
|
+
function buildLongitudeEdgeTree(lons) {
|
|
4832
|
+
const n = lons.length;
|
|
4833
|
+
const edges = [];
|
|
4834
|
+
for (let i = 0; i < n; i++) {
|
|
4835
|
+
if (lons[i] !== lons[i === 0 ? n - 1 : i - 1]) edges.push(i);
|
|
4836
|
+
}
|
|
4837
|
+
const lowOf = (i) => Math.min(lons[i], lons[i === 0 ? n - 1 : i - 1]);
|
|
4838
|
+
edges.sort((left, right) => lowOf(left) - lowOf(right));
|
|
4839
|
+
const count = edges.length;
|
|
4840
|
+
const tree = {
|
|
4841
|
+
from: new Int32Array(count),
|
|
4842
|
+
to: new Int32Array(count),
|
|
4843
|
+
low: new Float64Array(count),
|
|
4844
|
+
high: new Float64Array(count),
|
|
4845
|
+
subtreeHigh: new Float64Array(count)
|
|
4846
|
+
};
|
|
4847
|
+
edges.forEach((i, at) => {
|
|
4848
|
+
const j = i === 0 ? n - 1 : i - 1;
|
|
4849
|
+
tree.from[at] = i;
|
|
4850
|
+
tree.to[at] = j;
|
|
4851
|
+
tree.low[at] = Math.min(lons[i], lons[j]);
|
|
4852
|
+
tree.high[at] = Math.max(lons[i], lons[j]);
|
|
4853
|
+
});
|
|
4854
|
+
const fillSubtreeHigh = (start, end) => {
|
|
4855
|
+
if (start >= end) return Number.NEGATIVE_INFINITY;
|
|
4856
|
+
const mid = start + end >>> 1;
|
|
4857
|
+
const highest = Math.max(tree.high[mid], fillSubtreeHigh(start, mid), fillSubtreeHigh(mid + 1, end));
|
|
4858
|
+
tree.subtreeHigh[mid] = highest;
|
|
4859
|
+
return highest;
|
|
4860
|
+
};
|
|
4861
|
+
fillSubtreeHigh(0, count);
|
|
4862
|
+
return tree;
|
|
4863
|
+
}
|
|
4864
|
+
function forEachEdgeSpanning(tree, lon, visit) {
|
|
4865
|
+
const walk = (start, end) => {
|
|
4866
|
+
if (start >= end) return;
|
|
4867
|
+
const mid = start + end >>> 1;
|
|
4868
|
+
if (tree.subtreeHigh[mid] <= lon) return;
|
|
4869
|
+
walk(start, mid);
|
|
4870
|
+
if (tree.low[mid] > lon) return;
|
|
4871
|
+
if (lon < tree.high[mid]) visit(tree.from[mid], tree.to[mid]);
|
|
4872
|
+
walk(mid + 1, end);
|
|
4873
|
+
};
|
|
4874
|
+
walk(0, tree.from.length);
|
|
4875
|
+
}
|
|
4876
|
+
|
|
4552
4877
|
// src/geo/polygon.ts
|
|
4553
4878
|
function requirePolygonRing(points) {
|
|
4554
4879
|
if (!Array.isArray(points) || points.length < MIN_POLYGON_POINTS) {
|
|
@@ -4560,11 +4885,18 @@ function requirePolygonRing(points) {
|
|
|
4560
4885
|
}
|
|
4561
4886
|
points.forEach((point, position) => {
|
|
4562
4887
|
const { lat, lon } = point ?? {};
|
|
4563
|
-
if (typeof lat
|
|
4888
|
+
if (typeof lat !== "number" || !Number.isFinite(lat) || typeof lon !== "number" || !Number.isFinite(lon)) {
|
|
4889
|
+
throw new NarsilError(
|
|
4890
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4891
|
+
`A polygon point carries a finite "lat" and "lon", and the point at position ${position} does not`,
|
|
4892
|
+
{ position }
|
|
4893
|
+
);
|
|
4894
|
+
}
|
|
4895
|
+
if (isLatitude(lat) && isLongitude(lon)) return;
|
|
4564
4896
|
throw new NarsilError(
|
|
4565
4897
|
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
4566
|
-
`A polygon point
|
|
4567
|
-
{ position }
|
|
4898
|
+
`A polygon point must hold a "lat" from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES} and a "lon" from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, while the point at position ${position} holds ${lat}, ${lon}`,
|
|
4899
|
+
{ position, lat, lon }
|
|
4568
4900
|
);
|
|
4569
4901
|
});
|
|
4570
4902
|
}
|
|
@@ -4597,20 +4929,35 @@ function polygonContainment(polygon) {
|
|
|
4597
4929
|
lats[i] = polygon[i].lat;
|
|
4598
4930
|
lons[i] = unwrap ? eastward(polygon[i].lon) : polygon[i].lon;
|
|
4599
4931
|
}
|
|
4932
|
+
const edges = buildLongitudeEdgeTree(lons);
|
|
4600
4933
|
return (lat, rawLon) => {
|
|
4601
4934
|
const lon = unwrap ? eastward(rawLon) : rawLon;
|
|
4602
4935
|
let inside = false;
|
|
4603
|
-
|
|
4604
|
-
const
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
const intersectLat = lats[i] + (lats[j] - lats[i]) / (lonJ - lonI) * (lon - lonI);
|
|
4608
|
-
if (lat < intersectLat) inside = !inside;
|
|
4609
|
-
}
|
|
4610
|
-
}
|
|
4936
|
+
forEachEdgeSpanning(edges, lon, (i, j) => {
|
|
4937
|
+
const intersectLat = lats[i] + (lats[j] - lats[i]) / (lons[j] - lons[i]) * (lon - lons[i]);
|
|
4938
|
+
if (lat < intersectLat) inside = !inside;
|
|
4939
|
+
});
|
|
4611
4940
|
return inside;
|
|
4612
4941
|
};
|
|
4613
4942
|
}
|
|
4943
|
+
function polygonBoxes(polygon) {
|
|
4944
|
+
if (polygon.length < MIN_POLYGON_POINTS) return [];
|
|
4945
|
+
const unwrap = crossesAntimeridian(polygon);
|
|
4946
|
+
let south = Number.POSITIVE_INFINITY;
|
|
4947
|
+
let north = Number.NEGATIVE_INFINITY;
|
|
4948
|
+
let west = Number.POSITIVE_INFINITY;
|
|
4949
|
+
let east = Number.NEGATIVE_INFINITY;
|
|
4950
|
+
for (const point of polygon) {
|
|
4951
|
+
const lon = unwrap ? eastward(point.lon) : point.lon;
|
|
4952
|
+
south = Math.min(south, point.lat);
|
|
4953
|
+
north = Math.max(north, point.lat);
|
|
4954
|
+
west = Math.min(west, lon);
|
|
4955
|
+
east = Math.max(east, lon);
|
|
4956
|
+
}
|
|
4957
|
+
if (west >= MAX_LONGITUDE_DEGREES)
|
|
4958
|
+
return longitudeBoxes(south, north, west - FULL_TURN_DEGREES, east - FULL_TURN_DEGREES);
|
|
4959
|
+
return longitudeBoxes(south, north, west, east);
|
|
4960
|
+
}
|
|
4614
4961
|
|
|
4615
4962
|
// src/geo/vincenty.ts
|
|
4616
4963
|
var WGS84_A = 6378137;
|
|
@@ -4662,61 +5009,94 @@ function vincentyDistance(lat1, lon1, lat2, lon2) {
|
|
|
4662
5009
|
}
|
|
4663
5010
|
|
|
4664
5011
|
// src/geo/geo-index.ts
|
|
5012
|
+
function firstAtOrAfter(entries, code) {
|
|
5013
|
+
let lo = 0;
|
|
5014
|
+
let hi = entries.length;
|
|
5015
|
+
while (lo < hi) {
|
|
5016
|
+
const mid = lo + hi >>> 1;
|
|
5017
|
+
if (entries[mid].code < code) lo = mid + 1;
|
|
5018
|
+
else hi = mid;
|
|
5019
|
+
}
|
|
5020
|
+
return lo;
|
|
5021
|
+
}
|
|
4665
5022
|
function createGeoIndex() {
|
|
4666
5023
|
const entries = [];
|
|
4667
|
-
|
|
4668
|
-
|
|
4669
|
-
|
|
4670
|
-
|
|
5024
|
+
const byDocument = /* @__PURE__ */ new Map();
|
|
5025
|
+
let sorted = true;
|
|
5026
|
+
function ensureSorted() {
|
|
5027
|
+
if (sorted) return;
|
|
5028
|
+
entries.sort((left, right) => left.code - right.code || left.docId - right.docId);
|
|
5029
|
+
sorted = true;
|
|
5030
|
+
}
|
|
5031
|
+
function removeEntry(internalId) {
|
|
5032
|
+
const entry = byDocument.get(internalId);
|
|
5033
|
+
if (entry === void 0) return;
|
|
5034
|
+
byDocument.delete(internalId);
|
|
5035
|
+
ensureSorted();
|
|
5036
|
+
for (let at = firstAtOrAfter(entries, entry.code); at < entries.length; at++) {
|
|
5037
|
+
if (entries[at] !== entry) continue;
|
|
5038
|
+
entries.splice(at, 1);
|
|
5039
|
+
return;
|
|
5040
|
+
}
|
|
5041
|
+
}
|
|
5042
|
+
function matching(boxes, passes, inside) {
|
|
5043
|
+
ensureSorted();
|
|
5044
|
+
const within = /* @__PURE__ */ new Set();
|
|
5045
|
+
for (const range of coveringRanges(boxes)) {
|
|
5046
|
+
for (let at = firstAtOrAfter(entries, range.from); at < entries.length && entries[at].code < range.to; at++) {
|
|
5047
|
+
if (passes(entries[at])) within.add(entries[at].docId);
|
|
5048
|
+
}
|
|
5049
|
+
}
|
|
5050
|
+
if (inside) return within;
|
|
5051
|
+
const outside = /* @__PURE__ */ new Set();
|
|
5052
|
+
for (const entry of entries) {
|
|
5053
|
+
if (!within.has(entry.docId)) outside.add(entry.docId);
|
|
5054
|
+
}
|
|
5055
|
+
return outside;
|
|
5056
|
+
}
|
|
5057
|
+
const index = {
|
|
4671
5058
|
insert(internalId, lat, lon) {
|
|
4672
|
-
|
|
5059
|
+
removeEntry(internalId);
|
|
5060
|
+
const entry = { lat, lon, docId: internalId, code: locationCode(lat, lon) };
|
|
5061
|
+
entries.push(entry);
|
|
5062
|
+
byDocument.set(internalId, entry);
|
|
5063
|
+
sorted = false;
|
|
4673
5064
|
},
|
|
4674
5065
|
remove(internalId) {
|
|
4675
|
-
|
|
4676
|
-
if (entries[i].docId === internalId) {
|
|
4677
|
-
entries.splice(i, 1);
|
|
4678
|
-
return;
|
|
4679
|
-
}
|
|
4680
|
-
}
|
|
5066
|
+
removeEntry(internalId);
|
|
4681
5067
|
},
|
|
4682
5068
|
radiusQuery(lat, lon, distanceMeters, inside, highPrecision) {
|
|
4683
|
-
const result = /* @__PURE__ */ new Set();
|
|
4684
5069
|
const distanceFn = highPrecision ? vincentyDistance : haversineDistance;
|
|
4685
|
-
|
|
4686
|
-
|
|
4687
|
-
|
|
4688
|
-
|
|
4689
|
-
|
|
4690
|
-
result.add(entry.docId);
|
|
4691
|
-
}
|
|
4692
|
-
}
|
|
4693
|
-
return result;
|
|
5070
|
+
return matching(
|
|
5071
|
+
radiusBoxes(lat, lon, distanceMeters),
|
|
5072
|
+
(entry) => distanceFn(lat, lon, entry.lat, entry.lon) <= distanceMeters,
|
|
5073
|
+
inside
|
|
5074
|
+
);
|
|
4694
5075
|
},
|
|
4695
5076
|
polygonQuery(points, inside) {
|
|
4696
|
-
const result = /* @__PURE__ */ new Set();
|
|
4697
5077
|
const contains = polygonContainment(points);
|
|
4698
|
-
|
|
4699
|
-
|
|
4700
|
-
|
|
4701
|
-
|
|
4702
|
-
|
|
4703
|
-
|
|
4704
|
-
}
|
|
4705
|
-
return result;
|
|
5078
|
+
return matching(polygonBoxes(points), (entry) => contains(entry.lat, entry.lon), inside);
|
|
5079
|
+
},
|
|
5080
|
+
documentsBitset(capacity) {
|
|
5081
|
+
const bits = createBitSet(capacity);
|
|
5082
|
+
for (const entry of entries) bitsetSet(bits, entry.docId);
|
|
5083
|
+
return bits;
|
|
4706
5084
|
},
|
|
4707
5085
|
clear() {
|
|
4708
5086
|
entries.length = 0;
|
|
5087
|
+
byDocument.clear();
|
|
5088
|
+
sorted = true;
|
|
4709
5089
|
},
|
|
4710
5090
|
serialize() {
|
|
4711
|
-
|
|
5091
|
+
ensureSorted();
|
|
5092
|
+
return entries.map((entry) => ({ lat: entry.lat, lon: entry.lon, docId: entry.docId }));
|
|
4712
5093
|
},
|
|
4713
5094
|
deserialize(data) {
|
|
4714
|
-
|
|
4715
|
-
for (const entry of data)
|
|
4716
|
-
entries.push({ lat: entry.lat, lon: entry.lon, docId: entry.docId });
|
|
4717
|
-
}
|
|
5095
|
+
index.clear();
|
|
5096
|
+
for (const entry of data) index.insert(entry.docId, entry.lat, entry.lon);
|
|
4718
5097
|
}
|
|
4719
5098
|
};
|
|
5099
|
+
return index;
|
|
4720
5100
|
}
|
|
4721
5101
|
|
|
4722
5102
|
// src/core/field-index/boolean.ts
|
|
@@ -4790,9 +5170,90 @@ function createBooleanIndex() {
|
|
|
4790
5170
|
};
|
|
4791
5171
|
}
|
|
4792
5172
|
|
|
5173
|
+
// src/core/field-index/affixes.ts
|
|
5174
|
+
function compareUnits(left, right) {
|
|
5175
|
+
if (left < right) return -1;
|
|
5176
|
+
return left > right ? 1 : 0;
|
|
5177
|
+
}
|
|
5178
|
+
function reversedUnits(value) {
|
|
5179
|
+
let reversed = "";
|
|
5180
|
+
for (let at = value.length - 1; at >= 0; at--) reversed += value[at];
|
|
5181
|
+
return reversed;
|
|
5182
|
+
}
|
|
5183
|
+
function firstAtOrAfter2(sorted, key) {
|
|
5184
|
+
let lo = 0;
|
|
5185
|
+
let hi = sorted.length;
|
|
5186
|
+
while (lo < hi) {
|
|
5187
|
+
const mid = lo + hi >>> 1;
|
|
5188
|
+
if (sorted[mid] < key) lo = mid + 1;
|
|
5189
|
+
else hi = mid;
|
|
5190
|
+
}
|
|
5191
|
+
return lo;
|
|
5192
|
+
}
|
|
5193
|
+
function settledColumn(column, holds) {
|
|
5194
|
+
const { keys, values } = column;
|
|
5195
|
+
const order = [];
|
|
5196
|
+
for (let at = 0; at < keys.length; at++) if (holds(values[at])) order.push(at);
|
|
5197
|
+
order.sort((left, right) => compareUnits(keys[left], keys[right]) || compareUnits(values[left], values[right]));
|
|
5198
|
+
const settled2 = { keys: [], values: [] };
|
|
5199
|
+
for (const at of order) {
|
|
5200
|
+
const last = settled2.values.length - 1;
|
|
5201
|
+
if (last >= 0 && settled2.keys[last] === keys[at] && settled2.values[last] === values[at]) continue;
|
|
5202
|
+
settled2.keys.push(keys[at]);
|
|
5203
|
+
settled2.values.push(values[at]);
|
|
5204
|
+
}
|
|
5205
|
+
return settled2;
|
|
5206
|
+
}
|
|
5207
|
+
var ValueAffixes = class {
|
|
5208
|
+
constructor(holds, values) {
|
|
5209
|
+
this.holds = holds;
|
|
5210
|
+
for (const value of values) this.add(value);
|
|
5211
|
+
}
|
|
5212
|
+
forward = { keys: [], values: [] };
|
|
5213
|
+
backward = { keys: [], values: [] };
|
|
5214
|
+
settled = true;
|
|
5215
|
+
add(value) {
|
|
5216
|
+
this.forward.keys.push(value);
|
|
5217
|
+
this.forward.values.push(value);
|
|
5218
|
+
this.backward.keys.push(reversedUnits(value));
|
|
5219
|
+
this.backward.values.push(value);
|
|
5220
|
+
this.settled = false;
|
|
5221
|
+
}
|
|
5222
|
+
forget() {
|
|
5223
|
+
this.settled = false;
|
|
5224
|
+
}
|
|
5225
|
+
withPrefix(prefix) {
|
|
5226
|
+
this.settle();
|
|
5227
|
+
return this.matching(this.forward, prefix);
|
|
5228
|
+
}
|
|
5229
|
+
withSuffix(suffix) {
|
|
5230
|
+
this.settle();
|
|
5231
|
+
return this.matching(this.backward, reversedUnits(suffix));
|
|
5232
|
+
}
|
|
5233
|
+
settle() {
|
|
5234
|
+
if (this.settled) return;
|
|
5235
|
+
this.forward = settledColumn(this.forward, this.holds);
|
|
5236
|
+
this.backward = settledColumn(this.backward, this.holds);
|
|
5237
|
+
this.settled = true;
|
|
5238
|
+
}
|
|
5239
|
+
matching(column, start) {
|
|
5240
|
+
const found = [];
|
|
5241
|
+
for (let at = firstAtOrAfter2(column.keys, start); at < column.keys.length; at++) {
|
|
5242
|
+
if (!column.keys[at].startsWith(start)) break;
|
|
5243
|
+
found.push(column.values[at]);
|
|
5244
|
+
}
|
|
5245
|
+
return found;
|
|
5246
|
+
}
|
|
5247
|
+
};
|
|
5248
|
+
|
|
4793
5249
|
// src/core/field-index/enum.ts
|
|
4794
5250
|
function createEnumIndex() {
|
|
4795
5251
|
const valueMap = /* @__PURE__ */ new Map();
|
|
5252
|
+
let affixes = null;
|
|
5253
|
+
function affixesOf() {
|
|
5254
|
+
affixes ??= new ValueAffixes((value) => valueMap.has(value), valueMap.keys());
|
|
5255
|
+
return affixes;
|
|
5256
|
+
}
|
|
4796
5257
|
function allDocs() {
|
|
4797
5258
|
const result = /* @__PURE__ */ new Set();
|
|
4798
5259
|
for (const docSet of valueMap.values()) {
|
|
@@ -4806,6 +5267,7 @@ function createEnumIndex() {
|
|
|
4806
5267
|
if (!docSet) {
|
|
4807
5268
|
docSet = /* @__PURE__ */ new Set();
|
|
4808
5269
|
valueMap.set(value, docSet);
|
|
5270
|
+
affixes?.add(value);
|
|
4809
5271
|
}
|
|
4810
5272
|
docSet.add(internalId);
|
|
4811
5273
|
},
|
|
@@ -4813,7 +5275,15 @@ function createEnumIndex() {
|
|
|
4813
5275
|
const docSet = valueMap.get(value);
|
|
4814
5276
|
if (!docSet) return;
|
|
4815
5277
|
docSet.delete(internalId);
|
|
4816
|
-
if (docSet.size
|
|
5278
|
+
if (docSet.size > 0) return;
|
|
5279
|
+
valueMap.delete(value);
|
|
5280
|
+
affixes?.forget();
|
|
5281
|
+
},
|
|
5282
|
+
queryPrefixBitset(prefix, capacity) {
|
|
5283
|
+
return this.queryInBitset(affixesOf().withPrefix(prefix), capacity);
|
|
5284
|
+
},
|
|
5285
|
+
querySuffixBitset(suffix, capacity) {
|
|
5286
|
+
return this.queryInBitset(affixesOf().withSuffix(suffix), capacity);
|
|
4817
5287
|
},
|
|
4818
5288
|
queryEq(value) {
|
|
4819
5289
|
const docSet = valueMap.get(value);
|
|
@@ -4889,6 +5359,7 @@ function createEnumIndex() {
|
|
|
4889
5359
|
},
|
|
4890
5360
|
clear() {
|
|
4891
5361
|
valueMap.clear();
|
|
5362
|
+
affixes = null;
|
|
4892
5363
|
},
|
|
4893
5364
|
serialize() {
|
|
4894
5365
|
const result = /* @__PURE__ */ Object.create(null);
|
|
@@ -4899,6 +5370,7 @@ function createEnumIndex() {
|
|
|
4899
5370
|
},
|
|
4900
5371
|
deserialize(data) {
|
|
4901
5372
|
valueMap.clear();
|
|
5373
|
+
affixes = null;
|
|
4902
5374
|
for (const value of Object.keys(data)) {
|
|
4903
5375
|
valueMap.set(value, new Set(data[value]));
|
|
4904
5376
|
}
|
|
@@ -5080,6 +5552,44 @@ function createNumericIndex() {
|
|
|
5080
5552
|
};
|
|
5081
5553
|
}
|
|
5082
5554
|
|
|
5555
|
+
// src/core/partition/value-sizes.ts
|
|
5556
|
+
function keepsValueSize(base) {
|
|
5557
|
+
return base === "string" || base === "verbatim" || base.endsWith("[]");
|
|
5558
|
+
}
|
|
5559
|
+
function valueSizeOf(value) {
|
|
5560
|
+
if (Array.isArray(value)) return value.length;
|
|
5561
|
+
return value === "" ? 0 : 1;
|
|
5562
|
+
}
|
|
5563
|
+
function sizeIndexOf(indexes, fieldPath) {
|
|
5564
|
+
let index = indexes.get(fieldPath);
|
|
5565
|
+
if (index === void 0) {
|
|
5566
|
+
index = createNumericIndex();
|
|
5567
|
+
indexes.set(fieldPath, index);
|
|
5568
|
+
}
|
|
5569
|
+
return index;
|
|
5570
|
+
}
|
|
5571
|
+
function recordValueSize(indexes, internalId, fieldPath, base, value) {
|
|
5572
|
+
if (value === void 0 || value === null || !keepsValueSize(base)) return;
|
|
5573
|
+
sizeIndexOf(indexes, fieldPath).insert(internalId, valueSizeOf(value));
|
|
5574
|
+
}
|
|
5575
|
+
function forgetValueSize(indexes, internalId, fieldPath, base, value) {
|
|
5576
|
+
if (value === void 0 || value === null || !keepsValueSize(base)) return;
|
|
5577
|
+
indexes.get(fieldPath)?.remove(internalId, valueSizeOf(value));
|
|
5578
|
+
}
|
|
5579
|
+
function rebuildSizeIndexes(state, schema) {
|
|
5580
|
+
state.sizeIndexes.clear();
|
|
5581
|
+
const fields = schemaFieldsOf(schema).filter((field) => keepsValueSize(field.base));
|
|
5582
|
+
if (fields.length === 0) return;
|
|
5583
|
+
for (const [docId, stored] of state.docStore.all()) {
|
|
5584
|
+
const internalId = state.docStore.getInternalId(docId);
|
|
5585
|
+
if (internalId === void 0) continue;
|
|
5586
|
+
for (const { path, base } of fields) {
|
|
5587
|
+
const value = getNestedValue(stored.fields, path);
|
|
5588
|
+
recordValueSize(state.sizeIndexes, internalId, path, base, value);
|
|
5589
|
+
}
|
|
5590
|
+
}
|
|
5591
|
+
}
|
|
5592
|
+
|
|
5083
5593
|
// src/core/partition/indexing.ts
|
|
5084
5594
|
function ensureFieldIndex(state, fieldPath, base) {
|
|
5085
5595
|
if ((base === "number" || base === "number[]") && !state.numericIndexes.has(fieldPath)) {
|
|
@@ -5218,10 +5728,12 @@ function indexDocument(state, docId, document2, fields, language, options) {
|
|
|
5218
5728
|
const fieldLengths = {};
|
|
5219
5729
|
const tokensByField = {};
|
|
5220
5730
|
const internalId = resolveInternalId(state, docId);
|
|
5221
|
-
for (const { path: fieldPath, base } of fields) {
|
|
5731
|
+
for (const { path: fieldPath, base, pattern } of fields) {
|
|
5222
5732
|
const value = getNestedValue(document2, fieldPath);
|
|
5223
5733
|
if (value === void 0 || value === null) continue;
|
|
5734
|
+
recordValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
5224
5735
|
ensureFieldIndex(state, fieldPath, base);
|
|
5736
|
+
if (pattern) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
|
|
5225
5737
|
if (base === "string") {
|
|
5226
5738
|
indexStringField(state, internalId, fieldPath, value, language, options, fieldLengths, tokensByField);
|
|
5227
5739
|
} else if (base === "number") {
|
|
@@ -5265,9 +5777,11 @@ function removeFromIndexes(state, docId, storedDoc, schemaFields, language, opti
|
|
|
5265
5777
|
const fields = storedDoc.fields;
|
|
5266
5778
|
const internalId = resolveInternalId(state, docId);
|
|
5267
5779
|
const opts = tokenizeOptions(options);
|
|
5268
|
-
for (const { path: fieldPath, base } of schemaFields) {
|
|
5780
|
+
for (const { path: fieldPath, base, pattern } of schemaFields) {
|
|
5269
5781
|
const value = getNestedValue(fields, fieldPath);
|
|
5270
5782
|
if (value === void 0 || value === null) continue;
|
|
5783
|
+
forgetValueSize(state.sizeIndexes, internalId, fieldPath, base, value);
|
|
5784
|
+
if (pattern) state.patternIndexes.get(fieldPath)?.remove(internalId, value);
|
|
5271
5785
|
if (base === "string") {
|
|
5272
5786
|
const result = tokenize(value, language, opts);
|
|
5273
5787
|
fieldLengths[fieldPath] = result.tokens.length;
|
|
@@ -5339,16 +5853,24 @@ function resolveInternalId2(state, docId) {
|
|
|
5339
5853
|
}
|
|
5340
5854
|
function updateFieldIndexOnly(state, docId, oldFields, newDoc, fields) {
|
|
5341
5855
|
const internalId = resolveInternalId2(state, docId);
|
|
5342
|
-
for (const { path: fieldPath, base: fieldType } of fields) {
|
|
5856
|
+
for (const { path: fieldPath, base: fieldType, pattern } of fields) {
|
|
5343
5857
|
if (isWordIndexedBase(fieldType)) continue;
|
|
5344
5858
|
const oldVal = getNestedValue(oldFields, fieldPath);
|
|
5345
5859
|
const newVal = getNestedValue(newDoc, fieldPath);
|
|
5346
5860
|
if (oldVal === newVal) continue;
|
|
5347
5861
|
if (oldVal !== void 0 && oldVal !== null && newVal !== void 0 && newVal !== null && typeof oldVal === typeof newVal && JSON.stringify(oldVal) === JSON.stringify(newVal))
|
|
5348
5862
|
continue;
|
|
5863
|
+
forgetValueSize(state.sizeIndexes, internalId, fieldPath, fieldType, oldVal);
|
|
5864
|
+
recordValueSize(state.sizeIndexes, internalId, fieldPath, fieldType, newVal);
|
|
5349
5865
|
if (newVal !== void 0 && newVal !== null) {
|
|
5350
5866
|
ensureFieldIndex(state, fieldPath, fieldType);
|
|
5351
5867
|
}
|
|
5868
|
+
if (pattern) {
|
|
5869
|
+
if (oldVal !== void 0 && oldVal !== null) state.patternIndexes.get(fieldPath)?.remove(internalId, oldVal);
|
|
5870
|
+
if (newVal !== void 0 && newVal !== null) {
|
|
5871
|
+
patternIndexOf(state.patternIndexes, fieldPath).add(internalId, newVal);
|
|
5872
|
+
}
|
|
5873
|
+
}
|
|
5352
5874
|
if (fieldType === "number") {
|
|
5353
5875
|
const numIdx = state.numericIndexes.get(fieldPath);
|
|
5354
5876
|
if (numIdx) {
|
|
@@ -5410,27 +5932,224 @@ function updateFieldIndexOnly(state, docId, oldFields, newDoc, fields) {
|
|
|
5410
5932
|
}
|
|
5411
5933
|
}
|
|
5412
5934
|
|
|
5413
|
-
// src/filters/
|
|
5414
|
-
|
|
5415
|
-
|
|
5416
|
-
|
|
5417
|
-
|
|
5418
|
-
|
|
5419
|
-
|
|
5935
|
+
// src/filters/constants.ts
|
|
5936
|
+
var MAX_FILTER_DEPTH = 30;
|
|
5937
|
+
var MAX_FILTER_TESTS = 1024;
|
|
5938
|
+
var MAX_FILTER_ARRAY_SIZE = 65536;
|
|
5939
|
+
var GEO_UNIT_METRES = { km: 1e3, mi: 1609.344, m: 1 };
|
|
5940
|
+
|
|
5941
|
+
// src/filters/pattern-tests.ts
|
|
5942
|
+
function excluding(context, fieldPath, excluded) {
|
|
5943
|
+
const universe = bitsetAnd(context.presentBitset(fieldPath), context.allDocIdsBitset);
|
|
5944
|
+
return bitsetAnd(universe, bitsetNot(excluded, context.capacity));
|
|
5945
|
+
}
|
|
5946
|
+
function textsOf(operand) {
|
|
5947
|
+
const texts = /* @__PURE__ */ new Set();
|
|
5948
|
+
if (!Array.isArray(operand)) return texts;
|
|
5949
|
+
for (const text of operand) if (typeof text === "string") texts.add(text);
|
|
5950
|
+
return texts;
|
|
5951
|
+
}
|
|
5952
|
+
function evaluatePatternTests(fieldPath, filter, context, bitsets) {
|
|
5953
|
+
const caseFold = filter.caseFold === true;
|
|
5954
|
+
const matching = (operator, texts) => {
|
|
5955
|
+
const into = createBitSet(context.capacity);
|
|
5956
|
+
for (const text of texts) context.matchPattern(fieldPath, operator, text, caseFold, into);
|
|
5957
|
+
return into;
|
|
5958
|
+
};
|
|
5959
|
+
if (typeof filter.eq === "string") bitsets.push(matching("eq", [filter.eq]));
|
|
5960
|
+
if (typeof filter.ne === "string") bitsets.push(excluding(context, fieldPath, matching("eq", [filter.ne])));
|
|
5961
|
+
if (filter.in !== void 0) bitsets.push(matching("eq", textsOf(filter.in)));
|
|
5962
|
+
if (filter.nin !== void 0) bitsets.push(excluding(context, fieldPath, matching("eq", textsOf(filter.nin))));
|
|
5963
|
+
if (typeof filter.startsWith === "string") bitsets.push(matching("startsWith", [filter.startsWith]));
|
|
5964
|
+
if (typeof filter.endsWith === "string") bitsets.push(matching("endsWith", [filter.endsWith]));
|
|
5965
|
+
if (typeof filter.contains === "string") bitsets.push(matching("contains", [filter.contains]));
|
|
5966
|
+
}
|
|
5967
|
+
|
|
5968
|
+
// src/filters/field-tests.ts
|
|
5969
|
+
var COMPARISON_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
|
|
5970
|
+
var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
|
|
5971
|
+
var FieldBits = class {
|
|
5972
|
+
constructor(context, fieldPath) {
|
|
5973
|
+
this.context = context;
|
|
5974
|
+
this.fieldPath = fieldPath;
|
|
5420
5975
|
}
|
|
5421
|
-
|
|
5976
|
+
none() {
|
|
5977
|
+
return createBitSet(this.context.capacity);
|
|
5978
|
+
}
|
|
5979
|
+
present() {
|
|
5980
|
+
return this.context.presentBitset(this.fieldPath);
|
|
5981
|
+
}
|
|
5982
|
+
holdingSomething() {
|
|
5983
|
+
const empty = this.context.sizeIndex(this.fieldPath)?.queryEqBitset(0, this.context.capacity);
|
|
5984
|
+
return empty === void 0 ? this.present() : this.without(this.present(), empty);
|
|
5985
|
+
}
|
|
5986
|
+
without(from, removed) {
|
|
5987
|
+
const kept = from.slice();
|
|
5988
|
+
bitsetAndNotInto(kept, removed);
|
|
5989
|
+
return kept;
|
|
5990
|
+
}
|
|
5991
|
+
complement(bits) {
|
|
5992
|
+
return this.without(this.context.allDocIdsBitset, bits);
|
|
5993
|
+
}
|
|
5994
|
+
};
|
|
5995
|
+
function comparisonBits(index, operator, operand, bits, capacity) {
|
|
5996
|
+
if (operator === "between") {
|
|
5997
|
+
const [low, high] = operand;
|
|
5998
|
+
return index?.queryBetweenBitset(low, high, capacity) ?? bits.none();
|
|
5999
|
+
}
|
|
6000
|
+
const value = operand;
|
|
6001
|
+
if (operator === "ne") return bits.without(bits.present(), index?.queryEqBitset(value, capacity) ?? bits.none());
|
|
6002
|
+
if (index === void 0) return bits.none();
|
|
6003
|
+
if (operator === "eq") return index.queryEqBitset(value, capacity);
|
|
6004
|
+
if (operator === "gt") return index.queryGtBitset(value, capacity);
|
|
6005
|
+
if (operator === "gte") return index.queryGteBitset(value, capacity);
|
|
6006
|
+
if (operator === "lt") return index.queryLtBitset(value, capacity);
|
|
6007
|
+
return index.queryLteBitset(value, capacity);
|
|
6008
|
+
}
|
|
6009
|
+
function elementBits(context, fieldPath, parsed, value) {
|
|
6010
|
+
const { capacity } = context;
|
|
6011
|
+
if (parsed.pattern) {
|
|
6012
|
+
const matches = createBitSet(capacity);
|
|
6013
|
+
context.matchPattern(fieldPath, "eq", value, false, matches);
|
|
6014
|
+
return matches;
|
|
6015
|
+
}
|
|
6016
|
+
if (typeof value === "number")
|
|
6017
|
+
return context.numericIndex(fieldPath)?.queryEqBitset(value, capacity) ?? createBitSet(capacity);
|
|
6018
|
+
if (typeof value === "boolean")
|
|
6019
|
+
return context.booleanIndex(fieldPath)?.queryEqBitset(value, capacity) ?? createBitSet(capacity);
|
|
6020
|
+
return context.enumIndex(fieldPath)?.queryEqBitset(value, capacity) ?? createBitSet(capacity);
|
|
6021
|
+
}
|
|
6022
|
+
function anyElementBits(context, fieldPath, parsed, values) {
|
|
6023
|
+
const { capacity } = context;
|
|
6024
|
+
const union = createBitSet(capacity);
|
|
6025
|
+
if (parsed.pattern) {
|
|
6026
|
+
for (const value of values) context.matchPattern(fieldPath, "eq", value, false, union);
|
|
6027
|
+
return union;
|
|
6028
|
+
}
|
|
6029
|
+
if (parsed.base === "enum[]") {
|
|
6030
|
+
return context.enumIndex(fieldPath)?.queryInBitset(values, capacity) ?? union;
|
|
6031
|
+
}
|
|
6032
|
+
const numbers = context.numericIndex(fieldPath);
|
|
6033
|
+
const booleans = context.booleanIndex(fieldPath);
|
|
6034
|
+
for (const value of values) {
|
|
6035
|
+
if (typeof value === "number") for (const docId of numbers?.queryEq(value) ?? []) bitsetSet(union, docId);
|
|
6036
|
+
if (typeof value === "boolean") for (const docId of booleans?.queryEq(value) ?? []) bitsetSet(union, docId);
|
|
6037
|
+
}
|
|
6038
|
+
return union;
|
|
6039
|
+
}
|
|
6040
|
+
function distinctScalars(values) {
|
|
6041
|
+
return [...new Set(values)];
|
|
5422
6042
|
}
|
|
5423
|
-
function
|
|
5424
|
-
if (
|
|
5425
|
-
if (
|
|
5426
|
-
|
|
5427
|
-
|
|
5428
|
-
|
|
6043
|
+
function narrowByElements(fieldPath, f, parsed, context, bits, narrow) {
|
|
6044
|
+
if (f.matchesAny !== void 0) narrow(anyElementBits(context, fieldPath, parsed, distinctScalars(f.matchesAny)));
|
|
6045
|
+
if (f.containsAll !== void 0) {
|
|
6046
|
+
const all = bits.present().slice();
|
|
6047
|
+
for (const value of distinctScalars(f.containsAll)) {
|
|
6048
|
+
if (bitsetIsEmpty(all)) break;
|
|
6049
|
+
bitsetAndInto(all, elementBits(context, fieldPath, parsed, value));
|
|
6050
|
+
}
|
|
6051
|
+
narrow(all);
|
|
5429
6052
|
}
|
|
5430
|
-
|
|
6053
|
+
if (f.size !== void 0) {
|
|
6054
|
+
const sizes = context.sizeIndex(fieldPath);
|
|
6055
|
+
const comparison = f.size;
|
|
6056
|
+
for (const operator of COMPARISON_OPERATORS) {
|
|
6057
|
+
if (comparison[operator] === void 0) continue;
|
|
6058
|
+
narrow(comparisonBits(sizes, operator, comparison[operator], bits, context.capacity));
|
|
6059
|
+
}
|
|
6060
|
+
}
|
|
6061
|
+
}
|
|
6062
|
+
function narrowByValue(fieldPath, f, parsed, context, bits, narrow) {
|
|
6063
|
+
const { capacity } = context;
|
|
6064
|
+
const base = parsed.base;
|
|
6065
|
+
if (base === "number" || base === "number[]") {
|
|
6066
|
+
const index = context.numericIndex(fieldPath);
|
|
6067
|
+
for (const operator of COMPARISON_OPERATORS) {
|
|
6068
|
+
const operand = f[operator];
|
|
6069
|
+
if (operand === void 0 || typeof operand === "string") continue;
|
|
6070
|
+
if (operator === "between" && typeof operand[0] === "string") continue;
|
|
6071
|
+
narrow(comparisonBits(index, operator, operand, bits, capacity));
|
|
6072
|
+
}
|
|
6073
|
+
return;
|
|
6074
|
+
}
|
|
6075
|
+
if (base === "boolean" || base === "boolean[]") {
|
|
6076
|
+
const index = context.booleanIndex(fieldPath);
|
|
6077
|
+
const equal = (value) => index?.queryEqBitset(value, capacity) ?? bits.none();
|
|
6078
|
+
if (typeof f.eq === "boolean") narrow(equal(f.eq));
|
|
6079
|
+
if (typeof f.ne === "boolean") narrow(bits.without(bits.present(), equal(f.ne)));
|
|
6080
|
+
return;
|
|
6081
|
+
}
|
|
6082
|
+
if (base === "enum" || base === "enum[]") {
|
|
6083
|
+
const index = context.enumIndex(fieldPath);
|
|
6084
|
+
const equal = (value) => index?.queryEqBitset(value, capacity) ?? bits.none();
|
|
6085
|
+
const anyOf = (values) => index?.queryInBitset(values, capacity) ?? bits.none();
|
|
6086
|
+
if (typeof f.eq === "string") narrow(equal(f.eq));
|
|
6087
|
+
if (typeof f.ne === "string") narrow(bits.without(bits.present(), equal(f.ne)));
|
|
6088
|
+
if (Array.isArray(f.in)) narrow(anyOf(f.in));
|
|
6089
|
+
if (Array.isArray(f.nin)) narrow(bits.without(bits.present(), anyOf(f.nin)));
|
|
6090
|
+
if (typeof f.startsWith === "string") narrow(index?.queryPrefixBitset(f.startsWith, capacity) ?? bits.none());
|
|
6091
|
+
if (typeof f.endsWith === "string") narrow(index?.querySuffixBitset(f.endsWith, capacity) ?? bits.none());
|
|
6092
|
+
}
|
|
6093
|
+
}
|
|
6094
|
+
function narrowByLocation(fieldPath, f, context, narrow) {
|
|
6095
|
+
const index = context.geoIndex(fieldPath);
|
|
6096
|
+
if (f.radius !== void 0) {
|
|
6097
|
+
const circle = f.radius;
|
|
6098
|
+
const found = index?.radiusQuery(
|
|
6099
|
+
circle.lat,
|
|
6100
|
+
circle.lon,
|
|
6101
|
+
circle.distance * GEO_UNIT_METRES[circle.unit],
|
|
6102
|
+
circle.inside ?? true,
|
|
6103
|
+
circle.highPrecision ?? false
|
|
6104
|
+
);
|
|
6105
|
+
narrow(bitsetFromSet(found ?? /* @__PURE__ */ new Set(), context.capacity));
|
|
6106
|
+
}
|
|
6107
|
+
if (f.polygon !== void 0) {
|
|
6108
|
+
const polygon = f.polygon;
|
|
6109
|
+
narrow(bitsetFromSet(index?.polygonQuery(polygon.points, polygon.inside ?? true) ?? /* @__PURE__ */ new Set(), context.capacity));
|
|
6110
|
+
}
|
|
6111
|
+
}
|
|
6112
|
+
function narrowByPresence(f, bits, narrow) {
|
|
6113
|
+
if (typeof f.exists === "boolean") narrow(f.exists ? bits.present() : bits.complement(bits.present()));
|
|
6114
|
+
if (typeof f.notExists === "boolean") narrow(f.notExists ? bits.complement(bits.present()) : bits.present());
|
|
6115
|
+
if (typeof f.isNotEmpty === "boolean") {
|
|
6116
|
+
const holding = bits.holdingSomething();
|
|
6117
|
+
narrow(f.isNotEmpty ? holding : bits.complement(holding));
|
|
6118
|
+
}
|
|
6119
|
+
if (typeof f.isEmpty === "boolean") {
|
|
6120
|
+
const holding = bits.holdingSomething();
|
|
6121
|
+
narrow(f.isEmpty ? bits.complement(holding) : holding);
|
|
6122
|
+
}
|
|
6123
|
+
}
|
|
6124
|
+
function textBoundsOf(operator, bound) {
|
|
6125
|
+
if (operator === "gt" || operator === "gte") {
|
|
6126
|
+
return { lower: bound, upper: null, lowerInclusive: operator === "gte", upperInclusive: false };
|
|
6127
|
+
}
|
|
6128
|
+
return { lower: null, upper: bound, upperInclusive: operator === "lte", lowerInclusive: false };
|
|
5431
6129
|
}
|
|
5432
|
-
function
|
|
5433
|
-
|
|
6130
|
+
function narrowByTextRange(fieldPath, f, context, narrow) {
|
|
6131
|
+
for (const operator of TEXT_RANGE_OPERATORS) {
|
|
6132
|
+
const bound = f[operator];
|
|
6133
|
+
if (typeof bound === "string") narrow(context.textRangeBitset(fieldPath, textBoundsOf(operator, bound)));
|
|
6134
|
+
}
|
|
6135
|
+
if (!Array.isArray(f.between)) return;
|
|
6136
|
+
const [low, high] = f.between;
|
|
6137
|
+
if (typeof low !== "string" || typeof high !== "string") return;
|
|
6138
|
+
narrow(context.textRangeBitset(fieldPath, { lower: low, lowerInclusive: true, upper: high, upperInclusive: true }));
|
|
6139
|
+
}
|
|
6140
|
+
function narrowByFieldTests(fieldPath, f, parsed, context, narrow) {
|
|
6141
|
+
const bits = new FieldBits(context, fieldPath);
|
|
6142
|
+
narrowByPresence(f, bits, narrow);
|
|
6143
|
+
narrowByLocation(fieldPath, f, context, narrow);
|
|
6144
|
+
narrowByTextRange(fieldPath, f, context, narrow);
|
|
6145
|
+
if (parsed.pattern) {
|
|
6146
|
+
const patternBits = [];
|
|
6147
|
+
evaluatePatternTests(fieldPath, f, context, patternBits);
|
|
6148
|
+
for (const matched of patternBits) narrow(matched);
|
|
6149
|
+
} else {
|
|
6150
|
+
narrowByValue(fieldPath, f, parsed, context, bits, narrow);
|
|
6151
|
+
}
|
|
6152
|
+
narrowByElements(fieldPath, f, parsed, context, bits, narrow);
|
|
5434
6153
|
}
|
|
5435
6154
|
|
|
5436
6155
|
// src/filters/keys.ts
|
|
@@ -5447,6 +6166,8 @@ var FIELD_FILTER_OPERATORS = [
|
|
|
5447
6166
|
"nin",
|
|
5448
6167
|
"startsWith",
|
|
5449
6168
|
"endsWith",
|
|
6169
|
+
"contains",
|
|
6170
|
+
"caseFold",
|
|
5450
6171
|
"containsAll",
|
|
5451
6172
|
"matchesAny",
|
|
5452
6173
|
"size",
|
|
@@ -5458,14 +6179,7 @@ var FIELD_FILTER_OPERATORS = [
|
|
|
5458
6179
|
"polygon"
|
|
5459
6180
|
];
|
|
5460
6181
|
|
|
5461
|
-
// src/filters/
|
|
5462
|
-
var MAX_FILTER_DEPTH = 30;
|
|
5463
|
-
|
|
5464
|
-
// src/filters/operands.ts
|
|
5465
|
-
var RANGE_OPERATORS = ["gt", "lt", "gte", "lte"];
|
|
5466
|
-
var TEXT_OPERATORS = ["startsWith", "endsWith"];
|
|
5467
|
-
var EQUALITY_OPERATORS = ["eq", "ne"];
|
|
5468
|
-
var LIST_OPERATORS = ["in", "nin"];
|
|
6182
|
+
// src/filters/operands/checks.ts
|
|
5469
6183
|
function operandKindOf(type) {
|
|
5470
6184
|
if (type === void 0) return void 0;
|
|
5471
6185
|
if (type === "number" || type === "number[]") return "number";
|
|
@@ -5483,6 +6197,9 @@ function kindOfValue(value) {
|
|
|
5483
6197
|
if (typeof value === "string") return "text";
|
|
5484
6198
|
return void 0;
|
|
5485
6199
|
}
|
|
6200
|
+
function isObjectRecord(value) {
|
|
6201
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
6202
|
+
}
|
|
5486
6203
|
function invalid(message, fieldPath, operator) {
|
|
5487
6204
|
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message, { fieldPath, operator });
|
|
5488
6205
|
}
|
|
@@ -5505,6 +6222,20 @@ function requireTextRangeField(fieldPath, operator, type) {
|
|
|
5505
6222
|
operator
|
|
5506
6223
|
);
|
|
5507
6224
|
}
|
|
6225
|
+
function requireListLength(fieldPath, operator, values) {
|
|
6226
|
+
if (values.length <= MAX_FILTER_ARRAY_SIZE) return;
|
|
6227
|
+
invalid(
|
|
6228
|
+
`"${operator}" on field "${fieldPath}" holds ${values.length} values, while a filter list holds at most ${MAX_FILTER_ARRAY_SIZE}`,
|
|
6229
|
+
fieldPath,
|
|
6230
|
+
operator
|
|
6231
|
+
);
|
|
6232
|
+
}
|
|
6233
|
+
|
|
6234
|
+
// src/filters/operands/geo.ts
|
|
6235
|
+
function requireFlag(fieldPath, operator, flag, value) {
|
|
6236
|
+
if (value === void 0 || typeof value === "boolean") return;
|
|
6237
|
+
invalid(`"${operator}.${flag}" on field "${fieldPath}" must be true or false`, fieldPath, operator);
|
|
6238
|
+
}
|
|
5508
6239
|
function requireRadius(fieldPath, radius) {
|
|
5509
6240
|
const circle = radius;
|
|
5510
6241
|
if (circle === null || typeof circle !== "object") {
|
|
@@ -5515,6 +6246,15 @@ function requireRadius(fieldPath, radius) {
|
|
|
5515
6246
|
invalid(`"radius.${key}" on field "${fieldPath}" must be a finite number`, fieldPath, "radius");
|
|
5516
6247
|
}
|
|
5517
6248
|
}
|
|
6249
|
+
if (typeof circle.unit !== "string" || !Object.hasOwn(GEO_UNIT_METRES, circle.unit)) {
|
|
6250
|
+
invalid(
|
|
6251
|
+
`"radius.unit" on field "${fieldPath}" must be one of ${Object.keys(GEO_UNIT_METRES).join(", ")}, and this filter sets ${String(circle.unit)}`,
|
|
6252
|
+
fieldPath,
|
|
6253
|
+
"radius"
|
|
6254
|
+
);
|
|
6255
|
+
}
|
|
6256
|
+
requireFlag(fieldPath, "radius", "inside", circle.inside);
|
|
6257
|
+
requireFlag(fieldPath, "radius", "highPrecision", circle.highPrecision);
|
|
5518
6258
|
if (circle.distance < 0) {
|
|
5519
6259
|
invalid(
|
|
5520
6260
|
`"radius.distance" on field "${fieldPath}" must be zero or more, and this filter sets ${String(circle.distance)}`,
|
|
@@ -5522,13 +6262,197 @@ function requireRadius(fieldPath, radius) {
|
|
|
5522
6262
|
"radius"
|
|
5523
6263
|
);
|
|
5524
6264
|
}
|
|
6265
|
+
if (!isLatitude(circle.lat)) {
|
|
6266
|
+
invalid(
|
|
6267
|
+
`"radius.lat" on field "${fieldPath}" must be from -${MAX_LATITUDE_DEGREES} to ${MAX_LATITUDE_DEGREES}, and this filter sets ${String(circle.lat)}`,
|
|
6268
|
+
fieldPath,
|
|
6269
|
+
"radius"
|
|
6270
|
+
);
|
|
6271
|
+
}
|
|
6272
|
+
if (!isLongitude(circle.lon)) {
|
|
6273
|
+
invalid(
|
|
6274
|
+
`"radius.lon" on field "${fieldPath}" must be from -${MAX_LONGITUDE_DEGREES} to ${MAX_LONGITUDE_DEGREES}, and this filter sets ${String(circle.lon)}`,
|
|
6275
|
+
fieldPath,
|
|
6276
|
+
"radius"
|
|
6277
|
+
);
|
|
6278
|
+
}
|
|
6279
|
+
}
|
|
6280
|
+
function requireGeoOperands(fieldPath, filter) {
|
|
6281
|
+
if (filter.radius !== void 0) requireRadius(fieldPath, filter.radius);
|
|
6282
|
+
if (filter.polygon === void 0) return;
|
|
6283
|
+
if (!isObjectRecord(filter.polygon)) {
|
|
6284
|
+
invalid(`"polygon" on field "${fieldPath}" must be an object`, fieldPath, "polygon");
|
|
6285
|
+
}
|
|
6286
|
+
requireFlag(fieldPath, "polygon", "inside", filter.polygon.inside);
|
|
6287
|
+
const points = filter.polygon.points;
|
|
6288
|
+
if (Array.isArray(points)) requireListLength(fieldPath, "polygon", points);
|
|
6289
|
+
requirePolygonRing(points);
|
|
6290
|
+
}
|
|
6291
|
+
|
|
6292
|
+
// src/filters/operands/lists.ts
|
|
6293
|
+
var WHOLE_VALUE_OPERATORS = ["eq", "ne", "in", "nin", "startsWith", "endsWith", "matchesAny", "containsAll"];
|
|
6294
|
+
var ELEMENT_OPERATORS = ["matchesAny", "containsAll"];
|
|
6295
|
+
var PRESENCE_OPERATORS = ["exists", "notExists", "isEmpty", "isNotEmpty"];
|
|
6296
|
+
var SIZE_OPERATORS = ["eq", "ne", "gt", "gte", "lt", "lte", "between"];
|
|
6297
|
+
var SIZE_OPERATOR_SET = new Set(SIZE_OPERATORS);
|
|
6298
|
+
var ELEMENT_KIND_BY_BASE = {
|
|
6299
|
+
"number[]": "number",
|
|
6300
|
+
"boolean[]": "boolean",
|
|
6301
|
+
"enum[]": "text",
|
|
6302
|
+
"string[]": "text",
|
|
6303
|
+
"verbatim[]": "text"
|
|
6304
|
+
};
|
|
6305
|
+
function requireWholeValueIndex(fieldPath, filter, type) {
|
|
6306
|
+
const parsed = parsedTypeOf(type);
|
|
6307
|
+
if (parsed === void 0 || parsed.pattern || parsed.base !== "string" && parsed.base !== "string[]") return;
|
|
6308
|
+
for (const operator of WHOLE_VALUE_OPERATORS) {
|
|
6309
|
+
if (filter[operator] === void 0) continue;
|
|
6310
|
+
invalid(
|
|
6311
|
+
`"${operator}" compares whole values of field "${fieldPath}", which the engine finds only through a pattern index. A "${parsed.base}" field keeps a pattern index where its type includes "pattern", so declare the field as "${parsed.base}:pattern" in a new index to filter it this way`,
|
|
6312
|
+
fieldPath,
|
|
6313
|
+
operator
|
|
6314
|
+
);
|
|
6315
|
+
}
|
|
5525
6316
|
}
|
|
6317
|
+
function requirePresenceOperands(fieldPath, filter) {
|
|
6318
|
+
for (const operator of PRESENCE_OPERATORS) {
|
|
6319
|
+
const operand = filter[operator];
|
|
6320
|
+
if (operand !== void 0 && typeof operand !== "boolean") {
|
|
6321
|
+
invalid(`"${operator}" on field "${fieldPath}" must be true or false`, fieldPath, operator);
|
|
6322
|
+
}
|
|
6323
|
+
}
|
|
6324
|
+
}
|
|
6325
|
+
function listElementKind(fieldPath, operator, type) {
|
|
6326
|
+
const base = parsedTypeOf(type)?.base;
|
|
6327
|
+
const kind = base === void 0 ? void 0 : ELEMENT_KIND_BY_BASE[base];
|
|
6328
|
+
if (kind !== void 0) return kind;
|
|
6329
|
+
return invalid(
|
|
6330
|
+
`"${operator}" tests the elements of a list field, while the schema declares field "${fieldPath}" as "${type}"`,
|
|
6331
|
+
fieldPath,
|
|
6332
|
+
operator
|
|
6333
|
+
);
|
|
6334
|
+
}
|
|
6335
|
+
function requireElementOperands(fieldPath, filter, type) {
|
|
6336
|
+
for (const operator of ELEMENT_OPERATORS) {
|
|
6337
|
+
const values = filter[operator];
|
|
6338
|
+
if (values === void 0) continue;
|
|
6339
|
+
if (!Array.isArray(values)) invalid(`"${operator}" on field "${fieldPath}" must be a list`, fieldPath, operator);
|
|
6340
|
+
requireListLength(fieldPath, operator, values);
|
|
6341
|
+
const kind = listElementKind(fieldPath, operator, type);
|
|
6342
|
+
if (values.some((value) => kindOfValue(value) !== kind)) {
|
|
6343
|
+
invalid(
|
|
6344
|
+
`Every value of "${operator}" on field "${fieldPath}" must be a ${kind === "text" ? "string" : kind === "number" ? "finite number" : "boolean"}, because the field holds a list of them`,
|
|
6345
|
+
fieldPath,
|
|
6346
|
+
operator
|
|
6347
|
+
);
|
|
6348
|
+
}
|
|
6349
|
+
}
|
|
6350
|
+
}
|
|
6351
|
+
function requireSizeOperands(fieldPath, filter, type) {
|
|
6352
|
+
const size = filter.size;
|
|
6353
|
+
if (size === void 0) return;
|
|
6354
|
+
if (!isObjectRecord(size)) invalid(`"size" on field "${fieldPath}" must be a comparison object`, fieldPath, "size");
|
|
6355
|
+
listElementKind(fieldPath, "size", type);
|
|
6356
|
+
for (const [operator, operand] of Object.entries(size)) {
|
|
6357
|
+
if (!SIZE_OPERATOR_SET.has(operator)) {
|
|
6358
|
+
invalid(
|
|
6359
|
+
`"size" on field "${fieldPath}" takes ${SIZE_OPERATORS.join(", ")}, while "${operator}" is none of them`,
|
|
6360
|
+
fieldPath,
|
|
6361
|
+
"size"
|
|
6362
|
+
);
|
|
6363
|
+
}
|
|
6364
|
+
const bounds = operator === "between" ? operand : [operand];
|
|
6365
|
+
if (!Array.isArray(bounds) || bounds.length !== (operator === "between" ? 2 : 1) || !bounds.every(isFiniteNumber)) {
|
|
6366
|
+
invalid(
|
|
6367
|
+
`"size.${operator}" on field "${fieldPath}" must be ${operator === "between" ? "two finite numbers, the lower bound first" : "a finite number"}`,
|
|
6368
|
+
fieldPath,
|
|
6369
|
+
"size"
|
|
6370
|
+
);
|
|
6371
|
+
}
|
|
6372
|
+
}
|
|
6373
|
+
}
|
|
6374
|
+
|
|
6375
|
+
// src/filters/operands/pattern.ts
|
|
6376
|
+
function requirePatternField(fieldPath, operator, type) {
|
|
6377
|
+
if (parsedTypeOf(type)?.pattern === true) return;
|
|
6378
|
+
const declared = type === void 0 ? `The schema declares no field "${fieldPath}"` : `Field "${fieldPath}" has type "${type}"`;
|
|
6379
|
+
invalid(
|
|
6380
|
+
`"${operator}" applies only to a pattern field, because only a pattern field has a pattern index. A pattern field is a "verbatim" field, or a text field whose type includes "pattern". ${declared}`,
|
|
6381
|
+
fieldPath,
|
|
6382
|
+
operator
|
|
6383
|
+
);
|
|
6384
|
+
}
|
|
6385
|
+
function requirePatternText(fieldPath, operator, text, caseFold) {
|
|
6386
|
+
if (exceedsCodePoints(text, MAX_PATTERN_TEXT_CODE_POINTS)) {
|
|
6387
|
+
invalid(
|
|
6388
|
+
`The text of "${operator}" on pattern field "${fieldPath}" holds more than ${MAX_PATTERN_TEXT_CODE_POINTS} code points`,
|
|
6389
|
+
fieldPath,
|
|
6390
|
+
operator
|
|
6391
|
+
);
|
|
6392
|
+
}
|
|
6393
|
+
if (caseFold && foldedCodePointCount(text) > MAX_PATTERN_POSITIONS) {
|
|
6394
|
+
invalid(
|
|
6395
|
+
`The text of "${operator}" on pattern field "${fieldPath}" has a case fold of more than ${MAX_PATTERN_POSITIONS} code points, while a pattern holds at most ${MAX_PATTERN_POSITIONS} positions`,
|
|
6396
|
+
fieldPath,
|
|
6397
|
+
operator
|
|
6398
|
+
);
|
|
6399
|
+
}
|
|
6400
|
+
}
|
|
6401
|
+
var ELEMENT_OPERATORS2 = ["matchesAny", "containsAll"];
|
|
6402
|
+
function requirePatternOperands(fieldPath, filter, type) {
|
|
6403
|
+
if (filter.contains !== void 0) {
|
|
6404
|
+
if (typeof filter.contains !== "string") {
|
|
6405
|
+
invalid(`"contains" on field "${fieldPath}" must be a string`, fieldPath, "contains");
|
|
6406
|
+
}
|
|
6407
|
+
requirePatternField(fieldPath, "contains", type);
|
|
6408
|
+
}
|
|
6409
|
+
if (filter.caseFold !== void 0) {
|
|
6410
|
+
if (typeof filter.caseFold !== "boolean") {
|
|
6411
|
+
invalid(`"caseFold" on field "${fieldPath}" must be true or false`, fieldPath, "caseFold");
|
|
6412
|
+
}
|
|
6413
|
+
requirePatternField(fieldPath, "caseFold", type);
|
|
6414
|
+
}
|
|
6415
|
+
if (parsedTypeOf(type)?.pattern !== true) return;
|
|
6416
|
+
const caseFold = filter.caseFold === true;
|
|
6417
|
+
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
6418
|
+
const operand = filter[operator];
|
|
6419
|
+
if (typeof operand === "string") requirePatternText(fieldPath, operator, operand, caseFold);
|
|
6420
|
+
if (!Array.isArray(operand)) continue;
|
|
6421
|
+
for (const text of operand) {
|
|
6422
|
+
if (typeof text === "string") requirePatternText(fieldPath, operator, text, caseFold);
|
|
6423
|
+
}
|
|
6424
|
+
}
|
|
6425
|
+
for (const operator of ELEMENT_OPERATORS2) {
|
|
6426
|
+
const operand = filter[operator];
|
|
6427
|
+
if (!Array.isArray(operand)) continue;
|
|
6428
|
+
for (const text of operand) {
|
|
6429
|
+
if (typeof text === "string") requirePatternText(fieldPath, operator, text, false);
|
|
6430
|
+
}
|
|
6431
|
+
}
|
|
6432
|
+
}
|
|
6433
|
+
function requireWellFormedText(fieldPath, filter) {
|
|
6434
|
+
for (const [operator, operand] of Object.entries(filter)) {
|
|
6435
|
+
const texts = Array.isArray(operand) ? operand : [operand];
|
|
6436
|
+
for (const text of texts) {
|
|
6437
|
+
if (typeof text !== "string" || !holdsUnpairedSurrogate(text)) continue;
|
|
6438
|
+
invalid(
|
|
6439
|
+
`The text of "${operator}" on field "${fieldPath}" holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so no stored value can hold it`,
|
|
6440
|
+
fieldPath,
|
|
6441
|
+
operator
|
|
6442
|
+
);
|
|
6443
|
+
}
|
|
6444
|
+
}
|
|
6445
|
+
}
|
|
6446
|
+
|
|
6447
|
+
// src/filters/operands/index.ts
|
|
6448
|
+
var RANGE_OPERATORS = ["gt", "lt", "gte", "lte"];
|
|
6449
|
+
var TEXT_OPERATORS = ["startsWith", "endsWith"];
|
|
6450
|
+
var EQUALITY_OPERATORS = ["eq", "ne"];
|
|
6451
|
+
var LIST_OPERATORS = ["in", "nin"];
|
|
6452
|
+
var GEO_OPERATORS = ["radius", "polygon"];
|
|
5526
6453
|
function invalidShape(message) {
|
|
5527
6454
|
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message);
|
|
5528
6455
|
}
|
|
5529
|
-
function isObjectRecord(value) {
|
|
5530
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
5531
|
-
}
|
|
5532
6456
|
function undeclaredFieldOnStrictIndex(field, option) {
|
|
5533
6457
|
return new NarsilError(
|
|
5534
6458
|
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
@@ -5536,34 +6460,14 @@ function undeclaredFieldOnStrictIndex(field, option) {
|
|
|
5536
6460
|
{ field, option }
|
|
5537
6461
|
);
|
|
5538
6462
|
}
|
|
5539
|
-
function
|
|
5540
|
-
|
|
5541
|
-
|
|
5542
|
-
|
|
5543
|
-
|
|
5544
|
-
|
|
5545
|
-
for (const [fieldPath, filter] of Object.entries(fields)) {
|
|
5546
|
-
if (rules.declaredOnly && !Object.hasOwn(rules.fieldTypes, fieldPath)) {
|
|
5547
|
-
throw undeclaredFieldOnStrictIndex(fieldPath, "filter");
|
|
5548
|
-
}
|
|
5549
|
-
if (isObjectRecord(filter)) requireValidOperands(fieldPath, filter, rules.fieldTypes[fieldPath]);
|
|
5550
|
-
}
|
|
5551
|
-
}
|
|
5552
|
-
for (const [clause, list] of [
|
|
5553
|
-
["and", and],
|
|
5554
|
-
["or", or]
|
|
5555
|
-
]) {
|
|
5556
|
-
if (list === void 0) continue;
|
|
5557
|
-
if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
|
|
5558
|
-
for (const nested of list) requireValidExpression(nested, rules, depth + 1);
|
|
5559
|
-
}
|
|
5560
|
-
if (not !== void 0) requireValidExpression(not, rules, depth + 1);
|
|
5561
|
-
}
|
|
5562
|
-
function requireValidFilter(expression, fieldTypes, declaredOnly = false) {
|
|
5563
|
-
if (expression === void 0 || expression === null) return;
|
|
5564
|
-
requireValidExpression(expression, { fieldTypes, declaredOnly }, 0);
|
|
6463
|
+
function undeclaredFilterField(field) {
|
|
6464
|
+
return new NarsilError(
|
|
6465
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
6466
|
+
`The schema declares no field "${field}", while the engine answers a filter only from the indexes that it keeps for declared fields. Declare the field in the schema of a new index to filter by it`,
|
|
6467
|
+
{ field, option: "filter" }
|
|
6468
|
+
);
|
|
5565
6469
|
}
|
|
5566
|
-
function
|
|
6470
|
+
function requireRangeOperands(fieldPath, filter, type) {
|
|
5567
6471
|
for (const operator of RANGE_OPERATORS) {
|
|
5568
6472
|
const bound = filter[operator];
|
|
5569
6473
|
if (bound === void 0) continue;
|
|
@@ -5576,22 +6480,23 @@ function requireValidOperands(fieldPath, filter, type) {
|
|
|
5576
6480
|
}
|
|
5577
6481
|
requireFieldKind(fieldPath, operator, type, ["number"]);
|
|
5578
6482
|
}
|
|
5579
|
-
if (filter.between
|
|
5580
|
-
|
|
5581
|
-
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
5585
|
-
|
|
5586
|
-
|
|
5587
|
-
|
|
5588
|
-
|
|
5589
|
-
|
|
5590
|
-
|
|
5591
|
-
|
|
5592
|
-
requireFieldKind(fieldPath, "between", type, ["number"]);
|
|
5593
|
-
}
|
|
6483
|
+
if (filter.between === void 0) return;
|
|
6484
|
+
const bounds = filter.between;
|
|
6485
|
+
const isPair = Array.isArray(bounds) && bounds.length === 2;
|
|
6486
|
+
if (isPair && typeof bounds[0] === "string" && typeof bounds[1] === "string") {
|
|
6487
|
+
requireTextRangeField(fieldPath, "between", type);
|
|
6488
|
+
return;
|
|
6489
|
+
}
|
|
6490
|
+
if (!isPair || !isFiniteNumber(bounds[0]) || !isFiniteNumber(bounds[1])) {
|
|
6491
|
+
invalid(
|
|
6492
|
+
`"between" on field "${fieldPath}" must be two finite numbers or two strings, the lower bound first`,
|
|
6493
|
+
fieldPath,
|
|
6494
|
+
"between"
|
|
6495
|
+
);
|
|
5594
6496
|
}
|
|
6497
|
+
requireFieldKind(fieldPath, "between", type, ["number"]);
|
|
6498
|
+
}
|
|
6499
|
+
function requireValueOperands(fieldPath, filter, type) {
|
|
5595
6500
|
for (const operator of TEXT_OPERATORS) {
|
|
5596
6501
|
if (filter[operator] === void 0) continue;
|
|
5597
6502
|
if (typeof filter[operator] !== "string") {
|
|
@@ -5617,247 +6522,74 @@ function requireValidOperands(fieldPath, filter, type) {
|
|
|
5617
6522
|
if (!Array.isArray(values) || values.some((value) => typeof value !== "string")) {
|
|
5618
6523
|
invalid(`"${operator}" on field "${fieldPath}" must be a list of strings`, fieldPath, operator);
|
|
5619
6524
|
}
|
|
6525
|
+
requireListLength(fieldPath, operator, values);
|
|
5620
6526
|
requireFieldKind(fieldPath, operator, type, ["text"]);
|
|
5621
6527
|
}
|
|
5622
|
-
|
|
5623
|
-
|
|
5624
|
-
if (size === null || typeof size !== "object") {
|
|
5625
|
-
invalid(`"size" on field "${fieldPath}" must be a comparison object`, fieldPath, "size");
|
|
5626
|
-
}
|
|
5627
|
-
requireValidOperands(`${fieldPath}.size`, size, "number");
|
|
5628
|
-
}
|
|
5629
|
-
if (filter.radius !== void 0) requireRadius(fieldPath, filter.radius);
|
|
5630
|
-
if (filter.polygon !== void 0) {
|
|
5631
|
-
if (!isObjectRecord(filter.polygon)) {
|
|
5632
|
-
invalid(`"polygon" on field "${fieldPath}" must be an object`, fieldPath, "polygon");
|
|
5633
|
-
}
|
|
5634
|
-
requirePolygonRing(filter.polygon.points);
|
|
5635
|
-
}
|
|
5636
|
-
}
|
|
5637
|
-
|
|
5638
|
-
// src/filters/operators/shared.ts
|
|
5639
|
-
var METERS_PER_KM = 1e3;
|
|
5640
|
-
var METERS_PER_MI = 1609.344;
|
|
5641
|
-
function convertToMeters(distance, unit) {
|
|
5642
|
-
if (unit === "km") return distance * METERS_PER_KM;
|
|
5643
|
-
if (unit === "mi") return distance * METERS_PER_MI;
|
|
5644
|
-
if (unit === "m") return distance;
|
|
5645
|
-
throw new NarsilError(
|
|
5646
|
-
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
5647
|
-
`The engine measures a radius in "km", "mi", or "m", and "${String(unit)}" is none of them`,
|
|
5648
|
-
{ unit: String(unit) }
|
|
5649
|
-
);
|
|
5650
|
-
}
|
|
5651
|
-
function matchesNumericComparison(value, filter) {
|
|
5652
|
-
if (filter.eq !== void 0 && value !== filter.eq) return false;
|
|
5653
|
-
if (filter.ne !== void 0 && value === filter.ne) return false;
|
|
5654
|
-
if (filter.gt !== void 0 && !(value > filter.gt)) return false;
|
|
5655
|
-
if (filter.lt !== void 0 && !(value < filter.lt)) return false;
|
|
5656
|
-
if (filter.gte !== void 0 && !(value >= filter.gte)) return false;
|
|
5657
|
-
if (filter.lte !== void 0 && !(value <= filter.lte)) return false;
|
|
5658
|
-
if (filter.between !== void 0 && !(value >= filter.between[0] && value <= filter.between[1])) return false;
|
|
5659
|
-
return true;
|
|
5660
|
-
}
|
|
5661
|
-
function holdsValue(stored, value) {
|
|
5662
|
-
return Array.isArray(stored) ? stored.includes(value) : stored === value;
|
|
5663
|
-
}
|
|
5664
|
-
function lacksValue(stored, value) {
|
|
5665
|
-
if (stored === void 0 || stored === null) return false;
|
|
5666
|
-
return Array.isArray(stored) ? !stored.includes(value) : stored !== value;
|
|
5667
|
-
}
|
|
5668
|
-
function isStringIn(values) {
|
|
5669
|
-
return (element) => typeof element === "string" && values.has(element);
|
|
5670
|
-
}
|
|
5671
|
-
function holdsStringIn(stored, values) {
|
|
5672
|
-
return Array.isArray(stored) ? stored.some(isStringIn(values)) : isStringIn(values)(stored);
|
|
5673
|
-
}
|
|
5674
|
-
function holdsNoStringIn(stored, values) {
|
|
5675
|
-
if (Array.isArray(stored)) return !stored.some(isStringIn(values));
|
|
5676
|
-
return typeof stored === "string" && !values.has(stored);
|
|
5677
|
-
}
|
|
5678
|
-
function holdsStringWhere(stored, test) {
|
|
5679
|
-
const matches = (element) => typeof element === "string" && test(element);
|
|
5680
|
-
return Array.isArray(stored) ? stored.some(matches) : matches(stored);
|
|
5681
|
-
}
|
|
5682
|
-
|
|
5683
|
-
// src/filters/operators/bitset.ts
|
|
5684
|
-
function resolveBitset(bs) {
|
|
5685
|
-
return typeof bs === "function" ? bs() : bs;
|
|
5686
|
-
}
|
|
5687
|
-
function scanToBitset(docIds, capacity, getValue, predicate) {
|
|
5688
|
-
const resolved = resolveBitset(docIds);
|
|
5689
|
-
const result = createBitSet(capacity);
|
|
5690
|
-
for (let wi = 0; wi < resolved.length; wi++) {
|
|
5691
|
-
let word = resolved[wi];
|
|
5692
|
-
if (word === 0) continue;
|
|
5693
|
-
const base = wi << 5;
|
|
5694
|
-
while (word !== 0) {
|
|
5695
|
-
const tz = Math.clz32(word & -word) ^ 31;
|
|
5696
|
-
const id = base + tz;
|
|
5697
|
-
if (predicate(getValue(id))) {
|
|
5698
|
-
bitsetSet(result, id);
|
|
5699
|
-
}
|
|
5700
|
-
word &= word - 1;
|
|
5701
|
-
}
|
|
5702
|
-
}
|
|
5703
|
-
return result;
|
|
5704
|
-
}
|
|
5705
|
-
function applyEqBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5706
|
-
if (fieldIndex?.type === "numeric" && typeof value === "number") {
|
|
5707
|
-
return fieldIndex.index.eqBitset(value, capacity);
|
|
5708
|
-
}
|
|
5709
|
-
if (fieldIndex?.type === "boolean" && typeof value === "boolean") {
|
|
5710
|
-
return value ? fieldIndex.index.getTrueBitset(capacity) : fieldIndex.index.getFalseBitset(capacity);
|
|
5711
|
-
}
|
|
5712
|
-
if (fieldIndex?.type === "enum" && typeof value === "string") {
|
|
5713
|
-
return fieldIndex.index.getDocIdsBitset(value, capacity);
|
|
5714
|
-
}
|
|
5715
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsValue(v, value));
|
|
5716
|
-
}
|
|
5717
|
-
function applyNeBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5718
|
-
if (fieldIndex?.type === "numeric" && typeof value === "number") {
|
|
5719
|
-
const all = fieldIndex.index.allDocIdsBitset(capacity);
|
|
5720
|
-
const eq = fieldIndex.index.eqBitset(value, capacity);
|
|
5721
|
-
return bitsetAnd(all, bitsetNot(eq, capacity));
|
|
5722
|
-
}
|
|
5723
|
-
if (fieldIndex?.type === "boolean" && typeof value === "boolean") {
|
|
5724
|
-
return value ? fieldIndex.index.getFalseBitset(capacity) : fieldIndex.index.getTrueBitset(capacity);
|
|
5725
|
-
}
|
|
5726
|
-
if (fieldIndex?.type === "enum" && typeof value === "string") {
|
|
5727
|
-
const all = fieldIndex.index.allDocIdsBitset(capacity);
|
|
5728
|
-
const eq = fieldIndex.index.getDocIdsBitset(value, capacity);
|
|
5729
|
-
return bitsetAnd(all, bitsetNot(eq, capacity));
|
|
5730
|
-
}
|
|
5731
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => lacksValue(v, value));
|
|
5732
|
-
}
|
|
5733
|
-
function applyGtBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5734
|
-
if (fieldIndex?.type === "numeric") {
|
|
5735
|
-
return fieldIndex.index.gtBitset(value, capacity);
|
|
5736
|
-
}
|
|
5737
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v > value);
|
|
5738
|
-
}
|
|
5739
|
-
function applyLtBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5740
|
-
if (fieldIndex?.type === "numeric") {
|
|
5741
|
-
return fieldIndex.index.ltBitset(value, capacity);
|
|
5742
|
-
}
|
|
5743
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v < value);
|
|
5744
|
-
}
|
|
5745
|
-
function applyGteBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5746
|
-
if (fieldIndex?.type === "numeric") {
|
|
5747
|
-
return fieldIndex.index.gteBitset(value, capacity);
|
|
5748
|
-
}
|
|
5749
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v >= value);
|
|
5750
|
-
}
|
|
5751
|
-
function applyLteBitset(value, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5752
|
-
if (fieldIndex?.type === "numeric") {
|
|
5753
|
-
return fieldIndex.index.lteBitset(value, capacity);
|
|
5754
|
-
}
|
|
5755
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v <= value);
|
|
5756
|
-
}
|
|
5757
|
-
function applyBetweenBitset(range, allDocsBitset, capacity, getValue, fieldIndex) {
|
|
5758
|
-
if (fieldIndex?.type === "numeric") {
|
|
5759
|
-
return fieldIndex.index.betweenBitset(range[0], range[1], capacity);
|
|
6528
|
+
for (const operator of GEO_OPERATORS) {
|
|
6529
|
+
if (filter[operator] !== void 0) requireFieldKind(fieldPath, operator, type, ["geopoint"]);
|
|
5760
6530
|
}
|
|
5761
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => typeof v === "number" && v >= range[0] && v <= range[1]);
|
|
5762
6531
|
}
|
|
5763
|
-
function
|
|
5764
|
-
|
|
5765
|
-
|
|
6532
|
+
function requireValidOperands(fieldPath, filter, type) {
|
|
6533
|
+
requireWellFormedText(fieldPath, filter);
|
|
6534
|
+
requireRangeOperands(fieldPath, filter, type);
|
|
6535
|
+
requireValueOperands(fieldPath, filter, type);
|
|
6536
|
+
requireWholeValueIndex(fieldPath, filter, type);
|
|
6537
|
+
requirePresenceOperands(fieldPath, filter);
|
|
6538
|
+
requireElementOperands(fieldPath, filter, type);
|
|
6539
|
+
requireSizeOperands(fieldPath, filter, type);
|
|
6540
|
+
requirePatternOperands(fieldPath, filter, type);
|
|
6541
|
+
requireGeoOperands(fieldPath, filter);
|
|
6542
|
+
}
|
|
6543
|
+
function testsIn(filter) {
|
|
6544
|
+
let tests = 0;
|
|
6545
|
+
for (const [operator, operand] of Object.entries(filter)) {
|
|
6546
|
+
if (operator === "caseFold") continue;
|
|
6547
|
+
tests += operator === "size" && isObjectRecord(operand) ? Math.max(1, Object.keys(operand).length) : 1;
|
|
6548
|
+
}
|
|
6549
|
+
return Math.max(1, tests);
|
|
6550
|
+
}
|
|
6551
|
+
function countTests(walk, tests) {
|
|
6552
|
+
walk.tests += tests;
|
|
6553
|
+
if (walk.tests > MAX_FILTER_TESTS) {
|
|
6554
|
+
invalidShape(`A filter holds at most ${MAX_FILTER_TESTS} tests across all of its fields and nested expressions`);
|
|
6555
|
+
}
|
|
6556
|
+
}
|
|
6557
|
+
function holdsNestedExpressions(expression) {
|
|
6558
|
+
const { and, or, not } = expression;
|
|
6559
|
+
return Array.isArray(and) && and.length > 0 || Array.isArray(or) && or.length > 0 || not !== void 0;
|
|
6560
|
+
}
|
|
6561
|
+
function requireValidExpression(expression, walk, depth) {
|
|
6562
|
+
if (depth > MAX_FILTER_DEPTH) invalidShape(`A filter expression nests at most ${MAX_FILTER_DEPTH} levels deep`);
|
|
6563
|
+
if (!isObjectRecord(expression)) invalidShape("Each filter expression must be an object");
|
|
6564
|
+
const { fields, and, or, not } = expression;
|
|
6565
|
+
if (fields !== void 0 && !isObjectRecord(fields)) {
|
|
6566
|
+
invalidShape('"fields" in a filter expression must be an object keyed by field path');
|
|
6567
|
+
}
|
|
6568
|
+
const fieldFilters = Object.entries(fields ?? {});
|
|
6569
|
+
if (fieldFilters.length === 0 && !holdsNestedExpressions(expression)) countTests(walk, 1);
|
|
6570
|
+
for (const [fieldPath, filter] of fieldFilters) {
|
|
6571
|
+
const type = walk.fieldTypes[fieldPath];
|
|
6572
|
+
if (!Object.hasOwn(walk.fieldTypes, fieldPath) || type === void 0) throw undeclaredFilterField(fieldPath);
|
|
6573
|
+
if (!isObjectRecord(filter)) invalidShape(`The filter on field "${fieldPath}" must be an object of tests`);
|
|
6574
|
+
countTests(walk, testsIn(filter));
|
|
6575
|
+
requireValidOperands(fieldPath, filter, type);
|
|
5766
6576
|
}
|
|
5767
|
-
const
|
|
5768
|
-
|
|
5769
|
-
|
|
5770
|
-
|
|
5771
|
-
|
|
5772
|
-
|
|
5773
|
-
const
|
|
5774
|
-
return bitsetAnd(all, bitsetNot(matched, capacity));
|
|
6577
|
+
for (const [clause, list] of [
|
|
6578
|
+
["and", and],
|
|
6579
|
+
["or", or]
|
|
6580
|
+
]) {
|
|
6581
|
+
if (list === void 0) continue;
|
|
6582
|
+
if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
|
|
6583
|
+
for (const nested of list) requireValidExpression(nested, walk, depth + 1);
|
|
5775
6584
|
}
|
|
5776
|
-
|
|
5777
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsNoStringIn(v, valSet));
|
|
5778
|
-
}
|
|
5779
|
-
function applyStartsWithBitset(prefix, allDocsBitset, capacity, getValue) {
|
|
5780
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsStringWhere(v, (text) => text.startsWith(prefix)));
|
|
5781
|
-
}
|
|
5782
|
-
function applyEndsWithBitset(suffix, allDocsBitset, capacity, getValue) {
|
|
5783
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => holdsStringWhere(v, (text) => text.endsWith(suffix)));
|
|
5784
|
-
}
|
|
5785
|
-
function applyContainsAllBitset(values, allDocsBitset, capacity, getValue) {
|
|
5786
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
|
|
5787
|
-
if (!Array.isArray(v)) return false;
|
|
5788
|
-
const arr = v;
|
|
5789
|
-
for (const val of values) {
|
|
5790
|
-
if (!arr.includes(val)) return false;
|
|
5791
|
-
}
|
|
5792
|
-
return true;
|
|
5793
|
-
});
|
|
5794
|
-
}
|
|
5795
|
-
function applyMatchesAnyBitset(values, allDocsBitset, capacity, getValue) {
|
|
5796
|
-
const valSet = new Set(values);
|
|
5797
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
|
|
5798
|
-
if (!Array.isArray(v)) return false;
|
|
5799
|
-
for (const item of v) {
|
|
5800
|
-
if (valSet.has(item)) return true;
|
|
5801
|
-
}
|
|
5802
|
-
return false;
|
|
5803
|
-
});
|
|
5804
|
-
}
|
|
5805
|
-
function applySizeBitset(sizeFilter, allDocsBitset, capacity, getValue) {
|
|
5806
|
-
return scanToBitset(
|
|
5807
|
-
allDocsBitset,
|
|
5808
|
-
capacity,
|
|
5809
|
-
getValue,
|
|
5810
|
-
(v) => Array.isArray(v) && matchesNumericComparison(v.length, sizeFilter)
|
|
5811
|
-
);
|
|
5812
|
-
}
|
|
5813
|
-
function applyExistsBitset(allDocsBitset, capacity, getValue) {
|
|
5814
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => v !== void 0 && v !== null);
|
|
5815
|
-
}
|
|
5816
|
-
function applyNotExistsBitset(allDocsBitset, capacity, getValue) {
|
|
5817
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => v === void 0 || v === null);
|
|
5818
|
-
}
|
|
5819
|
-
function applyIsEmptyBitset(allDocsBitset, capacity, getValue) {
|
|
5820
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
|
|
5821
|
-
if (v === void 0 || v === null) return true;
|
|
5822
|
-
if (typeof v === "string" && v === "") return true;
|
|
5823
|
-
if (Array.isArray(v) && v.length === 0) return true;
|
|
5824
|
-
return false;
|
|
5825
|
-
});
|
|
5826
|
-
}
|
|
5827
|
-
function applyIsNotEmptyBitset(allDocsBitset, capacity, getValue) {
|
|
5828
|
-
return scanToBitset(allDocsBitset, capacity, getValue, (v) => {
|
|
5829
|
-
if (v === void 0 || v === null) return false;
|
|
5830
|
-
if (typeof v === "string" && v === "") return false;
|
|
5831
|
-
if (Array.isArray(v) && v.length === 0) return false;
|
|
5832
|
-
return true;
|
|
5833
|
-
});
|
|
5834
|
-
}
|
|
5835
|
-
function applyGeoRadiusBitset(filter, geoIndex, capacity) {
|
|
5836
|
-
const distanceMeters = convertToMeters(filter.distance, filter.unit);
|
|
5837
|
-
const setResult = geoIndex.radiusQuery(
|
|
5838
|
-
filter.lat,
|
|
5839
|
-
filter.lon,
|
|
5840
|
-
distanceMeters,
|
|
5841
|
-
filter.inside ?? true,
|
|
5842
|
-
filter.highPrecision ?? false
|
|
5843
|
-
);
|
|
5844
|
-
return bitsetFromSet(setResult, capacity);
|
|
6585
|
+
if (not !== void 0) requireValidExpression(not, walk, depth + 1);
|
|
5845
6586
|
}
|
|
5846
|
-
function
|
|
5847
|
-
|
|
5848
|
-
|
|
5849
|
-
return bitsetFromSet(setResult, capacity);
|
|
6587
|
+
function requireValidFilter(expression, fieldTypes) {
|
|
6588
|
+
if (expression === void 0 || expression === null) return;
|
|
6589
|
+
requireValidExpression(expression, { fieldTypes, tests: 0 }, 0);
|
|
5850
6590
|
}
|
|
5851
6591
|
|
|
5852
6592
|
// src/filters/evaluator.ts
|
|
5853
|
-
var UNBOUNDED = { lower: null, upper: null };
|
|
5854
|
-
function textBoundsOf(operator, bound) {
|
|
5855
|
-
if (operator === "gt" || operator === "gte") {
|
|
5856
|
-
return { ...UNBOUNDED, lower: bound, lowerInclusive: operator === "gte", upperInclusive: false };
|
|
5857
|
-
}
|
|
5858
|
-
return { ...UNBOUNDED, upper: bound, upperInclusive: operator === "lte", lowerInclusive: false };
|
|
5859
|
-
}
|
|
5860
|
-
var TEXT_RANGE_OPERATORS = ["gt", "gte", "lt", "lte"];
|
|
5861
6593
|
var EXPRESSION_KEYS = new Set(FILTER_EXPRESSION_KEYS);
|
|
5862
6594
|
var OPERATOR_KEYS = new Set(FIELD_FILTER_OPERATORS);
|
|
5863
6595
|
function requireKnownKeys2(value, known, describe) {
|
|
@@ -5867,160 +6599,66 @@ function requireKnownKeys2(value, known, describe) {
|
|
|
5867
6599
|
}
|
|
5868
6600
|
}
|
|
5869
6601
|
}
|
|
6602
|
+
var Narrowing = class {
|
|
6603
|
+
bits = null;
|
|
6604
|
+
narrow = (matched) => {
|
|
6605
|
+
if (this.bits === null) this.bits = matched.slice();
|
|
6606
|
+
else bitsetAndInto(this.bits, matched);
|
|
6607
|
+
};
|
|
6608
|
+
value(universe) {
|
|
6609
|
+
return this.bits ?? universe().slice();
|
|
6610
|
+
}
|
|
6611
|
+
};
|
|
6612
|
+
function narrowByField(fieldPath, filter, context, narrow) {
|
|
6613
|
+
requireKnownKeys2(
|
|
6614
|
+
filter,
|
|
6615
|
+
OPERATOR_KEYS,
|
|
6616
|
+
(key) => `Field "${fieldPath}" is tested with "${key}", which is no filter operator`
|
|
6617
|
+
);
|
|
6618
|
+
const type = context.fieldTypes[fieldPath];
|
|
6619
|
+
const parsed = parsedTypeOf(type);
|
|
6620
|
+
if (type === void 0 || parsed === void 0) throw undeclaredFilterField(fieldPath);
|
|
6621
|
+
const f = filter;
|
|
6622
|
+
requireValidOperands(fieldPath, f, type);
|
|
6623
|
+
narrowByFieldTests(fieldPath, f, parsed, context, narrow.narrow);
|
|
6624
|
+
}
|
|
5870
6625
|
function evaluateFilters(expression, context) {
|
|
5871
6626
|
requireKnownKeys2(
|
|
5872
6627
|
expression,
|
|
5873
6628
|
EXPRESSION_KEYS,
|
|
5874
6629
|
(key) => `A filter expression takes "fields", "and", "or", and "not", and "${key}" is none of them. A per-field test such as { ${key}: { eq: ... } } belongs under "fields"`
|
|
5875
6630
|
);
|
|
5876
|
-
const
|
|
6631
|
+
const narrowing = new Narrowing();
|
|
5877
6632
|
if (expression.fields) {
|
|
5878
6633
|
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
5879
|
-
|
|
6634
|
+
narrowByField(fieldPath, filter, context, narrowing);
|
|
5880
6635
|
}
|
|
5881
6636
|
}
|
|
5882
6637
|
if (expression.and?.length) {
|
|
5883
|
-
const
|
|
5884
|
-
bitsets.push(applyAndBitset(andBitsets));
|
|
6638
|
+
for (const nested of expression.and) narrowing.narrow(evaluateFilters(nested, context));
|
|
5885
6639
|
}
|
|
5886
6640
|
if (expression.or?.length) {
|
|
5887
|
-
const
|
|
5888
|
-
|
|
6641
|
+
const union = createBitSet(context.capacity);
|
|
6642
|
+
for (const nested of expression.or) bitsetOrInto(union, evaluateFilters(nested, context));
|
|
6643
|
+
narrowing.narrow(union);
|
|
5889
6644
|
}
|
|
5890
6645
|
if (expression.not) {
|
|
5891
6646
|
const excluded = evaluateFilters(expression.not, context);
|
|
5892
|
-
const
|
|
5893
|
-
|
|
6647
|
+
const kept = context.allDocIdsBitset.slice();
|
|
6648
|
+
bitsetAndNotInto(kept, excluded);
|
|
6649
|
+
narrowing.narrow(kept);
|
|
5894
6650
|
}
|
|
5895
|
-
|
|
5896
|
-
if (bitsets.length === 1) return bitsets[0];
|
|
5897
|
-
return applyAndBitset(bitsets);
|
|
5898
|
-
}
|
|
5899
|
-
function evaluateFieldFilter(fieldPath, filter, context) {
|
|
5900
|
-
requireKnownKeys2(
|
|
5901
|
-
filter,
|
|
5902
|
-
OPERATOR_KEYS,
|
|
5903
|
-
(key) => `Field "${fieldPath}" is tested with "${key}", which is no filter operator`
|
|
5904
|
-
);
|
|
5905
|
-
const f = filter;
|
|
5906
|
-
requireValidOperands(fieldPath, f, context.fieldTypes[fieldPath]);
|
|
5907
|
-
const fieldIndex = context.fieldIndexes[fieldPath];
|
|
5908
|
-
const getValue = (internalId) => context.getFieldValue(internalId, fieldPath);
|
|
5909
|
-
const bitsets = [];
|
|
5910
|
-
const { capacity } = context;
|
|
5911
|
-
let allDocsBitset = null;
|
|
5912
|
-
const getAllDocsBitset = () => {
|
|
5913
|
-
if (allDocsBitset === null) {
|
|
5914
|
-
allDocsBitset = context.allDocIdsBitset;
|
|
5915
|
-
}
|
|
5916
|
-
return allDocsBitset;
|
|
5917
|
-
};
|
|
5918
|
-
if ("radius" in f && f.radius) {
|
|
5919
|
-
const geoIndex = resolveGeoIndex(fieldPath, fieldIndex);
|
|
5920
|
-
bitsets.push(applyGeoRadiusBitset(filter.radius, geoIndex, capacity));
|
|
5921
|
-
}
|
|
5922
|
-
if ("polygon" in f && f.polygon) {
|
|
5923
|
-
const geoIndex = resolveGeoIndex(fieldPath, fieldIndex);
|
|
5924
|
-
bitsets.push(applyGeoPolygonBitset(filter.polygon, geoIndex, capacity));
|
|
5925
|
-
}
|
|
5926
|
-
if ("eq" in f && f.eq !== void 0) {
|
|
5927
|
-
bitsets.push(applyEqBitset(f.eq, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5928
|
-
}
|
|
5929
|
-
if ("ne" in f && f.ne !== void 0) {
|
|
5930
|
-
bitsets.push(applyNeBitset(f.ne, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5931
|
-
}
|
|
5932
|
-
for (const operator of TEXT_RANGE_OPERATORS) {
|
|
5933
|
-
const bound = f[operator];
|
|
5934
|
-
if (typeof bound === "string") bitsets.push(context.textRangeBitset(fieldPath, textBoundsOf(operator, bound)));
|
|
5935
|
-
}
|
|
5936
|
-
if (typeof f.gt === "number") {
|
|
5937
|
-
bitsets.push(applyGtBitset(f.gt, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5938
|
-
}
|
|
5939
|
-
if (typeof f.lt === "number") {
|
|
5940
|
-
bitsets.push(applyLtBitset(f.lt, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5941
|
-
}
|
|
5942
|
-
if (typeof f.gte === "number") {
|
|
5943
|
-
bitsets.push(applyGteBitset(f.gte, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5944
|
-
}
|
|
5945
|
-
if (typeof f.lte === "number") {
|
|
5946
|
-
bitsets.push(applyLteBitset(f.lte, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5947
|
-
}
|
|
5948
|
-
if (Array.isArray(f.between)) {
|
|
5949
|
-
const [low, high] = f.between;
|
|
5950
|
-
if (typeof low === "string" && typeof high === "string") {
|
|
5951
|
-
bitsets.push(
|
|
5952
|
-
context.textRangeBitset(fieldPath, { lower: low, lowerInclusive: true, upper: high, upperInclusive: true })
|
|
5953
|
-
);
|
|
5954
|
-
} else {
|
|
5955
|
-
bitsets.push(
|
|
5956
|
-
applyBetweenBitset([low, high], getAllDocsBitset, capacity, getValue, fieldIndex)
|
|
5957
|
-
);
|
|
5958
|
-
}
|
|
5959
|
-
}
|
|
5960
|
-
if ("in" in f && f.in !== void 0) {
|
|
5961
|
-
bitsets.push(applyInBitset(f.in, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5962
|
-
}
|
|
5963
|
-
if ("nin" in f && f.nin !== void 0) {
|
|
5964
|
-
bitsets.push(applyNinBitset(f.nin, getAllDocsBitset, capacity, getValue, fieldIndex));
|
|
5965
|
-
}
|
|
5966
|
-
if ("startsWith" in f && f.startsWith !== void 0) {
|
|
5967
|
-
bitsets.push(applyStartsWithBitset(f.startsWith, getAllDocsBitset, capacity, getValue));
|
|
5968
|
-
}
|
|
5969
|
-
if ("endsWith" in f && f.endsWith !== void 0) {
|
|
5970
|
-
bitsets.push(applyEndsWithBitset(f.endsWith, getAllDocsBitset, capacity, getValue));
|
|
5971
|
-
}
|
|
5972
|
-
if ("containsAll" in f && f.containsAll !== void 0) {
|
|
5973
|
-
bitsets.push(
|
|
5974
|
-
applyContainsAllBitset(f.containsAll, getAllDocsBitset, capacity, getValue)
|
|
5975
|
-
);
|
|
5976
|
-
}
|
|
5977
|
-
if ("matchesAny" in f && f.matchesAny !== void 0) {
|
|
5978
|
-
bitsets.push(
|
|
5979
|
-
applyMatchesAnyBitset(f.matchesAny, getAllDocsBitset, capacity, getValue)
|
|
5980
|
-
);
|
|
5981
|
-
}
|
|
5982
|
-
if ("size" in f && f.size !== void 0) {
|
|
5983
|
-
bitsets.push(applySizeBitset(f.size, getAllDocsBitset, capacity, getValue));
|
|
5984
|
-
}
|
|
5985
|
-
if ("exists" in f && f.exists !== void 0) {
|
|
5986
|
-
bitsets.push(
|
|
5987
|
-
f.exists ? applyExistsBitset(getAllDocsBitset, capacity, getValue) : applyNotExistsBitset(getAllDocsBitset, capacity, getValue)
|
|
5988
|
-
);
|
|
5989
|
-
}
|
|
5990
|
-
if ("notExists" in f && f.notExists !== void 0) {
|
|
5991
|
-
bitsets.push(
|
|
5992
|
-
f.notExists ? applyNotExistsBitset(getAllDocsBitset, capacity, getValue) : applyExistsBitset(getAllDocsBitset, capacity, getValue)
|
|
5993
|
-
);
|
|
5994
|
-
}
|
|
5995
|
-
if ("isEmpty" in f && f.isEmpty !== void 0) {
|
|
5996
|
-
bitsets.push(
|
|
5997
|
-
f.isEmpty ? applyIsEmptyBitset(getAllDocsBitset, capacity, getValue) : applyIsNotEmptyBitset(getAllDocsBitset, capacity, getValue)
|
|
5998
|
-
);
|
|
5999
|
-
}
|
|
6000
|
-
if ("isNotEmpty" in f && f.isNotEmpty !== void 0) {
|
|
6001
|
-
bitsets.push(
|
|
6002
|
-
f.isNotEmpty ? applyIsNotEmptyBitset(getAllDocsBitset, capacity, getValue) : applyIsEmptyBitset(getAllDocsBitset, capacity, getValue)
|
|
6003
|
-
);
|
|
6004
|
-
}
|
|
6005
|
-
if (bitsets.length === 0) return context.allDocIdsBitset;
|
|
6006
|
-
if (bitsets.length === 1) return bitsets[0];
|
|
6007
|
-
return applyAndBitset(bitsets);
|
|
6008
|
-
}
|
|
6009
|
-
function resolveGeoIndex(fieldPath, fieldIndex) {
|
|
6010
|
-
if (fieldIndex?.type !== "geopoint") {
|
|
6011
|
-
throw new NarsilError(
|
|
6012
|
-
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
6013
|
-
`Field "${fieldPath}" requires a geopoint index for geo filters`,
|
|
6014
|
-
{ fieldPath }
|
|
6015
|
-
);
|
|
6016
|
-
}
|
|
6017
|
-
return fieldIndex.index;
|
|
6651
|
+
return narrowing.value(() => context.allDocIdsBitset);
|
|
6018
6652
|
}
|
|
6019
6653
|
|
|
6020
6654
|
// src/core/partition/constants.ts
|
|
6021
6655
|
var TARGET_BLOCK_ENTRIES = 128;
|
|
6022
6656
|
var PRUNING_REJECTION_SLACK = 1e-12;
|
|
6023
6657
|
var SCORE_BUFFER_INITIAL_TOUCHED_CAPACITY = 1024;
|
|
6658
|
+
var SEGMENT_TOKEN_TABLE_START = 1 << 12;
|
|
6659
|
+
var SEGMENT_POSTING_LOG_START = 1 << 14;
|
|
6660
|
+
var SEGMENT_POSITION_POOL_START = 1 << 16;
|
|
6661
|
+
var SEGMENT_FIELD_SCRATCH_START = 1 << 10;
|
|
6024
6662
|
var SORT_COLUMN_MINIMUM_REBUILD_THRESHOLD = 1024;
|
|
6025
6663
|
var SORT_COLUMN_REBUILD_FRACTION_SHIFT = 2;
|
|
6026
6664
|
|
|
@@ -6637,99 +7275,72 @@ function sortColumnBytes(state) {
|
|
|
6637
7275
|
}
|
|
6638
7276
|
|
|
6639
7277
|
// src/core/partition/filters.ts
|
|
6640
|
-
|
|
6641
|
-
|
|
6642
|
-
|
|
6643
|
-
|
|
6644
|
-
|
|
7278
|
+
function presentBitsetOf(state, fieldPath, type, capacity) {
|
|
7279
|
+
const base = parsedTypeOf(type)?.base;
|
|
7280
|
+
let present;
|
|
7281
|
+
if (base === "number") present = state.numericIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
7282
|
+
else if (base === "boolean") present = state.booleanIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
7283
|
+
else if (base === "enum") present = state.enumIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
7284
|
+
else if (base === "geopoint") present = state.geoIndexes.get(fieldPath)?.documentsBitset(capacity);
|
|
7285
|
+
else if (base !== void 0 && keepsValueSize(base)) {
|
|
7286
|
+
present = state.sizeIndexes.get(fieldPath)?.getAllDocIdsBitset(capacity);
|
|
7287
|
+
}
|
|
7288
|
+
return present ?? createBitSet(capacity);
|
|
7289
|
+
}
|
|
7290
|
+
function liveDocumentsBitset(state, capacity) {
|
|
7291
|
+
const bits = createBitSet(capacity);
|
|
7292
|
+
for (const internalId of state.docStore.allInternalIds()) bitsetSet(bits, internalId);
|
|
7293
|
+
return bits;
|
|
7294
|
+
}
|
|
7295
|
+
function buildFilterContext(state, schema, meter) {
|
|
6645
7296
|
const flat = getFlatSchema(state, schema);
|
|
6646
|
-
const fieldIndexes = {};
|
|
6647
7297
|
const capacity = state.docStore.internalIdCapacity();
|
|
6648
|
-
|
|
6649
|
-
|
|
6650
|
-
|
|
6651
|
-
if (numIdx) {
|
|
6652
|
-
fieldIndexes[fieldPath] = {
|
|
6653
|
-
type: "numeric",
|
|
6654
|
-
index: {
|
|
6655
|
-
eq: (v) => numIdx.queryEq(v),
|
|
6656
|
-
gt: (v) => numIdx.queryGt(v),
|
|
6657
|
-
gte: (v) => numIdx.queryGte(v),
|
|
6658
|
-
lt: (v) => numIdx.queryLt(v),
|
|
6659
|
-
lte: (v) => numIdx.queryLte(v),
|
|
6660
|
-
between: (min, max) => numIdx.queryBetween(min, max),
|
|
6661
|
-
allDocIds: () => numIdx.getAllDocIds(),
|
|
6662
|
-
eqBitset: (v, cap) => numIdx.queryEqBitset(v, cap),
|
|
6663
|
-
gtBitset: (v, cap) => numIdx.queryGtBitset(v, cap),
|
|
6664
|
-
gteBitset: (v, cap) => numIdx.queryGteBitset(v, cap),
|
|
6665
|
-
ltBitset: (v, cap) => numIdx.queryLtBitset(v, cap),
|
|
6666
|
-
lteBitset: (v, cap) => numIdx.queryLteBitset(v, cap),
|
|
6667
|
-
betweenBitset: (min, max, cap) => numIdx.queryBetweenBitset(min, max, cap),
|
|
6668
|
-
allDocIdsBitset: (cap) => numIdx.getAllDocIdsBitset(cap)
|
|
6669
|
-
}
|
|
6670
|
-
};
|
|
6671
|
-
}
|
|
6672
|
-
} else if (fieldType === "boolean" || fieldType === "boolean[]") {
|
|
6673
|
-
const boolIdx = state.booleanIndexes.get(fieldPath);
|
|
6674
|
-
if (boolIdx) {
|
|
6675
|
-
fieldIndexes[fieldPath] = {
|
|
6676
|
-
type: "boolean",
|
|
6677
|
-
index: {
|
|
6678
|
-
getTrue: () => boolIdx.queryEq(true),
|
|
6679
|
-
getFalse: () => boolIdx.queryEq(false),
|
|
6680
|
-
allDocIds: () => boolIdx.getAllDocIds(),
|
|
6681
|
-
getTrueBitset: (cap) => boolIdx.queryEqBitset(true, cap),
|
|
6682
|
-
getFalseBitset: (cap) => boolIdx.queryEqBitset(false, cap),
|
|
6683
|
-
allDocIdsBitset: (cap) => boolIdx.getAllDocIdsBitset(cap)
|
|
6684
|
-
}
|
|
6685
|
-
};
|
|
6686
|
-
}
|
|
6687
|
-
} else if (fieldType === "enum" || fieldType === "enum[]") {
|
|
6688
|
-
const enumIdx = state.enumIndexes.get(fieldPath);
|
|
6689
|
-
if (enumIdx) {
|
|
6690
|
-
fieldIndexes[fieldPath] = {
|
|
6691
|
-
type: "enum",
|
|
6692
|
-
index: {
|
|
6693
|
-
getDocIds: (v) => enumIdx.queryEq(v),
|
|
6694
|
-
allDocIds: () => enumIdx.getAllDocIds(),
|
|
6695
|
-
getDocIdsBitset: (v, cap) => enumIdx.queryEqBitset(v, cap),
|
|
6696
|
-
getDocIdsInBitset: (values, cap) => enumIdx.queryInBitset(values, cap),
|
|
6697
|
-
allDocIdsBitset: (cap) => enumIdx.getAllDocIdsBitset(cap)
|
|
6698
|
-
}
|
|
6699
|
-
};
|
|
6700
|
-
}
|
|
6701
|
-
} else if (fieldType === "geopoint") {
|
|
6702
|
-
const geoIdx = state.geoIndexes.get(fieldPath);
|
|
6703
|
-
fieldIndexes[fieldPath] = {
|
|
6704
|
-
type: "geopoint",
|
|
6705
|
-
index: geoIdx === void 0 ? EMPTY_GEO_INDEX : geoIdx
|
|
6706
|
-
};
|
|
6707
|
-
}
|
|
7298
|
+
const patternFields = /* @__PURE__ */ new Set();
|
|
7299
|
+
for (const { path: fieldPath, pattern } of schemaFieldsOf(schema)) {
|
|
7300
|
+
if (pattern) patternFields.add(fieldPath);
|
|
6708
7301
|
}
|
|
6709
|
-
|
|
7302
|
+
const getFieldValue = (internalId, fieldPath) => getFieldValueByInternalId(state.docStore, internalId, fieldPath);
|
|
6710
7303
|
let cachedAllDocIdsBitset = null;
|
|
7304
|
+
const cachedPresentBitsets = /* @__PURE__ */ new Map();
|
|
6711
7305
|
return {
|
|
6712
7306
|
fieldTypes: flat,
|
|
6713
|
-
|
|
6714
|
-
|
|
7307
|
+
numericIndex: (fieldPath) => state.numericIndexes.get(fieldPath),
|
|
7308
|
+
booleanIndex: (fieldPath) => state.booleanIndexes.get(fieldPath),
|
|
7309
|
+
enumIndex: (fieldPath) => state.enumIndexes.get(fieldPath),
|
|
7310
|
+
geoIndex: (fieldPath) => state.geoIndexes.get(fieldPath),
|
|
7311
|
+
sizeIndex: (fieldPath) => state.sizeIndexes.get(fieldPath),
|
|
6715
7312
|
textRangeBitset: (fieldPath, bounds) => textRangeBitsetOf(state, fieldPath, flat[fieldPath], bounds, capacity),
|
|
6716
|
-
|
|
6717
|
-
|
|
6718
|
-
|
|
6719
|
-
|
|
6720
|
-
|
|
7313
|
+
isPatternField: (fieldPath) => patternFields.has(fieldPath),
|
|
7314
|
+
matchPattern: (fieldPath, operator, text, caseFold, into) => {
|
|
7315
|
+
const index = state.patternIndexes.get(fieldPath);
|
|
7316
|
+
if (index === void 0) return;
|
|
7317
|
+
requirePatternSearch(fieldPath).matchInto({
|
|
7318
|
+
operator,
|
|
7319
|
+
text,
|
|
7320
|
+
caseFold,
|
|
7321
|
+
index,
|
|
7322
|
+
matches: into,
|
|
7323
|
+
meter,
|
|
7324
|
+
host: PATTERN_HOST,
|
|
7325
|
+
valueOf: (internalId) => getFieldValue(internalId, fieldPath)
|
|
7326
|
+
});
|
|
7327
|
+
},
|
|
7328
|
+
presentBitset: (fieldPath) => {
|
|
7329
|
+
const cached = cachedPresentBitsets.get(fieldPath);
|
|
7330
|
+
if (cached !== void 0) return cached;
|
|
7331
|
+
const present = presentBitsetOf(state, fieldPath, flat[fieldPath], capacity);
|
|
7332
|
+
cachedPresentBitsets.set(fieldPath, present);
|
|
7333
|
+
return present;
|
|
6721
7334
|
},
|
|
6722
7335
|
capacity,
|
|
6723
7336
|
get allDocIdsBitset() {
|
|
6724
|
-
|
|
6725
|
-
cachedAllDocIdsBitset = bitsetFromSet(this.allDocIds, capacity);
|
|
6726
|
-
}
|
|
7337
|
+
cachedAllDocIdsBitset ??= liveDocumentsBitset(state, capacity);
|
|
6727
7338
|
return cachedAllDocIdsBitset;
|
|
6728
7339
|
}
|
|
6729
7340
|
};
|
|
6730
7341
|
}
|
|
6731
|
-
function applyPartitionFilters(state, filters, schema) {
|
|
6732
|
-
const context = buildFilterContext(state, schema);
|
|
7342
|
+
function applyPartitionFilters(state, filters, schema, meter) {
|
|
7343
|
+
const context = buildFilterContext(state, schema, meter);
|
|
6733
7344
|
const resultBitset = evaluateFilters(filters, context);
|
|
6734
7345
|
const resolver = state.docStore.resolver();
|
|
6735
7346
|
const externalResult = /* @__PURE__ */ new Set();
|
|
@@ -6749,12 +7360,12 @@ function applyPartitionFilters(state, filters, schema) {
|
|
|
6749
7360
|
}
|
|
6750
7361
|
return externalResult;
|
|
6751
7362
|
}
|
|
6752
|
-
function applyPartitionFiltersBitset(state, filters, schema) {
|
|
6753
|
-
const context = buildFilterContext(state, schema);
|
|
7363
|
+
function applyPartitionFiltersBitset(state, filters, schema, meter) {
|
|
7364
|
+
const context = buildFilterContext(state, schema, meter);
|
|
6754
7365
|
return evaluateFilters(filters, context);
|
|
6755
7366
|
}
|
|
6756
|
-
function partitionFilterMatches(state, filters, schema) {
|
|
6757
|
-
const accepted = applyPartitionFiltersBitset(state, filters, schema);
|
|
7367
|
+
function partitionFilterMatches(state, filters, schema, meter) {
|
|
7368
|
+
const accepted = applyPartitionFiltersBitset(state, filters, schema, meter);
|
|
6758
7369
|
const docStore = state.docStore;
|
|
6759
7370
|
const resolver = docStore.resolver();
|
|
6760
7371
|
let count = 0;
|
|
@@ -6911,6 +7522,8 @@ function estimatePartitionBytes(state) {
|
|
|
6911
7522
|
bytes += docCount * state.booleanIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
6912
7523
|
bytes += docCount * state.enumIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
6913
7524
|
bytes += docCount * state.geoIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
7525
|
+
bytes += docCount * state.sizeIndexes.size * FIELD_ENTRY_OVERHEAD_BYTES;
|
|
7526
|
+
for (const index of state.patternIndexes.values()) bytes += index.bytes();
|
|
6914
7527
|
return bytes + sortColumnBytes(state);
|
|
6915
7528
|
}
|
|
6916
7529
|
|
|
@@ -6970,12 +7583,12 @@ function mergeSurfaceForms(target, source) {
|
|
|
6970
7583
|
target.surfaceRegistry.add(surface, value[1], value[0]);
|
|
6971
7584
|
}
|
|
6972
7585
|
}
|
|
6973
|
-
function
|
|
6974
|
-
for (const [fieldPath, index] of source
|
|
6975
|
-
let targetIndex = target.
|
|
7586
|
+
function mergeNumericIndexes(target, source, idMapping) {
|
|
7587
|
+
for (const [fieldPath, index] of source) {
|
|
7588
|
+
let targetIndex = target.get(fieldPath);
|
|
6976
7589
|
if (targetIndex === void 0) {
|
|
6977
7590
|
targetIndex = createNumericIndex();
|
|
6978
|
-
target.
|
|
7591
|
+
target.set(fieldPath, targetIndex);
|
|
6979
7592
|
}
|
|
6980
7593
|
for (const entry of index.serialize()) {
|
|
6981
7594
|
const targetInternalId = idMapping.get(entry.docId);
|
|
@@ -6983,6 +7596,10 @@ function mergeFieldIndexes(target, source, idMapping) {
|
|
|
6983
7596
|
targetIndex.insert(targetInternalId, entry.value);
|
|
6984
7597
|
}
|
|
6985
7598
|
}
|
|
7599
|
+
}
|
|
7600
|
+
function mergeFieldIndexes(target, source, idMapping) {
|
|
7601
|
+
mergeNumericIndexes(target.numericIndexes, source.numericIndexes, idMapping);
|
|
7602
|
+
mergeNumericIndexes(target.sizeIndexes, source.sizeIndexes, idMapping);
|
|
6986
7603
|
for (const [fieldPath, index] of source.booleanIndexes) {
|
|
6987
7604
|
let targetIndex = target.booleanIndexes.get(fieldPath);
|
|
6988
7605
|
if (targetIndex === void 0) {
|
|
@@ -7024,6 +7641,12 @@ function mergeFieldIndexes(target, source, idMapping) {
|
|
|
7024
7641
|
targetIndex.insert(targetInternalId, entry.lat, entry.lon);
|
|
7025
7642
|
}
|
|
7026
7643
|
}
|
|
7644
|
+
for (const [fieldPath, index] of source.patternIndexes) {
|
|
7645
|
+
patternIndexOf(target.patternIndexes, fieldPath).addArrays(
|
|
7646
|
+
index.toArrays(fieldPath, null),
|
|
7647
|
+
(sourceInternalId) => idMapping.get(sourceInternalId) ?? -1
|
|
7648
|
+
);
|
|
7649
|
+
}
|
|
7027
7650
|
}
|
|
7028
7651
|
function mergeSegmentState(target, source) {
|
|
7029
7652
|
if (source.stats.totalDocuments === 0 && source.docStore.count() === 0) return;
|
|
@@ -8044,19 +8667,23 @@ function searchFulltext(state, params) {
|
|
|
8044
8667
|
}
|
|
8045
8668
|
|
|
8046
8669
|
// src/core/partition/segment-field-indexes.ts
|
|
8047
|
-
function
|
|
8048
|
-
const
|
|
8049
|
-
const
|
|
8050
|
-
|
|
8051
|
-
const entries = index.serialize().filter((entry) => survives(entry.docId));
|
|
8670
|
+
function encodeNumericIndexes(indexes, remap) {
|
|
8671
|
+
const encoded = [];
|
|
8672
|
+
for (const [fieldPath, index] of indexes) {
|
|
8673
|
+
const entries = index.serialize().filter((entry) => remap[entry.docId] >= 0);
|
|
8052
8674
|
const docIds = new Uint32Array(entries.length);
|
|
8053
8675
|
const values = new Float64Array(entries.length);
|
|
8054
8676
|
for (let i = 0; i < entries.length; i++) {
|
|
8055
8677
|
docIds[i] = remap[entries[i].docId];
|
|
8056
8678
|
values[i] = entries[i].value;
|
|
8057
8679
|
}
|
|
8058
|
-
|
|
8680
|
+
encoded.push({ fieldPath, docIds, values });
|
|
8059
8681
|
}
|
|
8682
|
+
return encoded;
|
|
8683
|
+
}
|
|
8684
|
+
function encodeFieldIndexes(state, remap) {
|
|
8685
|
+
const survives = (internalId) => remap[internalId] >= 0;
|
|
8686
|
+
const numeric = encodeNumericIndexes(state.numericIndexes, remap);
|
|
8060
8687
|
const booleans = [];
|
|
8061
8688
|
for (const [fieldPath, index] of state.booleanIndexes) {
|
|
8062
8689
|
const { trueDocs, falseDocs } = index.serialize();
|
|
@@ -8097,7 +8724,9 @@ function encodeFieldIndexes(state, remap) {
|
|
|
8097
8724
|
}
|
|
8098
8725
|
geo.push({ fieldPath, docIds, latitudes, longitudes });
|
|
8099
8726
|
}
|
|
8100
|
-
|
|
8727
|
+
const pattern = [];
|
|
8728
|
+
for (const [fieldPath, index] of state.patternIndexes) pattern.push(index.toArrays(fieldPath, remap));
|
|
8729
|
+
return { numeric, boolean: booleans, enums, geo, pattern, sizes: encodeNumericIndexes(state.sizeIndexes, remap) };
|
|
8101
8730
|
}
|
|
8102
8731
|
|
|
8103
8732
|
// src/core/partition/segment-payload.ts
|
|
@@ -8254,15 +8883,19 @@ function mergePayloadPostings(target, payload, ordinalBase) {
|
|
|
8254
8883
|
}
|
|
8255
8884
|
}
|
|
8256
8885
|
}
|
|
8257
|
-
function
|
|
8258
|
-
for (const entry of
|
|
8259
|
-
let index = target.
|
|
8886
|
+
function mergePayloadNumericIndexes(target, entries, ordinalBase) {
|
|
8887
|
+
for (const entry of entries) {
|
|
8888
|
+
let index = target.get(entry.fieldPath);
|
|
8260
8889
|
if (index === void 0) {
|
|
8261
8890
|
index = createNumericIndex();
|
|
8262
|
-
target.
|
|
8891
|
+
target.set(entry.fieldPath, index);
|
|
8263
8892
|
}
|
|
8264
8893
|
for (let i = 0; i < entry.docIds.length; i++) index.insert(ordinalBase + entry.docIds[i], entry.values[i]);
|
|
8265
8894
|
}
|
|
8895
|
+
}
|
|
8896
|
+
function mergePayloadFieldIndexes(target, payload, ordinalBase) {
|
|
8897
|
+
mergePayloadNumericIndexes(target.numericIndexes, payload.numeric, ordinalBase);
|
|
8898
|
+
mergePayloadNumericIndexes(target.sizeIndexes, payload.sizes, ordinalBase);
|
|
8266
8899
|
for (const entry of payload.boolean) {
|
|
8267
8900
|
let index = target.booleanIndexes.get(entry.fieldPath);
|
|
8268
8901
|
if (index === void 0) {
|
|
@@ -8293,6 +8926,9 @@ function mergePayloadFieldIndexes(target, payload, ordinalBase) {
|
|
|
8293
8926
|
index.insert(ordinalBase + entry.docIds[i], entry.latitudes[i], entry.longitudes[i]);
|
|
8294
8927
|
}
|
|
8295
8928
|
}
|
|
8929
|
+
for (const entry of payload.pattern) {
|
|
8930
|
+
patternIndexOf(target.patternIndexes, entry.fieldPath).addArrays(entry, (ordinal) => ordinalBase + ordinal);
|
|
8931
|
+
}
|
|
8296
8932
|
}
|
|
8297
8933
|
function mergeSegmentPayload(target, payload, documents) {
|
|
8298
8934
|
if (payload.documentCount === 0) return;
|
|
@@ -8316,25 +8952,142 @@ function mergeSegmentPayload(target, payload, documents) {
|
|
|
8316
8952
|
});
|
|
8317
8953
|
}
|
|
8318
8954
|
}
|
|
8319
|
-
mergePayloadPostings(target, payload, ordinalBase);
|
|
8320
|
-
mergePayloadFieldIndexes(target, payload, ordinalBase);
|
|
8321
|
-
target.stats.totalDocuments += payload.documentCount;
|
|
8322
|
-
for (const [fieldName, length] of Object.entries(payload.totalFieldLengths)) {
|
|
8323
|
-
target.stats.totalFieldLengths[fieldName] = (target.stats.totalFieldLengths[fieldName] ?? 0) + length;
|
|
8955
|
+
mergePayloadPostings(target, payload, ordinalBase);
|
|
8956
|
+
mergePayloadFieldIndexes(target, payload, ordinalBase);
|
|
8957
|
+
target.stats.totalDocuments += payload.documentCount;
|
|
8958
|
+
for (const [fieldName, length] of Object.entries(payload.totalFieldLengths)) {
|
|
8959
|
+
target.stats.totalFieldLengths[fieldName] = (target.stats.totalFieldLengths[fieldName] ?? 0) + length;
|
|
8960
|
+
}
|
|
8961
|
+
for (const [token, frequency] of Object.entries(payload.docFrequencies)) {
|
|
8962
|
+
target.stats.docFrequencies[token] = (target.stats.docFrequencies[token] ?? 0) + frequency;
|
|
8963
|
+
}
|
|
8964
|
+
target.stats.recalculateAverages();
|
|
8965
|
+
if (payload.surfaceForms !== null) {
|
|
8966
|
+
for (const surface of Object.keys(payload.surfaceForms)) {
|
|
8967
|
+
const value = payload.surfaceForms[surface];
|
|
8968
|
+
if (!Array.isArray(value)) continue;
|
|
8969
|
+
target.surfaceRegistry.add(surface, value[1], value[0]);
|
|
8970
|
+
}
|
|
8971
|
+
}
|
|
8972
|
+
target.sortColumns = null;
|
|
8973
|
+
target.scoreBuffer = null;
|
|
8974
|
+
}
|
|
8975
|
+
|
|
8976
|
+
// src/core/partition/pattern-snapshot.ts
|
|
8977
|
+
function loadFailed(fieldPath, problem) {
|
|
8978
|
+
return new NarsilError(
|
|
8979
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
8980
|
+
`The pattern index of field "${fieldPath}" in this partition is corrupt: ${problem}`,
|
|
8981
|
+
{ field: fieldPath }
|
|
8982
|
+
);
|
|
8983
|
+
}
|
|
8984
|
+
function serializePatternIndex(state, fieldPath, index) {
|
|
8985
|
+
const resolver = state.docStore.resolver();
|
|
8986
|
+
const arrays = index.toArrays(fieldPath, null);
|
|
8987
|
+
const positionOf = /* @__PURE__ */ new Map();
|
|
8988
|
+
const docIds = [];
|
|
8989
|
+
for (const ordinal of arrays.docIds) {
|
|
8990
|
+
const externalId = resolver.toExternal(ordinal);
|
|
8991
|
+
if (externalId === void 0) continue;
|
|
8992
|
+
positionOf.set(ordinal, docIds.length);
|
|
8993
|
+
docIds.push(externalId);
|
|
8994
|
+
}
|
|
8995
|
+
const runs = /* @__PURE__ */ Object.create(null);
|
|
8996
|
+
for (let run = 0; run + 1 < arrays.offsets.length; run++) {
|
|
8997
|
+
const positions = [];
|
|
8998
|
+
for (let at = arrays.offsets[run]; at < arrays.offsets[run + 1]; at++) {
|
|
8999
|
+
const position = positionOf.get(arrays.postings[at]);
|
|
9000
|
+
if (position !== void 0) positions.push(position);
|
|
9001
|
+
}
|
|
9002
|
+
if (positions.length === 0) continue;
|
|
9003
|
+
const base = run * PATTERN_RUN_CODE_POINTS;
|
|
9004
|
+
runs[String.fromCodePoint(arrays.runs[base], arrays.runs[base + 1], arrays.runs[base + 2])] = positions;
|
|
9005
|
+
}
|
|
9006
|
+
return { docIds, runs };
|
|
9007
|
+
}
|
|
9008
|
+
function serializePatternIndexes(state, schema) {
|
|
9009
|
+
const patternFields = schemaFieldsOf(schema).filter((field) => field.pattern);
|
|
9010
|
+
if (patternFields.length === 0) return void 0;
|
|
9011
|
+
const serialized = /* @__PURE__ */ Object.create(null);
|
|
9012
|
+
for (const { path: fieldPath } of patternFields) {
|
|
9013
|
+
const index = state.patternIndexes.get(fieldPath);
|
|
9014
|
+
serialized[fieldPath] = index === void 0 ? { docIds: [], runs: {} } : serializePatternIndex(state, fieldPath, index);
|
|
9015
|
+
}
|
|
9016
|
+
return serialized;
|
|
9017
|
+
}
|
|
9018
|
+
function appendRunCodePoints(fieldPath, key, out) {
|
|
9019
|
+
const start = out.length;
|
|
9020
|
+
for (const character of key) out.push(character.codePointAt(0) ?? 0);
|
|
9021
|
+
const count = out.length - start;
|
|
9022
|
+
if (count !== PATTERN_RUN_CODE_POINTS) {
|
|
9023
|
+
throw loadFailed(
|
|
9024
|
+
fieldPath,
|
|
9025
|
+
`the key "${key}" holds ${count} code points, while each key must hold ${PATTERN_RUN_CODE_POINTS}`
|
|
9026
|
+
);
|
|
9027
|
+
}
|
|
9028
|
+
}
|
|
9029
|
+
function requirePositions(fieldPath, key, list, documentCount) {
|
|
9030
|
+
if (!Array.isArray(list)) throw loadFailed(fieldPath, `the run "${key}" must map to a list of positions`);
|
|
9031
|
+
let previous = -1;
|
|
9032
|
+
for (const position of list) {
|
|
9033
|
+
if (!Number.isInteger(position) || position < 0 || position >= documentCount) {
|
|
9034
|
+
throw loadFailed(fieldPath, `the run "${key}" names position ${String(position)} outside its document list`);
|
|
9035
|
+
}
|
|
9036
|
+
if (position <= previous) {
|
|
9037
|
+
throw loadFailed(fieldPath, `the positions of the run "${key}" are out of ascending order`);
|
|
9038
|
+
}
|
|
9039
|
+
previous = position;
|
|
9040
|
+
}
|
|
9041
|
+
return list;
|
|
9042
|
+
}
|
|
9043
|
+
function requireDistinctDocuments(fieldPath, docIds) {
|
|
9044
|
+
const seen = /* @__PURE__ */ new Set();
|
|
9045
|
+
for (const docId of docIds) {
|
|
9046
|
+
if (seen.has(docId)) throw loadFailed(fieldPath, `the document list names "${docId}" twice`);
|
|
9047
|
+
seen.add(docId);
|
|
8324
9048
|
}
|
|
8325
|
-
|
|
8326
|
-
|
|
9049
|
+
}
|
|
9050
|
+
function decodePatternEntry(fieldPath, entry) {
|
|
9051
|
+
requireDistinctDocuments(fieldPath, entry.docIds);
|
|
9052
|
+
const runs = [];
|
|
9053
|
+
const offsets = [0];
|
|
9054
|
+
const postings = [];
|
|
9055
|
+
for (const [key, list] of Object.entries(entry.runs)) {
|
|
9056
|
+
appendRunCodePoints(fieldPath, key, runs);
|
|
9057
|
+
for (const position of requirePositions(fieldPath, key, list, entry.docIds.length)) postings.push(position);
|
|
9058
|
+
offsets.push(postings.length);
|
|
8327
9059
|
}
|
|
8328
|
-
|
|
8329
|
-
|
|
8330
|
-
|
|
8331
|
-
|
|
8332
|
-
|
|
8333
|
-
|
|
9060
|
+
const positions = new Uint32Array(entry.docIds.length);
|
|
9061
|
+
for (let position = 0; position < positions.length; position++) positions[position] = position;
|
|
9062
|
+
return {
|
|
9063
|
+
fieldPath,
|
|
9064
|
+
docIds: positions,
|
|
9065
|
+
runs: Uint32Array.from(runs),
|
|
9066
|
+
offsets: Uint32Array.from(offsets),
|
|
9067
|
+
postings: Uint32Array.from(postings)
|
|
9068
|
+
};
|
|
9069
|
+
}
|
|
9070
|
+
function rebuildPatternIndex(state, fieldPath) {
|
|
9071
|
+
for (const [docId, stored] of state.docStore.all()) {
|
|
9072
|
+
const value = getNestedValue(stored.fields, fieldPath);
|
|
9073
|
+
if (value === void 0 || value === null) continue;
|
|
9074
|
+
const internalId = state.docStore.getInternalId(docId);
|
|
9075
|
+
if (internalId !== void 0) patternIndexOf(state.patternIndexes, fieldPath).add(internalId, value);
|
|
9076
|
+
}
|
|
9077
|
+
}
|
|
9078
|
+
function loadPatternIndexes(state, serialized, schema) {
|
|
9079
|
+
const resolver = state.docStore.resolver();
|
|
9080
|
+
for (const { path: fieldPath, pattern } of schemaFieldsOf(schema)) {
|
|
9081
|
+
if (!pattern) continue;
|
|
9082
|
+
const entry = serialized !== void 0 && Object.hasOwn(serialized, fieldPath) ? serialized[fieldPath] : void 0;
|
|
9083
|
+
if (entry === void 0) {
|
|
9084
|
+
rebuildPatternIndex(state, fieldPath);
|
|
9085
|
+
continue;
|
|
8334
9086
|
}
|
|
9087
|
+
const decoded = decodePatternEntry(fieldPath, entry);
|
|
9088
|
+
const ordinals = entry.docIds.map((docId) => resolver.toInternal(docId) ?? -1);
|
|
9089
|
+
patternIndexOf(state.patternIndexes, fieldPath).addArrays(decoded, (position) => ordinals[position]);
|
|
8335
9090
|
}
|
|
8336
|
-
target.sortColumns = null;
|
|
8337
|
-
target.scoreBuffer = null;
|
|
8338
9091
|
}
|
|
8339
9092
|
|
|
8340
9093
|
// src/core/partition/serialization.ts
|
|
@@ -8409,6 +9162,7 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
|
|
|
8409
9162
|
}));
|
|
8410
9163
|
}
|
|
8411
9164
|
const serializedStats = state.stats.serialize();
|
|
9165
|
+
const serializedPattern = serializePatternIndexes(state, schema);
|
|
8412
9166
|
return {
|
|
8413
9167
|
indexName,
|
|
8414
9168
|
partitionId,
|
|
@@ -8423,7 +9177,8 @@ function serializePartition(state, partitionId, indexName, totalPartitions, lang
|
|
|
8423
9177
|
numeric: serializedNumeric,
|
|
8424
9178
|
boolean: serializedBoolean,
|
|
8425
9179
|
enum: serializedEnum,
|
|
8426
|
-
geopoint: serializedGeo
|
|
9180
|
+
geopoint: serializedGeo,
|
|
9181
|
+
...serializedPattern === void 0 ? {} : { pattern: serializedPattern }
|
|
8427
9182
|
},
|
|
8428
9183
|
surfaceForms: state.surfaceRegistry.serialize(),
|
|
8429
9184
|
statistics: serializedStats
|
|
@@ -8501,6 +9256,8 @@ function deserializePartition(state, data, clearFn, schema) {
|
|
|
8501
9256
|
throw new Error(`Schema mismatch: serialized data contains unknown field "${field}" not present in schema`);
|
|
8502
9257
|
}
|
|
8503
9258
|
}
|
|
9259
|
+
loadPatternIndexes(state, data.fieldIndexes.pattern, schema);
|
|
9260
|
+
rebuildSizeIndexes(state, schema);
|
|
8504
9261
|
}
|
|
8505
9262
|
|
|
8506
9263
|
// src/core/partition/suggestions.ts
|
|
@@ -8663,7 +9420,8 @@ function serializePartitionToWirePayloadV2(state, partitionId, indexName, totalP
|
|
|
8663
9420
|
numeric: wireNumeric,
|
|
8664
9421
|
boolean: wireBoolean,
|
|
8665
9422
|
enum: wireEnum,
|
|
8666
|
-
geopoint: wireGeo
|
|
9423
|
+
geopoint: wireGeo,
|
|
9424
|
+
...patternIndexesToWire(serializePatternIndexes(state, schema))
|
|
8667
9425
|
},
|
|
8668
9426
|
surface_forms: state.surfaceRegistry.serialize(),
|
|
8669
9427
|
statistics: {
|
|
@@ -8698,6 +9456,8 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8698
9456
|
booleanIndexes: /* @__PURE__ */ new Map(),
|
|
8699
9457
|
enumIndexes: /* @__PURE__ */ new Map(),
|
|
8700
9458
|
geoIndexes: /* @__PURE__ */ new Map(),
|
|
9459
|
+
patternIndexes: /* @__PURE__ */ new Map(),
|
|
9460
|
+
sizeIndexes: /* @__PURE__ */ new Map(),
|
|
8701
9461
|
fieldNameTable,
|
|
8702
9462
|
flatSchemaCache: null,
|
|
8703
9463
|
lastSchemaRef: null,
|
|
@@ -8717,6 +9477,8 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8717
9477
|
state.booleanIndexes.clear();
|
|
8718
9478
|
state.enumIndexes.clear();
|
|
8719
9479
|
state.geoIndexes.clear();
|
|
9480
|
+
state.patternIndexes.clear();
|
|
9481
|
+
state.sizeIndexes.clear();
|
|
8720
9482
|
state.stats.deserialize({ totalDocuments: 0, totalFieldLengths: {}, averageFieldLengths: {}, docFrequencies: {} });
|
|
8721
9483
|
state.flatSchemaCache = null;
|
|
8722
9484
|
state.lastSchemaRef = null;
|
|
@@ -8892,14 +9654,14 @@ function createPartitionIndex(partitionId, trackPositions = true) {
|
|
|
8892
9654
|
sortValues(docId, fields, fieldTypes, modes) {
|
|
8893
9655
|
return sortValuesOf(state, docId, fields, fieldTypes, modes);
|
|
8894
9656
|
},
|
|
8895
|
-
applyFilters(filters, schema) {
|
|
8896
|
-
return applyPartitionFilters(state, filters, schema);
|
|
9657
|
+
applyFilters(filters, schema, meter) {
|
|
9658
|
+
return applyPartitionFilters(state, filters, schema, meter);
|
|
8897
9659
|
},
|
|
8898
|
-
applyFiltersBitset(filters, schema) {
|
|
8899
|
-
return applyPartitionFiltersBitset(state, filters, schema);
|
|
9660
|
+
applyFiltersBitset(filters, schema, meter) {
|
|
9661
|
+
return applyPartitionFiltersBitset(state, filters, schema, meter);
|
|
8900
9662
|
},
|
|
8901
|
-
filterMatches(filters, schema) {
|
|
8902
|
-
return partitionFilterMatches(state, filters, schema);
|
|
9663
|
+
filterMatches(filters, schema, meter) {
|
|
9664
|
+
return partitionFilterMatches(state, filters, schema, meter);
|
|
8903
9665
|
},
|
|
8904
9666
|
computeFacets(matched, config, schema) {
|
|
8905
9667
|
return computeFacets(state, matched, config, schema);
|
|
@@ -9281,6 +10043,100 @@ function readHeapStatistics() {
|
|
|
9281
10043
|
return null;
|
|
9282
10044
|
}
|
|
9283
10045
|
|
|
10046
|
+
// src/core/pattern-index/work-meter.ts
|
|
10047
|
+
var COUNTER_BYTES = 8;
|
|
10048
|
+
function capExceeded(cap) {
|
|
10049
|
+
return new NarsilError(
|
|
10050
|
+
ErrorCodes.SEARCH_WORK_CAP_EXCEEDED,
|
|
10051
|
+
`The engine stopped this search before it checked every candidate, because the work on its pattern tests passed the cap of ${cap} units. Narrow the tests, or raise the patternWorkCap engine setting`,
|
|
10052
|
+
{ cap }
|
|
10053
|
+
);
|
|
10054
|
+
}
|
|
10055
|
+
function localMeter(cap, spent, matcherStateBytes, flushUnits) {
|
|
10056
|
+
let total = spent;
|
|
10057
|
+
let sharedCount = null;
|
|
10058
|
+
const spentSoFar = () => sharedCount === null ? total : Number(Atomics.load(sharedCount, 0));
|
|
10059
|
+
const meter = {
|
|
10060
|
+
cap,
|
|
10061
|
+
matcherStateBytes,
|
|
10062
|
+
flushUnits,
|
|
10063
|
+
add(units) {
|
|
10064
|
+
if (units <= 0) return;
|
|
10065
|
+
if (sharedCount === null) total += units;
|
|
10066
|
+
else total = Number(Atomics.add(sharedCount, 0, BigInt(units))) + units;
|
|
10067
|
+
if (total > cap) throw capExceeded(cap);
|
|
10068
|
+
},
|
|
10069
|
+
remaining() {
|
|
10070
|
+
return cap - spentSoFar();
|
|
10071
|
+
},
|
|
10072
|
+
fork(workerCount) {
|
|
10073
|
+
const copies = Math.max(1, workerCount);
|
|
10074
|
+
const seed = spentSoFar();
|
|
10075
|
+
let counter = null;
|
|
10076
|
+
if (typeof SharedArrayBuffer === "function") {
|
|
10077
|
+
counter = new SharedArrayBuffer(COUNTER_BYTES);
|
|
10078
|
+
new BigInt64Array(counter)[0] = BigInt(seed);
|
|
10079
|
+
}
|
|
10080
|
+
const shared2 = {
|
|
10081
|
+
cap,
|
|
10082
|
+
spent: seed,
|
|
10083
|
+
counter,
|
|
10084
|
+
matcherStateBytes: Math.floor(matcherStateBytes / copies),
|
|
10085
|
+
flushUnits: Math.max(1, Math.floor(flushUnits / copies))
|
|
10086
|
+
};
|
|
10087
|
+
return {
|
|
10088
|
+
shared: shared2,
|
|
10089
|
+
settle(reportedUnits) {
|
|
10090
|
+
if (counter === null) {
|
|
10091
|
+
meter.add(reportedUnits);
|
|
10092
|
+
return;
|
|
10093
|
+
}
|
|
10094
|
+
const unitsSinceFork = spentSoFar() - seed;
|
|
10095
|
+
const view = new BigInt64Array(counter);
|
|
10096
|
+
if (unitsSinceFork > 0) Atomics.add(view, 0, BigInt(unitsSinceFork));
|
|
10097
|
+
sharedCount = view;
|
|
10098
|
+
total = spentSoFar();
|
|
10099
|
+
if (total > cap) throw capExceeded(cap);
|
|
10100
|
+
}
|
|
10101
|
+
};
|
|
10102
|
+
}
|
|
10103
|
+
};
|
|
10104
|
+
return meter;
|
|
10105
|
+
}
|
|
10106
|
+
function createPatternWorkMeter(cap = DEFAULT_PATTERN_WORK_CAP) {
|
|
10107
|
+
return localMeter(cap, 0, MATCHER_STATE_BUDGET_BYTES, PATTERN_WORK_FLUSH_UNITS);
|
|
10108
|
+
}
|
|
10109
|
+
function sharedMeter(work, counter) {
|
|
10110
|
+
const view = new BigInt64Array(counter);
|
|
10111
|
+
return {
|
|
10112
|
+
matcherStateBytes: work.matcherStateBytes,
|
|
10113
|
+
flushUnits: work.flushUnits,
|
|
10114
|
+
add(units) {
|
|
10115
|
+
if (units <= 0) return;
|
|
10116
|
+
const total = Number(Atomics.add(view, 0, BigInt(units))) + units;
|
|
10117
|
+
if (total > work.cap) throw capExceeded(work.cap);
|
|
10118
|
+
},
|
|
10119
|
+
remaining() {
|
|
10120
|
+
return work.cap - Number(Atomics.load(view, 0));
|
|
10121
|
+
}
|
|
10122
|
+
};
|
|
10123
|
+
}
|
|
10124
|
+
function joinPatternWorkMeter(work) {
|
|
10125
|
+
const meter = work.counter === null ? localMeter(work.cap, work.spent, work.matcherStateBytes, work.flushUnits) : sharedMeter(work, work.counter);
|
|
10126
|
+
let spent = 0;
|
|
10127
|
+
return {
|
|
10128
|
+
matcherStateBytes: meter.matcherStateBytes,
|
|
10129
|
+
flushUnits: meter.flushUnits,
|
|
10130
|
+
add(units) {
|
|
10131
|
+
if (units <= 0) return;
|
|
10132
|
+
spent += units;
|
|
10133
|
+
meter.add(units);
|
|
10134
|
+
},
|
|
10135
|
+
remaining: () => meter.remaining(),
|
|
10136
|
+
spent: () => spent
|
|
10137
|
+
};
|
|
10138
|
+
}
|
|
10139
|
+
|
|
9284
10140
|
// src/partitioning/distributed-scoring.ts
|
|
9285
10141
|
function analysedQueryTokens(term, language, analysis) {
|
|
9286
10142
|
const { tokens } = tokenize(term, language, {
|
|
@@ -9938,7 +10794,11 @@ function prepareFulltextQuery(partition, params, language, schema, options) {
|
|
|
9938
10794
|
}
|
|
9939
10795
|
let filterBitset;
|
|
9940
10796
|
if (params.filters) {
|
|
9941
|
-
filterBitset = partition.applyFiltersBitset(
|
|
10797
|
+
filterBitset = partition.applyFiltersBitset(
|
|
10798
|
+
params.filters,
|
|
10799
|
+
schema,
|
|
10800
|
+
options?.patternWork ?? createPatternWorkMeter()
|
|
10801
|
+
);
|
|
9942
10802
|
if (bitsetIsEmpty(filterBitset)) return null;
|
|
9943
10803
|
}
|
|
9944
10804
|
return { queryTokens, prefixExpansion, filterBitset };
|
|
@@ -10181,7 +11041,8 @@ function resolveEffectiveMode(config) {
|
|
|
10181
11041
|
function buildSearchOptions(base, globalStats) {
|
|
10182
11042
|
return {
|
|
10183
11043
|
...base,
|
|
10184
|
-
globalStats: globalStats ?? base?.globalStats
|
|
11044
|
+
globalStats: globalStats ?? base?.globalStats,
|
|
11045
|
+
patternWork: base?.patternWork ?? createPatternWorkMeter()
|
|
10185
11046
|
};
|
|
10186
11047
|
}
|
|
10187
11048
|
function dispatchWithDispatcher(partitions, params, language, schema, options, dispatcher) {
|
|
@@ -10216,12 +11077,13 @@ function frozenSegmentBytes(segment) {
|
|
|
10216
11077
|
bytes += arrays.documentTable.blob.byteLength + arrays.documentTable.offsets.byteLength;
|
|
10217
11078
|
}
|
|
10218
11079
|
for (const column of arrays.fieldLengthColumns) bytes += column.byteLength;
|
|
10219
|
-
for (const entry of arrays.numeric) bytes += entry.docIds.byteLength + entry.values.byteLength;
|
|
11080
|
+
for (const entry of [...arrays.numeric, ...arrays.sizes]) bytes += entry.docIds.byteLength + entry.values.byteLength;
|
|
10220
11081
|
for (const entry of arrays.boolean) bytes += entry.trueDocs.byteLength + entry.falseDocs.byteLength;
|
|
10221
11082
|
for (const entry of arrays.enums) bytes += entry.offsets.byteLength + entry.docIds.byteLength;
|
|
10222
11083
|
for (const entry of arrays.geo) {
|
|
10223
11084
|
bytes += entry.docIds.byteLength + entry.latitudes.byteLength + entry.longitudes.byteLength;
|
|
10224
11085
|
}
|
|
11086
|
+
for (const reader of segment.patternIndexes.values()) bytes += reader.bytes();
|
|
10225
11087
|
return bytes;
|
|
10226
11088
|
}
|
|
10227
11089
|
|
|
@@ -10299,24 +11161,24 @@ function placeSubBitset(composite, layout, subIndex, subBits) {
|
|
|
10299
11161
|
if (words === 0) return;
|
|
10300
11162
|
composite.set(subBits.length > words ? subBits.subarray(0, words) : subBits, layout.bases[subIndex] >> 5);
|
|
10301
11163
|
}
|
|
10302
|
-
function compositeFiltersBitset(subs, layout, filters, schema) {
|
|
11164
|
+
function compositeFiltersBitset(subs, layout, filters, schema, meter) {
|
|
10303
11165
|
const composite = new Uint32Array(Math.ceil(layout.totalCapacity / WORD_BITS));
|
|
10304
11166
|
for (let i = 0; i < subs.length; i++) {
|
|
10305
|
-
placeSubBitset(composite, layout, i, applyPartitionFiltersBitset(subs[i], filters, schema));
|
|
11167
|
+
placeSubBitset(composite, layout, i, applyPartitionFiltersBitset(subs[i], filters, schema, meter));
|
|
10306
11168
|
}
|
|
10307
11169
|
return composite;
|
|
10308
11170
|
}
|
|
10309
|
-
function compositeFilters(subs, filters, schema) {
|
|
11171
|
+
function compositeFilters(subs, filters, schema, meter) {
|
|
10310
11172
|
const combined = /* @__PURE__ */ new Set();
|
|
10311
11173
|
for (const sub of subs) {
|
|
10312
|
-
for (const docId of applyPartitionFilters(sub, filters, schema)) {
|
|
11174
|
+
for (const docId of applyPartitionFilters(sub, filters, schema, meter)) {
|
|
10313
11175
|
combined.add(docId);
|
|
10314
11176
|
}
|
|
10315
11177
|
}
|
|
10316
11178
|
return combined;
|
|
10317
11179
|
}
|
|
10318
|
-
function compositeFilterMatches(subs, layout, filters, schema) {
|
|
10319
|
-
const bits = compositeFiltersBitset(subs, layout, filters, schema);
|
|
11180
|
+
function compositeFilterMatches(subs, layout, filters, schema, meter) {
|
|
11181
|
+
const bits = compositeFiltersBitset(subs, layout, filters, schema, meter);
|
|
10320
11182
|
let count = 0;
|
|
10321
11183
|
for (let i = 0; i < subs.length; i++) {
|
|
10322
11184
|
const resolver = subs[i].docStore.resolver();
|
|
@@ -10777,14 +11639,14 @@ function createCompositePartition(partitionId, trackPositions = true, existingLi
|
|
|
10777
11639
|
searchFulltextMatches(params) {
|
|
10778
11640
|
return compositeSearchMatches(subs(), layout(), params);
|
|
10779
11641
|
},
|
|
10780
|
-
applyFilters(filters, schema) {
|
|
10781
|
-
return compositeFilters(subs(), filters, schema);
|
|
11642
|
+
applyFilters(filters, schema, meter) {
|
|
11643
|
+
return compositeFilters(subs(), filters, schema, meter);
|
|
10782
11644
|
},
|
|
10783
|
-
applyFiltersBitset(filters, schema) {
|
|
10784
|
-
return compositeFiltersBitset(subs(), layout(), filters, schema);
|
|
11645
|
+
applyFiltersBitset(filters, schema, meter) {
|
|
11646
|
+
return compositeFiltersBitset(subs(), layout(), filters, schema, meter);
|
|
10785
11647
|
},
|
|
10786
|
-
filterMatches(filters, schema) {
|
|
10787
|
-
return compositeFilterMatches(subs(), layout(), filters, schema);
|
|
11648
|
+
filterMatches(filters, schema, meter) {
|
|
11649
|
+
return compositeFilterMatches(subs(), layout(), filters, schema, meter);
|
|
10788
11650
|
},
|
|
10789
11651
|
computeFacets(matched, config, schema) {
|
|
10790
11652
|
const everyValue = everyValueFacetConfig(config);
|
|
@@ -10954,6 +11816,7 @@ function setNestedValue(obj, path, value) {
|
|
|
10954
11816
|
|
|
10955
11817
|
// src/partitioning/manager/index.ts
|
|
10956
11818
|
function createPartitionManager(indexName, config, language, router, initialPartitionCount, vectorIndexes) {
|
|
11819
|
+
requirePatternSearchFor(config.schema);
|
|
10957
11820
|
const count = initialPartitionCount ?? 1;
|
|
10958
11821
|
const analysis = resolveIndexAnalysis(config);
|
|
10959
11822
|
const trackPositions = config.trackPositions ?? true;
|
|
@@ -11265,8 +12128,12 @@ function countsWithoutScores(params) {
|
|
|
11265
12128
|
}
|
|
11266
12129
|
function fanOutMatchCount(manager, params, language, schema, options) {
|
|
11267
12130
|
let count = 0;
|
|
12131
|
+
const searchOptions = {
|
|
12132
|
+
...options?.searchOptions,
|
|
12133
|
+
patternWork: options?.searchOptions?.patternWork ?? createPatternWorkMeter()
|
|
12134
|
+
};
|
|
11268
12135
|
for (const partition of partitionsIn(manager, options?.partitionIds)) {
|
|
11269
|
-
const matches = fulltextMatches(partition, params, language, schema,
|
|
12136
|
+
const matches = fulltextMatches(partition, params, language, schema, searchOptions);
|
|
11270
12137
|
if (matches !== null) count += matches.count;
|
|
11271
12138
|
}
|
|
11272
12139
|
return count;
|
|
@@ -17951,7 +18818,22 @@ function createFrozenEnumReader(entry) {
|
|
|
17951
18818
|
function rangeOf(value) {
|
|
17952
18819
|
return rangeByValue.get(value) ?? { from: 0, to: 0 };
|
|
17953
18820
|
}
|
|
18821
|
+
let affixes = null;
|
|
18822
|
+
function affixesOf() {
|
|
18823
|
+
affixes ??= new ValueAffixes(() => true, values);
|
|
18824
|
+
return affixes;
|
|
18825
|
+
}
|
|
18826
|
+
function bitsetOfValues(matching, capacity) {
|
|
18827
|
+
const bits = createBitSet(capacity);
|
|
18828
|
+
for (const value of matching) {
|
|
18829
|
+
const { from, to } = rangeOf(value);
|
|
18830
|
+
for (let i = from; i < to; i++) bitsetSet(bits, docIds[i]);
|
|
18831
|
+
}
|
|
18832
|
+
return bits;
|
|
18833
|
+
}
|
|
17954
18834
|
return {
|
|
18835
|
+
queryPrefixBitset: (prefix, capacity) => bitsetOfValues(affixesOf().withPrefix(prefix), capacity),
|
|
18836
|
+
querySuffixBitset: (suffix, capacity) => bitsetOfValues(affixesOf().withSuffix(suffix), capacity),
|
|
17955
18837
|
serialize: () => {
|
|
17956
18838
|
const byValue = /* @__PURE__ */ Object.create(null);
|
|
17957
18839
|
for (let i = 0; i < values.length; i++) {
|
|
@@ -18396,6 +19278,11 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
|
18396
19278
|
docIds: sharedUint32(entry.docIds),
|
|
18397
19279
|
values: sharedFloat64(entry.values)
|
|
18398
19280
|
})),
|
|
19281
|
+
sizes: payload.sizes.map((entry) => ({
|
|
19282
|
+
fieldPath: entry.fieldPath,
|
|
19283
|
+
docIds: sharedUint32(entry.docIds),
|
|
19284
|
+
values: sharedFloat64(entry.values)
|
|
19285
|
+
})),
|
|
18399
19286
|
boolean: payload.boolean.map((entry) => ({
|
|
18400
19287
|
fieldPath: entry.fieldPath,
|
|
18401
19288
|
trueDocs: sharedUint32(entry.trueDocs),
|
|
@@ -18413,6 +19300,13 @@ function freezeEncodedSegmentShared(payload, documentData, segmentId) {
|
|
|
18413
19300
|
latitudes: sharedFloat64(entry.latitudes),
|
|
18414
19301
|
longitudes: sharedFloat64(entry.longitudes)
|
|
18415
19302
|
})),
|
|
19303
|
+
pattern: payload.pattern.map((entry) => ({
|
|
19304
|
+
fieldPath: entry.fieldPath,
|
|
19305
|
+
docIds: sharedUint32(entry.docIds),
|
|
19306
|
+
runs: sharedUint32(entry.runs),
|
|
19307
|
+
offsets: sharedUint32(entry.offsets),
|
|
19308
|
+
postings: sharedUint32(entry.postings)
|
|
19309
|
+
})),
|
|
18416
19310
|
surfaceTable: sharedSurfaceTable(payload)
|
|
18417
19311
|
};
|
|
18418
19312
|
}
|
|
@@ -18459,10 +19353,16 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
18459
19353
|
const docStore = createFrozenDocTable(source.idTable, lengthColumns, documentSource, tombstones);
|
|
18460
19354
|
const numericIndexes = /* @__PURE__ */ new Map();
|
|
18461
19355
|
for (const entry of source.numeric) numericIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
19356
|
+
const sizeIndexes = /* @__PURE__ */ new Map();
|
|
19357
|
+
for (const entry of source.sizes) sizeIndexes.set(entry.fieldPath, createFrozenNumericReader(entry));
|
|
18462
19358
|
const booleanIndexes = /* @__PURE__ */ new Map();
|
|
18463
19359
|
for (const entry of source.boolean) booleanIndexes.set(entry.fieldPath, createFrozenBooleanReader(entry));
|
|
18464
19360
|
const enumIndexes = /* @__PURE__ */ new Map();
|
|
18465
19361
|
for (const entry of source.enums) enumIndexes.set(entry.fieldPath, createFrozenEnumReader(entry));
|
|
19362
|
+
const patternIndexes = /* @__PURE__ */ new Map();
|
|
19363
|
+
for (const entry of source.pattern) {
|
|
19364
|
+
patternIndexes.set(entry.fieldPath, requirePatternSearch(entry.fieldPath).readArrays(entry));
|
|
19365
|
+
}
|
|
18466
19366
|
const fieldNameTable = {
|
|
18467
19367
|
names: [...source.fieldNames],
|
|
18468
19368
|
indexMap: new Map(source.fieldNames.map((name, index) => [name, index]))
|
|
@@ -18480,6 +19380,8 @@ function assembleFrozenSegment(segmentId, source, documentSource, sharedSnapshot
|
|
|
18480
19380
|
booleanIndexes,
|
|
18481
19381
|
enumIndexes,
|
|
18482
19382
|
geoIndexes: buildGeoReaders(source.geo),
|
|
19383
|
+
patternIndexes,
|
|
19384
|
+
sizeIndexes,
|
|
18483
19385
|
fieldNameTable,
|
|
18484
19386
|
trackPositions: source.positionOffsets !== null,
|
|
18485
19387
|
flatSchemaCache: null,
|
|
@@ -18607,12 +19509,8 @@ function remapDocIds(inputs, pick) {
|
|
|
18607
19509
|
}
|
|
18608
19510
|
return out;
|
|
18609
19511
|
}
|
|
18610
|
-
function mergeNumeric(inputs) {
|
|
18611
|
-
const byFieldPath = rowsByFieldPath(
|
|
18612
|
-
inputs,
|
|
18613
|
-
(input) => input.segment.arrays.numeric,
|
|
18614
|
-
(entry, at) => [entry.values[at]]
|
|
18615
|
-
);
|
|
19512
|
+
function mergeNumeric(inputs, pick) {
|
|
19513
|
+
const byFieldPath = rowsByFieldPath(inputs, pick, (entry, at) => [entry.values[at]]);
|
|
18616
19514
|
const merged = [];
|
|
18617
19515
|
for (const [fieldPath, rows] of byFieldPath) {
|
|
18618
19516
|
rows.sort((a, b) => a.values[0] - b.values[0] || a.docId - b.docId);
|
|
@@ -18685,12 +19583,32 @@ function mergeEnums(inputs) {
|
|
|
18685
19583
|
}
|
|
18686
19584
|
return merged;
|
|
18687
19585
|
}
|
|
19586
|
+
function mergePattern(inputs) {
|
|
19587
|
+
const byFieldPath = /* @__PURE__ */ new Map();
|
|
19588
|
+
for (const input of inputs) {
|
|
19589
|
+
for (const arrays of input.segment.arrays.pattern) {
|
|
19590
|
+
let entries = byFieldPath.get(arrays.fieldPath);
|
|
19591
|
+
if (entries === void 0) {
|
|
19592
|
+
entries = [];
|
|
19593
|
+
byFieldPath.set(arrays.fieldPath, entries);
|
|
19594
|
+
}
|
|
19595
|
+
entries.push({ arrays, remap: input.remap });
|
|
19596
|
+
}
|
|
19597
|
+
}
|
|
19598
|
+
const merged = [];
|
|
19599
|
+
for (const [fieldPath, entries] of byFieldPath) {
|
|
19600
|
+
merged.push(requirePatternSearch(fieldPath).mergeArrays(fieldPath, entries));
|
|
19601
|
+
}
|
|
19602
|
+
return merged;
|
|
19603
|
+
}
|
|
18688
19604
|
function mergeFieldIndexes2(inputs) {
|
|
18689
19605
|
return {
|
|
18690
|
-
numeric: mergeNumeric(inputs),
|
|
19606
|
+
numeric: mergeNumeric(inputs, (input) => input.segment.arrays.numeric),
|
|
18691
19607
|
boolean: mergeBoolean(inputs),
|
|
18692
19608
|
enums: mergeEnums(inputs),
|
|
18693
|
-
geo: mergeGeo(inputs)
|
|
19609
|
+
geo: mergeGeo(inputs),
|
|
19610
|
+
pattern: mergePattern(inputs),
|
|
19611
|
+
sizes: mergeNumeric(inputs, (input) => input.segment.arrays.sizes)
|
|
18694
19612
|
};
|
|
18695
19613
|
}
|
|
18696
19614
|
|
|
@@ -19032,54 +19950,7 @@ var GrowableUint32 = class {
|
|
|
19032
19950
|
}
|
|
19033
19951
|
};
|
|
19034
19952
|
|
|
19035
|
-
// src/core/partition/segment-
|
|
19036
|
-
var TOKEN_TABLE_START = 1 << 12;
|
|
19037
|
-
var POSTING_LOG_START = 1 << 14;
|
|
19038
|
-
var POSITION_POOL_START = 1 << 16;
|
|
19039
|
-
var FIELD_SCRATCH_START = 1 << 10;
|
|
19040
|
-
function indexOf(indexes, fieldPath, create) {
|
|
19041
|
-
let index = indexes.get(fieldPath);
|
|
19042
|
-
if (index === void 0) {
|
|
19043
|
-
index = create();
|
|
19044
|
-
indexes.set(fieldPath, index);
|
|
19045
|
-
}
|
|
19046
|
-
return index;
|
|
19047
|
-
}
|
|
19048
|
-
function indexFilterValue(indexes, ordinal, fieldPath, fieldType, value) {
|
|
19049
|
-
if (fieldType === "number" || fieldType === "number[]") {
|
|
19050
|
-
const index = indexOf(indexes.numericIndexes, fieldPath, createNumericIndex);
|
|
19051
|
-
for (const entry of fieldType === "number" ? [value] : value) index.insert(ordinal, entry);
|
|
19052
|
-
} else if (fieldType === "boolean" || fieldType === "boolean[]") {
|
|
19053
|
-
const index = indexOf(indexes.booleanIndexes, fieldPath, createBooleanIndex);
|
|
19054
|
-
for (const entry of fieldType === "boolean" ? [value] : value)
|
|
19055
|
-
index.insert(ordinal, entry);
|
|
19056
|
-
} else if (fieldType === "enum" || fieldType === "enum[]") {
|
|
19057
|
-
const index = indexOf(indexes.enumIndexes, fieldPath, createEnumIndex);
|
|
19058
|
-
for (const entry of fieldType === "enum" ? [value] : value) index.insert(ordinal, entry);
|
|
19059
|
-
} else if (fieldType === "geopoint") {
|
|
19060
|
-
const point = value;
|
|
19061
|
-
indexOf(indexes.geoIndexes, fieldPath, createGeoIndex).insert(ordinal, point.lat, point.lon);
|
|
19062
|
-
}
|
|
19063
|
-
}
|
|
19064
|
-
function refuseRepeatedDocIds(documents) {
|
|
19065
|
-
const seen = /* @__PURE__ */ new Set();
|
|
19066
|
-
for (const { docId } of documents) {
|
|
19067
|
-
if (seen.has(docId)) {
|
|
19068
|
-
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists in partition 0`, {
|
|
19069
|
-
docId,
|
|
19070
|
-
partitionId: 0
|
|
19071
|
-
});
|
|
19072
|
-
}
|
|
19073
|
-
seen.add(docId);
|
|
19074
|
-
}
|
|
19075
|
-
}
|
|
19076
|
-
function validateAsAnInsertDoes(documents, schema, options) {
|
|
19077
|
-
if (options?.validate === false) return;
|
|
19078
|
-
for (const { document: document2 } of documents) {
|
|
19079
|
-
validateDocument(document2, schema, options?.patternValueLimit);
|
|
19080
|
-
if (options?.strict) validateDocumentStrict(document2, schema);
|
|
19081
|
-
}
|
|
19082
|
-
}
|
|
19953
|
+
// src/core/partition/segment-postings.ts
|
|
19083
19954
|
var TextPostings = class {
|
|
19084
19955
|
constructor(trackPositions) {
|
|
19085
19956
|
this.trackPositions = trackPositions;
|
|
@@ -19088,22 +19959,22 @@ var TextPostings = class {
|
|
|
19088
19959
|
fieldNames = [];
|
|
19089
19960
|
tokenIds = /* @__PURE__ */ new Map();
|
|
19090
19961
|
fieldNameIndexes = /* @__PURE__ */ new Map();
|
|
19091
|
-
postingsPerToken = new GrowableUint32(
|
|
19092
|
-
docFrequencyPerToken = new GrowableUint32(
|
|
19093
|
-
lastDocPerToken = new GrowableUint32(
|
|
19094
|
-
fieldStampPerToken = new GrowableUint32(
|
|
19095
|
-
fieldSlotPerToken = new GrowableUint32(
|
|
19096
|
-
logToken = new GrowableUint32(
|
|
19097
|
-
logDoc = new GrowableUint32(
|
|
19098
|
-
logFrequency = new GrowableUint32(
|
|
19099
|
-
logField = new GrowableUint32(
|
|
19100
|
-
logPositionStart = new GrowableUint32(
|
|
19101
|
-
positionPool = new GrowableUint32(
|
|
19102
|
-
slotToken = new GrowableUint32(
|
|
19103
|
-
slotCount = new GrowableUint32(
|
|
19104
|
-
slotCursor = new GrowableUint32(
|
|
19105
|
-
eventSlot = new GrowableUint32(
|
|
19106
|
-
eventPosition = new GrowableUint32(
|
|
19962
|
+
postingsPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19963
|
+
docFrequencyPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19964
|
+
lastDocPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19965
|
+
fieldStampPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19966
|
+
fieldSlotPerToken = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19967
|
+
logToken = new GrowableUint32(SEGMENT_POSTING_LOG_START);
|
|
19968
|
+
logDoc = new GrowableUint32(SEGMENT_POSTING_LOG_START);
|
|
19969
|
+
logFrequency = new GrowableUint32(SEGMENT_POSTING_LOG_START);
|
|
19970
|
+
logField = new GrowableUint32(SEGMENT_POSTING_LOG_START);
|
|
19971
|
+
logPositionStart = new GrowableUint32(SEGMENT_POSTING_LOG_START);
|
|
19972
|
+
positionPool = new GrowableUint32(SEGMENT_POSITION_POOL_START);
|
|
19973
|
+
slotToken = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
|
|
19974
|
+
slotCount = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
|
|
19975
|
+
slotCursor = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
|
|
19976
|
+
eventSlot = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
|
|
19977
|
+
eventPosition = new GrowableUint32(SEGMENT_FIELD_SCRATCH_START);
|
|
19107
19978
|
fieldStamp = 0;
|
|
19108
19979
|
beginField() {
|
|
19109
19980
|
this.fieldStamp++;
|
|
@@ -19231,11 +20102,56 @@ var TextPostings = class {
|
|
|
19231
20102
|
};
|
|
19232
20103
|
}
|
|
19233
20104
|
};
|
|
20105
|
+
|
|
20106
|
+
// src/core/partition/segment-builder.ts
|
|
20107
|
+
function indexOf(indexes, fieldPath, create) {
|
|
20108
|
+
let index = indexes.get(fieldPath);
|
|
20109
|
+
if (index === void 0) {
|
|
20110
|
+
index = create();
|
|
20111
|
+
indexes.set(fieldPath, index);
|
|
20112
|
+
}
|
|
20113
|
+
return index;
|
|
20114
|
+
}
|
|
20115
|
+
function indexFilterValue(indexes, ordinal, fieldPath, fieldType, value) {
|
|
20116
|
+
if (fieldType === "number" || fieldType === "number[]") {
|
|
20117
|
+
const index = indexOf(indexes.numericIndexes, fieldPath, createNumericIndex);
|
|
20118
|
+
for (const entry of fieldType === "number" ? [value] : value) index.insert(ordinal, entry);
|
|
20119
|
+
} else if (fieldType === "boolean" || fieldType === "boolean[]") {
|
|
20120
|
+
const index = indexOf(indexes.booleanIndexes, fieldPath, createBooleanIndex);
|
|
20121
|
+
for (const entry of fieldType === "boolean" ? [value] : value)
|
|
20122
|
+
index.insert(ordinal, entry);
|
|
20123
|
+
} else if (fieldType === "enum" || fieldType === "enum[]") {
|
|
20124
|
+
const index = indexOf(indexes.enumIndexes, fieldPath, createEnumIndex);
|
|
20125
|
+
for (const entry of fieldType === "enum" ? [value] : value) index.insert(ordinal, entry);
|
|
20126
|
+
} else if (fieldType === "geopoint") {
|
|
20127
|
+
const point = value;
|
|
20128
|
+
indexOf(indexes.geoIndexes, fieldPath, createGeoIndex).insert(ordinal, point.lat, point.lon);
|
|
20129
|
+
}
|
|
20130
|
+
}
|
|
20131
|
+
function refuseRepeatedDocIds(documents) {
|
|
20132
|
+
const seen = /* @__PURE__ */ new Set();
|
|
20133
|
+
for (const { docId } of documents) {
|
|
20134
|
+
if (seen.has(docId)) {
|
|
20135
|
+
throw new NarsilError(ErrorCodes.DOC_ALREADY_EXISTS, `Document "${docId}" already exists in partition 0`, {
|
|
20136
|
+
docId,
|
|
20137
|
+
partitionId: 0
|
|
20138
|
+
});
|
|
20139
|
+
}
|
|
20140
|
+
seen.add(docId);
|
|
20141
|
+
}
|
|
20142
|
+
}
|
|
20143
|
+
function validateAsAnInsertDoes(documents, schema, options) {
|
|
20144
|
+
if (options?.validate === false) return;
|
|
20145
|
+
for (const { document: document2 } of documents) {
|
|
20146
|
+
validateDocument(document2, schema, options?.patternValueLimit);
|
|
20147
|
+
if (options?.strict) validateDocumentStrict(document2, schema);
|
|
20148
|
+
}
|
|
20149
|
+
}
|
|
19234
20150
|
var SurfaceCounts = class {
|
|
19235
20151
|
ids = /* @__PURE__ */ new Map();
|
|
19236
20152
|
surfaces = [];
|
|
19237
20153
|
tokens = [];
|
|
19238
|
-
counts = new GrowableUint32(
|
|
20154
|
+
counts = new GrowableUint32(SEGMENT_TOKEN_TABLE_START);
|
|
19239
20155
|
count(surface, token) {
|
|
19240
20156
|
if (surface === void 0 || surface.length === 0) return;
|
|
19241
20157
|
const known = this.ids.get(surface);
|
|
@@ -19295,7 +20211,9 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
19295
20211
|
numericIndexes: /* @__PURE__ */ new Map(),
|
|
19296
20212
|
booleanIndexes: /* @__PURE__ */ new Map(),
|
|
19297
20213
|
enumIndexes: /* @__PURE__ */ new Map(),
|
|
19298
|
-
geoIndexes: /* @__PURE__ */ new Map()
|
|
20214
|
+
geoIndexes: /* @__PURE__ */ new Map(),
|
|
20215
|
+
patternIndexes: /* @__PURE__ */ new Map(),
|
|
20216
|
+
sizeIndexes: /* @__PURE__ */ new Map()
|
|
19299
20217
|
};
|
|
19300
20218
|
let positionOffset = 0;
|
|
19301
20219
|
const addToken = (token, position, surface) => {
|
|
@@ -19304,9 +20222,11 @@ function buildSegmentPayload(documents, schema, language, options, trackPosition
|
|
|
19304
20222
|
};
|
|
19305
20223
|
for (let ordinal = 0; ordinal < documents.length; ordinal++) {
|
|
19306
20224
|
const document2 = documents[ordinal].document;
|
|
19307
|
-
for (const { path: fieldPath, base: fieldType } of fields) {
|
|
20225
|
+
for (const { path: fieldPath, base: fieldType, pattern } of fields) {
|
|
19308
20226
|
const value = getNestedValue(document2, fieldPath);
|
|
19309
20227
|
if (value === void 0 || value === null) continue;
|
|
20228
|
+
recordValueSize(filters.sizeIndexes, ordinal, fieldPath, fieldType, value);
|
|
20229
|
+
if (pattern) patternIndexOf(filters.patternIndexes, fieldPath).add(ordinal, value);
|
|
19310
20230
|
if (!isWordIndexedBase(fieldType)) {
|
|
19311
20231
|
indexFilterValue(filters, ordinal, fieldPath, fieldType, value);
|
|
19312
20232
|
continue;
|
|
@@ -19831,6 +20751,7 @@ function createDirectExecutor(options) {
|
|
|
19831
20751
|
}
|
|
19832
20752
|
case "query": {
|
|
19833
20753
|
const entry = requireIndex(action.indexName);
|
|
20754
|
+
const patternWork = joinPatternWorkMeter(action.patternWork);
|
|
19834
20755
|
const result = await fanOutQuery(
|
|
19835
20756
|
entry.manager,
|
|
19836
20757
|
action.params,
|
|
@@ -19841,15 +20762,17 @@ function createDirectExecutor(options) {
|
|
|
19841
20762
|
partitionIds: action.partitionIds,
|
|
19842
20763
|
...action.globalStats !== void 0 ? { globalStats: sanitizeGlobalStats(action.globalStats) } : {}
|
|
19843
20764
|
},
|
|
19844
|
-
entry.searchOptions
|
|
20765
|
+
{ ...entry.searchOptions, patternWork }
|
|
19845
20766
|
);
|
|
20767
|
+
result.patternWorkSpent = patternWork.spent();
|
|
19846
20768
|
return result;
|
|
19847
20769
|
}
|
|
19848
20770
|
case "preflight": {
|
|
19849
20771
|
const entry = requireIndex(action.indexName);
|
|
20772
|
+
const searchOptions = { ...entry.searchOptions, patternWork: joinPatternWorkMeter(action.patternWork) };
|
|
19850
20773
|
if (countsWithoutScores(action.params)) {
|
|
19851
20774
|
const count = fanOutMatchCount(entry.manager, action.params, entry.language, entry.config.schema, {
|
|
19852
|
-
searchOptions
|
|
20775
|
+
searchOptions
|
|
19853
20776
|
});
|
|
19854
20777
|
return { count };
|
|
19855
20778
|
}
|
|
@@ -19859,7 +20782,7 @@ function createDirectExecutor(options) {
|
|
|
19859
20782
|
entry.language,
|
|
19860
20783
|
entry.config.schema,
|
|
19861
20784
|
{ scoringMode: entry.config.defaultScoring ?? "local" },
|
|
19862
|
-
|
|
20785
|
+
searchOptions
|
|
19863
20786
|
);
|
|
19864
20787
|
return { count: result.totalMatched };
|
|
19865
20788
|
}
|
|
@@ -25245,6 +26168,13 @@ function validateDocId(docId) {
|
|
|
25245
26168
|
if (docId.includes("\0")) {
|
|
25246
26169
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
|
|
25247
26170
|
}
|
|
26171
|
+
if (holdsUnpairedSurrogate(docId)) {
|
|
26172
|
+
throw new NarsilError(
|
|
26173
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
26174
|
+
"Document ID holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8",
|
|
26175
|
+
{ docId }
|
|
26176
|
+
);
|
|
26177
|
+
}
|
|
25248
26178
|
}
|
|
25249
26179
|
function validatePartitionConfig(partitions) {
|
|
25250
26180
|
const { maxDocsPerPartition, maxPartitions, watermark } = partitions;
|
|
@@ -27707,6 +28637,64 @@ function stopIdleSweep(state) {
|
|
|
27707
28637
|
state.idleSweep = null;
|
|
27708
28638
|
}
|
|
27709
28639
|
|
|
28640
|
+
// src/core/pattern-index/matcher-budget.ts
|
|
28641
|
+
var POSITIONS_PER_WORD = 32;
|
|
28642
|
+
function positionWords(positions) {
|
|
28643
|
+
return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
|
|
28644
|
+
}
|
|
28645
|
+
function automatonBytes(positions) {
|
|
28646
|
+
return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES;
|
|
28647
|
+
}
|
|
28648
|
+
function smallestMatcherBytes(positions) {
|
|
28649
|
+
return automatonBytes(positions) + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
|
|
28650
|
+
}
|
|
28651
|
+
function widestPatternSplit(positions) {
|
|
28652
|
+
return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / smallestMatcherBytes(positions)));
|
|
28653
|
+
}
|
|
28654
|
+
|
|
28655
|
+
// src/core/pattern-index/query-positions.ts
|
|
28656
|
+
function isRecord5(value) {
|
|
28657
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
28658
|
+
}
|
|
28659
|
+
function textPositions(text, caseFold) {
|
|
28660
|
+
if (typeof text !== "string") return 0;
|
|
28661
|
+
return caseFold ? foldedCodePointCount(text) : codePointCount(text);
|
|
28662
|
+
}
|
|
28663
|
+
function larger(left, right) {
|
|
28664
|
+
if (left === null) return right;
|
|
28665
|
+
if (right === null) return left;
|
|
28666
|
+
return Math.max(left, right);
|
|
28667
|
+
}
|
|
28668
|
+
function largestTestOf(filter) {
|
|
28669
|
+
const caseFold = filter.caseFold === true;
|
|
28670
|
+
let largest = null;
|
|
28671
|
+
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
28672
|
+
const operand = filter[operator];
|
|
28673
|
+
if (operand === void 0) continue;
|
|
28674
|
+
const texts = Array.isArray(operand) ? operand : [operand];
|
|
28675
|
+
for (const text of texts) largest = larger(largest, textPositions(text, caseFold));
|
|
28676
|
+
}
|
|
28677
|
+
return largest;
|
|
28678
|
+
}
|
|
28679
|
+
function largestIn(expression, patternFields) {
|
|
28680
|
+
let largest = null;
|
|
28681
|
+
if (isRecord5(expression.fields)) {
|
|
28682
|
+
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
28683
|
+
if (patternFields.has(fieldPath) && isRecord5(filter)) largest = larger(largest, largestTestOf(filter));
|
|
28684
|
+
}
|
|
28685
|
+
}
|
|
28686
|
+
const nested = [...expression.and ?? [], ...expression.or ?? []];
|
|
28687
|
+
if (expression.not !== void 0) nested.push(expression.not);
|
|
28688
|
+
for (const child of nested) largest = larger(largest, largestIn(child, patternFields));
|
|
28689
|
+
return largest;
|
|
28690
|
+
}
|
|
28691
|
+
function largestPatternTest(expression, schema) {
|
|
28692
|
+
if (expression === void 0) return null;
|
|
28693
|
+
const patternFields = /* @__PURE__ */ new Set();
|
|
28694
|
+
for (const { path, pattern } of schemaFieldsOf(schema)) if (pattern) patternFields.add(path);
|
|
28695
|
+
return patternFields.size === 0 ? null : largestIn(expression, patternFields);
|
|
28696
|
+
}
|
|
28697
|
+
|
|
27710
28698
|
// src/engine/orchestration/search.ts
|
|
27711
28699
|
function partitionsPerLease(scope, leaseCount) {
|
|
27712
28700
|
const assignments = Array.from({ length: leaseCount }, () => []);
|
|
@@ -27718,15 +28706,18 @@ function partitionsPerLease(scope, leaseCount) {
|
|
|
27718
28706
|
function mergeWorkerResults(results, facetConfig) {
|
|
27719
28707
|
const merged = kWayMerge(results.map((result) => result.scored));
|
|
27720
28708
|
let totalMatched = 0;
|
|
28709
|
+
let patternWorkSpent = 0;
|
|
27721
28710
|
const workerFacets = [];
|
|
27722
28711
|
for (const result of results) {
|
|
27723
28712
|
totalMatched += result.totalMatched;
|
|
28713
|
+
patternWorkSpent += result.patternWorkSpent ?? 0;
|
|
27724
28714
|
if (result.facets !== void 0) workerFacets.push(result.facets);
|
|
27725
28715
|
}
|
|
27726
28716
|
return {
|
|
27727
28717
|
scored: merged,
|
|
27728
28718
|
totalMatched,
|
|
27729
|
-
facets: facetConfig !== void 0 && workerFacets.length > 0 ? mergeFacets(workerFacets, facetConfig) : void 0
|
|
28719
|
+
facets: facetConfig !== void 0 && workerFacets.length > 0 ? mergeFacets(workerFacets, facetConfig) : void 0,
|
|
28720
|
+
patternWorkSpent
|
|
27730
28721
|
};
|
|
27731
28722
|
}
|
|
27732
28723
|
function scoresPerPartition(state, indexName, params, globalStats) {
|
|
@@ -27734,26 +28725,45 @@ function scoresPerPartition(state, indexName, params, globalStats) {
|
|
|
27734
28725
|
if (mode === "local") return true;
|
|
27735
28726
|
return mode === "broadcast" && globalStats !== void 0;
|
|
27736
28727
|
}
|
|
27737
|
-
function queryAction(indexName, params, globalStats, partitionIds) {
|
|
28728
|
+
function queryAction(indexName, params, globalStats, partitionIds, patternWork) {
|
|
27738
28729
|
return {
|
|
27739
28730
|
type: "query",
|
|
27740
28731
|
indexName,
|
|
27741
28732
|
params,
|
|
27742
28733
|
requestId: createRequestId(),
|
|
28734
|
+
patternWork,
|
|
27743
28735
|
...partitionIds !== void 0 ? { partitionIds } : {},
|
|
27744
28736
|
...globalStats !== void 0 ? { globalStats } : {}
|
|
27745
28737
|
};
|
|
27746
28738
|
}
|
|
27747
|
-
async function runSplit(leases, scope, indexName, params, globalStats) {
|
|
28739
|
+
async function runSplit(leases, scope, indexName, params, globalStats, patternWork) {
|
|
27748
28740
|
const assignments = partitionsPerLease(scope, leases.length);
|
|
27749
28741
|
const workerParams = params.facets !== void 0 ? { ...params, facets: oversampledFacetConfig(params.facets) } : params;
|
|
27750
|
-
const
|
|
28742
|
+
const outcomes = await Promise.allSettled(
|
|
27751
28743
|
assignments.map(
|
|
27752
|
-
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds)).finally(() => leases[at].release())
|
|
28744
|
+
(partitionIds, at) => leases[at].executor.execute(queryAction(indexName, workerParams, globalStats, partitionIds, patternWork)).finally(() => leases[at].release())
|
|
27753
28745
|
)
|
|
27754
28746
|
);
|
|
28747
|
+
const results = [];
|
|
28748
|
+
const failures = [];
|
|
28749
|
+
for (const outcome of outcomes) {
|
|
28750
|
+
if (outcome.status === "fulfilled") results.push(outcome.value);
|
|
28751
|
+
else failures.push(outcome.reason);
|
|
28752
|
+
}
|
|
28753
|
+
if (failures.length > 0) throw failures.find(isWorkCapError) ?? failures[0];
|
|
27755
28754
|
return mergeWorkerResults(results, params.facets);
|
|
27756
28755
|
}
|
|
28756
|
+
function isWorkCapError(err) {
|
|
28757
|
+
return err instanceof NarsilError && err.code === ErrorCodes.SEARCH_WORK_CAP_EXCEEDED;
|
|
28758
|
+
}
|
|
28759
|
+
function patternSplitWidth(state, indexName, params) {
|
|
28760
|
+
const schema = state.indexRegistry.get(indexName)?.config.schema;
|
|
28761
|
+
if (schema === void 0) return Number.POSITIVE_INFINITY;
|
|
28762
|
+
const largest = largestPatternTest(params.filters, schema);
|
|
28763
|
+
if (largest === null) return Number.POSITIVE_INFINITY;
|
|
28764
|
+
if (typeof SharedArrayBuffer !== "function") return 1;
|
|
28765
|
+
return widestPatternSplit(largest);
|
|
28766
|
+
}
|
|
27757
28767
|
function takeMainCopyTurn(state) {
|
|
27758
28768
|
if (state.mainCopyQueries === "none" || state.mainCopyTurnTaken) return false;
|
|
27759
28769
|
state.mainCopyTurnTaken = true;
|
|
@@ -27767,7 +28777,7 @@ function answersFasterOnMainCopy(state, pool, manager) {
|
|
|
27767
28777
|
if (manager.countDocuments() > MAIN_COPY_LONE_QUERY_DOCUMENTS) return false;
|
|
27768
28778
|
return takeMainCopyTurn(state);
|
|
27769
28779
|
}
|
|
27770
|
-
async function searchViaWorker(state, indexName, params, globalStats, partitionIds) {
|
|
28780
|
+
async function searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork) {
|
|
27771
28781
|
const pool = state.workerPool;
|
|
27772
28782
|
if (!pool) return null;
|
|
27773
28783
|
if (!state.scaledOutIndexes.has(indexName) || state.copyLoadBuffers.has(indexName)) return null;
|
|
@@ -27775,19 +28785,30 @@ async function searchViaWorker(state, indexName, params, globalStats, partitionI
|
|
|
27775
28785
|
if (!manager) return null;
|
|
27776
28786
|
if (answersFasterOnMainCopy(state, pool, manager)) return null;
|
|
27777
28787
|
const scope = partitionIds ?? Array.from({ length: manager.partitionCount }, (_, partitionId) => partitionId);
|
|
27778
|
-
const idle = pool.leaseIdle(scope.length);
|
|
28788
|
+
const idle = pool.leaseIdle(Math.min(scope.length, patternSplitWidth(state, indexName, params)));
|
|
27779
28789
|
const leases = [];
|
|
28790
|
+
let fork = null;
|
|
27780
28791
|
try {
|
|
27781
28792
|
if (idle.length >= 2 && scoresPerPartition(state, indexName, params, globalStats)) {
|
|
27782
28793
|
leases.push(...idle);
|
|
27783
|
-
|
|
28794
|
+
fork = patternWork.fork(leases.length);
|
|
28795
|
+
const merged = await runSplit(leases, scope, indexName, params, globalStats, fork.shared);
|
|
28796
|
+
fork.settle(merged.patternWorkSpent ?? 0);
|
|
28797
|
+
return merged;
|
|
27784
28798
|
}
|
|
27785
28799
|
for (const lease2 of idle.slice(1)) lease2.release();
|
|
27786
28800
|
const lease = idle[0] ?? (takeMainCopyTurn(state) ? null : pool.leaseLeastBusy());
|
|
27787
28801
|
if (lease === null) return null;
|
|
27788
28802
|
leases.push(lease);
|
|
27789
|
-
|
|
28803
|
+
fork = patternWork.fork(1);
|
|
28804
|
+
const result = await lease.executor.execute(
|
|
28805
|
+
queryAction(indexName, params, globalStats, partitionIds, fork.shared)
|
|
28806
|
+
);
|
|
28807
|
+
fork.settle(result.patternWorkSpent ?? 0);
|
|
28808
|
+
return result;
|
|
27790
28809
|
} catch (err) {
|
|
28810
|
+
if (isWorkCapError(err)) throw err;
|
|
28811
|
+
fork?.settle(0);
|
|
27791
28812
|
console.warn("Worker search failed, falling back to local:", err);
|
|
27792
28813
|
return null;
|
|
27793
28814
|
} finally {
|
|
@@ -28016,7 +29037,7 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
28016
29037
|
segmentBuildConcurrency: (indexName) => segmentBuildConcurrency(state, indexName),
|
|
28017
29038
|
holdUnbroadcastSegments: (indexName, segmentIds) => holdUnbroadcastSegments(state, indexName, segmentIds),
|
|
28018
29039
|
releaseUnbroadcastSegments: (indexName, segmentIds) => releaseUnbroadcastSegments(state, indexName, segmentIds),
|
|
28019
|
-
searchViaWorker: (indexName, params, globalStats, partitionIds) => searchViaWorker(state, indexName, params, globalStats, partitionIds),
|
|
29040
|
+
searchViaWorker: (indexName, params, globalStats, partitionIds, patternWork) => searchViaWorker(state, indexName, params, globalStats, partitionIds, patternWork),
|
|
28020
29041
|
hasWorkerPool: () => state.workerPool !== null,
|
|
28021
29042
|
mainCopyQueries: () => state.mainCopyQueries ?? "lone",
|
|
28022
29043
|
shareMainThread: () => {
|
|
@@ -28040,6 +29061,30 @@ function createWorkerOrchestrator(config, executor, indexRegistry, callbacks, ve
|
|
|
28040
29061
|
};
|
|
28041
29062
|
}
|
|
28042
29063
|
|
|
29064
|
+
// src/engine/mutations/write-visibility.ts
|
|
29065
|
+
async function awaitWriteVisibility(ctx, indexName) {
|
|
29066
|
+
await ctx.awaitRebalanceReplay(indexName);
|
|
29067
|
+
await ctx.orchestrator.awaitWrites(indexName);
|
|
29068
|
+
}
|
|
29069
|
+
|
|
29070
|
+
// src/engine/write-events.ts
|
|
29071
|
+
var WRITE_EVENT = "write";
|
|
29072
|
+
function emitWriteEvent(source, indexName) {
|
|
29073
|
+
const handlers = source.eventHandlers.get(WRITE_EVENT);
|
|
29074
|
+
if (handlers === void 0 || handlers.size === 0) return;
|
|
29075
|
+
const emit = () => {
|
|
29076
|
+
emitEngineEvent(source.eventHandlers, WRITE_EVENT, { indexName });
|
|
29077
|
+
};
|
|
29078
|
+
awaitWriteVisibility(source.mutationCtx, indexName).then(emit, emit);
|
|
29079
|
+
}
|
|
29080
|
+
async function emitWriteEventAfter(source, indexName, write) {
|
|
29081
|
+
try {
|
|
29082
|
+
return await write;
|
|
29083
|
+
} finally {
|
|
29084
|
+
emitWriteEvent(source, indexName);
|
|
29085
|
+
}
|
|
29086
|
+
}
|
|
29087
|
+
|
|
28043
29088
|
// src/engine/core.ts
|
|
28044
29089
|
function createEngineCore(config, hooks, capacityScope = "index") {
|
|
28045
29090
|
validateWorkerConfig(config?.workers, config?.lifecycle);
|
|
@@ -28179,7 +29224,11 @@ function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
28179
29224
|
}
|
|
28180
29225
|
return names;
|
|
28181
29226
|
},
|
|
28182
|
-
reloadIndex: (indexName) =>
|
|
29227
|
+
reloadIndex: (indexName) => emitWriteEventAfter(
|
|
29228
|
+
{ eventHandlers, mutationCtx },
|
|
29229
|
+
indexName,
|
|
29230
|
+
durability?.manager.reloadIndex?.(indexName) ?? Promise.resolve()
|
|
29231
|
+
),
|
|
28183
29232
|
onError(error) {
|
|
28184
29233
|
if (emitEngineEvent(eventHandlers, "invalidationError", { error }) === 0) {
|
|
28185
29234
|
console.warn("Invalidation error:", error);
|
|
@@ -28277,7 +29326,8 @@ function createEngineCore(config, hooks, capacityScope = "index") {
|
|
|
28277
29326
|
analysisRebuild,
|
|
28278
29327
|
indexState,
|
|
28279
29328
|
mutationCtx,
|
|
28280
|
-
rebalanceCtx
|
|
29329
|
+
rebalanceCtx,
|
|
29330
|
+
patternWorkCap: config?.patternWorkCap ?? DEFAULT_PATTERN_WORK_CAP
|
|
28281
29331
|
};
|
|
28282
29332
|
}
|
|
28283
29333
|
|
|
@@ -28376,6 +29426,7 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
28376
29426
|
}
|
|
28377
29427
|
const schema = validateSchema(requestedConfig.schema);
|
|
28378
29428
|
const indexConfig = schema === requestedConfig.schema ? requestedConfig : { ...requestedConfig, schema };
|
|
29429
|
+
requirePatternSearchFor(schema);
|
|
28379
29430
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
28380
29431
|
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
28381
29432
|
validateVectorStorage(indexConfig.vectorPromotion, core.filesystemDurability);
|
|
@@ -28406,10 +29457,10 @@ async function createEngineIndex(core, config, name, requestedConfig, creation =
|
|
|
28406
29457
|
const fields = indexConfig.embedding.fields;
|
|
28407
29458
|
if (typeof configuredAdapter === "string") {
|
|
28408
29459
|
embeddingAdapterName = configuredAdapter;
|
|
28409
|
-
const
|
|
28410
|
-
if (
|
|
29460
|
+
const registered2 = core.embeddingAdapters.get(configuredAdapter);
|
|
29461
|
+
if (registered2 !== void 0) {
|
|
28411
29462
|
resolvedEmbeddingAdapter = validateEmbeddingConfig(
|
|
28412
|
-
{ fields, adapter:
|
|
29463
|
+
{ fields, adapter: registered2 },
|
|
28413
29464
|
indexConfig.schema,
|
|
28414
29465
|
config?.embedding
|
|
28415
29466
|
);
|
|
@@ -28763,9 +29814,30 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
28763
29814
|
|
|
28764
29815
|
// src/engine/mutations/insert-admission.ts
|
|
28765
29816
|
function providedDocId(document2) {
|
|
29817
|
+
if (document2 === null || typeof document2 !== "object") return void 0;
|
|
28766
29818
|
const id = document2.id;
|
|
28767
29819
|
return typeof id === "string" && id.length > 0 ? id : void 0;
|
|
28768
29820
|
}
|
|
29821
|
+
function requireDocumentObject(document2) {
|
|
29822
|
+
if (document2 !== null && typeof document2 === "object" && !Array.isArray(document2)) return;
|
|
29823
|
+
const received = Array.isArray(document2) ? "a list" : document2 === null ? "null" : `a ${typeof document2}`;
|
|
29824
|
+
throw new NarsilError(
|
|
29825
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
29826
|
+
`Each document must be an object, while this entry is ${received}`,
|
|
29827
|
+
{
|
|
29828
|
+
received
|
|
29829
|
+
}
|
|
29830
|
+
);
|
|
29831
|
+
}
|
|
29832
|
+
function requireMatchingDocId(docId, document2) {
|
|
29833
|
+
const embeddedDocId = providedDocId(document2);
|
|
29834
|
+
if (embeddedDocId === void 0 || embeddedDocId === docId) return;
|
|
29835
|
+
throw new NarsilError(
|
|
29836
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
29837
|
+
`The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
|
|
29838
|
+
{ docId, documentId: embeddedDocId }
|
|
29839
|
+
);
|
|
29840
|
+
}
|
|
28769
29841
|
function asBatchInsertError(err) {
|
|
28770
29842
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
28771
29843
|
}
|
|
@@ -28872,25 +29944,12 @@ async function embedChunkDocuments(entry, documents, chunkStart, chunkEnd, signa
|
|
|
28872
29944
|
}
|
|
28873
29945
|
}
|
|
28874
29946
|
|
|
28875
|
-
// src/engine/mutations/write-visibility.ts
|
|
28876
|
-
async function awaitWriteVisibility(ctx, indexName) {
|
|
28877
|
-
await ctx.awaitRebalanceReplay(indexName);
|
|
28878
|
-
await ctx.orchestrator.awaitWrites(indexName);
|
|
28879
|
-
}
|
|
28880
|
-
|
|
28881
29947
|
// src/engine/mutations/insert.ts
|
|
28882
29948
|
async function insertDocument(ctx, indexName, document2, docId, options) {
|
|
28883
29949
|
ctx.guardShutdown();
|
|
28884
29950
|
const entry = ctx.requireIndex(indexName);
|
|
28885
|
-
|
|
28886
|
-
|
|
28887
|
-
throw new NarsilError(
|
|
28888
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
28889
|
-
`The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
|
|
28890
|
-
{ docId, documentId: embeddedDocId }
|
|
28891
|
-
);
|
|
28892
|
-
}
|
|
28893
|
-
const resolvedDocId = docId ?? embeddedDocId ?? ctx.idGenerator();
|
|
29951
|
+
if (docId !== void 0) requireMatchingDocId(docId, document2);
|
|
29952
|
+
const resolvedDocId = docId ?? providedDocId(document2) ?? ctx.idGenerator();
|
|
28894
29953
|
validateDocId(resolvedDocId);
|
|
28895
29954
|
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: resolvedDocId, document: document2 });
|
|
28896
29955
|
if (entry.config.required && entry.config.required.length > 0) {
|
|
@@ -29127,6 +30186,7 @@ async function admitBatchDocuments(ctx, indexName, documents, failed) {
|
|
|
29127
30186
|
if (chunkFailedIndexes.has(i)) continue;
|
|
29128
30187
|
const docId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
29129
30188
|
try {
|
|
30189
|
+
requireDocumentObject(documents[i]);
|
|
29130
30190
|
validateDocId(docId);
|
|
29131
30191
|
if (hasBeforeHook) {
|
|
29132
30192
|
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId, document: documents[i] });
|
|
@@ -29625,6 +30685,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
29625
30685
|
if (chunkFailedIndexes.has(i)) continue;
|
|
29626
30686
|
const batchDocId = providedDocId(documents[i]) ?? ctx.idGenerator();
|
|
29627
30687
|
try {
|
|
30688
|
+
requireDocumentObject(documents[i]);
|
|
29628
30689
|
validateDocId(batchDocId);
|
|
29629
30690
|
if (hasBeforeHook) {
|
|
29630
30691
|
await ctx.pluginRegistry.runHook("beforeInsert", { indexName, docId: batchDocId, document: documents[i] });
|
|
@@ -29847,6 +30908,7 @@ async function settleUpdate(ctx, indexName, prepared) {
|
|
|
29847
30908
|
async function prepareUpdate(ctx, indexName, entry, docId, document2) {
|
|
29848
30909
|
ctx.guardShutdown();
|
|
29849
30910
|
validateDocId(docId);
|
|
30911
|
+
requireMatchingDocId(docId, document2);
|
|
29850
30912
|
const manager = ctx.requireManager(indexName);
|
|
29851
30913
|
if (ctx.pluginRegistry.hasHooks("beforeUpdate")) {
|
|
29852
30914
|
await ctx.pluginRegistry.runHook("beforeUpdate", {
|
|
@@ -29956,6 +31018,22 @@ async function updateDocument(ctx, indexName, docId, document2, options) {
|
|
|
29956
31018
|
ctx.checkHeapPressure(indexName);
|
|
29957
31019
|
if (options?.wait === true) await awaitWriteVisibility(ctx, indexName);
|
|
29958
31020
|
}
|
|
31021
|
+
function reportedDocIdOf(update) {
|
|
31022
|
+
const docId = update !== null && typeof update === "object" ? update.docId : void 0;
|
|
31023
|
+
return typeof docId === "string" ? docId : "";
|
|
31024
|
+
}
|
|
31025
|
+
function requireUpdateEntry(update) {
|
|
31026
|
+
if (update !== null && typeof update === "object" && !Array.isArray(update)) {
|
|
31027
|
+
const { docId, document: document2 } = update;
|
|
31028
|
+
if (typeof docId === "string" && document2 !== null && typeof document2 === "object" && !Array.isArray(document2)) {
|
|
31029
|
+
return { document: document2 };
|
|
31030
|
+
}
|
|
31031
|
+
}
|
|
31032
|
+
throw new NarsilError(
|
|
31033
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
31034
|
+
"Each entry of a batch update must be an object whose docId is a string and whose document is an object"
|
|
31035
|
+
);
|
|
31036
|
+
}
|
|
29959
31037
|
async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
29960
31038
|
ctx.guardShutdown();
|
|
29961
31039
|
const entry = ctx.requireIndex(indexName);
|
|
@@ -29968,10 +31046,13 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
29968
31046
|
const chunkEnd = Math.min(chunkStart + BATCH_CHUNK_SIZE, updates.length);
|
|
29969
31047
|
const prepared = [];
|
|
29970
31048
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
31049
|
+
const update = updates[i];
|
|
31050
|
+
const docId = reportedDocIdOf(update);
|
|
29971
31051
|
try {
|
|
29972
|
-
|
|
31052
|
+
const { document: document2 } = requireUpdateEntry(update);
|
|
31053
|
+
prepared.push(await prepareUpdate(ctx, indexName, entry, docId, document2));
|
|
29973
31054
|
} catch (err) {
|
|
29974
|
-
failed.push({ docId
|
|
31055
|
+
failed.push({ docId, error: asUpdateError(err) });
|
|
29975
31056
|
}
|
|
29976
31057
|
}
|
|
29977
31058
|
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
@@ -30956,6 +32037,9 @@ function requireValidTypoTolerance(params) {
|
|
|
30956
32037
|
}
|
|
30957
32038
|
|
|
30958
32039
|
// src/engine/query/shared.ts
|
|
32040
|
+
function withPatternWork(context) {
|
|
32041
|
+
return { ...context, patternWork: createPatternWorkMeter(context.patternWorkCap) };
|
|
32042
|
+
}
|
|
30959
32043
|
function partitionsFor(manager, partitionIds) {
|
|
30960
32044
|
return partitionsIn(manager, partitionIds);
|
|
30961
32045
|
}
|
|
@@ -31002,18 +32086,19 @@ function broadcastStatsForWorker(params, context, scoring) {
|
|
|
31002
32086
|
}
|
|
31003
32087
|
return pruneStatsToQueryTerms(scoring.globalStats, term, context.language, context.manager.analysis);
|
|
31004
32088
|
}
|
|
31005
|
-
function searchOptionsFor(manager) {
|
|
32089
|
+
function searchOptionsFor(manager, patternWork) {
|
|
31006
32090
|
return {
|
|
31007
32091
|
bm25Params: manager.config.bm25,
|
|
31008
32092
|
stopWords: manager.analysis.stopWords,
|
|
31009
|
-
customTokenizer: manager.analysis.customTokenizer
|
|
32093
|
+
customTokenizer: manager.analysis.customTokenizer,
|
|
32094
|
+
patternWork
|
|
31010
32095
|
};
|
|
31011
32096
|
}
|
|
31012
|
-
function collectFilterDocIds(manager, params, schema, partitionIds) {
|
|
32097
|
+
function collectFilterDocIds(manager, params, schema, partitionIds, patternWork) {
|
|
31013
32098
|
const filterDocIds = /* @__PURE__ */ new Set();
|
|
31014
32099
|
if (!params.filters) return filterDocIds;
|
|
31015
32100
|
for (const partition of partitionsFor(manager, partitionIds)) {
|
|
31016
|
-
const partitionFiltered = partition.applyFilters(params.filters, schema);
|
|
32101
|
+
const partitionFiltered = partition.applyFilters(params.filters, schema, patternWork);
|
|
31017
32102
|
for (const docId of partitionFiltered) {
|
|
31018
32103
|
filterDocIds.add(docId);
|
|
31019
32104
|
}
|
|
@@ -31048,11 +32133,11 @@ function requireValidQueryOptions(params, config) {
|
|
|
31048
32133
|
const strict = config.strict === true;
|
|
31049
32134
|
requireValidTypoTolerance(params);
|
|
31050
32135
|
requireSortableFields(params.sort, config.schema, strict);
|
|
31051
|
-
requireValidQueryFilter(params.filters, config.schema
|
|
32136
|
+
requireValidQueryFilter(params.filters, config.schema);
|
|
31052
32137
|
requireUsableQueryFields(params, config.schema, strict);
|
|
31053
32138
|
}
|
|
31054
|
-
function requireValidQueryFilter(filters, schema
|
|
31055
|
-
if (filters !== void 0) requireValidFilter(filters, flattenSchema(schema)
|
|
32139
|
+
function requireValidQueryFilter(filters, schema) {
|
|
32140
|
+
if (filters !== void 0) requireValidFilter(filters, flattenSchema(schema));
|
|
31056
32141
|
}
|
|
31057
32142
|
function requireVectorSearchable(params, context, needsVector) {
|
|
31058
32143
|
if (!needsVector) return;
|
|
@@ -31088,8 +32173,9 @@ function firstIdAfter(sorted, anchor) {
|
|
|
31088
32173
|
}
|
|
31089
32174
|
return low;
|
|
31090
32175
|
}
|
|
31091
|
-
function collectFilterMatches(partitions, filters, schema) {
|
|
31092
|
-
const
|
|
32176
|
+
function collectFilterMatches(partitions, filters, schema, patternWorkCap) {
|
|
32177
|
+
const meter = createPatternWorkMeter(patternWorkCap);
|
|
32178
|
+
const perPartition = partitions.map((partition) => partition.filterMatches(filters, schema, meter));
|
|
31093
32179
|
let total = 0;
|
|
31094
32180
|
for (const matches of perPartition) total += matches.count;
|
|
31095
32181
|
return { matchesFor: (partitionIndex) => perPartition[partitionIndex] ?? null, total };
|
|
@@ -31167,7 +32253,7 @@ function executeListDocuments(params, context) {
|
|
|
31167
32253
|
const limit = clampListLimit(params.limit);
|
|
31168
32254
|
const signature = sortSignatureOf(params.sort);
|
|
31169
32255
|
requireSortableFields(params.sort, schema, context.strict === true);
|
|
31170
|
-
requireValidQueryFilter(params.filters, schema
|
|
32256
|
+
requireValidQueryFilter(params.filters, schema);
|
|
31171
32257
|
const binding = listBindingOf(params.filters);
|
|
31172
32258
|
let cursor = null;
|
|
31173
32259
|
if (params.cursor !== void 0) {
|
|
@@ -31175,7 +32261,7 @@ function executeListDocuments(params, context) {
|
|
|
31175
32261
|
requireMatchingCursor(cursor, params.cursor, signature, false, binding);
|
|
31176
32262
|
}
|
|
31177
32263
|
const partitions = context.partitionIds === void 0 ? manager.getAllPartitions() : context.partitionIds.map((partitionId) => manager.partitionAt(partitionId)).filter((partition) => partition !== void 0);
|
|
31178
|
-
const filtered = params.filters === void 0 ? null : collectFilterMatches(partitions, params.filters, schema);
|
|
32264
|
+
const filtered = params.filters === void 0 ? null : collectFilterMatches(partitions, params.filters, schema, context.patternWorkCap);
|
|
31179
32265
|
const total = filtered === null ? countAcrossPartitions(manager, context.partitionIds, partitions) : filtered.total;
|
|
31180
32266
|
const page = params.sort === void 0 || signature === null ? pageInIdOrder(partitions, cursor, filtered, limit, binding) : pageInSortOrder(partitions, cursor, filtered, limit, params.sort, signature, schema, binding);
|
|
31181
32267
|
const projection = resolveProjection(params.document);
|
|
@@ -31542,7 +32628,7 @@ function executeSortedQueryPage(params, context, limit, offset, sortSignature) {
|
|
|
31542
32628
|
const flatSchema = flattenSchema(schema);
|
|
31543
32629
|
const fieldTypes = fields.map((field) => flatSchema[field]);
|
|
31544
32630
|
const modes = sortModesOf(normalized);
|
|
31545
|
-
const options = searchOptionsFor(manager);
|
|
32631
|
+
const options = searchOptionsFor(manager, context.patternWork);
|
|
31546
32632
|
const partitions = partitionsFor(manager, context.partitionIds);
|
|
31547
32633
|
const partitionLimit = offset + limit + 1;
|
|
31548
32634
|
const merged = [];
|
|
@@ -31743,7 +32829,7 @@ async function executeVectorSearch(params, context, limit, offset, cursorDepth =
|
|
|
31743
32829
|
let filterDocIds;
|
|
31744
32830
|
let filterPartitions;
|
|
31745
32831
|
if (params.filters) {
|
|
31746
|
-
filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds);
|
|
32832
|
+
filterDocIds = collectFilterDocIds(manager, params, config.schema, partitionIds, context.patternWork);
|
|
31747
32833
|
if (filterDocIds.size === 0) {
|
|
31748
32834
|
return { scored: [], totalMatched: 0 };
|
|
31749
32835
|
}
|
|
@@ -31771,11 +32857,19 @@ async function textLeg(textOnlyParams, context) {
|
|
|
31771
32857
|
indexName,
|
|
31772
32858
|
textOnlyParams,
|
|
31773
32859
|
broadcastStatsForWorker(textOnlyParams, context, scoring),
|
|
31774
|
-
context.partitionIds
|
|
32860
|
+
context.partitionIds,
|
|
32861
|
+
context.patternWork
|
|
31775
32862
|
);
|
|
31776
32863
|
if (viaWorker) return viaWorker;
|
|
31777
32864
|
}
|
|
31778
|
-
return fanOutQuery(
|
|
32865
|
+
return fanOutQuery(
|
|
32866
|
+
manager,
|
|
32867
|
+
textOnlyParams,
|
|
32868
|
+
language,
|
|
32869
|
+
config.schema,
|
|
32870
|
+
scoring,
|
|
32871
|
+
searchOptionsFor(manager, context.patternWork)
|
|
32872
|
+
);
|
|
31779
32873
|
}
|
|
31780
32874
|
async function vectorLeg(vectorConfig, queryVector, context, filterDocIds, k) {
|
|
31781
32875
|
const { manager } = context;
|
|
@@ -31801,7 +32895,7 @@ async function executeHybridSearch(params, context, limit, offset, cursorDepth =
|
|
|
31801
32895
|
const fusion = resolveHybridFusion(params.hybrid);
|
|
31802
32896
|
let filterDocIds;
|
|
31803
32897
|
if (params.filters) {
|
|
31804
|
-
filterDocIds = collectFilterDocIds(manager, params, config.schema, context.partitionIds);
|
|
32898
|
+
filterDocIds = collectFilterDocIds(manager, params, config.schema, context.partitionIds, context.patternWork);
|
|
31805
32899
|
if (filterDocIds.size === 0) {
|
|
31806
32900
|
return { scored: [], totalMatched: 0 };
|
|
31807
32901
|
}
|
|
@@ -31845,7 +32939,8 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
31845
32939
|
requireMatchingCursor(decoded, cursor, signature, true, binding);
|
|
31846
32940
|
return decoded.depth ?? 0;
|
|
31847
32941
|
}
|
|
31848
|
-
async function executeQuery(params,
|
|
32942
|
+
async function executeQuery(params, callerContext) {
|
|
32943
|
+
const context = withPatternWork(callerContext);
|
|
31849
32944
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
31850
32945
|
const startTime = now();
|
|
31851
32946
|
const limit = clampLimit(params.limit);
|
|
@@ -31891,11 +32986,24 @@ async function executeQuery(params, context) {
|
|
|
31891
32986
|
fanOutResult = await executeHybridSearch(params, context, limit, offset, depth);
|
|
31892
32987
|
} else {
|
|
31893
32988
|
const scoring = scoringConfigFor(params, context);
|
|
31894
|
-
const workerResult = workerSearch ? await workerSearch(
|
|
32989
|
+
const workerResult = workerSearch ? await workerSearch(
|
|
32990
|
+
indexName,
|
|
32991
|
+
params,
|
|
32992
|
+
broadcastStatsForWorker(params, context, scoring),
|
|
32993
|
+
context.partitionIds,
|
|
32994
|
+
context.patternWork
|
|
32995
|
+
) : null;
|
|
31895
32996
|
if (workerResult) {
|
|
31896
32997
|
fanOutResult = workerResult;
|
|
31897
32998
|
} else {
|
|
31898
|
-
fanOutResult = await fanOutQuery(
|
|
32999
|
+
fanOutResult = await fanOutQuery(
|
|
33000
|
+
manager,
|
|
33001
|
+
params,
|
|
33002
|
+
language,
|
|
33003
|
+
config.schema,
|
|
33004
|
+
scoring,
|
|
33005
|
+
searchOptionsFor(manager, context.patternWork)
|
|
33006
|
+
);
|
|
31899
33007
|
}
|
|
31900
33008
|
}
|
|
31901
33009
|
const needsFullHits = params.sort !== void 0 || params.group !== void 0 || params.pinned !== void 0 || params.searchAfter !== void 0;
|
|
@@ -32015,7 +33123,8 @@ async function executeQuery(params, context) {
|
|
|
32015
33123
|
coverage: coverageFor(manager, context.partitionIds)
|
|
32016
33124
|
};
|
|
32017
33125
|
}
|
|
32018
|
-
async function executePreflight(params,
|
|
33126
|
+
async function executePreflight(params, callerContext) {
|
|
33127
|
+
const context = withPatternWork(callerContext);
|
|
32019
33128
|
const { manager, language, config, workerSearch, indexName } = context;
|
|
32020
33129
|
const startTime = now();
|
|
32021
33130
|
requireKnownMode(params);
|
|
@@ -32041,12 +33150,18 @@ async function executePreflight(params, context) {
|
|
|
32041
33150
|
countExact = result.matchedExact !== false;
|
|
32042
33151
|
} else if (countsWithoutScores(params)) {
|
|
32043
33152
|
totalMatched = fanOutMatchCount(manager, params, language, config.schema, {
|
|
32044
|
-
searchOptions: searchOptionsFor(manager),
|
|
33153
|
+
searchOptions: searchOptionsFor(manager, context.patternWork),
|
|
32045
33154
|
partitionIds: context.partitionIds
|
|
32046
33155
|
});
|
|
32047
33156
|
} else {
|
|
32048
33157
|
const scoring = scoringConfigFor(params, context);
|
|
32049
|
-
const workerResult = workerSearch ? await workerSearch(
|
|
33158
|
+
const workerResult = workerSearch ? await workerSearch(
|
|
33159
|
+
indexName,
|
|
33160
|
+
params,
|
|
33161
|
+
broadcastStatsForWorker(params, context, scoring),
|
|
33162
|
+
context.partitionIds,
|
|
33163
|
+
context.patternWork
|
|
33164
|
+
) : null;
|
|
32050
33165
|
if (workerResult) {
|
|
32051
33166
|
totalMatched = workerResult.totalMatched;
|
|
32052
33167
|
} else {
|
|
@@ -32056,7 +33171,7 @@ async function executePreflight(params, context) {
|
|
|
32056
33171
|
language,
|
|
32057
33172
|
config.schema,
|
|
32058
33173
|
scoring,
|
|
32059
|
-
searchOptionsFor(manager)
|
|
33174
|
+
searchOptionsFor(manager, context.patternWork)
|
|
32060
33175
|
);
|
|
32061
33176
|
totalMatched = fanOutResult.totalMatched;
|
|
32062
33177
|
}
|
|
@@ -32204,7 +33319,8 @@ async function runEngineQuery(core, indexName, params, options) {
|
|
|
32204
33319
|
indexName,
|
|
32205
33320
|
broadcastStats: broadcastStatsFor(core, options),
|
|
32206
33321
|
partitionIds: options?.partitionIds,
|
|
32207
|
-
cursorBinding: queryBindingOf(params)
|
|
33322
|
+
cursorBinding: queryBindingOf(params),
|
|
33323
|
+
patternWorkCap: core.patternWorkCap
|
|
32208
33324
|
});
|
|
32209
33325
|
if (core.pluginRegistry.hasHooks("afterSearch")) {
|
|
32210
33326
|
const observed = { ...result, hits: [...result.hits] };
|
|
@@ -32245,7 +33361,8 @@ async function runEnginePreflight(core, indexName, params, options) {
|
|
|
32245
33361
|
indexName,
|
|
32246
33362
|
broadcastStats: broadcastStatsFor(core, options),
|
|
32247
33363
|
partitionIds: options?.partitionIds,
|
|
32248
|
-
cursorBinding: queryBindingOf(params)
|
|
33364
|
+
cursorBinding: queryBindingOf(params),
|
|
33365
|
+
patternWorkCap: core.patternWorkCap
|
|
32249
33366
|
});
|
|
32250
33367
|
if (core.analysisRebuild.isStale(indexName)) {
|
|
32251
33368
|
result.analysisStale = true;
|
|
@@ -32282,7 +33399,8 @@ async function runEngineListDocuments(core, indexName, params, partitionIds) {
|
|
|
32282
33399
|
manager: core.requireManager(indexName),
|
|
32283
33400
|
schema: entry.config.schema,
|
|
32284
33401
|
strict: entry.config.strict,
|
|
32285
|
-
partitionIds
|
|
33402
|
+
partitionIds,
|
|
33403
|
+
patternWorkCap: core.patternWorkCap
|
|
32286
33404
|
});
|
|
32287
33405
|
} finally {
|
|
32288
33406
|
release();
|
|
@@ -32316,15 +33434,18 @@ function createNarsilFromCore(core, config) {
|
|
|
32316
33434
|
release();
|
|
32317
33435
|
}
|
|
32318
33436
|
}
|
|
33437
|
+
function withOpenIndexWrite(indexName, action) {
|
|
33438
|
+
return emitWriteEventAfter(core, indexName, withOpenIndex(indexName, action));
|
|
33439
|
+
}
|
|
32319
33440
|
const narsil = {
|
|
32320
33441
|
createIndex(name, indexConfig) {
|
|
32321
|
-
return createEngineIndex(core, config, name, indexConfig);
|
|
33442
|
+
return emitWriteEventAfter(core, name, createEngineIndex(core, config, name, indexConfig));
|
|
32322
33443
|
},
|
|
32323
33444
|
registerEmbeddingAdapter(name, adapter) {
|
|
32324
33445
|
registerEngineEmbeddingAdapter(core, name, adapter);
|
|
32325
33446
|
},
|
|
32326
33447
|
dropIndex(name) {
|
|
32327
|
-
return dropEngineIndex(core, name);
|
|
33448
|
+
return emitWriteEventAfter(core, name, dropEngineIndex(core, name));
|
|
32328
33449
|
},
|
|
32329
33450
|
open(indexName) {
|
|
32330
33451
|
guardShutdown();
|
|
@@ -32370,22 +33491,22 @@ function createNarsilFromCore(core, config) {
|
|
|
32370
33491
|
return executor.getManager(indexName)?.getPartitionStats() ?? [];
|
|
32371
33492
|
},
|
|
32372
33493
|
insert(indexName, document2, docId, options) {
|
|
32373
|
-
return
|
|
33494
|
+
return withOpenIndexWrite(indexName, () => insertDocument(mutationCtx, indexName, document2, docId, options));
|
|
32374
33495
|
},
|
|
32375
33496
|
insertBatch(indexName, documents, options) {
|
|
32376
|
-
return
|
|
33497
|
+
return withOpenIndexWrite(indexName, () => insertDocumentBatch(mutationCtx, indexName, documents, options));
|
|
32377
33498
|
},
|
|
32378
33499
|
remove(indexName, docId, options) {
|
|
32379
|
-
return
|
|
33500
|
+
return withOpenIndexWrite(indexName, () => removeDocument(mutationCtx, indexName, docId, options));
|
|
32380
33501
|
},
|
|
32381
33502
|
removeBatch(indexName, docIds, options) {
|
|
32382
|
-
return
|
|
33503
|
+
return withOpenIndexWrite(indexName, () => removeDocumentBatch(mutationCtx, indexName, docIds, options));
|
|
32383
33504
|
},
|
|
32384
33505
|
update(indexName, docId, document2, options) {
|
|
32385
|
-
return
|
|
33506
|
+
return withOpenIndexWrite(indexName, () => updateDocument(mutationCtx, indexName, docId, document2, options));
|
|
32386
33507
|
},
|
|
32387
33508
|
updateBatch(indexName, updates, options) {
|
|
32388
|
-
return
|
|
33509
|
+
return withOpenIndexWrite(indexName, () => updateDocumentBatch(mutationCtx, indexName, updates, options));
|
|
32389
33510
|
},
|
|
32390
33511
|
waitForWrites(indexName) {
|
|
32391
33512
|
guardShutdown();
|
|
@@ -32433,7 +33554,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32433
33554
|
async rebuildAnalysis(indexName) {
|
|
32434
33555
|
guardShutdown();
|
|
32435
33556
|
requireIndex(indexName);
|
|
32436
|
-
await
|
|
33557
|
+
await withOpenIndexWrite(indexName, () => core.analysisRebuild.rebuild(indexName));
|
|
32437
33558
|
},
|
|
32438
33559
|
async snapshot(indexName) {
|
|
32439
33560
|
guardShutdown();
|
|
@@ -32441,11 +33562,11 @@ function createNarsilFromCore(core, config) {
|
|
|
32441
33562
|
},
|
|
32442
33563
|
async restore(indexName, data) {
|
|
32443
33564
|
guardShutdown();
|
|
32444
|
-
|
|
33565
|
+
const restored = restoreFromSnapshot(indexName, data, {
|
|
32445
33566
|
executor,
|
|
32446
33567
|
indexRegistry,
|
|
32447
33568
|
getVectorFieldPaths,
|
|
32448
|
-
dropIndex:
|
|
33569
|
+
dropIndex: (name) => dropEngineIndex(core, name),
|
|
32449
33570
|
announceIndexCreated: (name, restoredConfig) => announceIndexCreated(core, name, restoredConfig),
|
|
32450
33571
|
requireManager,
|
|
32451
33572
|
durability,
|
|
@@ -32456,6 +33577,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32456
33577
|
clearAnalysisStale: core.analysisRebuild.clearStale,
|
|
32457
33578
|
indexState: core.indexState
|
|
32458
33579
|
});
|
|
33580
|
+
await emitWriteEventAfter(core, indexName, restored);
|
|
32459
33581
|
core.heapPressureNotifier.check(indexName);
|
|
32460
33582
|
},
|
|
32461
33583
|
async checkpoint(indexName) {
|
|
@@ -32468,7 +33590,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32468
33590
|
async clear(indexName) {
|
|
32469
33591
|
guardShutdown();
|
|
32470
33592
|
requireIndex(indexName);
|
|
32471
|
-
await
|
|
33593
|
+
await withOpenIndexWrite(indexName, async () => {
|
|
32472
33594
|
await executor.execute({ type: "clear", indexName, requestId: indexName });
|
|
32473
33595
|
await orchestrator.replicateToWorkers({ type: "clear", indexName, requestId: `replicate-clear-${indexName}` });
|
|
32474
33596
|
core.watermarkNotifier.forget(indexName);
|
|
@@ -32477,7 +33599,7 @@ function createNarsilFromCore(core, config) {
|
|
|
32477
33599
|
async rebalance(indexName, targetPartitionCount) {
|
|
32478
33600
|
guardShutdown();
|
|
32479
33601
|
requireIndex(indexName);
|
|
32480
|
-
return
|
|
33602
|
+
return withOpenIndexWrite(
|
|
32481
33603
|
indexName,
|
|
32482
33604
|
() => executeRebalance(requireManager(indexName), indexName, targetPartitionCount, rebalanceCtx)
|
|
32483
33605
|
);
|
|
@@ -32543,14 +33665,14 @@ function createNarsilFromCore(core, config) {
|
|
|
32543
33665
|
async compactVectors(indexName, fieldName) {
|
|
32544
33666
|
guardShutdown();
|
|
32545
33667
|
requireIndex(indexName);
|
|
32546
|
-
await
|
|
33668
|
+
await withOpenIndexWrite(indexName, async () => {
|
|
32547
33669
|
compactVectors(requireManager(indexName), indexName, fieldName);
|
|
32548
33670
|
});
|
|
32549
33671
|
},
|
|
32550
33672
|
async optimizeVectors(indexName, fieldName) {
|
|
32551
33673
|
guardShutdown();
|
|
32552
33674
|
requireIndex(indexName);
|
|
32553
|
-
await
|
|
33675
|
+
await withOpenIndexWrite(indexName, () => optimizeVectors(requireManager(indexName), indexName, fieldName));
|
|
32554
33676
|
},
|
|
32555
33677
|
vectorMaintenanceStatus(indexName) {
|
|
32556
33678
|
guardShutdown();
|
|
@@ -32598,4 +33720,4 @@ async function createNarsil(config) {
|
|
|
32598
33720
|
// src/index.ts
|
|
32599
33721
|
var VERSION = "0.1.0";
|
|
32600
33722
|
|
|
32601
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNarsilError, isSimdAvailable, registerLanguage, registerStopWords, registerTokenizer, resetNormalizationCache };
|
|
33723
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isNarsilError, isSimdAvailable, registerLanguage, registerPatternSearch, registerStopWords, registerTokenizer, resetNormalizationCache };
|