@delali/narsil 0.2.4-next.51 → 0.2.4-next.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -27
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-BWm4IITU.d.ts → capabilities-DlYw0Ov0.d.ts} +13 -2
- package/dist/{chunk-KMMJMDUA.mjs → chunk-3L4U6EJQ.mjs} +15 -10
- package/dist/chunk-4NVPQ6S5.mjs +366 -0
- package/dist/{chunk-CMVG2UEF.mjs → chunk-5T6AOR37.mjs} +56 -518
- package/dist/{chunk-ND34I3SV.mjs → chunk-BH5SYB5W.mjs} +3295 -2149
- package/dist/{chunk-XE6XR2OE.mjs → chunk-D2S642IV.mjs} +4 -2
- package/dist/{chunk-P4ELXVJQ.mjs → chunk-FRQVMFY7.mjs} +7 -1
- package/dist/chunk-IFBKLBGB.mjs +7 -0
- package/dist/chunk-L225SHNB.mjs +513 -0
- package/dist/{chunk-MHE7YWAU.mjs → chunk-NTZYA3SJ.mjs} +3 -3
- package/dist/{chunk-WYJXSLER.mjs → chunk-POWOTUKJ.mjs} +7 -1
- package/dist/{chunk-CV5H7XL7.mjs → chunk-RGS7FMCB.mjs} +53 -10
- package/dist/{chunk-ZHUQKZBE.mjs → chunk-TVT5V4E7.mjs} +101 -34
- package/dist/chunk-U2AL3SON.mjs +4198 -0
- package/dist/chunk-UTOFOJOR.mjs +135 -0
- package/dist/{chunk-BEIK3EOZ.mjs → chunk-XNQAIO74.mjs} +5523 -3847
- package/dist/chunk-Y3L5MDRV.mjs +96 -0
- package/dist/chunk-ZJ4CSAB2.mjs +13 -0
- package/dist/{client-B_X6_AbT.d.ts → client-CQUqt9Ir.d.ts} +13 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +20 -14
- package/dist/{config-Czh2YNDi.d.ts → config-CNas6wJH.d.ts} +2 -2
- package/dist/{config-OON7IJ_B.d.ts → config-bXxnJrQc.d.ts} +80 -35
- package/dist/direct-executor-RBA5O7EN.mjs +12 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +32 -17
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-9PtFY50q.d.ts → engine-Dj4I91Jv.d.ts} +21 -8
- package/dist/index.browser.mjs +12962 -7786
- package/dist/index.d.ts +9 -21
- package/dist/index.mjs +14 -13
- package/dist/{memory-Bzd5xcEW.d.ts → memory-xrXOZMbw.d.ts} +68 -13
- package/dist/persistence/durability/checkpoint-worker.d.ts +138 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +32 -18
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +16 -4
- package/dist/{schema-BqRx9t4E.d.ts → schema-mLpXZBEh.d.ts} +43 -12
- package/dist/serialization/crc32-worker.mjs +2 -1
- package/dist/server.d.ts +13 -10
- package/dist/server.mjs +136 -47
- package/dist/simd-DW5AtJ6B.d.ts +41 -0
- package/dist/{tasks-DT7L1p6w.d.ts → tasks-Df73Tjgq.d.ts} +9 -5
- package/dist/{thread-PZLEO6JL.mjs → thread-5UP6MGED.mjs} +131 -43
- package/dist/{types-DXPxaXPO.d.ts → types-BTIVO8DI.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +292 -128
- package/dist/vector/search-worker.mjs +70 -54
- package/dist/workers/entry.mjs +30 -14
- package/package.json +52 -31
- package/dist/brute-force-DbHYCTc3.d.ts +0 -8
- package/dist/chunk-CBLS46IE.mjs +0 -10
- package/dist/chunk-PIWKZPP7.mjs +0 -64
- package/dist/chunk-PMBHH44R.mjs +0 -492
- package/dist/chunk-RP23PWMV.mjs +0 -11
- package/dist/chunk-X6YZO4MQ.mjs +0 -1572
- package/dist/direct-executor-OREXU72C.mjs +0 -12
- package/dist/vector/hnsw-build-worker.d.ts +0 -105
- package/dist/vector/hnsw-build-worker.mjs +0 -123
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, RESULT_WINDOW, DEFAULT_PAGE_SIZE, PREFIX_MAX_EXPANSIONS, MAX_CURSOR_ANCHOR_CODE_POINTS } from './chunk-
|
|
2
|
-
import { MAX_M } from './chunk-RP23PWMV.mjs';
|
|
1
|
+
import { flattenSchema, SORTABLE_TEXT_FIELD_TYPE, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, RESULT_WINDOW, DEFAULT_PAGE_SIZE, PROTOTYPE_POLLUTION_KEYS, isTextFieldType, PREFIX_MAX_EXPANSIONS, MAX_CURSOR_ANCHOR_CODE_POINTS } from './chunk-L225SHNB.mjs';
|
|
3
2
|
import { compareCodePoints, compareComparableKeys, toComparableSortValue, readSortField, SORT_VALUE_MAX_CODE_POINTS, compareSortValues } from './chunk-2GWVQWGA.mjs';
|
|
4
|
-
import { createNarsilError, ErrorCodes, NarsilError } from './chunk-
|
|
3
|
+
import { createNarsilError, ErrorCodes, NarsilError } from './chunk-FRQVMFY7.mjs';
|
|
5
4
|
|
|
6
5
|
// src/core/tokenizer/constants.ts
|
|
7
6
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
@@ -70,7 +69,7 @@ function computeDefaultCacheSize() {
|
|
|
70
69
|
try {
|
|
71
70
|
if (typeof process !== "undefined" && typeof process.versions?.node === "string" && typeof window === "undefined") {
|
|
72
71
|
const constrainedMemory = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
|
|
73
|
-
if (constrainedMemory > 0) {
|
|
72
|
+
if (constrainedMemory > 0 && constrainedMemory < Number.MAX_SAFE_INTEGER) {
|
|
74
73
|
const budget = constrainedMemory * CONSTRAINED_MEMORY_TOKEN_CACHE_FRACTION;
|
|
75
74
|
return clampCacheSize(Math.floor(budget / TOKEN_CACHE_BYTES_PER_ENTRY));
|
|
76
75
|
}
|
|
@@ -329,6 +328,10 @@ function expandNgrams(part, size, out) {
|
|
|
329
328
|
}
|
|
330
329
|
|
|
331
330
|
// src/core/tokenizer/tokenize.ts
|
|
331
|
+
function producesSurfaceForms(language, options) {
|
|
332
|
+
if (!options?.collectSurfaces || options.customTokenizer) return false;
|
|
333
|
+
return (options.stem ?? true) && language.stemmer !== null;
|
|
334
|
+
}
|
|
332
335
|
var DEFAULT_SPLIT_PATTERN = /[^\p{L}\p{M}\p{N}_'-]+/u;
|
|
333
336
|
function resolveStopWords(language, override) {
|
|
334
337
|
if (!override) return language.stopWords;
|
|
@@ -359,7 +362,7 @@ function stripPossessive(token) {
|
|
|
359
362
|
}
|
|
360
363
|
return token;
|
|
361
364
|
}
|
|
362
|
-
function
|
|
365
|
+
function visitTokens(text, language, options, visit) {
|
|
363
366
|
const {
|
|
364
367
|
stem = true,
|
|
365
368
|
removeStopWords = true,
|
|
@@ -369,45 +372,42 @@ function tokenize(text, language, options) {
|
|
|
369
372
|
stopWordOverride
|
|
370
373
|
} = options ?? {};
|
|
371
374
|
if (customTokenizer) {
|
|
372
|
-
const
|
|
373
|
-
|
|
374
|
-
return {
|
|
375
|
-
tokens: customResult,
|
|
376
|
-
originalTokens: originals
|
|
377
|
-
};
|
|
375
|
+
for (const entry of customTokenizer.tokenize(text)) visit(entry.token, entry.position, void 0, entry.token);
|
|
376
|
+
return;
|
|
378
377
|
}
|
|
379
|
-
const
|
|
380
|
-
const rawParts = splitText(normalized, language);
|
|
378
|
+
const rawParts = splitText(normalizeForSplitting(text), language);
|
|
381
379
|
const minLength = language.tokenizer?.minTokenLength ?? DEFAULT_MIN_TOKEN_LENGTH;
|
|
382
380
|
const effectiveDiacritics = removeDiacritics || (language.tokenizer?.normalizeDiacritics ?? false);
|
|
383
|
-
const
|
|
384
|
-
const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) :
|
|
381
|
+
const wantSurfaces = collectSurfaces && stem && language.stemmer !== null;
|
|
382
|
+
const stopWords = removeStopWords ? resolveStopWords(language, stopWordOverride) : null;
|
|
385
383
|
const stripPossessives = language.tokenizer?.stripPossessive ?? false;
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
const surfaces = collectSurfaces && stemsTokens ? [] : void 0;
|
|
389
|
-
let position = 0;
|
|
390
|
-
for (const part of rawParts) {
|
|
384
|
+
for (let position = 0; position < rawParts.length; position++) {
|
|
385
|
+
const part = rawParts[position];
|
|
391
386
|
const candidate = stripPossessives ? stripPossessive(part) : part;
|
|
392
|
-
if (candidate.length < minLength)
|
|
393
|
-
position++;
|
|
394
|
-
continue;
|
|
395
|
-
}
|
|
396
|
-
if (stopWords.has(candidate)) {
|
|
397
|
-
position++;
|
|
398
|
-
continue;
|
|
399
|
-
}
|
|
387
|
+
if (candidate.length < minLength || stopWords?.has(candidate)) continue;
|
|
400
388
|
const processed = transformToken(candidate, language, stem, effectiveDiacritics);
|
|
401
|
-
if (processed.length
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
|
|
406
|
-
surfaces.push(surface === processed ? void 0 : surface);
|
|
407
|
-
}
|
|
389
|
+
if (processed.length === 0) continue;
|
|
390
|
+
if (!wantSurfaces) {
|
|
391
|
+
visit(processed, position, void 0, part);
|
|
392
|
+
continue;
|
|
408
393
|
}
|
|
409
|
-
|
|
394
|
+
const surface = effectiveDiacritics ? transformToken(candidate, language, false, true) : candidate;
|
|
395
|
+
visit(processed, position, surface === processed ? void 0 : surface, part);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
function tokenize(text, language, options) {
|
|
399
|
+
if (options?.customTokenizer) {
|
|
400
|
+
const customResult = options.customTokenizer.tokenize(text);
|
|
401
|
+
return { tokens: customResult, originalTokens: customResult.map((entry) => entry.token) };
|
|
410
402
|
}
|
|
403
|
+
const tokens = [];
|
|
404
|
+
const originalTokens = [];
|
|
405
|
+
const surfaces = producesSurfaceForms(language, options) ? [] : void 0;
|
|
406
|
+
visitTokens(text, language, options, (token, position, surface, original) => {
|
|
407
|
+
tokens.push({ token, position });
|
|
408
|
+
originalTokens.push(original);
|
|
409
|
+
surfaces?.push(surface);
|
|
410
|
+
});
|
|
411
411
|
return { tokens, originalTokens, surfaces };
|
|
412
412
|
}
|
|
413
413
|
function* tokenizeIterator(text, language, options) {
|
|
@@ -458,487 +458,6 @@ function* tokenizeIterator(text, language, options) {
|
|
|
458
458
|
}
|
|
459
459
|
}
|
|
460
460
|
|
|
461
|
-
// src/schema/constants.ts
|
|
462
|
-
var MAX_NESTING_DEPTH = 4;
|
|
463
|
-
var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
464
|
-
|
|
465
|
-
// src/schema/validator/shared.ts
|
|
466
|
-
var SORTABLE_TEXT_FIELD_TYPE = "string:sortable";
|
|
467
|
-
var SCALAR_FIELD_TYPES = /* @__PURE__ */ new Set([
|
|
468
|
-
"string",
|
|
469
|
-
SORTABLE_TEXT_FIELD_TYPE,
|
|
470
|
-
"number",
|
|
471
|
-
"boolean",
|
|
472
|
-
"enum",
|
|
473
|
-
"geopoint",
|
|
474
|
-
"string[]",
|
|
475
|
-
"number[]",
|
|
476
|
-
"boolean[]",
|
|
477
|
-
"enum[]"
|
|
478
|
-
]);
|
|
479
|
-
function isTextFieldType(fieldType) {
|
|
480
|
-
return fieldType === "string" || fieldType === SORTABLE_TEXT_FIELD_TYPE;
|
|
481
|
-
}
|
|
482
|
-
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
483
|
-
var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
|
|
484
|
-
var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
|
|
485
|
-
var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
486
|
-
function isPlainObject(value) {
|
|
487
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
488
|
-
}
|
|
489
|
-
|
|
490
|
-
// src/schema/validator/schema.ts
|
|
491
|
-
function validateSchemaFields(schema, depth, prefix) {
|
|
492
|
-
if (depth > MAX_NESTING_DEPTH) {
|
|
493
|
-
throw new NarsilError(
|
|
494
|
-
ErrorCodes.SCHEMA_DEPTH_EXCEEDED,
|
|
495
|
-
`Schema nesting exceeds the maximum depth of ${MAX_NESTING_DEPTH} at "${prefix}"`,
|
|
496
|
-
{ path: prefix, maxDepth: MAX_NESTING_DEPTH }
|
|
497
|
-
);
|
|
498
|
-
}
|
|
499
|
-
for (const [field, type] of Object.entries(schema)) {
|
|
500
|
-
const path = prefix ? `${prefix}.${field}` : field;
|
|
501
|
-
if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
|
|
502
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field name "${field}" is not allowed in a schema`, {
|
|
503
|
-
field,
|
|
504
|
-
path
|
|
505
|
-
});
|
|
506
|
-
}
|
|
507
|
-
if (!FIELD_NAME_PATTERN.test(field)) {
|
|
508
|
-
throw new NarsilError(
|
|
509
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
510
|
-
`Field name "${field}" contains characters that are not allowed; use letters, digits, and underscores only`,
|
|
511
|
-
{ field, path }
|
|
512
|
-
);
|
|
513
|
-
}
|
|
514
|
-
if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
|
|
515
|
-
throw new NarsilError(
|
|
516
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
517
|
-
`Field "${field}" is reserved and cannot be defined in a schema`,
|
|
518
|
-
{ field, path }
|
|
519
|
-
);
|
|
520
|
-
}
|
|
521
|
-
if (isPlainObject(type)) {
|
|
522
|
-
validateSchemaFields(type, depth + 1, path);
|
|
523
|
-
continue;
|
|
524
|
-
}
|
|
525
|
-
if (typeof type !== "string") {
|
|
526
|
-
throw new NarsilError(
|
|
527
|
-
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
528
|
-
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
529
|
-
{
|
|
530
|
-
field: path,
|
|
531
|
-
type: String(type)
|
|
532
|
-
}
|
|
533
|
-
);
|
|
534
|
-
}
|
|
535
|
-
if (SCALAR_FIELD_TYPES.has(type)) continue;
|
|
536
|
-
const vectorMatch = VECTOR_PATTERN.exec(type);
|
|
537
|
-
if (vectorMatch) {
|
|
538
|
-
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
539
|
-
if (dimension <= 0) {
|
|
540
|
-
throw new NarsilError(
|
|
541
|
-
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
542
|
-
`Vector field "${path}" must have a positive dimension, got ${dimension}`,
|
|
543
|
-
{ field: path, dimension }
|
|
544
|
-
);
|
|
545
|
-
}
|
|
546
|
-
continue;
|
|
547
|
-
}
|
|
548
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field "${path}" has an unsupported type: "${type}"`, {
|
|
549
|
-
field: path,
|
|
550
|
-
type
|
|
551
|
-
});
|
|
552
|
-
}
|
|
553
|
-
}
|
|
554
|
-
function validateSchema(schema) {
|
|
555
|
-
if (!isPlainObject(schema)) {
|
|
556
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must be a plain object", {
|
|
557
|
-
received: typeof schema
|
|
558
|
-
});
|
|
559
|
-
}
|
|
560
|
-
if (Object.keys(schema).length === 0) {
|
|
561
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
|
|
562
|
-
}
|
|
563
|
-
validateSchemaFields(schema, 1, "");
|
|
564
|
-
}
|
|
565
|
-
function flattenRecursive(schema, prefix, result) {
|
|
566
|
-
for (const [field, type] of Object.entries(schema)) {
|
|
567
|
-
const path = prefix ? `${prefix}.${field}` : field;
|
|
568
|
-
if (isPlainObject(type)) {
|
|
569
|
-
flattenRecursive(type, path, result);
|
|
570
|
-
} else {
|
|
571
|
-
result[path] = type;
|
|
572
|
-
}
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
function flattenSchema(schema) {
|
|
576
|
-
const result = {};
|
|
577
|
-
flattenRecursive(schema, "", result);
|
|
578
|
-
return result;
|
|
579
|
-
}
|
|
580
|
-
function extractVectorFieldsFromSchema(schema) {
|
|
581
|
-
const flat = flattenSchema(schema);
|
|
582
|
-
const result = /* @__PURE__ */ new Map();
|
|
583
|
-
for (const [fieldPath, fieldType] of Object.entries(flat)) {
|
|
584
|
-
const match = VECTOR_PATTERN.exec(fieldType);
|
|
585
|
-
if (match) {
|
|
586
|
-
result.set(fieldPath, Number.parseInt(match[1], 10));
|
|
587
|
-
}
|
|
588
|
-
}
|
|
589
|
-
return result;
|
|
590
|
-
}
|
|
591
|
-
|
|
592
|
-
// src/schema/validator/field-values.ts
|
|
593
|
-
function validateGeopoint(path, value) {
|
|
594
|
-
if (!isPlainObject(value)) {
|
|
595
|
-
throw new NarsilError(
|
|
596
|
-
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
597
|
-
`Field "${path}" expected a geopoint object with lat and lon`,
|
|
598
|
-
{
|
|
599
|
-
field: path,
|
|
600
|
-
received: Array.isArray(value) ? "array" : typeof value
|
|
601
|
-
}
|
|
602
|
-
);
|
|
603
|
-
}
|
|
604
|
-
if (typeof value.lat !== "number" || typeof value.lon !== "number" || Number.isNaN(value.lat) || Number.isNaN(value.lon)) {
|
|
605
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_GEOPOINT, `Field "${path}" geopoint requires numeric lat and lon`, {
|
|
606
|
-
field: path,
|
|
607
|
-
lat: Number.isNaN(value.lat) ? "NaN" : typeof value.lat,
|
|
608
|
-
lon: Number.isNaN(value.lon) ? "NaN" : typeof value.lon
|
|
609
|
-
});
|
|
610
|
-
}
|
|
611
|
-
if (value.lat < -90 || value.lat > 90) {
|
|
612
|
-
throw new NarsilError(
|
|
613
|
-
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
614
|
-
`Field "${path}" geopoint lat must be between -90 and 90`,
|
|
615
|
-
{
|
|
616
|
-
field: path,
|
|
617
|
-
lat: value.lat
|
|
618
|
-
}
|
|
619
|
-
);
|
|
620
|
-
}
|
|
621
|
-
if (value.lon < -180 || value.lon > 180) {
|
|
622
|
-
throw new NarsilError(
|
|
623
|
-
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
624
|
-
`Field "${path}" geopoint lon must be between -180 and 180`,
|
|
625
|
-
{
|
|
626
|
-
field: path,
|
|
627
|
-
lon: value.lon
|
|
628
|
-
}
|
|
629
|
-
);
|
|
630
|
-
}
|
|
631
|
-
}
|
|
632
|
-
function validateVector(path, value, expectedDimension) {
|
|
633
|
-
const isArray = Array.isArray(value);
|
|
634
|
-
const isFloat32 = value instanceof Float32Array;
|
|
635
|
-
if (!isArray && !isFloat32) {
|
|
636
|
-
throw new NarsilError(
|
|
637
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
638
|
-
`Field "${path}" expected a vector (Array or Float32Array)`,
|
|
639
|
-
{ field: path, received: typeof value }
|
|
640
|
-
);
|
|
641
|
-
}
|
|
642
|
-
const arr = value;
|
|
643
|
-
if (arr.length !== expectedDimension) {
|
|
644
|
-
throw new NarsilError(
|
|
645
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
646
|
-
`Field "${path}" expected vector of dimension ${expectedDimension}, got ${arr.length}`,
|
|
647
|
-
{ field: path, expected: expectedDimension, received: arr.length }
|
|
648
|
-
);
|
|
649
|
-
}
|
|
650
|
-
for (let i = 0; i < arr.length; i++) {
|
|
651
|
-
if (typeof arr[i] !== "number" || Number.isNaN(arr[i])) {
|
|
652
|
-
throw new NarsilError(
|
|
653
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
654
|
-
`Field "${path}" vector element at index ${i} is not a valid number`,
|
|
655
|
-
{ field: path, index: i, value: arr[i] }
|
|
656
|
-
);
|
|
657
|
-
}
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
function validateTypedArray(path, value, elementType) {
|
|
661
|
-
if (!Array.isArray(value)) {
|
|
662
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an array`, {
|
|
663
|
-
field: path,
|
|
664
|
-
received: typeof value
|
|
665
|
-
});
|
|
666
|
-
}
|
|
667
|
-
for (let i = 0; i < value.length; i++) {
|
|
668
|
-
const el = value[i];
|
|
669
|
-
if (typeof el !== elementType) {
|
|
670
|
-
throw new NarsilError(
|
|
671
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
672
|
-
`Field "${path}" array element at index ${i} expected ${elementType}, got ${typeof el}`,
|
|
673
|
-
{ field: path, index: i, expected: elementType, received: typeof el }
|
|
674
|
-
);
|
|
675
|
-
}
|
|
676
|
-
if (elementType === "number" && Number.isNaN(el)) {
|
|
677
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" array element at index ${i} is NaN`, {
|
|
678
|
-
field: path,
|
|
679
|
-
index: i
|
|
680
|
-
});
|
|
681
|
-
}
|
|
682
|
-
}
|
|
683
|
-
}
|
|
684
|
-
function validateFieldValue(path, value, type) {
|
|
685
|
-
switch (type) {
|
|
686
|
-
case "string":
|
|
687
|
-
case SORTABLE_TEXT_FIELD_TYPE:
|
|
688
|
-
if (typeof value !== "string") {
|
|
689
|
-
throw new NarsilError(
|
|
690
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
691
|
-
`Field "${path}" expected string, got ${typeof value}`,
|
|
692
|
-
{
|
|
693
|
-
field: path,
|
|
694
|
-
expected: "string",
|
|
695
|
-
received: typeof value
|
|
696
|
-
}
|
|
697
|
-
);
|
|
698
|
-
}
|
|
699
|
-
return;
|
|
700
|
-
case "number":
|
|
701
|
-
if (typeof value !== "number" || Number.isNaN(value)) {
|
|
702
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a valid number`, {
|
|
703
|
-
field: path,
|
|
704
|
-
expected: "number",
|
|
705
|
-
received: Number.isNaN(value) ? "NaN" : typeof value
|
|
706
|
-
});
|
|
707
|
-
}
|
|
708
|
-
return;
|
|
709
|
-
case "boolean":
|
|
710
|
-
if (typeof value !== "boolean") {
|
|
711
|
-
throw new NarsilError(
|
|
712
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
713
|
-
`Field "${path}" expected boolean, got ${typeof value}`,
|
|
714
|
-
{
|
|
715
|
-
field: path,
|
|
716
|
-
expected: "boolean",
|
|
717
|
-
received: typeof value
|
|
718
|
-
}
|
|
719
|
-
);
|
|
720
|
-
}
|
|
721
|
-
return;
|
|
722
|
-
case "enum":
|
|
723
|
-
if (typeof value !== "string") {
|
|
724
|
-
throw new NarsilError(
|
|
725
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
726
|
-
`Field "${path}" expected enum (string), got ${typeof value}`,
|
|
727
|
-
{
|
|
728
|
-
field: path,
|
|
729
|
-
expected: "enum",
|
|
730
|
-
received: typeof value
|
|
731
|
-
}
|
|
732
|
-
);
|
|
733
|
-
}
|
|
734
|
-
return;
|
|
735
|
-
case "geopoint":
|
|
736
|
-
validateGeopoint(path, value);
|
|
737
|
-
return;
|
|
738
|
-
case "string[]":
|
|
739
|
-
validateTypedArray(path, value, "string");
|
|
740
|
-
return;
|
|
741
|
-
case "number[]":
|
|
742
|
-
validateTypedArray(path, value, "number");
|
|
743
|
-
return;
|
|
744
|
-
case "boolean[]":
|
|
745
|
-
validateTypedArray(path, value, "boolean");
|
|
746
|
-
return;
|
|
747
|
-
case "enum[]":
|
|
748
|
-
validateTypedArray(path, value, "string");
|
|
749
|
-
return;
|
|
750
|
-
default: {
|
|
751
|
-
const vectorMatch = VECTOR_PATTERN.exec(type);
|
|
752
|
-
if (vectorMatch) {
|
|
753
|
-
validateVector(path, value, Number.parseInt(vectorMatch[1], 10));
|
|
754
|
-
}
|
|
755
|
-
}
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
|
|
759
|
-
// src/schema/validator/storable.ts
|
|
760
|
-
var RESERVED_OBJECT_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
761
|
-
function reject(path, reason) {
|
|
762
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
763
|
-
field: path
|
|
764
|
-
});
|
|
765
|
-
}
|
|
766
|
-
function isPlainObject2(value) {
|
|
767
|
-
const proto = Object.getPrototypeOf(value);
|
|
768
|
-
return proto === Object.prototype || proto === null;
|
|
769
|
-
}
|
|
770
|
-
function walkStorable(value, path, depth, ancestors, skip) {
|
|
771
|
-
if (value === null || value === void 0) return;
|
|
772
|
-
const kind = typeof value;
|
|
773
|
-
if (kind === "string" || kind === "boolean") return;
|
|
774
|
-
if (kind === "number") return;
|
|
775
|
-
if (kind === "bigint" || kind === "symbol" || kind === "function") {
|
|
776
|
-
reject(path, `holds a ${kind}, which cannot be stored or recovered`);
|
|
777
|
-
}
|
|
778
|
-
const container = value;
|
|
779
|
-
if (container instanceof Date || container instanceof Uint8Array) return;
|
|
780
|
-
const isArray = Array.isArray(container);
|
|
781
|
-
if (!isArray && !isPlainObject2(container)) {
|
|
782
|
-
reject(
|
|
783
|
-
path,
|
|
784
|
-
"holds a value that does not survive storage; use plain objects, arrays, strings, numbers, booleans, dates, or Uint8Array bytes"
|
|
785
|
-
);
|
|
786
|
-
}
|
|
787
|
-
if (depth >= MAX_DOCUMENT_NESTING_DEPTH) {
|
|
788
|
-
reject(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
|
|
789
|
-
}
|
|
790
|
-
if (ancestors.has(container)) {
|
|
791
|
-
reject(path, "refers back to one of its own ancestors");
|
|
792
|
-
}
|
|
793
|
-
ancestors.add(container);
|
|
794
|
-
if (isArray) {
|
|
795
|
-
const items = container;
|
|
796
|
-
for (let i = 0; i < items.length; i++) {
|
|
797
|
-
walkStorable(items[i], `${path}[${i}]`, depth + 1, ancestors, skip);
|
|
798
|
-
}
|
|
799
|
-
} else {
|
|
800
|
-
const record = container;
|
|
801
|
-
for (const key of Object.keys(record)) {
|
|
802
|
-
const keyPath = path === "" ? key : `${path}.${key}`;
|
|
803
|
-
if (RESERVED_OBJECT_KEYS.has(key)) {
|
|
804
|
-
reject(keyPath, "uses a reserved object key");
|
|
805
|
-
}
|
|
806
|
-
if (skip.has(keyPath)) continue;
|
|
807
|
-
walkStorable(record[key], keyPath, depth + 1, ancestors, skip);
|
|
808
|
-
}
|
|
809
|
-
}
|
|
810
|
-
ancestors.delete(container);
|
|
811
|
-
}
|
|
812
|
-
function assertStorableDocument(document, skipFieldPaths) {
|
|
813
|
-
walkStorable(document, "", 0, /* @__PURE__ */ new Set(), skipFieldPaths ?? /* @__PURE__ */ new Set());
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
// src/schema/validator/document.ts
|
|
817
|
-
var vectorPathCache = /* @__PURE__ */ new WeakMap();
|
|
818
|
-
function vectorFieldPaths(schema) {
|
|
819
|
-
let paths = vectorPathCache.get(schema);
|
|
820
|
-
if (paths !== void 0) return paths;
|
|
821
|
-
paths = /* @__PURE__ */ new Set();
|
|
822
|
-
const collect = (level, prefix) => {
|
|
823
|
-
for (const [field, type] of Object.entries(level)) {
|
|
824
|
-
const path = prefix === "" ? field : `${prefix}.${field}`;
|
|
825
|
-
if (typeof type === "string" && VECTOR_PATTERN.test(type)) {
|
|
826
|
-
paths?.add(path);
|
|
827
|
-
} else if (isPlainObject(type)) {
|
|
828
|
-
collect(type, path);
|
|
829
|
-
}
|
|
830
|
-
}
|
|
831
|
-
};
|
|
832
|
-
collect(schema, "");
|
|
833
|
-
vectorPathCache.set(schema, paths);
|
|
834
|
-
return paths;
|
|
835
|
-
}
|
|
836
|
-
function validateDocumentFields(doc, schema, prefix) {
|
|
837
|
-
for (const [field, type] of Object.entries(schema)) {
|
|
838
|
-
const path = prefix ? `${prefix}.${field}` : field;
|
|
839
|
-
const value = doc[field];
|
|
840
|
-
if (value === void 0 || value === null) continue;
|
|
841
|
-
if (isPlainObject(type)) {
|
|
842
|
-
if (!isPlainObject(value)) {
|
|
843
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an object`, {
|
|
844
|
-
field: path,
|
|
845
|
-
received: Array.isArray(value) ? "array" : typeof value
|
|
846
|
-
});
|
|
847
|
-
}
|
|
848
|
-
validateDocumentFields(value, type, path);
|
|
849
|
-
continue;
|
|
850
|
-
}
|
|
851
|
-
validateFieldValue(path, value, type);
|
|
852
|
-
}
|
|
853
|
-
}
|
|
854
|
-
function validateDocument(document, schema) {
|
|
855
|
-
if (!isPlainObject(document)) {
|
|
856
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document must be a plain object", {
|
|
857
|
-
received: typeof document
|
|
858
|
-
});
|
|
859
|
-
}
|
|
860
|
-
assertStorableDocument(document, vectorFieldPaths(schema));
|
|
861
|
-
validateDocumentFields(document, schema, "");
|
|
862
|
-
}
|
|
863
|
-
function collectExtraFields(doc, schema, prefix, extras) {
|
|
864
|
-
for (const key of Object.keys(doc)) {
|
|
865
|
-
if (key === "id") continue;
|
|
866
|
-
const path = prefix ? `${prefix}.${key}` : key;
|
|
867
|
-
if (!(key in schema)) {
|
|
868
|
-
extras.push(path);
|
|
869
|
-
continue;
|
|
870
|
-
}
|
|
871
|
-
const schemaType = schema[key];
|
|
872
|
-
if (isPlainObject(schemaType) && isPlainObject(doc[key])) {
|
|
873
|
-
collectExtraFields(doc[key], schemaType, path, extras);
|
|
874
|
-
}
|
|
875
|
-
}
|
|
876
|
-
}
|
|
877
|
-
function validateDocumentStrict(document, schema) {
|
|
878
|
-
const extras = [];
|
|
879
|
-
collectExtraFields(document, schema, "", extras);
|
|
880
|
-
if (extras.length > 0) {
|
|
881
|
-
throw new NarsilError(
|
|
882
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
883
|
-
`Document contains fields not defined in schema: ${extras.join(", ")}`,
|
|
884
|
-
{ extraFields: extras }
|
|
885
|
-
);
|
|
886
|
-
}
|
|
887
|
-
}
|
|
888
|
-
function resolveNestedValue(document, path) {
|
|
889
|
-
if (!path.includes(".")) return document[path];
|
|
890
|
-
const segments = path.split(".");
|
|
891
|
-
let current = document;
|
|
892
|
-
for (const segment of segments) {
|
|
893
|
-
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
894
|
-
current = current[segment];
|
|
895
|
-
}
|
|
896
|
-
return current;
|
|
897
|
-
}
|
|
898
|
-
function validateRequiredFields(document, required) {
|
|
899
|
-
if (required.length === 0) return;
|
|
900
|
-
for (const field of required) {
|
|
901
|
-
const value = resolveNestedValue(document, field);
|
|
902
|
-
if (value === void 0 || value === null) {
|
|
903
|
-
throw new NarsilError(ErrorCodes.DOC_MISSING_REQUIRED_FIELD, `Document is missing required field "${field}"`, {
|
|
904
|
-
field
|
|
905
|
-
});
|
|
906
|
-
}
|
|
907
|
-
}
|
|
908
|
-
}
|
|
909
|
-
|
|
910
|
-
// src/schema/validator/vector-promotion.ts
|
|
911
|
-
function fail(message, details) {
|
|
912
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, message, details);
|
|
913
|
-
}
|
|
914
|
-
function isPositiveInteger(value) {
|
|
915
|
-
return Number.isInteger(value) && value >= 1;
|
|
916
|
-
}
|
|
917
|
-
function validateVectorPromotion(config) {
|
|
918
|
-
if (config === void 0) return;
|
|
919
|
-
if (config.threshold !== void 0 && !isPositiveInteger(config.threshold)) {
|
|
920
|
-
fail("vectorPromotion.threshold must be a positive integer", { threshold: config.threshold });
|
|
921
|
-
}
|
|
922
|
-
const quantization = config.quantization;
|
|
923
|
-
if (quantization !== void 0 && quantization !== "sq8" && quantization !== "none") {
|
|
924
|
-
fail("vectorPromotion.quantization must be 'sq8' or 'none'", { quantization });
|
|
925
|
-
}
|
|
926
|
-
const hnsw = config.hnswConfig;
|
|
927
|
-
if (hnsw === void 0) return;
|
|
928
|
-
if (hnsw.m !== void 0 && (!isPositiveInteger(hnsw.m) || hnsw.m > MAX_M)) {
|
|
929
|
-
fail(`vectorPromotion.hnswConfig.m must be a positive integer no greater than ${MAX_M}`, { m: hnsw.m });
|
|
930
|
-
}
|
|
931
|
-
if (hnsw.efConstruction !== void 0 && !isPositiveInteger(hnsw.efConstruction)) {
|
|
932
|
-
fail("vectorPromotion.hnswConfig.efConstruction must be a positive integer", {
|
|
933
|
-
efConstruction: hnsw.efConstruction
|
|
934
|
-
});
|
|
935
|
-
}
|
|
936
|
-
const metric = hnsw.metric;
|
|
937
|
-
if (metric !== void 0 && metric !== "cosine" && metric !== "dotProduct" && metric !== "euclidean") {
|
|
938
|
-
fail("vectorPromotion.hnswConfig.metric must be 'cosine', 'dotProduct', or 'euclidean'", { metric });
|
|
939
|
-
}
|
|
940
|
-
}
|
|
941
|
-
|
|
942
461
|
// src/partitioning/distributed-scoring.ts
|
|
943
462
|
function analysedQueryTokens(term, language, analysis) {
|
|
944
463
|
const { tokens } = tokenize(term, language, {
|
|
@@ -2035,4 +1554,23 @@ function fnv1a(input) {
|
|
|
2035
1554
|
return fnv1aBytes(encoder.encode(input));
|
|
2036
1555
|
}
|
|
2037
1556
|
|
|
2038
|
-
|
|
1557
|
+
// src/partitioning/manager/nested-values.ts
|
|
1558
|
+
function setNestedValue(obj, path, value) {
|
|
1559
|
+
if (!path.includes(".")) {
|
|
1560
|
+
obj[path] = value;
|
|
1561
|
+
return;
|
|
1562
|
+
}
|
|
1563
|
+
const segments = path.split(".");
|
|
1564
|
+
let current = obj;
|
|
1565
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
1566
|
+
let next = current[segments[i]];
|
|
1567
|
+
if (next === null || next === void 0 || typeof next !== "object") {
|
|
1568
|
+
next = {};
|
|
1569
|
+
current[segments[i]] = next;
|
|
1570
|
+
}
|
|
1571
|
+
current = next;
|
|
1572
|
+
}
|
|
1573
|
+
current[segments[segments.length - 1]] = value;
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
export { applyPagination, applyProjection, applySorting, bitsetAnd, bitsetFromSet, bitsetHas, bitsetNot, bitsetOr, bitsetSet, boundedLevenshtein, clampRowCount, clearNormalizationCache, cloneProjected, collectQueryTermStats, configureNormalizationCache, countsWithoutScores, createBitSet, decodePageCursor, encodePageCursor, fanOutMatchCount, fanOutQuery, fnv1a, fnv1aBytes, fulltextMatches, getNormalizationCacheSize, kWayMerge, mergeFacets, mergePartitionStats, normalizeSort, partitionsIn, projectionKeepsField, pruneStatsToQueryTerms, readSortValues, requireMatchingCursor, requireSortableFields, requireWithinResultWindow, resetNormalizationCache, resolveProjection, sanitizeGlobalStats, setNestedValue, sortSignatureOf, tokenize, tokenizeIterator, visitTokens };
|