@delali/narsil 0.3.1-next.71 → 0.3.1-next.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-Dt-7B4Wz.d.ts → capabilities-CYeAWYxQ.d.ts} +19 -4
- package/dist/{chunk-3JGWULDF.mjs → chunk-2WLD2NFW.mjs} +5 -5
- package/dist/{chunk-LMO5TJUS.mjs → chunk-3YKWSO54.mjs} +642 -196
- package/dist/{chunk-HWQLTLNG.mjs → chunk-4MQ5F4IL.mjs} +1 -1
- package/dist/{chunk-FVLQ63B7.mjs → chunk-AFKH4ZK3.mjs} +30 -6
- package/dist/{chunk-653BNMAF.mjs → chunk-AK3BLGCI.mjs} +195 -25
- package/dist/{chunk-IOR2K26O.mjs → chunk-CVFAFZSN.mjs} +8000 -4391
- package/dist/{chunk-USKFU7J5.mjs → chunk-I64LXKO3.mjs} +1 -1
- package/dist/{chunk-OI42GWLR.mjs → chunk-LJYTW7GT.mjs} +57 -11
- package/dist/{chunk-2STVUTNA.mjs → chunk-LPBWCHBE.mjs} +194 -13
- package/dist/{chunk-YNVIUGF5.mjs → chunk-PU43WDY4.mjs} +4 -2
- package/dist/{chunk-M76V3OGJ.mjs → chunk-THMSFOJO.mjs} +1289 -1405
- package/dist/{chunk-YJDQ7NLA.mjs → chunk-WVDFG45M.mjs} +93 -95
- package/dist/chunk-XW25X6AU.mjs +614 -0
- package/dist/chunk-ZKRV5M2P.mjs +75 -0
- package/dist/{chunk-P4DSQ6M4.mjs → chunk-ZULSEZCE.mjs} +68 -28
- package/dist/{client-COek5GXK.d.ts → client-D698E5ge.d.ts} +77 -26
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +45 -22
- package/dist/{config-rBpbgTw8.d.ts → config-Bv8etwUZ.d.ts} +11 -4
- package/dist/{config-Dfanyqqt.d.ts → config-QXU3HMpq.d.ts} +26 -2
- package/dist/direct-executor-5SRDFZAS.mjs +13 -0
- package/dist/distribution/coordinator/etcd.d.ts +29 -16
- package/dist/distribution/coordinator/etcd.mjs +163 -76
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +22 -2
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +6 -3
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +40 -16
- package/dist/distribution.d.ts +9 -9
- package/dist/distribution.mjs +682 -388
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-igk0aMpQ.d.ts → engine-D_OzKZ-6.d.ts} +97 -188
- package/dist/index.browser.mjs +11671 -7427
- package/dist/index.d.ts +27 -8
- package/dist/index.mjs +13 -14
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-C7UMCMH3.d.ts → memory-B4jMmygw.d.ts} +279 -27
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +7 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +41 -17
- package/dist/persistence/durability/checkpoint-worker.mjs +24 -16
- package/dist/react.d.ts +7 -7
- package/dist/react.mjs +1 -1
- package/dist/{schema-Bgzr4g7b.d.ts → schema-BBOCdhND.d.ts} +9 -7
- package/dist/server.d.ts +11 -11
- package/dist/server.mjs +213 -34
- package/dist/{tasks-BZt_zes6.d.ts → tasks-DpgCpW1x.d.ts} +3 -3
- package/dist/{thread-JEVAYETR.mjs → thread-3LNVA43V.mjs} +94 -27
- package/dist/{types-BQDlIvfS.d.ts → types-B3b68nfF.d.ts} +1 -1
- package/dist/{types-gOUqhs2F.d.ts → types-BiUmaiWT.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cb9kiY5P.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +4 -4
- package/package.json +15 -7
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-GIVAGD3Q.mjs +0 -25
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/direct-executor-GYJEKNV5.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { validateRequiredFields } from './chunk-
|
|
2
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
1
|
+
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-AK3BLGCI.mjs';
|
|
2
|
+
import { ErrorCodes, NarsilError } from './chunk-I64LXKO3.mjs';
|
|
3
3
|
|
|
4
4
|
// src/narsil/internals.ts
|
|
5
5
|
var engineCores = /* @__PURE__ */ new WeakMap();
|
|
@@ -239,9 +239,33 @@ async function embedBatchDocumentFields(documents, embeddingConfig, adapter, sig
|
|
|
239
239
|
|
|
240
240
|
// src/engine/mutations/insert-admission.ts
|
|
241
241
|
function providedDocId(document) {
|
|
242
|
+
if (document === null || typeof document !== "object") return void 0;
|
|
242
243
|
const id = document.id;
|
|
243
244
|
return typeof id === "string" && id.length > 0 ? id : void 0;
|
|
244
245
|
}
|
|
246
|
+
function requireDocumentObject(document) {
|
|
247
|
+
if (document !== null && typeof document === "object" && !Array.isArray(document)) {
|
|
248
|
+
requireDocumentWithinSizeLimit(document);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
const received = Array.isArray(document) ? "a list" : document === null ? "null" : `a ${typeof document}`;
|
|
252
|
+
throw new NarsilError(
|
|
253
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
254
|
+
`Each document must be an object, while this entry is ${received}`,
|
|
255
|
+
{
|
|
256
|
+
received
|
|
257
|
+
}
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
function requireMatchingDocId(docId, document) {
|
|
261
|
+
const embeddedDocId = providedDocId(document);
|
|
262
|
+
if (embeddedDocId === void 0 || embeddedDocId === docId) return;
|
|
263
|
+
throw new NarsilError(
|
|
264
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
265
|
+
`The call names document id "${docId}" while the document's own id field is "${embeddedDocId}", so the stored document would contradict the key that it is stored under`,
|
|
266
|
+
{ docId, documentId: embeddedDocId }
|
|
267
|
+
);
|
|
268
|
+
}
|
|
245
269
|
function asBatchInsertError(err) {
|
|
246
270
|
return err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err));
|
|
247
271
|
}
|
|
@@ -557,13 +581,13 @@ function readSharedConfigFields(source, reject) {
|
|
|
557
581
|
...typeof source.patternValueLimit === "number" ? { patternValueLimit: source.patternValueLimit } : {}
|
|
558
582
|
};
|
|
559
583
|
}
|
|
560
|
-
function restoredConfigFields(envelope) {
|
|
561
|
-
const shared = readSharedConfigFields(envelope,
|
|
562
|
-
const partitions = restoredPartitionConfig(envelope.partitionConfig);
|
|
584
|
+
function restoredConfigFields(envelope, reject = rejectSnapshot) {
|
|
585
|
+
const shared = readSharedConfigFields(envelope, reject);
|
|
586
|
+
const partitions = restoredPartitionConfig(envelope.partitionConfig, reject);
|
|
563
587
|
return {
|
|
564
588
|
...shared,
|
|
565
589
|
...partitions !== void 0 ? { partitions } : {}
|
|
566
590
|
};
|
|
567
591
|
}
|
|
568
592
|
|
|
569
|
-
export { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, readSharedConfigFields, recordPendingAdmission, resolveVectorText, restoredConfigFields, restoredEmbedding, restoredPartitionConfig };
|
|
593
|
+
export { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, admitBufferedUpsert, admitInsert, asBatchInsertError, assertDocumentCarriesMappedVectors, bindEngineCore, collectRequiredFieldFailures, createPendingAdmissions, embedChunkDocuments, embedDocumentFields, engineCoreOf, providedDocId, readSharedConfigFields, recordPendingAdmission, requireDocumentObject, requireMatchingDocId, resolveVectorText, restoredConfigFields, restoredEmbedding, restoredPartitionConfig };
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { MAX_M } from './chunk-
|
|
2
|
-
import { exceedsCodePoints } from './chunk-
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { EARLIER_RELEASE_MIN_M, MIN_M, MAX_M } from './chunk-PU43WDY4.mjs';
|
|
2
|
+
import { holdsUnpairedSurrogate, exceedsCodePoints } from './chunk-LJYTW7GT.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
|
|
4
|
+
import { encode } from '@msgpack/msgpack';
|
|
4
5
|
|
|
5
6
|
// src/schema/constants.ts
|
|
6
7
|
var MAX_NESTING_DEPTH = 4;
|
|
@@ -8,6 +9,10 @@ var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
|
8
9
|
var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
|
|
9
10
|
var MIN_PATTERN_VALUE_LIMIT = 1;
|
|
10
11
|
var MAX_PATTERN_VALUE_LIMIT = 65536;
|
|
12
|
+
var MAX_VECTOR_DIMENSION = 4096;
|
|
13
|
+
var MAX_STORED_VECTOR_DIMENSION = 65535;
|
|
14
|
+
var MAX_TEXT_FIELDS = 65536;
|
|
15
|
+
var MAX_DOCUMENT_BYTES = 16777216;
|
|
11
16
|
|
|
12
17
|
// src/schema/validator/shared.ts
|
|
13
18
|
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
@@ -57,11 +62,11 @@ function parseBase(baseName, path, type) {
|
|
|
57
62
|
invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
|
|
58
63
|
}
|
|
59
64
|
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
60
|
-
if (dimension <= 0) {
|
|
65
|
+
if (dimension <= 0 || dimension > MAX_STORED_VECTOR_DIMENSION) {
|
|
61
66
|
throw new NarsilError(
|
|
62
67
|
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
63
|
-
`Vector field "${path}"
|
|
64
|
-
{ field: path, dimension }
|
|
68
|
+
`Vector field "${path}" has ${dimension} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
|
|
69
|
+
{ field: path, dimension, maximum: MAX_STORED_VECTOR_DIMENSION }
|
|
65
70
|
);
|
|
66
71
|
}
|
|
67
72
|
return { base: "vector", dimension };
|
|
@@ -194,9 +199,30 @@ function validateSchema(schema) {
|
|
|
194
199
|
if (Object.keys(schema).length === 0) {
|
|
195
200
|
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
|
|
196
201
|
}
|
|
197
|
-
|
|
202
|
+
const validated = validateSchemaFields(schema, 1, "");
|
|
203
|
+
requireTextFieldCount(schemaFieldsOf(validated).filter((field) => isWordIndexedBase(field.base)).length);
|
|
204
|
+
return validated;
|
|
205
|
+
}
|
|
206
|
+
function requireTextFieldCount(count) {
|
|
207
|
+
if (count <= MAX_TEXT_FIELDS) return;
|
|
208
|
+
throw new NarsilError(
|
|
209
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
210
|
+
`The schema declares ${count} string and string[] fields, but a schema holds at most ${MAX_TEXT_FIELDS}`,
|
|
211
|
+
{ textFields: count, maximum: MAX_TEXT_FIELDS }
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
function requireCreatableSchema(schema) {
|
|
215
|
+
for (const field of schemaFieldsOf(schema)) {
|
|
216
|
+
if (field.base !== "vector" || field.dimension <= MAX_VECTOR_DIMENSION) continue;
|
|
217
|
+
throw new NarsilError(
|
|
218
|
+
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
219
|
+
`Vector field "${field.path}" has ${field.dimension} dimensions, but a new index takes at most ${MAX_VECTOR_DIMENSION}`,
|
|
220
|
+
{ field: field.path, dimension: field.dimension, maximum: MAX_VECTOR_DIMENSION }
|
|
221
|
+
);
|
|
222
|
+
}
|
|
198
223
|
}
|
|
199
224
|
function requireValidFieldTypes(flatSchema) {
|
|
225
|
+
let textFields = 0;
|
|
200
226
|
for (const [path, type] of Object.entries(flatSchema)) {
|
|
201
227
|
if (typeof type !== "string") {
|
|
202
228
|
throw new NarsilError(
|
|
@@ -205,8 +231,9 @@ function requireValidFieldTypes(flatSchema) {
|
|
|
205
231
|
{ field: path, type: String(type) }
|
|
206
232
|
);
|
|
207
233
|
}
|
|
208
|
-
parseFieldType(type, path);
|
|
234
|
+
if (isWordIndexedBase(parseFieldType(type, path).base)) textFields += 1;
|
|
209
235
|
}
|
|
236
|
+
requireTextFieldCount(textFields);
|
|
210
237
|
}
|
|
211
238
|
function flattenRecursive(schema, prefix, result) {
|
|
212
239
|
for (const [field, type] of Object.entries(schema)) {
|
|
@@ -247,6 +274,29 @@ function extractVectorFieldsFromSchema(schema) {
|
|
|
247
274
|
return result;
|
|
248
275
|
}
|
|
249
276
|
|
|
277
|
+
// src/geo/constants.ts
|
|
278
|
+
var DEG_TO_RAD = Math.PI / 180;
|
|
279
|
+
var MAX_LATITUDE_DEGREES = 90;
|
|
280
|
+
var MAX_LONGITUDE_DEGREES = 180;
|
|
281
|
+
var MIN_POLYGON_POINTS = 3;
|
|
282
|
+
var ANTIMERIDIAN_SPAN_DEGREES = 180;
|
|
283
|
+
var FULL_TURN_DEGREES = 360;
|
|
284
|
+
var EARTH_RADIUS_METERS = 63710088e-1;
|
|
285
|
+
var LOCATION_CODE_BITS_PER_AXIS = 26;
|
|
286
|
+
var MAX_COVERING_CELLS = 16;
|
|
287
|
+
var RADIUS_BOX_MARGIN_RATIO = 1.02;
|
|
288
|
+
var RADIUS_BOX_MARGIN_RADIANS = 1e-9;
|
|
289
|
+
var VINCENTY_MAX_ITERATIONS = 200;
|
|
290
|
+
var VINCENTY_CONVERGENCE_THRESHOLD = 1e-12;
|
|
291
|
+
|
|
292
|
+
// src/geo/coordinates.ts
|
|
293
|
+
function isLatitude(value) {
|
|
294
|
+
return value >= -MAX_LATITUDE_DEGREES && value <= MAX_LATITUDE_DEGREES;
|
|
295
|
+
}
|
|
296
|
+
function isLongitude(value) {
|
|
297
|
+
return value >= -MAX_LONGITUDE_DEGREES && value <= MAX_LONGITUDE_DEGREES;
|
|
298
|
+
}
|
|
299
|
+
|
|
250
300
|
// src/schema/validator/field-values.ts
|
|
251
301
|
function validateGeopoint(path, value) {
|
|
252
302
|
if (!isPlainObject(value)) {
|
|
@@ -266,7 +316,7 @@ function validateGeopoint(path, value) {
|
|
|
266
316
|
lon: Number.isNaN(value.lon) ? "NaN" : typeof value.lon
|
|
267
317
|
});
|
|
268
318
|
}
|
|
269
|
-
if (value.lat
|
|
319
|
+
if (!isLatitude(value.lat)) {
|
|
270
320
|
throw new NarsilError(
|
|
271
321
|
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
272
322
|
`Field "${path}" geopoint lat must be between -90 and 90`,
|
|
@@ -276,7 +326,7 @@ function validateGeopoint(path, value) {
|
|
|
276
326
|
}
|
|
277
327
|
);
|
|
278
328
|
}
|
|
279
|
-
if (value.lon
|
|
329
|
+
if (!isLongitude(value.lon)) {
|
|
280
330
|
throw new NarsilError(
|
|
281
331
|
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
282
332
|
`Field "${path}" geopoint lon must be between -180 and 180`,
|
|
@@ -306,7 +356,7 @@ function validateVector(path, value, expectedDimension) {
|
|
|
306
356
|
);
|
|
307
357
|
}
|
|
308
358
|
for (let i = 0; i < arr.length; i++) {
|
|
309
|
-
if (typeof arr[i] !== "number" || Number.
|
|
359
|
+
if (typeof arr[i] !== "number" || !Number.isFinite(arr[i])) {
|
|
310
360
|
throw new NarsilError(
|
|
311
361
|
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
312
362
|
`Field "${path}" vector element at index ${i} is not a valid number`,
|
|
@@ -331,11 +381,12 @@ function validateTypedArray(path, value, elementType) {
|
|
|
331
381
|
{ field: path, index: i, expected: elementType, received: typeof el }
|
|
332
382
|
);
|
|
333
383
|
}
|
|
334
|
-
if (elementType === "number" && Number.
|
|
335
|
-
throw new NarsilError(
|
|
336
|
-
|
|
337
|
-
index
|
|
338
|
-
|
|
384
|
+
if (elementType === "number" && !Number.isFinite(el)) {
|
|
385
|
+
throw new NarsilError(
|
|
386
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
387
|
+
`Field "${path}" array element at index ${i} is ${String(el)}, while a number field holds finite numbers alone`,
|
|
388
|
+
{ field: path, index: i }
|
|
389
|
+
);
|
|
339
390
|
}
|
|
340
391
|
}
|
|
341
392
|
}
|
|
@@ -366,11 +417,11 @@ function validateFieldValue(path, value, type, patternValueLimit) {
|
|
|
366
417
|
if (type.pattern) validatePatternValue(path, value, patternValueLimit);
|
|
367
418
|
return;
|
|
368
419
|
case "number":
|
|
369
|
-
if (typeof value !== "number" || Number.
|
|
370
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a
|
|
420
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
421
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a finite number`, {
|
|
371
422
|
field: path,
|
|
372
423
|
expected: "number",
|
|
373
|
-
received:
|
|
424
|
+
received: typeof value === "number" ? String(value) : typeof value
|
|
374
425
|
});
|
|
375
426
|
}
|
|
376
427
|
return;
|
|
@@ -428,6 +479,7 @@ function validateFieldValue(path, value, type, patternValueLimit) {
|
|
|
428
479
|
|
|
429
480
|
// src/schema/validator/storable.ts
|
|
430
481
|
var RESERVED_OBJECT_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
482
|
+
var UNPAIRED_SURROGATE_REASON = "holds an unpaired surrogate, the half of a character such as an emoji whose other half is missing, so the engine cannot store it as UTF-8";
|
|
431
483
|
function reject(path, reason) {
|
|
432
484
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
433
485
|
field: path
|
|
@@ -440,13 +492,20 @@ function isPlainObject2(value) {
|
|
|
440
492
|
function walkStorable(value, path, depth, ancestors, skip) {
|
|
441
493
|
if (value === null || value === void 0) return;
|
|
442
494
|
const kind = typeof value;
|
|
443
|
-
if (kind === "string"
|
|
444
|
-
|
|
495
|
+
if (kind === "string") {
|
|
496
|
+
if (holdsUnpairedSurrogate(value)) reject(path, UNPAIRED_SURROGATE_REASON);
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
if (kind === "boolean" || kind === "number") return;
|
|
445
500
|
if (kind === "bigint" || kind === "symbol" || kind === "function") {
|
|
446
501
|
reject(path, `holds a ${kind}, which cannot be stored or recovered`);
|
|
447
502
|
}
|
|
448
503
|
const container = value;
|
|
449
|
-
if (container instanceof Date
|
|
504
|
+
if (container instanceof Date) {
|
|
505
|
+
if (Number.isNaN(container.getTime())) reject(path, "holds an invalid date, which the engine cannot store");
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
if (container instanceof Uint8Array) return;
|
|
450
509
|
const isArray = Array.isArray(container);
|
|
451
510
|
if (!isArray && !isPlainObject2(container)) {
|
|
452
511
|
reject(
|
|
@@ -473,6 +532,7 @@ function walkStorable(value, path, depth, ancestors, skip) {
|
|
|
473
532
|
if (RESERVED_OBJECT_KEYS.has(key)) {
|
|
474
533
|
reject(keyPath, "uses a reserved object key");
|
|
475
534
|
}
|
|
535
|
+
if (holdsUnpairedSurrogate(key)) reject(keyPath, `has a name that ${UNPAIRED_SURROGATE_REASON}`);
|
|
476
536
|
if (skip.has(keyPath)) continue;
|
|
477
537
|
walkStorable(record[key], keyPath, depth + 1, ancestors, skip);
|
|
478
538
|
}
|
|
@@ -539,9 +599,20 @@ function validateDocument(document, schema, patternValueLimit = DEFAULT_PATTERN_
|
|
|
539
599
|
received: typeof document
|
|
540
600
|
});
|
|
541
601
|
}
|
|
602
|
+
requireIdFieldKey(document);
|
|
542
603
|
assertStorableDocument(document, vectorFieldPaths(schema));
|
|
543
604
|
validateDocumentFields(document, schema, "", patternValueLimit);
|
|
544
605
|
}
|
|
606
|
+
function requireIdFieldKey(document) {
|
|
607
|
+
if (!Object.hasOwn(document, "id")) return;
|
|
608
|
+
const id = document.id;
|
|
609
|
+
if (typeof id === "string" && id.length > 0) return;
|
|
610
|
+
throw new NarsilError(
|
|
611
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
612
|
+
`Field "id" holds ${typeof id === "string" ? "an empty string" : id === null ? "null" : `a ${typeof id}`}, while the engine stores a document under its id field, so that field must hold a non-empty string`,
|
|
613
|
+
{ field: "id", received: id === null ? "null" : typeof id }
|
|
614
|
+
);
|
|
615
|
+
}
|
|
545
616
|
function collectExtraFields(doc, schema, prefix, extras) {
|
|
546
617
|
for (const key of Object.keys(doc)) {
|
|
547
618
|
if (key === "id") continue;
|
|
@@ -588,6 +659,71 @@ function validateRequiredFields(document, required) {
|
|
|
588
659
|
}
|
|
589
660
|
}
|
|
590
661
|
}
|
|
662
|
+
function holdsNegativeZero(value) {
|
|
663
|
+
const pending = [value];
|
|
664
|
+
while (pending.length > 0) {
|
|
665
|
+
const next = pending.pop();
|
|
666
|
+
if (typeof next === "number") {
|
|
667
|
+
if (Object.is(next, -0)) return true;
|
|
668
|
+
continue;
|
|
669
|
+
}
|
|
670
|
+
if (next === null || typeof next !== "object" || ArrayBuffer.isView(next)) continue;
|
|
671
|
+
for (const child of Array.isArray(next) ? next : Object.values(next)) pending.push(child);
|
|
672
|
+
}
|
|
673
|
+
return false;
|
|
674
|
+
}
|
|
675
|
+
function encodeDocument(document) {
|
|
676
|
+
return encode(document, holdsNegativeZero(document) ? { forceIntegerToFloat: true } : void 0);
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
// src/schema/validator/document-size.ts
|
|
680
|
+
var LARGEST_HEADER_BYTES = 5;
|
|
681
|
+
var LARGEST_NUMBER_BYTES = 9;
|
|
682
|
+
var LARGEST_TIMESTAMP_BYTES = 15;
|
|
683
|
+
var LARGEST_UTF8_BYTES_PER_UNIT = 3;
|
|
684
|
+
function upperBoundBytes(value, budget, depth) {
|
|
685
|
+
if (value === null || value === void 0 || typeof value === "boolean") return 1;
|
|
686
|
+
if (typeof value === "number") return LARGEST_NUMBER_BYTES;
|
|
687
|
+
if (typeof value === "string") return LARGEST_HEADER_BYTES + value.length * LARGEST_UTF8_BYTES_PER_UNIT;
|
|
688
|
+
if (value instanceof Date) return LARGEST_TIMESTAMP_BYTES;
|
|
689
|
+
if (ArrayBuffer.isView(value)) return LARGEST_HEADER_BYTES + value.byteLength;
|
|
690
|
+
if (typeof value !== "object" || depth > MAX_DOCUMENT_NESTING_DEPTH) return LARGEST_NUMBER_BYTES;
|
|
691
|
+
let total = LARGEST_HEADER_BYTES;
|
|
692
|
+
if (Array.isArray(value)) {
|
|
693
|
+
for (const item of value) {
|
|
694
|
+
total += upperBoundBytes(item, budget - total, depth + 1);
|
|
695
|
+
if (total > budget) return total;
|
|
696
|
+
}
|
|
697
|
+
return total;
|
|
698
|
+
}
|
|
699
|
+
const record = value;
|
|
700
|
+
for (const key of Object.keys(record)) {
|
|
701
|
+
total += LARGEST_HEADER_BYTES + key.length * LARGEST_UTF8_BYTES_PER_UNIT;
|
|
702
|
+
total += upperBoundBytes(record[key], budget - total, depth + 1);
|
|
703
|
+
if (total > budget) return total;
|
|
704
|
+
}
|
|
705
|
+
return total;
|
|
706
|
+
}
|
|
707
|
+
function messagePackUpperBoundBytes(value) {
|
|
708
|
+
return upperBoundBytes(value, Number.POSITIVE_INFINITY, 0);
|
|
709
|
+
}
|
|
710
|
+
function encodedBytesOf(document) {
|
|
711
|
+
try {
|
|
712
|
+
return encodeDocument(document).byteLength;
|
|
713
|
+
} catch {
|
|
714
|
+
return null;
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
function requireDocumentWithinSizeLimit(document) {
|
|
718
|
+
if (upperBoundBytes(document, MAX_DOCUMENT_BYTES, 0) <= MAX_DOCUMENT_BYTES) return;
|
|
719
|
+
const encodedBytes = encodedBytesOf(document);
|
|
720
|
+
if (encodedBytes === null || encodedBytes <= MAX_DOCUMENT_BYTES) return;
|
|
721
|
+
throw new NarsilError(
|
|
722
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
723
|
+
`The document takes ${encodedBytes} bytes as MessagePack, vector values included, but a document may take at most ${MAX_DOCUMENT_BYTES}`,
|
|
724
|
+
{ bytes: encodedBytes, maximum: MAX_DOCUMENT_BYTES }
|
|
725
|
+
);
|
|
726
|
+
}
|
|
591
727
|
|
|
592
728
|
// src/schema/validator/vector-promotion.ts
|
|
593
729
|
var QUANTIZATION_MODES = /* @__PURE__ */ new Set(["osq8", "osq4", "osq2", "osq1", "none"]);
|
|
@@ -611,6 +747,13 @@ function validateVectorStorage(config, filesystemDurability) {
|
|
|
611
747
|
});
|
|
612
748
|
}
|
|
613
749
|
}
|
|
750
|
+
function raiseEarlierReleaseGraphSetting(config) {
|
|
751
|
+
if (typeof config !== "object" || config === null) return config;
|
|
752
|
+
const hnsw = config.hnswConfig;
|
|
753
|
+
if (typeof hnsw !== "object" || hnsw === null) return config;
|
|
754
|
+
if (hnsw.m !== EARLIER_RELEASE_MIN_M) return config;
|
|
755
|
+
return { ...config, hnswConfig: { ...hnsw, m: MIN_M } };
|
|
756
|
+
}
|
|
614
757
|
function validateVectorPromotion(config) {
|
|
615
758
|
if (config === void 0) return;
|
|
616
759
|
if (config.threshold !== void 0 && !isPositiveInteger(config.threshold)) {
|
|
@@ -626,8 +769,8 @@ function validateVectorPromotion(config) {
|
|
|
626
769
|
}
|
|
627
770
|
const hnsw = config.hnswConfig;
|
|
628
771
|
if (hnsw === void 0) return;
|
|
629
|
-
if (hnsw.m !== void 0 && (!
|
|
630
|
-
fail(`vectorPromotion.hnswConfig.m must be a
|
|
772
|
+
if (hnsw.m !== void 0 && (!Number.isInteger(hnsw.m) || hnsw.m < MIN_M || hnsw.m > MAX_M)) {
|
|
773
|
+
fail(`vectorPromotion.hnswConfig.m must be a whole number from ${MIN_M} to ${MAX_M}`, { m: hnsw.m });
|
|
631
774
|
}
|
|
632
775
|
if (hnsw.efConstruction !== void 0 && !isPositiveInteger(hnsw.efConstruction)) {
|
|
633
776
|
fail("vectorPromotion.hnswConfig.efConstruction must be a positive integer", {
|
|
@@ -658,4 +801,31 @@ var DEFAULT_LINEAR_ALPHA = 0.5;
|
|
|
658
801
|
var MIN_LINEAR_ALPHA = 0;
|
|
659
802
|
var MAX_LINEAR_ALPHA = 1;
|
|
660
803
|
|
|
661
|
-
|
|
804
|
+
// src/engine/constants.ts
|
|
805
|
+
var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
|
|
806
|
+
var BATCH_CHUNK_SIZE = 1e3;
|
|
807
|
+
var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
|
|
808
|
+
var DEFAULT_OFFSET = 0;
|
|
809
|
+
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
810
|
+
var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
|
|
811
|
+
var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
|
|
812
|
+
var MAX_INDEX_NAME_LENGTH = 255;
|
|
813
|
+
var MAX_DOC_ID_LENGTH = 512;
|
|
814
|
+
var EXPORT_CHUNK_BYTES = 1048576;
|
|
815
|
+
var MAX_EXPORT_LINE_BYTES = 117443592;
|
|
816
|
+
var MAX_INDEX_CONFIGURATION_BYTES = 16777216;
|
|
817
|
+
var IMPORT_BATCH_DOCUMENTS = 1e3;
|
|
818
|
+
var MAX_IMPORT_REFUSALS_REPORTED = 1e3;
|
|
819
|
+
var MIN_SWEEP_INTERVAL_MS = 100;
|
|
820
|
+
var MAX_SWEEP_INTERVAL_MS = 6e4;
|
|
821
|
+
var HEAP_PRESSURE_WARNING_FRACTION = 0.9;
|
|
822
|
+
var HEAP_PRESSURE_REARM_FRACTION = 0.8;
|
|
823
|
+
var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
|
|
824
|
+
var FOREIGN_STATS_TTL_MS = 6e4;
|
|
825
|
+
var DEFAULT_SUGGEST_LIMIT = 10;
|
|
826
|
+
var MAX_SUGGEST_LIMIT = 100;
|
|
827
|
+
var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
|
|
828
|
+
var SUGGEST_OVERSAMPLE_PADDING = 10;
|
|
829
|
+
var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
|
|
830
|
+
|
|
831
|
+
export { ABSOLUTE_WINDOWS_PATH_PATTERN, ANTIMERIDIAN_SPAN_DEGREES, BATCH_CHUNK_SIZE, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LIMIT, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_OFFSET, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, DEFAULT_SUGGEST_LIMIT, DEG_TO_RAD, EARTH_RADIUS_METERS, EXPORT_CHUNK_BYTES, FOREIGN_STATS_TTL_MS, FULL_TURN_DEGREES, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, IMPORT_BATCH_DOCUMENTS, INDEX_NAME_PATTERN, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_DOCUMENT_NESTING_DEPTH, MAX_DOC_ID_LENGTH, MAX_EXPORT_LINE_BYTES, MAX_IMPORT_REFUSALS_REPORTED, MAX_INDEX_CONFIGURATION_BYTES, MAX_INDEX_NAME_LENGTH, MAX_LATITUDE_DEGREES, MAX_LINEAR_ALPHA, MAX_LONGITUDE_DEGREES, MAX_NESTING_DEPTH, MAX_PATTERN_VALUE_LIMIT, MAX_PREFIX_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_STORED_VECTOR_DIMENSION, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MAX_TOLERANCE, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_LINEAR_ALPHA, MIN_PATTERN_VALUE_LIMIT, MIN_POLYGON_POINTS, MIN_RANK_CONSTANT, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, PREFIX_MAX_EXPANSIONS, PROTOTYPE_POLLUTION_KEYS, RADIUS_BOX_MARGIN_RADIANS, RADIUS_BOX_MARGIN_RATIO, RESULT_WINDOW, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, VECTOR_PATTERN, VINCENTY_CONVERGENCE_THRESHOLD, VINCENTY_MAX_ITERATIONS, encodeDocument, extractVectorFieldsFromSchema, flattenSchema, isGeopointOrVectorType, isLatitude, isListBase, isLongitude, isQuantizationMode, isSingleStringFieldType, isStorageMode, isStringValuedBase, isWordIndexedBase, isWordIndexedFieldType, messagePackUpperBoundBytes, parsedTypeOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireValidFieldTypes, schemaFieldsOf, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateVectorPromotion, validateVectorStorage, withSortableOption };
|