@delali/narsil 0.2.4-next.51 → 0.2.4-next.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -27
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-BWm4IITU.d.ts → capabilities-DlYw0Ov0.d.ts} +13 -2
- package/dist/{chunk-KMMJMDUA.mjs → chunk-3L4U6EJQ.mjs} +15 -10
- package/dist/chunk-4NVPQ6S5.mjs +366 -0
- package/dist/{chunk-CMVG2UEF.mjs → chunk-5T6AOR37.mjs} +56 -518
- package/dist/{chunk-ND34I3SV.mjs → chunk-BH5SYB5W.mjs} +3295 -2149
- package/dist/{chunk-XE6XR2OE.mjs → chunk-D2S642IV.mjs} +4 -2
- package/dist/{chunk-P4ELXVJQ.mjs → chunk-FRQVMFY7.mjs} +7 -1
- package/dist/chunk-IFBKLBGB.mjs +7 -0
- package/dist/chunk-L225SHNB.mjs +513 -0
- package/dist/{chunk-MHE7YWAU.mjs → chunk-NTZYA3SJ.mjs} +3 -3
- package/dist/{chunk-WYJXSLER.mjs → chunk-POWOTUKJ.mjs} +7 -1
- package/dist/{chunk-CV5H7XL7.mjs → chunk-RGS7FMCB.mjs} +53 -10
- package/dist/{chunk-ZHUQKZBE.mjs → chunk-TVT5V4E7.mjs} +101 -34
- package/dist/chunk-U2AL3SON.mjs +4198 -0
- package/dist/chunk-UTOFOJOR.mjs +135 -0
- package/dist/{chunk-BEIK3EOZ.mjs → chunk-XNQAIO74.mjs} +5523 -3847
- package/dist/chunk-Y3L5MDRV.mjs +96 -0
- package/dist/chunk-ZJ4CSAB2.mjs +13 -0
- package/dist/{client-B_X6_AbT.d.ts → client-CQUqt9Ir.d.ts} +13 -4
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +20 -14
- package/dist/{config-Czh2YNDi.d.ts → config-CNas6wJH.d.ts} +2 -2
- package/dist/{config-OON7IJ_B.d.ts → config-bXxnJrQc.d.ts} +80 -35
- package/dist/direct-executor-RBA5O7EN.mjs +12 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +32 -17
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-9PtFY50q.d.ts → engine-Dj4I91Jv.d.ts} +21 -8
- package/dist/index.browser.mjs +12962 -7786
- package/dist/index.d.ts +9 -21
- package/dist/index.mjs +14 -13
- package/dist/{memory-Bzd5xcEW.d.ts → memory-xrXOZMbw.d.ts} +68 -13
- package/dist/persistence/durability/checkpoint-worker.d.ts +138 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +32 -18
- package/dist/react.d.ts +11 -11
- package/dist/react.mjs +16 -4
- package/dist/{schema-BqRx9t4E.d.ts → schema-mLpXZBEh.d.ts} +43 -12
- package/dist/serialization/crc32-worker.mjs +2 -1
- package/dist/server.d.ts +13 -10
- package/dist/server.mjs +136 -47
- package/dist/simd-DW5AtJ6B.d.ts +41 -0
- package/dist/{tasks-DT7L1p6w.d.ts → tasks-Df73Tjgq.d.ts} +9 -5
- package/dist/{thread-PZLEO6JL.mjs → thread-5UP6MGED.mjs} +131 -43
- package/dist/{types-DXPxaXPO.d.ts → types-BTIVO8DI.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +292 -128
- package/dist/vector/search-worker.mjs +70 -54
- package/dist/workers/entry.mjs +30 -14
- package/package.json +52 -31
- package/dist/brute-force-DbHYCTc3.d.ts +0 -8
- package/dist/chunk-CBLS46IE.mjs +0 -10
- package/dist/chunk-PIWKZPP7.mjs +0 -64
- package/dist/chunk-PMBHH44R.mjs +0 -492
- package/dist/chunk-RP23PWMV.mjs +0 -11
- package/dist/chunk-X6YZO4MQ.mjs +0 -1572
- package/dist/direct-executor-OREXU72C.mjs +0 -12
- package/dist/vector/hnsw-build-worker.d.ts +0 -105
- package/dist/vector/hnsw-build-worker.mjs +0 -123
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DEFAULT_PAGE_SIZE } from './chunk-
|
|
1
|
+
import { DEFAULT_PAGE_SIZE } from './chunk-L225SHNB.mjs';
|
|
2
2
|
|
|
3
3
|
// src/engine/constants.ts
|
|
4
4
|
var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
|
|
@@ -6,6 +6,8 @@ var BATCH_CHUNK_SIZE = 1e3;
|
|
|
6
6
|
var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
|
|
7
7
|
var DEFAULT_OFFSET = 0;
|
|
8
8
|
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
9
|
+
var ABSOLUTE_WINDOWS_PATH_PATTERN = /^[a-zA-Z]:[\\/]/;
|
|
10
|
+
var MODULE_URL_SCHEME_PATTERN = /^(file|https?|data|node):/;
|
|
9
11
|
var MAX_INDEX_NAME_LENGTH = 255;
|
|
10
12
|
var MAX_DOC_ID_LENGTH = 512;
|
|
11
13
|
var MIN_SWEEP_INTERVAL_MS = 100;
|
|
@@ -20,4 +22,4 @@ var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
|
|
|
20
22
|
var SUGGEST_OVERSAMPLE_PADDING = 10;
|
|
21
23
|
var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
|
|
22
24
|
|
|
23
|
-
export { BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_INDEX_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING };
|
|
25
|
+
export { ABSOLUTE_WINDOWS_PATH_PATTERN, BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_INDEX_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, MODULE_URL_SCHEME_PATTERN, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING };
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// src/errors.ts
|
|
2
|
+
var NARSIL_ERROR_MARKER = /* @__PURE__ */ Symbol.for("@delali/narsil.NarsilError");
|
|
2
3
|
var ErrorCodes = {
|
|
3
4
|
SCHEMA_INVALID_TYPE: "SCHEMA_INVALID_TYPE",
|
|
4
5
|
SCHEMA_DEPTH_EXCEEDED: "SCHEMA_DEPTH_EXCEEDED",
|
|
@@ -16,6 +17,7 @@ var ErrorCodes = {
|
|
|
16
17
|
WORKER_CRASHED: "WORKER_CRASHED",
|
|
17
18
|
WORKER_BUSY: "WORKER_BUSY",
|
|
18
19
|
WORKER_TIMEOUT: "WORKER_TIMEOUT",
|
|
20
|
+
WORKER_ACTION_FAILED: "WORKER_ACTION_FAILED",
|
|
19
21
|
PERSISTENCE_SAVE_FAILED: "PERSISTENCE_SAVE_FAILED",
|
|
20
22
|
PERSISTENCE_LOAD_FAILED: "PERSISTENCE_LOAD_FAILED",
|
|
21
23
|
PERSISTENCE_DELETE_FAILED: "PERSISTENCE_DELETE_FAILED",
|
|
@@ -135,8 +137,12 @@ var NarsilError = class extends Error {
|
|
|
135
137
|
this.name = "NarsilError";
|
|
136
138
|
this.code = code;
|
|
137
139
|
this.details = details ?? {};
|
|
140
|
+
Object.defineProperty(this, NARSIL_ERROR_MARKER, { value: true });
|
|
138
141
|
}
|
|
139
142
|
};
|
|
143
|
+
function isNarsilError(value) {
|
|
144
|
+
return typeof value === "object" && value !== null && NARSIL_ERROR_MARKER in value;
|
|
145
|
+
}
|
|
140
146
|
function createNarsilError(code, message, details) {
|
|
141
147
|
return new NarsilError(code, message, details);
|
|
142
148
|
}
|
|
@@ -144,4 +150,4 @@ function describeError(error) {
|
|
|
144
150
|
return error instanceof Error ? error.message : String(error);
|
|
145
151
|
}
|
|
146
152
|
|
|
147
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, createNarsilError, describeError };
|
|
153
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, createNarsilError, describeError, isNarsilError };
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// src/serialization/constants.ts
|
|
2
|
+
var CHECKSUM_TIMEOUT_MS = 12e4;
|
|
3
|
+
var CHECKSUM_YIELD_CHUNK_BYTES = 4 * 1024 * 1024;
|
|
4
|
+
var NATIVE_CRC32_MIN_BYTES = 16;
|
|
5
|
+
var VALID_HNSW_METRICS = /* @__PURE__ */ new Set(["cosine", "dotProduct", "euclidean"]);
|
|
6
|
+
|
|
7
|
+
export { CHECKSUM_TIMEOUT_MS, CHECKSUM_YIELD_CHUNK_BYTES, NATIVE_CRC32_MIN_BYTES, VALID_HNSW_METRICS };
|
|
@@ -0,0 +1,513 @@
|
|
|
1
|
+
import { MAX_M } from './chunk-UTOFOJOR.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-FRQVMFY7.mjs';
|
|
3
|
+
|
|
4
|
+
// src/schema/constants.ts
|
|
5
|
+
var MAX_NESTING_DEPTH = 4;
|
|
6
|
+
var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
7
|
+
|
|
8
|
+
// src/schema/validator/shared.ts
|
|
9
|
+
var SORTABLE_TEXT_FIELD_TYPE = "string:sortable";
|
|
10
|
+
var SCALAR_FIELD_TYPES = /* @__PURE__ */ new Set([
|
|
11
|
+
"string",
|
|
12
|
+
SORTABLE_TEXT_FIELD_TYPE,
|
|
13
|
+
"number",
|
|
14
|
+
"boolean",
|
|
15
|
+
"enum",
|
|
16
|
+
"geopoint",
|
|
17
|
+
"string[]",
|
|
18
|
+
"number[]",
|
|
19
|
+
"boolean[]",
|
|
20
|
+
"enum[]"
|
|
21
|
+
]);
|
|
22
|
+
function isTextFieldType(fieldType) {
|
|
23
|
+
return fieldType === "string" || fieldType === SORTABLE_TEXT_FIELD_TYPE;
|
|
24
|
+
}
|
|
25
|
+
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
26
|
+
var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
|
|
27
|
+
var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
|
|
28
|
+
var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
29
|
+
function isPlainObject(value) {
|
|
30
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// src/schema/validator/schema.ts
|
|
34
|
+
function validateSchemaFields(schema, depth, prefix) {
|
|
35
|
+
if (depth > MAX_NESTING_DEPTH) {
|
|
36
|
+
throw new NarsilError(
|
|
37
|
+
ErrorCodes.SCHEMA_DEPTH_EXCEEDED,
|
|
38
|
+
`Schema nesting exceeds the maximum depth of ${MAX_NESTING_DEPTH} at "${prefix}"`,
|
|
39
|
+
{ path: prefix, maxDepth: MAX_NESTING_DEPTH }
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
43
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
44
|
+
if (PROTOTYPE_POLLUTION_KEYS.has(field)) {
|
|
45
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field name "${field}" is not allowed in a schema`, {
|
|
46
|
+
field,
|
|
47
|
+
path
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
if (!FIELD_NAME_PATTERN.test(field)) {
|
|
51
|
+
throw new NarsilError(
|
|
52
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
53
|
+
`Field name "${field}" contains characters that are not allowed; use letters, digits, and underscores only`,
|
|
54
|
+
{ field, path }
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
if (depth === 1 && RESERVED_ROOT_FIELDS.has(field)) {
|
|
58
|
+
throw new NarsilError(
|
|
59
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
60
|
+
`Field "${field}" is reserved and cannot be defined in a schema`,
|
|
61
|
+
{ field, path }
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
if (isPlainObject(type)) {
|
|
65
|
+
validateSchemaFields(type, depth + 1, path);
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
if (typeof type !== "string") {
|
|
69
|
+
throw new NarsilError(
|
|
70
|
+
ErrorCodes.SCHEMA_INVALID_TYPE,
|
|
71
|
+
`Field "${path}" has an invalid type definition: ${String(type)}`,
|
|
72
|
+
{
|
|
73
|
+
field: path,
|
|
74
|
+
type: String(type)
|
|
75
|
+
}
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
if (SCALAR_FIELD_TYPES.has(type)) continue;
|
|
79
|
+
const vectorMatch = VECTOR_PATTERN.exec(type);
|
|
80
|
+
if (vectorMatch) {
|
|
81
|
+
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
82
|
+
if (dimension <= 0) {
|
|
83
|
+
throw new NarsilError(
|
|
84
|
+
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
85
|
+
`Vector field "${path}" must have a positive dimension, got ${dimension}`,
|
|
86
|
+
{ field: path, dimension }
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, `Field "${path}" has an unsupported type: "${type}"`, {
|
|
92
|
+
field: path,
|
|
93
|
+
type
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
function validateSchema(schema) {
|
|
98
|
+
if (!isPlainObject(schema)) {
|
|
99
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must be a plain object", {
|
|
100
|
+
received: typeof schema
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
if (Object.keys(schema).length === 0) {
|
|
104
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, "Schema must define at least one field");
|
|
105
|
+
}
|
|
106
|
+
validateSchemaFields(schema, 1, "");
|
|
107
|
+
}
|
|
108
|
+
function flattenRecursive(schema, prefix, result) {
|
|
109
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
110
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
111
|
+
if (isPlainObject(type)) {
|
|
112
|
+
flattenRecursive(type, path, result);
|
|
113
|
+
} else {
|
|
114
|
+
result[path] = type;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
function flattenSchema(schema) {
|
|
119
|
+
const result = {};
|
|
120
|
+
flattenRecursive(schema, "", result);
|
|
121
|
+
return result;
|
|
122
|
+
}
|
|
123
|
+
function extractVectorFieldsFromSchema(schema) {
|
|
124
|
+
const flat = flattenSchema(schema);
|
|
125
|
+
const result = /* @__PURE__ */ new Map();
|
|
126
|
+
for (const [fieldPath, fieldType] of Object.entries(flat)) {
|
|
127
|
+
const match = VECTOR_PATTERN.exec(fieldType);
|
|
128
|
+
if (match) {
|
|
129
|
+
result.set(fieldPath, Number.parseInt(match[1], 10));
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return result;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// src/schema/validator/field-values.ts
|
|
136
|
+
function validateGeopoint(path, value) {
|
|
137
|
+
if (!isPlainObject(value)) {
|
|
138
|
+
throw new NarsilError(
|
|
139
|
+
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
140
|
+
`Field "${path}" expected a geopoint object with lat and lon`,
|
|
141
|
+
{
|
|
142
|
+
field: path,
|
|
143
|
+
received: Array.isArray(value) ? "array" : typeof value
|
|
144
|
+
}
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
if (typeof value.lat !== "number" || typeof value.lon !== "number" || Number.isNaN(value.lat) || Number.isNaN(value.lon)) {
|
|
148
|
+
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_GEOPOINT, `Field "${path}" geopoint requires numeric lat and lon`, {
|
|
149
|
+
field: path,
|
|
150
|
+
lat: Number.isNaN(value.lat) ? "NaN" : typeof value.lat,
|
|
151
|
+
lon: Number.isNaN(value.lon) ? "NaN" : typeof value.lon
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
if (value.lat < -90 || value.lat > 90) {
|
|
155
|
+
throw new NarsilError(
|
|
156
|
+
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
157
|
+
`Field "${path}" geopoint lat must be between -90 and 90`,
|
|
158
|
+
{
|
|
159
|
+
field: path,
|
|
160
|
+
lat: value.lat
|
|
161
|
+
}
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
if (value.lon < -180 || value.lon > 180) {
|
|
165
|
+
throw new NarsilError(
|
|
166
|
+
ErrorCodes.SCHEMA_INVALID_GEOPOINT,
|
|
167
|
+
`Field "${path}" geopoint lon must be between -180 and 180`,
|
|
168
|
+
{
|
|
169
|
+
field: path,
|
|
170
|
+
lon: value.lon
|
|
171
|
+
}
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
function validateVector(path, value, expectedDimension) {
|
|
176
|
+
const isArray = Array.isArray(value);
|
|
177
|
+
const isFloat32 = value instanceof Float32Array;
|
|
178
|
+
if (!isArray && !isFloat32) {
|
|
179
|
+
throw new NarsilError(
|
|
180
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
181
|
+
`Field "${path}" expected a vector (Array or Float32Array)`,
|
|
182
|
+
{ field: path, received: typeof value }
|
|
183
|
+
);
|
|
184
|
+
}
|
|
185
|
+
const arr = value;
|
|
186
|
+
if (arr.length !== expectedDimension) {
|
|
187
|
+
throw new NarsilError(
|
|
188
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
189
|
+
`Field "${path}" expected vector of dimension ${expectedDimension}, got ${arr.length}`,
|
|
190
|
+
{ field: path, expected: expectedDimension, received: arr.length }
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
for (let i = 0; i < arr.length; i++) {
|
|
194
|
+
if (typeof arr[i] !== "number" || Number.isNaN(arr[i])) {
|
|
195
|
+
throw new NarsilError(
|
|
196
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
197
|
+
`Field "${path}" vector element at index ${i} is not a valid number`,
|
|
198
|
+
{ field: path, index: i, value: arr[i] }
|
|
199
|
+
);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
function validateTypedArray(path, value, elementType) {
|
|
204
|
+
if (!Array.isArray(value)) {
|
|
205
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an array`, {
|
|
206
|
+
field: path,
|
|
207
|
+
received: typeof value
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
for (let i = 0; i < value.length; i++) {
|
|
211
|
+
const el = value[i];
|
|
212
|
+
if (typeof el !== elementType) {
|
|
213
|
+
throw new NarsilError(
|
|
214
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
215
|
+
`Field "${path}" array element at index ${i} expected ${elementType}, got ${typeof el}`,
|
|
216
|
+
{ field: path, index: i, expected: elementType, received: typeof el }
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
if (elementType === "number" && Number.isNaN(el)) {
|
|
220
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" array element at index ${i} is NaN`, {
|
|
221
|
+
field: path,
|
|
222
|
+
index: i
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
function validateFieldValue(path, value, type) {
|
|
228
|
+
switch (type) {
|
|
229
|
+
case "string":
|
|
230
|
+
case SORTABLE_TEXT_FIELD_TYPE:
|
|
231
|
+
if (typeof value !== "string") {
|
|
232
|
+
throw new NarsilError(
|
|
233
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
234
|
+
`Field "${path}" expected string, got ${typeof value}`,
|
|
235
|
+
{
|
|
236
|
+
field: path,
|
|
237
|
+
expected: "string",
|
|
238
|
+
received: typeof value
|
|
239
|
+
}
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
return;
|
|
243
|
+
case "number":
|
|
244
|
+
if (typeof value !== "number" || Number.isNaN(value)) {
|
|
245
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected a valid number`, {
|
|
246
|
+
field: path,
|
|
247
|
+
expected: "number",
|
|
248
|
+
received: Number.isNaN(value) ? "NaN" : typeof value
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
return;
|
|
252
|
+
case "boolean":
|
|
253
|
+
if (typeof value !== "boolean") {
|
|
254
|
+
throw new NarsilError(
|
|
255
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
256
|
+
`Field "${path}" expected boolean, got ${typeof value}`,
|
|
257
|
+
{
|
|
258
|
+
field: path,
|
|
259
|
+
expected: "boolean",
|
|
260
|
+
received: typeof value
|
|
261
|
+
}
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
return;
|
|
265
|
+
case "enum":
|
|
266
|
+
if (typeof value !== "string") {
|
|
267
|
+
throw new NarsilError(
|
|
268
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
269
|
+
`Field "${path}" expected enum (string), got ${typeof value}`,
|
|
270
|
+
{
|
|
271
|
+
field: path,
|
|
272
|
+
expected: "enum",
|
|
273
|
+
received: typeof value
|
|
274
|
+
}
|
|
275
|
+
);
|
|
276
|
+
}
|
|
277
|
+
return;
|
|
278
|
+
case "geopoint":
|
|
279
|
+
validateGeopoint(path, value);
|
|
280
|
+
return;
|
|
281
|
+
case "string[]":
|
|
282
|
+
validateTypedArray(path, value, "string");
|
|
283
|
+
return;
|
|
284
|
+
case "number[]":
|
|
285
|
+
validateTypedArray(path, value, "number");
|
|
286
|
+
return;
|
|
287
|
+
case "boolean[]":
|
|
288
|
+
validateTypedArray(path, value, "boolean");
|
|
289
|
+
return;
|
|
290
|
+
case "enum[]":
|
|
291
|
+
validateTypedArray(path, value, "string");
|
|
292
|
+
return;
|
|
293
|
+
default: {
|
|
294
|
+
const vectorMatch = VECTOR_PATTERN.exec(type);
|
|
295
|
+
if (vectorMatch) {
|
|
296
|
+
validateVector(path, value, Number.parseInt(vectorMatch[1], 10));
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// src/schema/validator/storable.ts
|
|
303
|
+
var RESERVED_OBJECT_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
304
|
+
function reject(path, reason) {
|
|
305
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path === "" ? "(document)" : path}" ${reason}`, {
|
|
306
|
+
field: path
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
function isPlainObject2(value) {
|
|
310
|
+
const proto = Object.getPrototypeOf(value);
|
|
311
|
+
return proto === Object.prototype || proto === null;
|
|
312
|
+
}
|
|
313
|
+
function walkStorable(value, path, depth, ancestors, skip) {
|
|
314
|
+
if (value === null || value === void 0) return;
|
|
315
|
+
const kind = typeof value;
|
|
316
|
+
if (kind === "string" || kind === "boolean") return;
|
|
317
|
+
if (kind === "number") return;
|
|
318
|
+
if (kind === "bigint" || kind === "symbol" || kind === "function") {
|
|
319
|
+
reject(path, `holds a ${kind}, which cannot be stored or recovered`);
|
|
320
|
+
}
|
|
321
|
+
const container = value;
|
|
322
|
+
if (container instanceof Date || container instanceof Uint8Array) return;
|
|
323
|
+
const isArray = Array.isArray(container);
|
|
324
|
+
if (!isArray && !isPlainObject2(container)) {
|
|
325
|
+
reject(
|
|
326
|
+
path,
|
|
327
|
+
"holds a value that does not survive storage; use plain objects, arrays, strings, numbers, booleans, dates, or Uint8Array bytes"
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
if (depth >= MAX_DOCUMENT_NESTING_DEPTH) {
|
|
331
|
+
reject(path, `nests deeper than ${MAX_DOCUMENT_NESTING_DEPTH} levels`);
|
|
332
|
+
}
|
|
333
|
+
if (ancestors.has(container)) {
|
|
334
|
+
reject(path, "refers back to one of its own ancestors");
|
|
335
|
+
}
|
|
336
|
+
ancestors.add(container);
|
|
337
|
+
if (isArray) {
|
|
338
|
+
const items = container;
|
|
339
|
+
for (let i = 0; i < items.length; i++) {
|
|
340
|
+
walkStorable(items[i], `${path}[${i}]`, depth + 1, ancestors, skip);
|
|
341
|
+
}
|
|
342
|
+
} else {
|
|
343
|
+
const record = container;
|
|
344
|
+
for (const key of Object.keys(record)) {
|
|
345
|
+
const keyPath = path === "" ? key : `${path}.${key}`;
|
|
346
|
+
if (RESERVED_OBJECT_KEYS.has(key)) {
|
|
347
|
+
reject(keyPath, "uses a reserved object key");
|
|
348
|
+
}
|
|
349
|
+
if (skip.has(keyPath)) continue;
|
|
350
|
+
walkStorable(record[key], keyPath, depth + 1, ancestors, skip);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
ancestors.delete(container);
|
|
354
|
+
}
|
|
355
|
+
function assertStorableDocument(document, skipFieldPaths) {
|
|
356
|
+
walkStorable(document, "", 0, /* @__PURE__ */ new Set(), skipFieldPaths ?? /* @__PURE__ */ new Set());
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
// src/schema/validator/document.ts
|
|
360
|
+
var vectorPathCache = /* @__PURE__ */ new WeakMap();
|
|
361
|
+
function vectorFieldPaths(schema) {
|
|
362
|
+
let paths = vectorPathCache.get(schema);
|
|
363
|
+
if (paths !== void 0) return paths;
|
|
364
|
+
paths = /* @__PURE__ */ new Set();
|
|
365
|
+
const collect = (level, prefix) => {
|
|
366
|
+
for (const [field, type] of Object.entries(level)) {
|
|
367
|
+
const path = prefix === "" ? field : `${prefix}.${field}`;
|
|
368
|
+
if (typeof type === "string" && VECTOR_PATTERN.test(type)) {
|
|
369
|
+
paths?.add(path);
|
|
370
|
+
} else if (isPlainObject(type)) {
|
|
371
|
+
collect(type, path);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
collect(schema, "");
|
|
376
|
+
vectorPathCache.set(schema, paths);
|
|
377
|
+
return paths;
|
|
378
|
+
}
|
|
379
|
+
function validateDocumentFields(doc, schema, prefix) {
|
|
380
|
+
for (const [field, type] of Object.entries(schema)) {
|
|
381
|
+
const path = prefix ? `${prefix}.${field}` : field;
|
|
382
|
+
const value = doc[field];
|
|
383
|
+
if (value === void 0 || value === null) continue;
|
|
384
|
+
if (isPlainObject(type)) {
|
|
385
|
+
if (!isPlainObject(value)) {
|
|
386
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, `Field "${path}" expected an object`, {
|
|
387
|
+
field: path,
|
|
388
|
+
received: Array.isArray(value) ? "array" : typeof value
|
|
389
|
+
});
|
|
390
|
+
}
|
|
391
|
+
validateDocumentFields(value, type, path);
|
|
392
|
+
continue;
|
|
393
|
+
}
|
|
394
|
+
validateFieldValue(path, value, type);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
function validateDocument(document, schema) {
|
|
398
|
+
if (!isPlainObject(document)) {
|
|
399
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document must be a plain object", {
|
|
400
|
+
received: typeof document
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
assertStorableDocument(document, vectorFieldPaths(schema));
|
|
404
|
+
validateDocumentFields(document, schema, "");
|
|
405
|
+
}
|
|
406
|
+
function collectExtraFields(doc, schema, prefix, extras) {
|
|
407
|
+
for (const key of Object.keys(doc)) {
|
|
408
|
+
if (key === "id") continue;
|
|
409
|
+
const path = prefix ? `${prefix}.${key}` : key;
|
|
410
|
+
if (!(key in schema)) {
|
|
411
|
+
extras.push(path);
|
|
412
|
+
continue;
|
|
413
|
+
}
|
|
414
|
+
const schemaType = schema[key];
|
|
415
|
+
if (isPlainObject(schemaType) && isPlainObject(doc[key])) {
|
|
416
|
+
collectExtraFields(doc[key], schemaType, path, extras);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
function validateDocumentStrict(document, schema) {
|
|
421
|
+
const extras = [];
|
|
422
|
+
collectExtraFields(document, schema, "", extras);
|
|
423
|
+
if (extras.length > 0) {
|
|
424
|
+
throw new NarsilError(
|
|
425
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
426
|
+
`Document contains fields not defined in schema: ${extras.join(", ")}`,
|
|
427
|
+
{ extraFields: extras }
|
|
428
|
+
);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
function resolveNestedValue(document, path) {
|
|
432
|
+
if (!path.includes(".")) return document[path];
|
|
433
|
+
const segments = path.split(".");
|
|
434
|
+
let current = document;
|
|
435
|
+
for (const segment of segments) {
|
|
436
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
437
|
+
current = current[segment];
|
|
438
|
+
}
|
|
439
|
+
return current;
|
|
440
|
+
}
|
|
441
|
+
function validateRequiredFields(document, required) {
|
|
442
|
+
if (required.length === 0) return;
|
|
443
|
+
for (const field of required) {
|
|
444
|
+
const value = resolveNestedValue(document, field);
|
|
445
|
+
if (value === void 0 || value === null) {
|
|
446
|
+
throw new NarsilError(ErrorCodes.DOC_MISSING_REQUIRED_FIELD, `Document is missing required field "${field}"`, {
|
|
447
|
+
field
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
// src/schema/validator/vector-promotion.ts
|
|
454
|
+
var QUANTIZATION_MODES = /* @__PURE__ */ new Set(["osq8", "osq4", "osq2", "osq1", "none"]);
|
|
455
|
+
var STORAGE_MODES = /* @__PURE__ */ new Set(["memory", "disk"]);
|
|
456
|
+
function fail(message, details) {
|
|
457
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, message, details);
|
|
458
|
+
}
|
|
459
|
+
function isPositiveInteger(value) {
|
|
460
|
+
return Number.isInteger(value) && value >= 1;
|
|
461
|
+
}
|
|
462
|
+
function isQuantizationMode(value) {
|
|
463
|
+
return typeof value === "string" && QUANTIZATION_MODES.has(value);
|
|
464
|
+
}
|
|
465
|
+
function isStorageMode(value) {
|
|
466
|
+
return typeof value === "string" && STORAGE_MODES.has(value);
|
|
467
|
+
}
|
|
468
|
+
function validateVectorStorage(config, filesystemDurability) {
|
|
469
|
+
if (config?.storage === "disk" && !filesystemDurability) {
|
|
470
|
+
fail("vectorPromotion.storage 'disk' needs filesystem durability, because a checkpoint file holds the vectors", {
|
|
471
|
+
storage: config.storage
|
|
472
|
+
});
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
function validateVectorPromotion(config) {
|
|
476
|
+
if (config === void 0) return;
|
|
477
|
+
if (config.threshold !== void 0 && !isPositiveInteger(config.threshold)) {
|
|
478
|
+
fail("vectorPromotion.threshold must be a positive integer", { threshold: config.threshold });
|
|
479
|
+
}
|
|
480
|
+
const quantization = config.quantization;
|
|
481
|
+
if (quantization !== void 0 && !isQuantizationMode(quantization)) {
|
|
482
|
+
fail("vectorPromotion.quantization must be 'osq8', 'osq4', 'osq2', 'osq1', or 'none'", { quantization });
|
|
483
|
+
}
|
|
484
|
+
const storage = config.storage;
|
|
485
|
+
if (storage !== void 0 && !isStorageMode(storage)) {
|
|
486
|
+
fail("vectorPromotion.storage must be 'memory' or 'disk'", { storage });
|
|
487
|
+
}
|
|
488
|
+
const hnsw = config.hnswConfig;
|
|
489
|
+
if (hnsw === void 0) return;
|
|
490
|
+
if (hnsw.m !== void 0 && (!isPositiveInteger(hnsw.m) || hnsw.m > MAX_M)) {
|
|
491
|
+
fail(`vectorPromotion.hnswConfig.m must be a positive integer no greater than ${MAX_M}`, { m: hnsw.m });
|
|
492
|
+
}
|
|
493
|
+
if (hnsw.efConstruction !== void 0 && !isPositiveInteger(hnsw.efConstruction)) {
|
|
494
|
+
fail("vectorPromotion.hnswConfig.efConstruction must be a positive integer", {
|
|
495
|
+
efConstruction: hnsw.efConstruction
|
|
496
|
+
});
|
|
497
|
+
}
|
|
498
|
+
const metric = hnsw.metric;
|
|
499
|
+
if (metric !== void 0 && metric !== "cosine" && metric !== "dotProduct" && metric !== "euclidean") {
|
|
500
|
+
fail("vectorPromotion.hnswConfig.metric must be 'cosine', 'dotProduct', or 'euclidean'", { metric });
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// src/search/constants.ts
|
|
505
|
+
var RESULT_WINDOW = 1e4;
|
|
506
|
+
var DEFAULT_PAGE_SIZE = 10;
|
|
507
|
+
var MAX_CURSOR_LENGTH = 40960;
|
|
508
|
+
var MAX_SORT_FIELDS = 8;
|
|
509
|
+
var MAX_SORT_FIELD_NAME_LENGTH = 255;
|
|
510
|
+
var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
|
|
511
|
+
var PREFIX_MAX_EXPANSIONS = 50;
|
|
512
|
+
|
|
513
|
+
export { DEFAULT_PAGE_SIZE, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_NESTING_DEPTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, PREFIX_MAX_EXPANSIONS, PROTOTYPE_POLLUTION_KEYS, RESULT_WINDOW, SORTABLE_TEXT_FIELD_TYPE, VECTOR_PATTERN, extractVectorFieldsFromSchema, flattenSchema, isQuantizationMode, isStorageMode, isTextFieldType, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateVectorPromotion, validateVectorStorage };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-
|
|
2
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-
|
|
3
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-4NVPQ6S5.mjs';
|
|
2
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN } from './chunk-D2S642IV.mjs';
|
|
3
|
+
import { NarsilError, ErrorCodes } from './chunk-FRQVMFY7.mjs';
|
|
4
4
|
import { encode, decode } from '@msgpack/msgpack';
|
|
5
5
|
|
|
6
6
|
var INDEX_CONFIG_PREFIX = "_narsil/index/";
|
|
@@ -11,6 +11,7 @@ var KNOWN_ACTION_TYPES = [
|
|
|
11
11
|
"createIndex",
|
|
12
12
|
"loadVectorCopy",
|
|
13
13
|
"dropVectorCopy",
|
|
14
|
+
"insertVectorOrdinals",
|
|
14
15
|
"serveRequests",
|
|
15
16
|
"stopServing",
|
|
16
17
|
"dropIndex",
|
|
@@ -57,4 +58,9 @@ function isValidWorkerResponse(msg) {
|
|
|
57
58
|
return true;
|
|
58
59
|
}
|
|
59
60
|
|
|
60
|
-
|
|
61
|
+
// src/workers/thread-slot.ts
|
|
62
|
+
function threadSlotOfWorker(workerId) {
|
|
63
|
+
return workerId + 1;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export { createRequestId, isValidWorkerAction, isValidWorkerResponse, threadSlotOfWorker };
|