@delali/narsil 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +777 -130
- package/dist/{chunk-GQEHCZG7.mjs → chunk-CU6BVZEU.mjs} +643 -447
- package/dist/{chunk-FS7XLIUS.mjs → chunk-Q4OVNESD.mjs} +52 -4
- package/dist/{config-cQLlYR3U.d.ts → config-DX--2Owg.d.ts} +3 -1
- package/dist/{direct-executor-FX56DYD3.mjs → direct-executor-6FOKJKLD.mjs} +1 -1
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +2 -2
- package/dist/embeddings/openai.mjs +5 -0
- package/dist/index.browser.mjs +476 -244
- package/dist/index.d.ts +3 -3
- package/dist/index.mjs +2 -2
- package/dist/{narsil-ChTdkwPu.d.ts → narsil-B3jKyXCr.d.ts} +6 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +2 -2
- package/dist/{schema-BVaw7nqz.d.ts → schema-C3VhNvfw.d.ts} +3 -1
- package/dist/server.d.ts +3 -3
- package/dist/server.mjs +11 -12
- package/dist/{types-Bt7c4hDF.d.ts → types-CTVVm4GG.d.ts} +1 -1
- package/dist/workers/entry.mjs +1 -1
- package/package.json +11 -2
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter,
|
|
1
|
+
import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, createPartitionManager, createVectorIndex, validateSchema, deserializePayloadV2, tokenize, fanOutQuery, validateRequiredFields, validateVectorPromotion, createPartitionIndex, flattenSchema, fnv1a, detectRuntime, isNodeMainThread, spawnNodeWorker, kWayMerge } from './chunk-Q4OVNESD.mjs';
|
|
2
2
|
import { crc32, crc32Init, crc32Update, crc32Final } from './chunk-KMMJMDUA.mjs';
|
|
3
3
|
import { ErrorCodes, NarsilError } from './chunk-ALB35PRQ.mjs';
|
|
4
4
|
import { generateId, createRequestId, isValidWorkerResponse } from './chunk-HRABJLPB.mjs';
|
|
@@ -132,6 +132,123 @@ function createPluginRegistry() {
|
|
|
132
132
|
};
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
+
// src/schema/embedding-validator.ts
|
|
136
|
+
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
137
|
+
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
138
|
+
const fieldEntries = Object.keys(embeddingConfig.fields);
|
|
139
|
+
if (fieldEntries.length === 0) {
|
|
140
|
+
throw new NarsilError(
|
|
141
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
142
|
+
"Embedding config must define at least one field mapping"
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
if (typeof embeddingConfig.adapter === "string") {
|
|
146
|
+
throw new NarsilError(
|
|
147
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
148
|
+
`Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
|
|
149
|
+
{ adapter: embeddingConfig.adapter }
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
|
|
153
|
+
if (!resolvedAdapter) {
|
|
154
|
+
throw new NarsilError(
|
|
155
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
156
|
+
"Embedding config requires an adapter (set on the index or the instance)"
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
|
|
160
|
+
throw new NarsilError(
|
|
161
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
162
|
+
`Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
const flatFields = flattenSchema(schema);
|
|
166
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
167
|
+
const targetType = flatFields[targetField];
|
|
168
|
+
if (targetType === void 0) {
|
|
169
|
+
throw new NarsilError(
|
|
170
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
171
|
+
`Embedding target field "${targetField}" does not exist in the schema`,
|
|
172
|
+
{ field: targetField }
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
|
|
176
|
+
if (!vectorMatch) {
|
|
177
|
+
throw new NarsilError(
|
|
178
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
179
|
+
`Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
|
|
180
|
+
{ field: targetField, type: targetType }
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
const schemaDimension = Number.parseInt(vectorMatch[1], 10);
|
|
184
|
+
if (schemaDimension !== resolvedAdapter.dimensions) {
|
|
185
|
+
throw new NarsilError(
|
|
186
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
187
|
+
`Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
|
|
188
|
+
{ field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
192
|
+
if (sources.length === 0) {
|
|
193
|
+
throw new NarsilError(
|
|
194
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
195
|
+
`Embedding target field "${targetField}" must have at least one source field`,
|
|
196
|
+
{ field: targetField }
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
for (const source of sources) {
|
|
200
|
+
const sourceType = flatFields[source];
|
|
201
|
+
if (sourceType === void 0) {
|
|
202
|
+
throw new NarsilError(
|
|
203
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
204
|
+
`Embedding source field "${source}" does not exist in the schema`,
|
|
205
|
+
{ field: source, targetField }
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
if (sourceType !== "string") {
|
|
209
|
+
throw new NarsilError(
|
|
210
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
211
|
+
`Embedding source field "${source}" must be a string type, got "${sourceType}"`,
|
|
212
|
+
{ field: source, type: sourceType, targetField }
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return resolvedAdapter;
|
|
218
|
+
}
|
|
219
|
+
function validateRegisteredAdapter(name, adapter) {
|
|
220
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
221
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
|
|
222
|
+
}
|
|
223
|
+
if (typeof adapter?.embed !== "function") {
|
|
224
|
+
throw new NarsilError(
|
|
225
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
226
|
+
`Embedding adapter "${name}" must provide an embed function`,
|
|
227
|
+
{ adapter: name }
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
|
|
231
|
+
throw new NarsilError(
|
|
232
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
233
|
+
`Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
|
|
234
|
+
{ adapter: name, dimensions: adapter.dimensions }
|
|
235
|
+
);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
function validateRequiredFieldsInSchema(required, schema) {
|
|
239
|
+
if (required.length === 0) return;
|
|
240
|
+
const flatFields = flattenSchema(schema);
|
|
241
|
+
for (const field of required) {
|
|
242
|
+
if (flatFields[field] === void 0) {
|
|
243
|
+
throw new NarsilError(
|
|
244
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
245
|
+
`Required field "${field}" does not exist in the schema`,
|
|
246
|
+
{ field }
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
135
252
|
// src/workers/promoter.ts
|
|
136
253
|
var DEFAULT_PER_INDEX_THRESHOLD = 1e4;
|
|
137
254
|
var DEFAULT_TOTAL_THRESHOLD = 5e4;
|
|
@@ -170,7 +287,7 @@ function createExecutionPromoter(config) {
|
|
|
170
287
|
return { check, markPromoted, isPromoted };
|
|
171
288
|
}
|
|
172
289
|
|
|
173
|
-
// src/persistence/durability/
|
|
290
|
+
// src/persistence/durability/node-fs.ts
|
|
174
291
|
var cachedFs = null;
|
|
175
292
|
var cachedPath = null;
|
|
176
293
|
async function getFs() {
|
|
@@ -185,6 +302,8 @@ async function getPath() {
|
|
|
185
302
|
}
|
|
186
303
|
return cachedPath;
|
|
187
304
|
}
|
|
305
|
+
|
|
306
|
+
// src/persistence/durability/durable-filesystem.ts
|
|
188
307
|
function validateRelativeKey(key) {
|
|
189
308
|
if (key.length === 0) {
|
|
190
309
|
throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: empty");
|
|
@@ -452,12 +571,7 @@ var mainThreadResolved = false;
|
|
|
452
571
|
var onMainThread = true;
|
|
453
572
|
async function isOnMainThread() {
|
|
454
573
|
if (!mainThreadResolved) {
|
|
455
|
-
|
|
456
|
-
const workerThreads = await import('worker_threads');
|
|
457
|
-
onMainThread = workerThreads.isMainThread;
|
|
458
|
-
} catch {
|
|
459
|
-
onMainThread = true;
|
|
460
|
-
}
|
|
574
|
+
onMainThread = await isNodeMainThread();
|
|
461
575
|
mainThreadResolved = true;
|
|
462
576
|
}
|
|
463
577
|
return onMainThread;
|
|
@@ -472,8 +586,7 @@ function resolveWorkerEntryPoint() {
|
|
|
472
586
|
}
|
|
473
587
|
async function spawnWorker() {
|
|
474
588
|
try {
|
|
475
|
-
|
|
476
|
-
return new workerThreads.Worker(new URL(resolveWorkerEntryPoint()));
|
|
589
|
+
return await spawnNodeWorker(new URL(resolveWorkerEntryPoint()));
|
|
477
590
|
} catch {
|
|
478
591
|
return null;
|
|
479
592
|
}
|
|
@@ -739,6 +852,9 @@ function metadataToWire(meta) {
|
|
|
739
852
|
if (meta.vectorFields) {
|
|
740
853
|
wire.vector_fields = meta.vectorFields;
|
|
741
854
|
}
|
|
855
|
+
if (meta.embedding) {
|
|
856
|
+
wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
|
|
857
|
+
}
|
|
742
858
|
return wire;
|
|
743
859
|
}
|
|
744
860
|
function wireToMetadata(raw) {
|
|
@@ -754,6 +870,9 @@ function wireToMetadata(raw) {
|
|
|
754
870
|
if (raw.vector_fields) {
|
|
755
871
|
meta.vectorFields = raw.vector_fields;
|
|
756
872
|
}
|
|
873
|
+
if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
|
|
874
|
+
meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
|
|
875
|
+
}
|
|
757
876
|
return meta;
|
|
758
877
|
}
|
|
759
878
|
function serializeMetadata(meta) {
|
|
@@ -1146,8 +1265,10 @@ function resolveWorkerEntryPoint2() {
|
|
|
1146
1265
|
}
|
|
1147
1266
|
async function spawnWorker2() {
|
|
1148
1267
|
try {
|
|
1149
|
-
const
|
|
1150
|
-
|
|
1268
|
+
const worker = await spawnNodeWorker(new URL(resolveWorkerEntryPoint2()));
|
|
1269
|
+
if (worker === null) {
|
|
1270
|
+
return null;
|
|
1271
|
+
}
|
|
1151
1272
|
if (typeof worker.unref === "function") {
|
|
1152
1273
|
worker.unref();
|
|
1153
1274
|
}
|
|
@@ -2115,11 +2236,15 @@ function unflattenSchema(flat) {
|
|
|
2115
2236
|
return root;
|
|
2116
2237
|
}
|
|
2117
2238
|
function reconstructSchemaFromMetadata(metadata) {
|
|
2118
|
-
|
|
2239
|
+
const config = {
|
|
2119
2240
|
schema: unflattenSchema(metadata.schema),
|
|
2120
2241
|
language: metadata.language,
|
|
2121
2242
|
bm25: { k1: metadata.bm25Params.k1, b: metadata.bm25Params.b }
|
|
2122
2243
|
};
|
|
2244
|
+
if (metadata.embedding) {
|
|
2245
|
+
config.embedding = metadata.embedding.adapter !== void 0 ? { adapter: metadata.embedding.adapter, fields: metadata.embedding.fields } : { fields: metadata.embedding.fields };
|
|
2246
|
+
}
|
|
2247
|
+
return config;
|
|
2123
2248
|
}
|
|
2124
2249
|
|
|
2125
2250
|
// src/persistence/durability/segment/build-segment.ts
|
|
@@ -3693,7 +3818,7 @@ function buildMetadata(indexName, hooks) {
|
|
|
3693
3818
|
if (config === void 0 || manager === void 0) {
|
|
3694
3819
|
return void 0;
|
|
3695
3820
|
}
|
|
3696
|
-
|
|
3821
|
+
const metadata = {
|
|
3697
3822
|
indexName,
|
|
3698
3823
|
schema: config.schema,
|
|
3699
3824
|
language: config.language,
|
|
@@ -3702,6 +3827,10 @@ function buildMetadata(indexName, hooks) {
|
|
|
3702
3827
|
createdAt: Date.now(),
|
|
3703
3828
|
engineVersion: VERSION
|
|
3704
3829
|
};
|
|
3830
|
+
if (config.embedding) {
|
|
3831
|
+
metadata.embedding = config.embedding;
|
|
3832
|
+
}
|
|
3833
|
+
return metadata;
|
|
3705
3834
|
}
|
|
3706
3835
|
function createDurabilityIntegration(tier, hooks) {
|
|
3707
3836
|
const managerHooks = {
|
|
@@ -4211,11 +4340,16 @@ function createDurabilityFromConfig(config, wiring) {
|
|
|
4211
4340
|
if (entry === void 0) {
|
|
4212
4341
|
return void 0;
|
|
4213
4342
|
}
|
|
4343
|
+
const embedding = entry.config.embedding ? {
|
|
4344
|
+
fields: entry.config.embedding.fields,
|
|
4345
|
+
...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
|
|
4346
|
+
} : void 0;
|
|
4214
4347
|
return {
|
|
4215
4348
|
schema: flattenSchema(entry.config.schema),
|
|
4216
4349
|
language: entry.language.name,
|
|
4217
4350
|
k1: entry.config.bm25?.k1 ?? 1.2,
|
|
4218
|
-
b: entry.config.bm25?.b ?? 0.75
|
|
4351
|
+
b: entry.config.bm25?.b ?? 0.75,
|
|
4352
|
+
...embedding !== void 0 ? { embedding } : {}
|
|
4219
4353
|
};
|
|
4220
4354
|
},
|
|
4221
4355
|
createIndexFromMetadata: wiring.createIndexFromMetadata,
|
|
@@ -4234,6 +4368,13 @@ function createEngineCore(config) {
|
|
|
4234
4368
|
}
|
|
4235
4369
|
const idGenerator = config?.idGenerator ?? generateId;
|
|
4236
4370
|
const indexRegistry = /* @__PURE__ */ new Map();
|
|
4371
|
+
const embeddingAdapters = /* @__PURE__ */ new Map();
|
|
4372
|
+
if (config?.embeddingAdapters) {
|
|
4373
|
+
for (const [name, adapter] of Object.entries(config.embeddingAdapters)) {
|
|
4374
|
+
validateRegisteredAdapter(name, adapter);
|
|
4375
|
+
embeddingAdapters.set(name, adapter);
|
|
4376
|
+
}
|
|
4377
|
+
}
|
|
4237
4378
|
const eventHandlers = /* @__PURE__ */ new Map();
|
|
4238
4379
|
const shutdownState = { isShutdown: false };
|
|
4239
4380
|
const abortController = new AbortController();
|
|
@@ -4283,11 +4424,35 @@ function createEngineCore(config) {
|
|
|
4283
4424
|
}
|
|
4284
4425
|
const indexConfig = reconstructSchemaFromMetadata(metadata);
|
|
4285
4426
|
const language = getLanguage(indexConfig.language ?? "english");
|
|
4427
|
+
const adapterName = metadata.embedding?.adapter ?? null;
|
|
4428
|
+
let embeddingAdapter = null;
|
|
4429
|
+
if (metadata.embedding) {
|
|
4430
|
+
const candidate = adapterName !== null ? embeddingAdapters.get(adapterName) ?? null : config?.embedding ?? null;
|
|
4431
|
+
if (candidate) {
|
|
4432
|
+
try {
|
|
4433
|
+
validateEmbeddingConfig(
|
|
4434
|
+
{ fields: metadata.embedding.fields, adapter: candidate },
|
|
4435
|
+
indexConfig.schema,
|
|
4436
|
+
void 0
|
|
4437
|
+
);
|
|
4438
|
+
} catch (err) {
|
|
4439
|
+
if (err instanceof NarsilError) {
|
|
4440
|
+
throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
|
|
4441
|
+
indexName: metadata.indexName,
|
|
4442
|
+
adapter: adapterName ?? void 0
|
|
4443
|
+
});
|
|
4444
|
+
}
|
|
4445
|
+
throw err;
|
|
4446
|
+
}
|
|
4447
|
+
embeddingAdapter = candidate;
|
|
4448
|
+
}
|
|
4449
|
+
}
|
|
4286
4450
|
executor.createIndex(metadata.indexName, indexConfig, language);
|
|
4287
4451
|
indexRegistry.set(metadata.indexName, {
|
|
4288
4452
|
config: indexConfig,
|
|
4289
4453
|
language,
|
|
4290
|
-
embeddingAdapter
|
|
4454
|
+
embeddingAdapter,
|
|
4455
|
+
embeddingAdapterName: adapterName,
|
|
4291
4456
|
vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
|
|
4292
4457
|
});
|
|
4293
4458
|
}
|
|
@@ -4330,6 +4495,7 @@ function createEngineCore(config) {
|
|
|
4330
4495
|
durability,
|
|
4331
4496
|
idGenerator,
|
|
4332
4497
|
indexRegistry,
|
|
4498
|
+
embeddingAdapters,
|
|
4333
4499
|
eventHandlers,
|
|
4334
4500
|
shutdownState,
|
|
4335
4501
|
abortController,
|
|
@@ -4347,285 +4513,50 @@ function createEngineCore(config) {
|
|
|
4347
4513
|
rebalanceCtx
|
|
4348
4514
|
};
|
|
4349
4515
|
}
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
}
|
|
4361
|
-
}
|
|
4362
|
-
if (durability) {
|
|
4363
|
-
await durability.manager.shutdown();
|
|
4364
|
-
}
|
|
4365
|
-
const adaptersToShutdown = /* @__PURE__ */ new Set();
|
|
4366
|
-
for (const [, entry] of indexRegistry) {
|
|
4367
|
-
if (entry.embeddingAdapter?.shutdown) {
|
|
4368
|
-
adaptersToShutdown.add(entry.embeddingAdapter);
|
|
4369
|
-
}
|
|
4370
|
-
}
|
|
4371
|
-
for (const adapter of adaptersToShutdown) {
|
|
4372
|
-
try {
|
|
4373
|
-
await adapter.shutdown?.();
|
|
4374
|
-
} catch {
|
|
4375
|
-
}
|
|
4516
|
+
var INDEX_CONFIG_PREFIX = "_narsil/index/";
|
|
4517
|
+
var INDEX_CONFIG_SUFFIX = "/config";
|
|
4518
|
+
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
4519
|
+
var MAX_INDEX_NAME_LENGTH = 255;
|
|
4520
|
+
var MAX_PARTITION_COUNT = 65536;
|
|
4521
|
+
var MAX_REPLICATION_FACTOR = 255;
|
|
4522
|
+
function truncateForDisplay(value) {
|
|
4523
|
+
const str = String(value);
|
|
4524
|
+
if (str.length > 100) {
|
|
4525
|
+
return `${str.slice(0, 100)}...`;
|
|
4376
4526
|
}
|
|
4377
|
-
|
|
4378
|
-
await executor.shutdown();
|
|
4379
|
-
eventHandlers.clear();
|
|
4380
|
-
indexRegistry.clear();
|
|
4527
|
+
return str;
|
|
4381
4528
|
}
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
function validateVector(vector, targetField, expectedDimensions) {
|
|
4385
|
-
if (!(vector instanceof Float32Array)) {
|
|
4529
|
+
function validateIndexName(indexName) {
|
|
4530
|
+
if (indexName.length === 0 || indexName.length > MAX_INDEX_NAME_LENGTH) {
|
|
4386
4531
|
throw new NarsilError(
|
|
4387
|
-
ErrorCodes.
|
|
4388
|
-
`
|
|
4389
|
-
{
|
|
4532
|
+
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4533
|
+
`Index name must be between 1 and ${MAX_INDEX_NAME_LENGTH} characters`,
|
|
4534
|
+
{ indexName: truncateForDisplay(indexName) }
|
|
4390
4535
|
);
|
|
4391
4536
|
}
|
|
4392
|
-
if (
|
|
4537
|
+
if (indexName.includes("\0") || indexName.includes("/") || indexName.includes("\\") || indexName.includes("..")) {
|
|
4393
4538
|
throw new NarsilError(
|
|
4394
|
-
ErrorCodes.
|
|
4395
|
-
|
|
4396
|
-
{
|
|
4539
|
+
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4540
|
+
"Index name contains forbidden characters (/, \\, .., or null bytes)",
|
|
4541
|
+
{ indexName: truncateForDisplay(indexName) }
|
|
4397
4542
|
);
|
|
4398
4543
|
}
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
);
|
|
4406
|
-
}
|
|
4544
|
+
if (!INDEX_NAME_PATTERN.test(indexName)) {
|
|
4545
|
+
throw new NarsilError(
|
|
4546
|
+
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4547
|
+
"Index name must start with an alphanumeric character and contain only alphanumeric characters, hyphens, underscores, or dots",
|
|
4548
|
+
{ indexName: truncateForDisplay(indexName) }
|
|
4549
|
+
);
|
|
4407
4550
|
}
|
|
4408
|
-
return vector;
|
|
4409
4551
|
}
|
|
4410
|
-
function
|
|
4411
|
-
|
|
4412
|
-
const segments = path.split(".");
|
|
4413
|
-
let current = document;
|
|
4414
|
-
for (const segment of segments) {
|
|
4415
|
-
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
4416
|
-
current = current[segment];
|
|
4417
|
-
}
|
|
4418
|
-
return current;
|
|
4552
|
+
function isValidInteger(value) {
|
|
4553
|
+
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
4419
4554
|
}
|
|
4420
|
-
function
|
|
4421
|
-
|
|
4422
|
-
document[path] = value;
|
|
4423
|
-
return;
|
|
4424
|
-
}
|
|
4425
|
-
const segments = path.split(".");
|
|
4426
|
-
let current = document;
|
|
4427
|
-
for (let i = 0; i < segments.length - 1; i++) {
|
|
4428
|
-
let next = current[segments[i]];
|
|
4429
|
-
if (next === null || next === void 0 || typeof next !== "object") {
|
|
4430
|
-
next = {};
|
|
4431
|
-
current[segments[i]] = next;
|
|
4432
|
-
}
|
|
4433
|
-
current = next;
|
|
4434
|
-
}
|
|
4435
|
-
current[segments[segments.length - 1]] = value;
|
|
4555
|
+
function indexConfigKey(indexName) {
|
|
4556
|
+
return `${INDEX_CONFIG_PREFIX}${indexName}${INDEX_CONFIG_SUFFIX}`;
|
|
4436
4557
|
}
|
|
4437
|
-
function
|
|
4438
|
-
|
|
4439
|
-
delete document[path];
|
|
4440
|
-
return;
|
|
4441
|
-
}
|
|
4442
|
-
const segments = path.split(".");
|
|
4443
|
-
let current = document;
|
|
4444
|
-
for (let i = 0; i < segments.length - 1; i++) {
|
|
4445
|
-
const next = current[segments[i]];
|
|
4446
|
-
if (next === null || next === void 0 || typeof next !== "object") return;
|
|
4447
|
-
current = next;
|
|
4448
|
-
}
|
|
4449
|
-
delete current[segments[segments.length - 1]];
|
|
4450
|
-
}
|
|
4451
|
-
function collectSourceText(document, sourceFields, targetField, documentIndex) {
|
|
4452
|
-
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
4453
|
-
const parts = [];
|
|
4454
|
-
for (const source of sources) {
|
|
4455
|
-
const value = resolveFieldValue(document, source);
|
|
4456
|
-
if (value === void 0 || value === null || value === "") continue;
|
|
4457
|
-
if (typeof value !== "string") {
|
|
4458
|
-
throw new NarsilError(
|
|
4459
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
4460
|
-
`Embedding source field "${source}" must be a string, got ${typeof value}`,
|
|
4461
|
-
{ field: source, targetField }
|
|
4462
|
-
);
|
|
4463
|
-
}
|
|
4464
|
-
parts.push(value);
|
|
4465
|
-
}
|
|
4466
|
-
if (parts.length === 0) {
|
|
4467
|
-
const details = { field: targetField };
|
|
4468
|
-
if (documentIndex !== void 0) {
|
|
4469
|
-
details.documentIndex = documentIndex;
|
|
4470
|
-
}
|
|
4471
|
-
throw new NarsilError(
|
|
4472
|
-
ErrorCodes.EMBEDDING_NO_SOURCE,
|
|
4473
|
-
`All source fields for embedding target "${targetField}" are missing or empty`,
|
|
4474
|
-
details
|
|
4475
|
-
);
|
|
4476
|
-
}
|
|
4477
|
-
return parts.join("\n");
|
|
4478
|
-
}
|
|
4479
|
-
function wrapAdapterError(err, targetField) {
|
|
4480
|
-
if (err instanceof NarsilError) return err;
|
|
4481
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
4482
|
-
return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
|
|
4483
|
-
field: targetField,
|
|
4484
|
-
cause: message
|
|
4485
|
-
});
|
|
4486
|
-
}
|
|
4487
|
-
async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
|
|
4488
|
-
const embeddedFields = /* @__PURE__ */ new Set();
|
|
4489
|
-
try {
|
|
4490
|
-
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
4491
|
-
const existing = resolveFieldValue(document, targetField);
|
|
4492
|
-
if (existing !== void 0 && existing !== null) continue;
|
|
4493
|
-
const text = collectSourceText(document, sourceFields, targetField);
|
|
4494
|
-
let vector;
|
|
4495
|
-
try {
|
|
4496
|
-
const raw = await adapter.embed(text, "document", signal);
|
|
4497
|
-
vector = validateVector(raw, targetField, adapter.dimensions);
|
|
4498
|
-
} catch (err) {
|
|
4499
|
-
throw wrapAdapterError(err, targetField);
|
|
4500
|
-
}
|
|
4501
|
-
setFieldValue(document, targetField, vector);
|
|
4502
|
-
embeddedFields.add(targetField);
|
|
4503
|
-
}
|
|
4504
|
-
} catch (err) {
|
|
4505
|
-
for (const field of embeddedFields) {
|
|
4506
|
-
deleteFieldValue(document, field);
|
|
4507
|
-
}
|
|
4508
|
-
throw err;
|
|
4509
|
-
}
|
|
4510
|
-
return embeddedFields;
|
|
4511
|
-
}
|
|
4512
|
-
var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([ErrorCodes.EMBEDDING_NO_SOURCE, ErrorCodes.DOC_VALIDATION_FAILED]);
|
|
4513
|
-
function isDocumentSourceError(err) {
|
|
4514
|
-
return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
|
|
4515
|
-
}
|
|
4516
|
-
async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
|
|
4517
|
-
const embedded = /* @__PURE__ */ new Map();
|
|
4518
|
-
const failed = /* @__PURE__ */ new Map();
|
|
4519
|
-
try {
|
|
4520
|
-
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
4521
|
-
const needsEmbedding = [];
|
|
4522
|
-
for (let i = 0; i < documents.length; i++) {
|
|
4523
|
-
if (failed.has(i)) continue;
|
|
4524
|
-
const doc = documents[i];
|
|
4525
|
-
const existing = resolveFieldValue(doc, targetField);
|
|
4526
|
-
if (existing !== void 0 && existing !== null) continue;
|
|
4527
|
-
try {
|
|
4528
|
-
const text = collectSourceText(doc, sourceFields, targetField, i);
|
|
4529
|
-
needsEmbedding.push({ docIndex: i, text });
|
|
4530
|
-
} catch (err) {
|
|
4531
|
-
if (isDocumentSourceError(err)) {
|
|
4532
|
-
failed.set(i, err);
|
|
4533
|
-
} else {
|
|
4534
|
-
throw err;
|
|
4535
|
-
}
|
|
4536
|
-
}
|
|
4537
|
-
}
|
|
4538
|
-
if (needsEmbedding.length === 0) continue;
|
|
4539
|
-
const texts = needsEmbedding.map((entry) => entry.text);
|
|
4540
|
-
let vectors;
|
|
4541
|
-
try {
|
|
4542
|
-
if (adapter.embedBatch) {
|
|
4543
|
-
vectors = await adapter.embedBatch(texts, "document", signal);
|
|
4544
|
-
} else {
|
|
4545
|
-
vectors = [];
|
|
4546
|
-
const concurrency = 8;
|
|
4547
|
-
for (let start = 0; start < texts.length; start += concurrency) {
|
|
4548
|
-
const chunk = texts.slice(start, start + concurrency);
|
|
4549
|
-
const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
|
|
4550
|
-
vectors.push(...chunkResults);
|
|
4551
|
-
}
|
|
4552
|
-
}
|
|
4553
|
-
} catch (err) {
|
|
4554
|
-
throw wrapAdapterError(err, targetField);
|
|
4555
|
-
}
|
|
4556
|
-
if (vectors.length !== needsEmbedding.length) {
|
|
4557
|
-
throw new NarsilError(
|
|
4558
|
-
ErrorCodes.EMBEDDING_FAILED,
|
|
4559
|
-
`Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
|
|
4560
|
-
{ field: targetField, expected: needsEmbedding.length, actual: vectors.length }
|
|
4561
|
-
);
|
|
4562
|
-
}
|
|
4563
|
-
for (let j = 0; j < needsEmbedding.length; j++) {
|
|
4564
|
-
const { docIndex } = needsEmbedding[j];
|
|
4565
|
-
const validated = validateVector(vectors[j], targetField, adapter.dimensions);
|
|
4566
|
-
setFieldValue(documents[docIndex], targetField, validated);
|
|
4567
|
-
let fieldSet = embedded.get(docIndex);
|
|
4568
|
-
if (!fieldSet) {
|
|
4569
|
-
fieldSet = /* @__PURE__ */ new Set();
|
|
4570
|
-
embedded.set(docIndex, fieldSet);
|
|
4571
|
-
}
|
|
4572
|
-
fieldSet.add(targetField);
|
|
4573
|
-
}
|
|
4574
|
-
}
|
|
4575
|
-
} catch (err) {
|
|
4576
|
-
for (const [docIndex, fields] of embedded) {
|
|
4577
|
-
for (const field of fields) {
|
|
4578
|
-
deleteFieldValue(documents[docIndex], field);
|
|
4579
|
-
}
|
|
4580
|
-
}
|
|
4581
|
-
throw err;
|
|
4582
|
-
}
|
|
4583
|
-
return { embedded, failed };
|
|
4584
|
-
}
|
|
4585
|
-
var INDEX_CONFIG_PREFIX = "_narsil/index/";
|
|
4586
|
-
var INDEX_CONFIG_SUFFIX = "/config";
|
|
4587
|
-
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
4588
|
-
var MAX_INDEX_NAME_LENGTH = 255;
|
|
4589
|
-
var MAX_PARTITION_COUNT = 65536;
|
|
4590
|
-
var MAX_REPLICATION_FACTOR = 255;
|
|
4591
|
-
function truncateForDisplay(value) {
|
|
4592
|
-
const str = String(value);
|
|
4593
|
-
if (str.length > 100) {
|
|
4594
|
-
return `${str.slice(0, 100)}...`;
|
|
4595
|
-
}
|
|
4596
|
-
return str;
|
|
4597
|
-
}
|
|
4598
|
-
function validateIndexName(indexName) {
|
|
4599
|
-
if (indexName.length === 0 || indexName.length > MAX_INDEX_NAME_LENGTH) {
|
|
4600
|
-
throw new NarsilError(
|
|
4601
|
-
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4602
|
-
`Index name must be between 1 and ${MAX_INDEX_NAME_LENGTH} characters`,
|
|
4603
|
-
{ indexName: truncateForDisplay(indexName) }
|
|
4604
|
-
);
|
|
4605
|
-
}
|
|
4606
|
-
if (indexName.includes("\0") || indexName.includes("/") || indexName.includes("\\") || indexName.includes("..")) {
|
|
4607
|
-
throw new NarsilError(
|
|
4608
|
-
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4609
|
-
"Index name contains forbidden characters (/, \\, .., or null bytes)",
|
|
4610
|
-
{ indexName: truncateForDisplay(indexName) }
|
|
4611
|
-
);
|
|
4612
|
-
}
|
|
4613
|
-
if (!INDEX_NAME_PATTERN.test(indexName)) {
|
|
4614
|
-
throw new NarsilError(
|
|
4615
|
-
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4616
|
-
"Index name must start with an alphanumeric character and contain only alphanumeric characters, hyphens, underscores, or dots",
|
|
4617
|
-
{ indexName: truncateForDisplay(indexName) }
|
|
4618
|
-
);
|
|
4619
|
-
}
|
|
4620
|
-
}
|
|
4621
|
-
function isValidInteger(value) {
|
|
4622
|
-
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
4623
|
-
}
|
|
4624
|
-
function indexConfigKey(indexName) {
|
|
4625
|
-
return `${INDEX_CONFIG_PREFIX}${indexName}${INDEX_CONFIG_SUFFIX}`;
|
|
4626
|
-
}
|
|
4627
|
-
function isRecord2(value) {
|
|
4628
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
4558
|
+
function isRecord2(value) {
|
|
4559
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
4629
4560
|
}
|
|
4630
4561
|
function validateDecodedMetadata(decoded, indexName) {
|
|
4631
4562
|
if (!isRecord2(decoded)) {
|
|
@@ -4743,27 +4674,361 @@ function validateDocId(docId) {
|
|
|
4743
4674
|
if (!docId || docId.length === 0) {
|
|
4744
4675
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
|
|
4745
4676
|
}
|
|
4746
|
-
if (docId.length > MAX_DOC_ID_LENGTH) {
|
|
4677
|
+
if (docId.length > MAX_DOC_ID_LENGTH) {
|
|
4678
|
+
throw new NarsilError(
|
|
4679
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
4680
|
+
`Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
|
|
4681
|
+
{
|
|
4682
|
+
docId,
|
|
4683
|
+
length: docId.length
|
|
4684
|
+
}
|
|
4685
|
+
);
|
|
4686
|
+
}
|
|
4687
|
+
if (docId.includes("\0")) {
|
|
4688
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
|
|
4689
|
+
}
|
|
4690
|
+
}
|
|
4691
|
+
function clampLimit(limit) {
|
|
4692
|
+
if (limit === void 0) return DEFAULT_LIMIT;
|
|
4693
|
+
return Math.max(0, Math.min(limit, MAX_LIMIT));
|
|
4694
|
+
}
|
|
4695
|
+
function clampOffset(offset) {
|
|
4696
|
+
if (offset === void 0) return DEFAULT_OFFSET;
|
|
4697
|
+
return Math.max(0, Math.min(offset, MAX_OFFSET));
|
|
4698
|
+
}
|
|
4699
|
+
|
|
4700
|
+
// src/engine/index-lifecycle.ts
|
|
4701
|
+
async function createEngineIndex(core, config, name, indexConfig) {
|
|
4702
|
+
core.guardShutdown();
|
|
4703
|
+
validateIndexName2(name);
|
|
4704
|
+
if (core.indexRegistry.has(name)) {
|
|
4705
|
+
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
|
|
4706
|
+
}
|
|
4707
|
+
validateSchema(indexConfig.schema);
|
|
4708
|
+
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
4709
|
+
let resolvedEmbeddingAdapter = null;
|
|
4710
|
+
let embeddingAdapterName = null;
|
|
4711
|
+
if (indexConfig.embedding) {
|
|
4712
|
+
let configuredAdapter = indexConfig.embedding.adapter;
|
|
4713
|
+
if (typeof configuredAdapter === "string") {
|
|
4714
|
+
embeddingAdapterName = configuredAdapter;
|
|
4715
|
+
const registered = core.embeddingAdapters.get(configuredAdapter);
|
|
4716
|
+
if (!registered) {
|
|
4717
|
+
throw new NarsilError(
|
|
4718
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4719
|
+
`Embedding adapter "${configuredAdapter}" is not registered on this engine`,
|
|
4720
|
+
{ adapter: configuredAdapter, available: [...core.embeddingAdapters.keys()] }
|
|
4721
|
+
);
|
|
4722
|
+
}
|
|
4723
|
+
configuredAdapter = registered;
|
|
4724
|
+
}
|
|
4725
|
+
resolvedEmbeddingAdapter = validateEmbeddingConfig(
|
|
4726
|
+
{ fields: indexConfig.embedding.fields, adapter: configuredAdapter },
|
|
4727
|
+
indexConfig.schema,
|
|
4728
|
+
config?.embedding
|
|
4729
|
+
);
|
|
4730
|
+
}
|
|
4731
|
+
if (indexConfig.required && indexConfig.required.length > 0) {
|
|
4732
|
+
validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
|
|
4733
|
+
}
|
|
4734
|
+
const language = getLanguage(indexConfig.language ?? "english");
|
|
4735
|
+
core.executor.createIndex(name, indexConfig, language);
|
|
4736
|
+
core.indexRegistry.set(name, {
|
|
4737
|
+
config: indexConfig,
|
|
4738
|
+
language,
|
|
4739
|
+
embeddingAdapter: resolvedEmbeddingAdapter,
|
|
4740
|
+
embeddingAdapterName,
|
|
4741
|
+
vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
|
|
4742
|
+
});
|
|
4743
|
+
if (core.durability) {
|
|
4744
|
+
await core.durability.manager.persistMetadata(name);
|
|
4745
|
+
}
|
|
4746
|
+
await core.pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
|
|
4747
|
+
}
|
|
4748
|
+
async function dropEngineIndex(core, name) {
|
|
4749
|
+
core.guardShutdown();
|
|
4750
|
+
const entry = core.requireIndex(name);
|
|
4751
|
+
core.executor.dropIndex(name);
|
|
4752
|
+
core.indexRegistry.delete(name);
|
|
4753
|
+
if (core.durability) {
|
|
4754
|
+
await core.durability.manager.removeIndex(name);
|
|
4755
|
+
}
|
|
4756
|
+
await core.pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
|
|
4757
|
+
}
|
|
4758
|
+
function registerEngineEmbeddingAdapter(core, name, adapter) {
|
|
4759
|
+
core.guardShutdown();
|
|
4760
|
+
validateRegisteredAdapter(name, adapter);
|
|
4761
|
+
const affected = [];
|
|
4762
|
+
for (const [indexName, entry] of core.indexRegistry) {
|
|
4763
|
+
if (entry.embeddingAdapterName !== name || !entry.config.embedding) continue;
|
|
4764
|
+
try {
|
|
4765
|
+
validateEmbeddingConfig({ fields: entry.config.embedding.fields, adapter }, entry.config.schema, void 0);
|
|
4766
|
+
} catch (err) {
|
|
4767
|
+
if (err instanceof NarsilError) {
|
|
4768
|
+
throw new NarsilError(
|
|
4769
|
+
err.code,
|
|
4770
|
+
`Cannot bind embedding adapter "${name}" to index "${indexName}": ${err.message}`,
|
|
4771
|
+
{
|
|
4772
|
+
indexName,
|
|
4773
|
+
adapter: name
|
|
4774
|
+
}
|
|
4775
|
+
);
|
|
4776
|
+
}
|
|
4777
|
+
throw err;
|
|
4778
|
+
}
|
|
4779
|
+
affected.push(entry);
|
|
4780
|
+
}
|
|
4781
|
+
core.embeddingAdapters.set(name, adapter);
|
|
4782
|
+
for (const entry of affected) {
|
|
4783
|
+
entry.embeddingAdapter = adapter;
|
|
4784
|
+
}
|
|
4785
|
+
}
|
|
4786
|
+
|
|
4787
|
+
// src/engine/lifecycle.ts
|
|
4788
|
+
async function shutdownEngine(core) {
|
|
4789
|
+
const { executor, durability, indexRegistry, eventHandlers, orchestrator } = core;
|
|
4790
|
+
for (const [name] of indexRegistry) {
|
|
4791
|
+
const manager = executor.getManager(name);
|
|
4792
|
+
if (manager) {
|
|
4793
|
+
for (const [, vectorIndex] of manager.getVectorIndexes()) {
|
|
4794
|
+
vectorIndex.dispose();
|
|
4795
|
+
}
|
|
4796
|
+
}
|
|
4797
|
+
}
|
|
4798
|
+
if (durability) {
|
|
4799
|
+
await durability.manager.shutdown();
|
|
4800
|
+
}
|
|
4801
|
+
const adaptersToShutdown = /* @__PURE__ */ new Set();
|
|
4802
|
+
for (const [, entry] of indexRegistry) {
|
|
4803
|
+
if (entry.embeddingAdapter?.shutdown) {
|
|
4804
|
+
adaptersToShutdown.add(entry.embeddingAdapter);
|
|
4805
|
+
}
|
|
4806
|
+
}
|
|
4807
|
+
for (const adapter of adaptersToShutdown) {
|
|
4808
|
+
try {
|
|
4809
|
+
await adapter.shutdown?.();
|
|
4810
|
+
} catch {
|
|
4811
|
+
}
|
|
4812
|
+
}
|
|
4813
|
+
await orchestrator.shutdown();
|
|
4814
|
+
await executor.shutdown();
|
|
4815
|
+
eventHandlers.clear();
|
|
4816
|
+
indexRegistry.clear();
|
|
4817
|
+
}
|
|
4818
|
+
|
|
4819
|
+
// src/engine/embed.ts
|
|
4820
|
+
function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterName) {
|
|
4821
|
+
for (const targetField of Object.keys(embeddingConfig.fields)) {
|
|
4822
|
+
const existing = resolveFieldValue(document, targetField);
|
|
4823
|
+
if (existing === void 0 || existing === null) {
|
|
4824
|
+
throw new NarsilError(
|
|
4825
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4826
|
+
adapterName ? `The index embeds "${targetField}" via adapter "${adapterName}", which is not registered on this engine. Register the adapter or supply the vector on the document.` : `The index embeds "${targetField}" automatically but no embedding adapter is bound. Bind an adapter or supply the vector on the document.`,
|
|
4827
|
+
adapterName ? { field: targetField, adapter: adapterName } : { field: targetField }
|
|
4828
|
+
);
|
|
4829
|
+
}
|
|
4830
|
+
}
|
|
4831
|
+
}
|
|
4832
|
+
function validateVector(vector, targetField, expectedDimensions) {
|
|
4833
|
+
if (!(vector instanceof Float32Array)) {
|
|
4834
|
+
throw new NarsilError(
|
|
4835
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
4836
|
+
`Adapter returned ${typeof vector} for field "${targetField}", expected Float32Array`,
|
|
4837
|
+
{ field: targetField, returnedType: typeof vector }
|
|
4838
|
+
);
|
|
4839
|
+
}
|
|
4840
|
+
if (vector.length !== expectedDimensions) {
|
|
4841
|
+
throw new NarsilError(
|
|
4842
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
4843
|
+
`Adapter returned ${vector.length}-dimensional vector for field "${targetField}", expected ${expectedDimensions}`,
|
|
4844
|
+
{ field: targetField, expected: expectedDimensions, actual: vector.length }
|
|
4845
|
+
);
|
|
4846
|
+
}
|
|
4847
|
+
for (let i = 0; i < vector.length; i++) {
|
|
4848
|
+
if (Number.isNaN(vector[i])) {
|
|
4849
|
+
throw new NarsilError(
|
|
4850
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
4851
|
+
`Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
|
|
4852
|
+
{ field: targetField, index: i }
|
|
4853
|
+
);
|
|
4854
|
+
}
|
|
4855
|
+
}
|
|
4856
|
+
return vector;
|
|
4857
|
+
}
|
|
4858
|
+
function resolveFieldValue(document, path) {
|
|
4859
|
+
if (!path.includes(".")) return document[path];
|
|
4860
|
+
const segments = path.split(".");
|
|
4861
|
+
let current = document;
|
|
4862
|
+
for (const segment of segments) {
|
|
4863
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
4864
|
+
current = current[segment];
|
|
4865
|
+
}
|
|
4866
|
+
return current;
|
|
4867
|
+
}
|
|
4868
|
+
function setFieldValue(document, path, value) {
|
|
4869
|
+
if (!path.includes(".")) {
|
|
4870
|
+
document[path] = value;
|
|
4871
|
+
return;
|
|
4872
|
+
}
|
|
4873
|
+
const segments = path.split(".");
|
|
4874
|
+
let current = document;
|
|
4875
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
4876
|
+
let next = current[segments[i]];
|
|
4877
|
+
if (next === null || next === void 0 || typeof next !== "object") {
|
|
4878
|
+
next = {};
|
|
4879
|
+
current[segments[i]] = next;
|
|
4880
|
+
}
|
|
4881
|
+
current = next;
|
|
4882
|
+
}
|
|
4883
|
+
current[segments[segments.length - 1]] = value;
|
|
4884
|
+
}
|
|
4885
|
+
function deleteFieldValue(document, path) {
|
|
4886
|
+
if (!path.includes(".")) {
|
|
4887
|
+
delete document[path];
|
|
4888
|
+
return;
|
|
4889
|
+
}
|
|
4890
|
+
const segments = path.split(".");
|
|
4891
|
+
let current = document;
|
|
4892
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
4893
|
+
const next = current[segments[i]];
|
|
4894
|
+
if (next === null || next === void 0 || typeof next !== "object") return;
|
|
4895
|
+
current = next;
|
|
4896
|
+
}
|
|
4897
|
+
delete current[segments[segments.length - 1]];
|
|
4898
|
+
}
|
|
4899
|
+
function collectSourceText(document, sourceFields, targetField, documentIndex) {
|
|
4900
|
+
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
4901
|
+
const parts = [];
|
|
4902
|
+
for (const source of sources) {
|
|
4903
|
+
const value = resolveFieldValue(document, source);
|
|
4904
|
+
if (value === void 0 || value === null || value === "") continue;
|
|
4905
|
+
if (typeof value !== "string") {
|
|
4906
|
+
throw new NarsilError(
|
|
4907
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
4908
|
+
`Embedding source field "${source}" must be a string, got ${typeof value}`,
|
|
4909
|
+
{ field: source, targetField }
|
|
4910
|
+
);
|
|
4911
|
+
}
|
|
4912
|
+
parts.push(value);
|
|
4913
|
+
}
|
|
4914
|
+
if (parts.length === 0) {
|
|
4915
|
+
const details = { field: targetField };
|
|
4916
|
+
if (documentIndex !== void 0) {
|
|
4917
|
+
details.documentIndex = documentIndex;
|
|
4918
|
+
}
|
|
4747
4919
|
throw new NarsilError(
|
|
4748
|
-
ErrorCodes.
|
|
4749
|
-
`
|
|
4750
|
-
|
|
4751
|
-
docId,
|
|
4752
|
-
length: docId.length
|
|
4753
|
-
}
|
|
4920
|
+
ErrorCodes.EMBEDDING_NO_SOURCE,
|
|
4921
|
+
`All source fields for embedding target "${targetField}" are missing or empty`,
|
|
4922
|
+
details
|
|
4754
4923
|
);
|
|
4755
4924
|
}
|
|
4756
|
-
|
|
4757
|
-
|
|
4925
|
+
return parts.join("\n");
|
|
4926
|
+
}
|
|
4927
|
+
function wrapAdapterError(err, targetField) {
|
|
4928
|
+
if (err instanceof NarsilError) return err;
|
|
4929
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4930
|
+
return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
|
|
4931
|
+
field: targetField,
|
|
4932
|
+
cause: message
|
|
4933
|
+
});
|
|
4934
|
+
}
|
|
4935
|
+
async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
|
|
4936
|
+
const embeddedFields = /* @__PURE__ */ new Set();
|
|
4937
|
+
try {
|
|
4938
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
4939
|
+
const existing = resolveFieldValue(document, targetField);
|
|
4940
|
+
if (existing !== void 0 && existing !== null) continue;
|
|
4941
|
+
const text = collectSourceText(document, sourceFields, targetField);
|
|
4942
|
+
let vector;
|
|
4943
|
+
try {
|
|
4944
|
+
const raw = await adapter.embed(text, "document", signal);
|
|
4945
|
+
vector = validateVector(raw, targetField, adapter.dimensions);
|
|
4946
|
+
} catch (err) {
|
|
4947
|
+
throw wrapAdapterError(err, targetField);
|
|
4948
|
+
}
|
|
4949
|
+
setFieldValue(document, targetField, vector);
|
|
4950
|
+
embeddedFields.add(targetField);
|
|
4951
|
+
}
|
|
4952
|
+
} catch (err) {
|
|
4953
|
+
for (const field of embeddedFields) {
|
|
4954
|
+
deleteFieldValue(document, field);
|
|
4955
|
+
}
|
|
4956
|
+
throw err;
|
|
4758
4957
|
}
|
|
4958
|
+
return embeddedFields;
|
|
4759
4959
|
}
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
return
|
|
4960
|
+
var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([ErrorCodes.EMBEDDING_NO_SOURCE, ErrorCodes.DOC_VALIDATION_FAILED]);
|
|
4961
|
+
function isDocumentSourceError(err) {
|
|
4962
|
+
return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
|
|
4763
4963
|
}
|
|
4764
|
-
function
|
|
4765
|
-
|
|
4766
|
-
|
|
4964
|
+
async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
|
|
4965
|
+
const embedded = /* @__PURE__ */ new Map();
|
|
4966
|
+
const failed = /* @__PURE__ */ new Map();
|
|
4967
|
+
try {
|
|
4968
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
4969
|
+
const needsEmbedding = [];
|
|
4970
|
+
for (let i = 0; i < documents.length; i++) {
|
|
4971
|
+
if (failed.has(i)) continue;
|
|
4972
|
+
const doc = documents[i];
|
|
4973
|
+
const existing = resolveFieldValue(doc, targetField);
|
|
4974
|
+
if (existing !== void 0 && existing !== null) continue;
|
|
4975
|
+
try {
|
|
4976
|
+
const text = collectSourceText(doc, sourceFields, targetField, i);
|
|
4977
|
+
needsEmbedding.push({ docIndex: i, text });
|
|
4978
|
+
} catch (err) {
|
|
4979
|
+
if (isDocumentSourceError(err)) {
|
|
4980
|
+
failed.set(i, err);
|
|
4981
|
+
} else {
|
|
4982
|
+
throw err;
|
|
4983
|
+
}
|
|
4984
|
+
}
|
|
4985
|
+
}
|
|
4986
|
+
if (needsEmbedding.length === 0) continue;
|
|
4987
|
+
const texts = needsEmbedding.map((entry) => entry.text);
|
|
4988
|
+
let vectors;
|
|
4989
|
+
try {
|
|
4990
|
+
if (adapter.embedBatch) {
|
|
4991
|
+
vectors = await adapter.embedBatch(texts, "document", signal);
|
|
4992
|
+
} else {
|
|
4993
|
+
vectors = [];
|
|
4994
|
+
const concurrency = 8;
|
|
4995
|
+
for (let start = 0; start < texts.length; start += concurrency) {
|
|
4996
|
+
const chunk = texts.slice(start, start + concurrency);
|
|
4997
|
+
const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
|
|
4998
|
+
vectors.push(...chunkResults);
|
|
4999
|
+
}
|
|
5000
|
+
}
|
|
5001
|
+
} catch (err) {
|
|
5002
|
+
throw wrapAdapterError(err, targetField);
|
|
5003
|
+
}
|
|
5004
|
+
if (vectors.length !== needsEmbedding.length) {
|
|
5005
|
+
throw new NarsilError(
|
|
5006
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
5007
|
+
`Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
|
|
5008
|
+
{ field: targetField, expected: needsEmbedding.length, actual: vectors.length }
|
|
5009
|
+
);
|
|
5010
|
+
}
|
|
5011
|
+
for (let j = 0; j < needsEmbedding.length; j++) {
|
|
5012
|
+
const { docIndex } = needsEmbedding[j];
|
|
5013
|
+
const validated = validateVector(vectors[j], targetField, adapter.dimensions);
|
|
5014
|
+
setFieldValue(documents[docIndex], targetField, validated);
|
|
5015
|
+
let fieldSet = embedded.get(docIndex);
|
|
5016
|
+
if (!fieldSet) {
|
|
5017
|
+
fieldSet = /* @__PURE__ */ new Set();
|
|
5018
|
+
embedded.set(docIndex, fieldSet);
|
|
5019
|
+
}
|
|
5020
|
+
fieldSet.add(targetField);
|
|
5021
|
+
}
|
|
5022
|
+
}
|
|
5023
|
+
} catch (err) {
|
|
5024
|
+
for (const [docIndex, fields] of embedded) {
|
|
5025
|
+
for (const field of fields) {
|
|
5026
|
+
deleteFieldValue(documents[docIndex], field);
|
|
5027
|
+
}
|
|
5028
|
+
}
|
|
5029
|
+
throw err;
|
|
5030
|
+
}
|
|
5031
|
+
return { embedded, failed };
|
|
4767
5032
|
}
|
|
4768
5033
|
|
|
4769
5034
|
// src/engine/mutations/durable-rollback.ts
|
|
@@ -4827,13 +5092,21 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
4827
5092
|
if (entry.config.required && entry.config.required.length > 0) {
|
|
4828
5093
|
validateRequiredFields(document, entry.config.required);
|
|
4829
5094
|
}
|
|
4830
|
-
if (entry.
|
|
4831
|
-
|
|
4832
|
-
|
|
4833
|
-
|
|
4834
|
-
|
|
4835
|
-
|
|
4836
|
-
|
|
5095
|
+
if (entry.config.embedding) {
|
|
5096
|
+
if (entry.embeddingAdapter) {
|
|
5097
|
+
await embedDocumentFields(
|
|
5098
|
+
document,
|
|
5099
|
+
entry.config.embedding,
|
|
5100
|
+
entry.embeddingAdapter,
|
|
5101
|
+
ctx.abortController.signal
|
|
5102
|
+
);
|
|
5103
|
+
} else {
|
|
5104
|
+
assertDocumentCarriesMappedVectors(
|
|
5105
|
+
document,
|
|
5106
|
+
entry.config.embedding,
|
|
5107
|
+
entry.embeddingAdapterName
|
|
5108
|
+
);
|
|
5109
|
+
}
|
|
4837
5110
|
}
|
|
4838
5111
|
const insertManager = ctx.requireManager(indexName);
|
|
4839
5112
|
const insertVecIndexes = insertManager.getVectorIndexes();
|
|
@@ -4913,6 +5186,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
4913
5186
|
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
4914
5187
|
const hasRequired = entry.config.required && entry.config.required.length > 0;
|
|
4915
5188
|
const hasEmbedding = entry.embeddingAdapter && entry.config.embedding;
|
|
5189
|
+
const embeddingUnbound = !entry.embeddingAdapter && entry.config.embedding;
|
|
4916
5190
|
const batchManager = ctx.requireManager(indexName);
|
|
4917
5191
|
const batchVecIndexes = batchManager.getVectorIndexes();
|
|
4918
5192
|
const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
@@ -4928,7 +5202,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
4928
5202
|
} catch (err) {
|
|
4929
5203
|
chunkFailedIndexes.add(i);
|
|
4930
5204
|
failed.push({
|
|
4931
|
-
docId: "",
|
|
5205
|
+
docId: providedDocId(documents[i]) ?? "",
|
|
4932
5206
|
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err))
|
|
4933
5207
|
});
|
|
4934
5208
|
}
|
|
@@ -4953,16 +5227,33 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
4953
5227
|
for (const [sliceIndex, error] of embedResult.failed) {
|
|
4954
5228
|
const originalIdx = embeddableOriginalIndexes[sliceIndex];
|
|
4955
5229
|
chunkFailedIndexes.add(originalIdx);
|
|
4956
|
-
failed.push({ docId: "", error });
|
|
5230
|
+
failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error });
|
|
4957
5231
|
}
|
|
4958
5232
|
} catch (err) {
|
|
4959
5233
|
const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
4960
5234
|
for (const originalIdx of embeddableOriginalIndexes) {
|
|
4961
5235
|
chunkFailedIndexes.add(originalIdx);
|
|
4962
|
-
failed.push({ docId: "", error: embeddingError });
|
|
5236
|
+
failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error: embeddingError });
|
|
4963
5237
|
}
|
|
4964
5238
|
}
|
|
4965
5239
|
}
|
|
5240
|
+
} else if (embeddingUnbound) {
|
|
5241
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
5242
|
+
if (chunkFailedIndexes.has(i)) continue;
|
|
5243
|
+
try {
|
|
5244
|
+
assertDocumentCarriesMappedVectors(
|
|
5245
|
+
documents[i],
|
|
5246
|
+
entry.config.embedding,
|
|
5247
|
+
entry.embeddingAdapterName
|
|
5248
|
+
);
|
|
5249
|
+
} catch (err) {
|
|
5250
|
+
chunkFailedIndexes.add(i);
|
|
5251
|
+
failed.push({
|
|
5252
|
+
docId: providedDocId(documents[i]) ?? "",
|
|
5253
|
+
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
|
|
5254
|
+
});
|
|
5255
|
+
}
|
|
5256
|
+
}
|
|
4966
5257
|
}
|
|
4967
5258
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
4968
5259
|
if (ctx.abortController.signal.aborted) break;
|
|
@@ -5156,13 +5447,21 @@ async function updateDocument(ctx, indexName, docId, document) {
|
|
|
5156
5447
|
if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
|
|
5157
5448
|
return;
|
|
5158
5449
|
}
|
|
5159
|
-
if (entry.
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5450
|
+
if (entry.config.embedding) {
|
|
5451
|
+
if (entry.embeddingAdapter) {
|
|
5452
|
+
await embedDocumentFields(
|
|
5453
|
+
document,
|
|
5454
|
+
entry.config.embedding,
|
|
5455
|
+
entry.embeddingAdapter,
|
|
5456
|
+
ctx.abortController.signal
|
|
5457
|
+
);
|
|
5458
|
+
} else {
|
|
5459
|
+
assertDocumentCarriesMappedVectors(
|
|
5460
|
+
document,
|
|
5461
|
+
entry.config.embedding,
|
|
5462
|
+
entry.embeddingAdapterName
|
|
5463
|
+
);
|
|
5464
|
+
}
|
|
5166
5465
|
}
|
|
5167
5466
|
const updateManager = ctx.requireManager(indexName);
|
|
5168
5467
|
const oldDocument = updateManager.get(docId);
|
|
@@ -6212,7 +6511,7 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
|
6212
6511
|
}
|
|
6213
6512
|
|
|
6214
6513
|
// src/engine/resolve-vector-text.ts
|
|
6215
|
-
async function resolveVectorText(params, embeddingAdapter, signal) {
|
|
6514
|
+
async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
|
|
6216
6515
|
if (!params.vector) return params;
|
|
6217
6516
|
if (params.vector.text === void 0 && params.vector.value === void 0) return params;
|
|
6218
6517
|
if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
|
|
@@ -6222,7 +6521,8 @@ async function resolveVectorText(params, embeddingAdapter, signal) {
|
|
|
6222
6521
|
if (!embeddingAdapter) {
|
|
6223
6522
|
throw new NarsilError(
|
|
6224
6523
|
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6225
|
-
"Vector query with 'text' requires an embedding adapter on the index or instance"
|
|
6524
|
+
embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
|
|
6525
|
+
embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
|
|
6226
6526
|
);
|
|
6227
6527
|
}
|
|
6228
6528
|
if (typeof params.vector.text !== "string") {
|
|
@@ -6304,7 +6604,13 @@ async function restoreFromSnapshot(indexName, data, executor, indexRegistry, get
|
|
|
6304
6604
|
const indexConfig = { schema, language: envelope.language };
|
|
6305
6605
|
executor.createIndex(indexName, indexConfig, language);
|
|
6306
6606
|
const vectorFieldPaths = getVectorFieldPaths2(schema);
|
|
6307
|
-
indexRegistry.set(indexName, {
|
|
6607
|
+
indexRegistry.set(indexName, {
|
|
6608
|
+
config: indexConfig,
|
|
6609
|
+
language,
|
|
6610
|
+
embeddingAdapter: null,
|
|
6611
|
+
embeddingAdapterName: null,
|
|
6612
|
+
vectorFieldPaths
|
|
6613
|
+
});
|
|
6308
6614
|
try {
|
|
6309
6615
|
const manager = requireManager(indexName);
|
|
6310
6616
|
while (manager.partitionCount < envelope.partitions.length) {
|
|
@@ -6435,97 +6741,6 @@ function readProcessMemory() {
|
|
|
6435
6741
|
}
|
|
6436
6742
|
}
|
|
6437
6743
|
|
|
6438
|
-
// src/schema/embedding-validator.ts
|
|
6439
|
-
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
6440
|
-
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
6441
|
-
const fieldEntries = Object.keys(embeddingConfig.fields);
|
|
6442
|
-
if (fieldEntries.length === 0) {
|
|
6443
|
-
throw new NarsilError(
|
|
6444
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6445
|
-
"Embedding config must define at least one field mapping"
|
|
6446
|
-
);
|
|
6447
|
-
}
|
|
6448
|
-
const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
|
|
6449
|
-
if (!resolvedAdapter) {
|
|
6450
|
-
throw new NarsilError(
|
|
6451
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6452
|
-
"Embedding config requires an adapter (set on the index or the instance)"
|
|
6453
|
-
);
|
|
6454
|
-
}
|
|
6455
|
-
if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
|
|
6456
|
-
throw new NarsilError(
|
|
6457
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6458
|
-
`Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
|
|
6459
|
-
);
|
|
6460
|
-
}
|
|
6461
|
-
const flatFields = flattenSchema(schema);
|
|
6462
|
-
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
6463
|
-
const targetType = flatFields[targetField];
|
|
6464
|
-
if (targetType === void 0) {
|
|
6465
|
-
throw new NarsilError(
|
|
6466
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6467
|
-
`Embedding target field "${targetField}" does not exist in the schema`,
|
|
6468
|
-
{ field: targetField }
|
|
6469
|
-
);
|
|
6470
|
-
}
|
|
6471
|
-
const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
|
|
6472
|
-
if (!vectorMatch) {
|
|
6473
|
-
throw new NarsilError(
|
|
6474
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6475
|
-
`Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
|
|
6476
|
-
{ field: targetField, type: targetType }
|
|
6477
|
-
);
|
|
6478
|
-
}
|
|
6479
|
-
const schemaDimension = Number.parseInt(vectorMatch[1], 10);
|
|
6480
|
-
if (schemaDimension !== resolvedAdapter.dimensions) {
|
|
6481
|
-
throw new NarsilError(
|
|
6482
|
-
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
6483
|
-
`Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
|
|
6484
|
-
{ field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
|
|
6485
|
-
);
|
|
6486
|
-
}
|
|
6487
|
-
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
6488
|
-
if (sources.length === 0) {
|
|
6489
|
-
throw new NarsilError(
|
|
6490
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6491
|
-
`Embedding target field "${targetField}" must have at least one source field`,
|
|
6492
|
-
{ field: targetField }
|
|
6493
|
-
);
|
|
6494
|
-
}
|
|
6495
|
-
for (const source of sources) {
|
|
6496
|
-
const sourceType = flatFields[source];
|
|
6497
|
-
if (sourceType === void 0) {
|
|
6498
|
-
throw new NarsilError(
|
|
6499
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6500
|
-
`Embedding source field "${source}" does not exist in the schema`,
|
|
6501
|
-
{ field: source, targetField }
|
|
6502
|
-
);
|
|
6503
|
-
}
|
|
6504
|
-
if (sourceType !== "string") {
|
|
6505
|
-
throw new NarsilError(
|
|
6506
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6507
|
-
`Embedding source field "${source}" must be a string type, got "${sourceType}"`,
|
|
6508
|
-
{ field: source, type: sourceType, targetField }
|
|
6509
|
-
);
|
|
6510
|
-
}
|
|
6511
|
-
}
|
|
6512
|
-
}
|
|
6513
|
-
return resolvedAdapter;
|
|
6514
|
-
}
|
|
6515
|
-
function validateRequiredFieldsInSchema(required, schema) {
|
|
6516
|
-
if (required.length === 0) return;
|
|
6517
|
-
const flatFields = flattenSchema(schema);
|
|
6518
|
-
for (const field of required) {
|
|
6519
|
-
if (flatFields[field] === void 0) {
|
|
6520
|
-
throw new NarsilError(
|
|
6521
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6522
|
-
`Required field "${field}" does not exist in the schema`,
|
|
6523
|
-
{ field }
|
|
6524
|
-
);
|
|
6525
|
-
}
|
|
6526
|
-
}
|
|
6527
|
-
}
|
|
6528
|
-
|
|
6529
6744
|
// src/narsil.ts
|
|
6530
6745
|
async function createNarsil(config) {
|
|
6531
6746
|
const core = createEngineCore(config);
|
|
@@ -6552,43 +6767,14 @@ function createNarsilFromCore(core, config) {
|
|
|
6552
6767
|
rebalanceCtx
|
|
6553
6768
|
} = core;
|
|
6554
6769
|
const narsil = {
|
|
6555
|
-
|
|
6556
|
-
|
|
6557
|
-
validateIndexName2(name);
|
|
6558
|
-
if (indexRegistry.has(name)) {
|
|
6559
|
-
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
|
|
6560
|
-
}
|
|
6561
|
-
validateSchema(indexConfig.schema);
|
|
6562
|
-
let resolvedEmbeddingAdapter = null;
|
|
6563
|
-
if (indexConfig.embedding) {
|
|
6564
|
-
resolvedEmbeddingAdapter = validateEmbeddingConfig(indexConfig.embedding, indexConfig.schema, config?.embedding);
|
|
6565
|
-
}
|
|
6566
|
-
if (indexConfig.required && indexConfig.required.length > 0) {
|
|
6567
|
-
validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
|
|
6568
|
-
}
|
|
6569
|
-
const language = getLanguage(indexConfig.language ?? "english");
|
|
6570
|
-
executor.createIndex(name, indexConfig, language);
|
|
6571
|
-
const vectorFieldPaths = getVectorFieldPaths(indexConfig.schema);
|
|
6572
|
-
indexRegistry.set(name, {
|
|
6573
|
-
config: indexConfig,
|
|
6574
|
-
language,
|
|
6575
|
-
embeddingAdapter: resolvedEmbeddingAdapter,
|
|
6576
|
-
vectorFieldPaths
|
|
6577
|
-
});
|
|
6578
|
-
if (durability) {
|
|
6579
|
-
await durability.manager.persistMetadata(name);
|
|
6580
|
-
}
|
|
6581
|
-
await pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
|
|
6770
|
+
createIndex(name, indexConfig) {
|
|
6771
|
+
return createEngineIndex(core, config, name, indexConfig);
|
|
6582
6772
|
},
|
|
6583
|
-
|
|
6584
|
-
|
|
6585
|
-
|
|
6586
|
-
|
|
6587
|
-
|
|
6588
|
-
if (durability) {
|
|
6589
|
-
await durability.manager.removeIndex(name);
|
|
6590
|
-
}
|
|
6591
|
-
await pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
|
|
6773
|
+
registerEmbeddingAdapter(name, adapter) {
|
|
6774
|
+
registerEngineEmbeddingAdapter(core, name, adapter);
|
|
6775
|
+
},
|
|
6776
|
+
dropIndex(name) {
|
|
6777
|
+
return dropEngineIndex(core, name);
|
|
6592
6778
|
},
|
|
6593
6779
|
listIndexes() {
|
|
6594
6780
|
const infos = [];
|
|
@@ -6667,7 +6853,12 @@ function createNarsilFromCore(core, config) {
|
|
|
6667
6853
|
guardShutdown();
|
|
6668
6854
|
const entry = requireIndex(indexName);
|
|
6669
6855
|
const manager = requireManager(indexName);
|
|
6670
|
-
const resolvedParams = await resolveVectorText(
|
|
6856
|
+
const resolvedParams = await resolveVectorText(
|
|
6857
|
+
params,
|
|
6858
|
+
entry.embeddingAdapter,
|
|
6859
|
+
abortController.signal,
|
|
6860
|
+
entry.embeddingAdapterName
|
|
6861
|
+
);
|
|
6671
6862
|
await pluginRegistry.runHook("beforeSearch", { indexName, params: resolvedParams });
|
|
6672
6863
|
const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
|
|
6673
6864
|
const result = await executeQuery(resolvedParams, {
|
|
@@ -6692,7 +6883,12 @@ function createNarsilFromCore(core, config) {
|
|
|
6692
6883
|
guardShutdown();
|
|
6693
6884
|
const entry = requireIndex(indexName);
|
|
6694
6885
|
const manager = requireManager(indexName);
|
|
6695
|
-
const resolvedParams = await resolveVectorText(
|
|
6886
|
+
const resolvedParams = await resolveVectorText(
|
|
6887
|
+
params,
|
|
6888
|
+
entry.embeddingAdapter,
|
|
6889
|
+
abortController.signal,
|
|
6890
|
+
entry.embeddingAdapterName
|
|
6891
|
+
);
|
|
6696
6892
|
const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
|
|
6697
6893
|
return executePreflight(resolvedParams, {
|
|
6698
6894
|
manager,
|