@delali/narsil 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +777 -130
- package/dist/{chunk-GQEHCZG7.mjs → chunk-SVDEEMO6.mjs} +648 -450
- package/dist/{chunk-FS7XLIUS.mjs → chunk-VWF7RKSF.mjs} +33 -1
- package/dist/{config-cQLlYR3U.d.ts → config-DX--2Owg.d.ts} +3 -1
- package/dist/{direct-executor-FX56DYD3.mjs → direct-executor-KC5O5D2R.mjs} +1 -1
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +2 -2
- package/dist/embeddings/openai.mjs +5 -0
- package/dist/index.browser.mjs +450 -220
- package/dist/index.d.ts +3 -3
- package/dist/index.mjs +2 -2
- package/dist/{narsil-ChTdkwPu.d.ts → narsil-B3jKyXCr.d.ts} +6 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +2 -2
- package/dist/{schema-BVaw7nqz.d.ts → schema-C3VhNvfw.d.ts} +3 -1
- package/dist/server.d.ts +3 -3
- package/dist/server.mjs +11 -12
- package/dist/{types-Bt7c4hDF.d.ts → types-CTVVm4GG.d.ts} +1 -1
- package/dist/workers/entry.mjs +1 -1
- package/package.json +2 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter,
|
|
1
|
+
import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, createPartitionManager, createVectorIndex, validateSchema, deserializePayloadV2, tokenize, fanOutQuery, validateRequiredFields, validateVectorPromotion, createPartitionIndex, flattenSchema, fnv1a, detectRuntime, kWayMerge } from './chunk-VWF7RKSF.mjs';
|
|
2
2
|
import { crc32, crc32Init, crc32Update, crc32Final } from './chunk-KMMJMDUA.mjs';
|
|
3
3
|
import { ErrorCodes, NarsilError } from './chunk-ALB35PRQ.mjs';
|
|
4
4
|
import { generateId, createRequestId, isValidWorkerResponse } from './chunk-HRABJLPB.mjs';
|
|
@@ -132,6 +132,123 @@ function createPluginRegistry() {
|
|
|
132
132
|
};
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
+
// src/schema/embedding-validator.ts
|
|
136
|
+
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
137
|
+
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
138
|
+
const fieldEntries = Object.keys(embeddingConfig.fields);
|
|
139
|
+
if (fieldEntries.length === 0) {
|
|
140
|
+
throw new NarsilError(
|
|
141
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
142
|
+
"Embedding config must define at least one field mapping"
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
if (typeof embeddingConfig.adapter === "string") {
|
|
146
|
+
throw new NarsilError(
|
|
147
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
148
|
+
`Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
|
|
149
|
+
{ adapter: embeddingConfig.adapter }
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
|
|
153
|
+
if (!resolvedAdapter) {
|
|
154
|
+
throw new NarsilError(
|
|
155
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
156
|
+
"Embedding config requires an adapter (set on the index or the instance)"
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
|
|
160
|
+
throw new NarsilError(
|
|
161
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
162
|
+
`Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
const flatFields = flattenSchema(schema);
|
|
166
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
167
|
+
const targetType = flatFields[targetField];
|
|
168
|
+
if (targetType === void 0) {
|
|
169
|
+
throw new NarsilError(
|
|
170
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
171
|
+
`Embedding target field "${targetField}" does not exist in the schema`,
|
|
172
|
+
{ field: targetField }
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
|
|
176
|
+
if (!vectorMatch) {
|
|
177
|
+
throw new NarsilError(
|
|
178
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
179
|
+
`Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
|
|
180
|
+
{ field: targetField, type: targetType }
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
const schemaDimension = Number.parseInt(vectorMatch[1], 10);
|
|
184
|
+
if (schemaDimension !== resolvedAdapter.dimensions) {
|
|
185
|
+
throw new NarsilError(
|
|
186
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
187
|
+
`Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
|
|
188
|
+
{ field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
192
|
+
if (sources.length === 0) {
|
|
193
|
+
throw new NarsilError(
|
|
194
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
195
|
+
`Embedding target field "${targetField}" must have at least one source field`,
|
|
196
|
+
{ field: targetField }
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
for (const source of sources) {
|
|
200
|
+
const sourceType = flatFields[source];
|
|
201
|
+
if (sourceType === void 0) {
|
|
202
|
+
throw new NarsilError(
|
|
203
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
204
|
+
`Embedding source field "${source}" does not exist in the schema`,
|
|
205
|
+
{ field: source, targetField }
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
if (sourceType !== "string") {
|
|
209
|
+
throw new NarsilError(
|
|
210
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
211
|
+
`Embedding source field "${source}" must be a string type, got "${sourceType}"`,
|
|
212
|
+
{ field: source, type: sourceType, targetField }
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return resolvedAdapter;
|
|
218
|
+
}
|
|
219
|
+
function validateRegisteredAdapter(name, adapter) {
|
|
220
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
221
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
|
|
222
|
+
}
|
|
223
|
+
if (typeof adapter?.embed !== "function") {
|
|
224
|
+
throw new NarsilError(
|
|
225
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
226
|
+
`Embedding adapter "${name}" must provide an embed function`,
|
|
227
|
+
{ adapter: name }
|
|
228
|
+
);
|
|
229
|
+
}
|
|
230
|
+
if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
|
|
231
|
+
throw new NarsilError(
|
|
232
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
233
|
+
`Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
|
|
234
|
+
{ adapter: name, dimensions: adapter.dimensions }
|
|
235
|
+
);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
function validateRequiredFieldsInSchema(required, schema) {
|
|
239
|
+
if (required.length === 0) return;
|
|
240
|
+
const flatFields = flattenSchema(schema);
|
|
241
|
+
for (const field of required) {
|
|
242
|
+
if (flatFields[field] === void 0) {
|
|
243
|
+
throw new NarsilError(
|
|
244
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
245
|
+
`Required field "${field}" does not exist in the schema`,
|
|
246
|
+
{ field }
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
135
252
|
// src/workers/promoter.ts
|
|
136
253
|
var DEFAULT_PER_INDEX_THRESHOLD = 1e4;
|
|
137
254
|
var DEFAULT_TOTAL_THRESHOLD = 5e4;
|
|
@@ -739,6 +856,9 @@ function metadataToWire(meta) {
|
|
|
739
856
|
if (meta.vectorFields) {
|
|
740
857
|
wire.vector_fields = meta.vectorFields;
|
|
741
858
|
}
|
|
859
|
+
if (meta.embedding) {
|
|
860
|
+
wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
|
|
861
|
+
}
|
|
742
862
|
return wire;
|
|
743
863
|
}
|
|
744
864
|
function wireToMetadata(raw) {
|
|
@@ -754,6 +874,9 @@ function wireToMetadata(raw) {
|
|
|
754
874
|
if (raw.vector_fields) {
|
|
755
875
|
meta.vectorFields = raw.vector_fields;
|
|
756
876
|
}
|
|
877
|
+
if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
|
|
878
|
+
meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
|
|
879
|
+
}
|
|
757
880
|
return meta;
|
|
758
881
|
}
|
|
759
882
|
function serializeMetadata(meta) {
|
|
@@ -2115,11 +2238,15 @@ function unflattenSchema(flat) {
|
|
|
2115
2238
|
return root;
|
|
2116
2239
|
}
|
|
2117
2240
|
function reconstructSchemaFromMetadata(metadata) {
|
|
2118
|
-
|
|
2241
|
+
const config = {
|
|
2119
2242
|
schema: unflattenSchema(metadata.schema),
|
|
2120
2243
|
language: metadata.language,
|
|
2121
2244
|
bm25: { k1: metadata.bm25Params.k1, b: metadata.bm25Params.b }
|
|
2122
2245
|
};
|
|
2246
|
+
if (metadata.embedding) {
|
|
2247
|
+
config.embedding = metadata.embedding.adapter !== void 0 ? { adapter: metadata.embedding.adapter, fields: metadata.embedding.fields } : { fields: metadata.embedding.fields };
|
|
2248
|
+
}
|
|
2249
|
+
return config;
|
|
2123
2250
|
}
|
|
2124
2251
|
|
|
2125
2252
|
// src/persistence/durability/segment/build-segment.ts
|
|
@@ -3693,7 +3820,7 @@ function buildMetadata(indexName, hooks) {
|
|
|
3693
3820
|
if (config === void 0 || manager === void 0) {
|
|
3694
3821
|
return void 0;
|
|
3695
3822
|
}
|
|
3696
|
-
|
|
3823
|
+
const metadata = {
|
|
3697
3824
|
indexName,
|
|
3698
3825
|
schema: config.schema,
|
|
3699
3826
|
language: config.language,
|
|
@@ -3702,6 +3829,10 @@ function buildMetadata(indexName, hooks) {
|
|
|
3702
3829
|
createdAt: Date.now(),
|
|
3703
3830
|
engineVersion: VERSION
|
|
3704
3831
|
};
|
|
3832
|
+
if (config.embedding) {
|
|
3833
|
+
metadata.embedding = config.embedding;
|
|
3834
|
+
}
|
|
3835
|
+
return metadata;
|
|
3705
3836
|
}
|
|
3706
3837
|
function createDurabilityIntegration(tier, hooks) {
|
|
3707
3838
|
const managerHooks = {
|
|
@@ -4211,11 +4342,16 @@ function createDurabilityFromConfig(config, wiring) {
|
|
|
4211
4342
|
if (entry === void 0) {
|
|
4212
4343
|
return void 0;
|
|
4213
4344
|
}
|
|
4345
|
+
const embedding = entry.config.embedding ? {
|
|
4346
|
+
fields: entry.config.embedding.fields,
|
|
4347
|
+
...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
|
|
4348
|
+
} : void 0;
|
|
4214
4349
|
return {
|
|
4215
4350
|
schema: flattenSchema(entry.config.schema),
|
|
4216
4351
|
language: entry.language.name,
|
|
4217
4352
|
k1: entry.config.bm25?.k1 ?? 1.2,
|
|
4218
|
-
b: entry.config.bm25?.b ?? 0.75
|
|
4353
|
+
b: entry.config.bm25?.b ?? 0.75,
|
|
4354
|
+
...embedding !== void 0 ? { embedding } : {}
|
|
4219
4355
|
};
|
|
4220
4356
|
},
|
|
4221
4357
|
createIndexFromMetadata: wiring.createIndexFromMetadata,
|
|
@@ -4234,6 +4370,13 @@ function createEngineCore(config) {
|
|
|
4234
4370
|
}
|
|
4235
4371
|
const idGenerator = config?.idGenerator ?? generateId;
|
|
4236
4372
|
const indexRegistry = /* @__PURE__ */ new Map();
|
|
4373
|
+
const embeddingAdapters = /* @__PURE__ */ new Map();
|
|
4374
|
+
if (config?.embeddingAdapters) {
|
|
4375
|
+
for (const [name, adapter] of Object.entries(config.embeddingAdapters)) {
|
|
4376
|
+
validateRegisteredAdapter(name, adapter);
|
|
4377
|
+
embeddingAdapters.set(name, adapter);
|
|
4378
|
+
}
|
|
4379
|
+
}
|
|
4237
4380
|
const eventHandlers = /* @__PURE__ */ new Map();
|
|
4238
4381
|
const shutdownState = { isShutdown: false };
|
|
4239
4382
|
const abortController = new AbortController();
|
|
@@ -4283,11 +4426,35 @@ function createEngineCore(config) {
|
|
|
4283
4426
|
}
|
|
4284
4427
|
const indexConfig = reconstructSchemaFromMetadata(metadata);
|
|
4285
4428
|
const language = getLanguage(indexConfig.language ?? "english");
|
|
4429
|
+
const adapterName = metadata.embedding?.adapter ?? null;
|
|
4430
|
+
let embeddingAdapter = null;
|
|
4431
|
+
if (metadata.embedding) {
|
|
4432
|
+
const candidate = adapterName !== null ? embeddingAdapters.get(adapterName) ?? null : config?.embedding ?? null;
|
|
4433
|
+
if (candidate) {
|
|
4434
|
+
try {
|
|
4435
|
+
validateEmbeddingConfig(
|
|
4436
|
+
{ fields: metadata.embedding.fields, adapter: candidate },
|
|
4437
|
+
indexConfig.schema,
|
|
4438
|
+
void 0
|
|
4439
|
+
);
|
|
4440
|
+
} catch (err) {
|
|
4441
|
+
if (err instanceof NarsilError) {
|
|
4442
|
+
throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
|
|
4443
|
+
indexName: metadata.indexName,
|
|
4444
|
+
adapter: adapterName ?? void 0
|
|
4445
|
+
});
|
|
4446
|
+
}
|
|
4447
|
+
throw err;
|
|
4448
|
+
}
|
|
4449
|
+
embeddingAdapter = candidate;
|
|
4450
|
+
}
|
|
4451
|
+
}
|
|
4286
4452
|
executor.createIndex(metadata.indexName, indexConfig, language);
|
|
4287
4453
|
indexRegistry.set(metadata.indexName, {
|
|
4288
4454
|
config: indexConfig,
|
|
4289
4455
|
language,
|
|
4290
|
-
embeddingAdapter
|
|
4456
|
+
embeddingAdapter,
|
|
4457
|
+
embeddingAdapterName: adapterName,
|
|
4291
4458
|
vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
|
|
4292
4459
|
});
|
|
4293
4460
|
}
|
|
@@ -4330,6 +4497,7 @@ function createEngineCore(config) {
|
|
|
4330
4497
|
durability,
|
|
4331
4498
|
idGenerator,
|
|
4332
4499
|
indexRegistry,
|
|
4500
|
+
embeddingAdapters,
|
|
4333
4501
|
eventHandlers,
|
|
4334
4502
|
shutdownState,
|
|
4335
4503
|
abortController,
|
|
@@ -4347,300 +4515,65 @@ function createEngineCore(config) {
|
|
|
4347
4515
|
rebalanceCtx
|
|
4348
4516
|
};
|
|
4349
4517
|
}
|
|
4350
|
-
|
|
4351
|
-
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4355
|
-
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
}
|
|
4361
|
-
}
|
|
4362
|
-
if (durability) {
|
|
4363
|
-
await durability.manager.shutdown();
|
|
4364
|
-
}
|
|
4365
|
-
const adaptersToShutdown = /* @__PURE__ */ new Set();
|
|
4366
|
-
for (const [, entry] of indexRegistry) {
|
|
4367
|
-
if (entry.embeddingAdapter?.shutdown) {
|
|
4368
|
-
adaptersToShutdown.add(entry.embeddingAdapter);
|
|
4369
|
-
}
|
|
4370
|
-
}
|
|
4371
|
-
for (const adapter of adaptersToShutdown) {
|
|
4372
|
-
try {
|
|
4373
|
-
await adapter.shutdown?.();
|
|
4374
|
-
} catch {
|
|
4375
|
-
}
|
|
4518
|
+
var INDEX_CONFIG_PREFIX = "_narsil/index/";
|
|
4519
|
+
var INDEX_CONFIG_SUFFIX = "/config";
|
|
4520
|
+
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
4521
|
+
var MAX_INDEX_NAME_LENGTH = 255;
|
|
4522
|
+
var MAX_PARTITION_COUNT = 65536;
|
|
4523
|
+
var MAX_REPLICATION_FACTOR = 255;
|
|
4524
|
+
function truncateForDisplay(value) {
|
|
4525
|
+
const str = String(value);
|
|
4526
|
+
if (str.length > 100) {
|
|
4527
|
+
return `${str.slice(0, 100)}...`;
|
|
4376
4528
|
}
|
|
4377
|
-
|
|
4378
|
-
await executor.shutdown();
|
|
4379
|
-
eventHandlers.clear();
|
|
4380
|
-
indexRegistry.clear();
|
|
4529
|
+
return str;
|
|
4381
4530
|
}
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
function validateVector(vector, targetField, expectedDimensions) {
|
|
4385
|
-
if (!(vector instanceof Float32Array)) {
|
|
4531
|
+
function validateIndexName(indexName) {
|
|
4532
|
+
if (indexName.length === 0 || indexName.length > MAX_INDEX_NAME_LENGTH) {
|
|
4386
4533
|
throw new NarsilError(
|
|
4387
|
-
ErrorCodes.
|
|
4388
|
-
`
|
|
4389
|
-
{
|
|
4534
|
+
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4535
|
+
`Index name must be between 1 and ${MAX_INDEX_NAME_LENGTH} characters`,
|
|
4536
|
+
{ indexName: truncateForDisplay(indexName) }
|
|
4390
4537
|
);
|
|
4391
4538
|
}
|
|
4392
|
-
if (
|
|
4539
|
+
if (indexName.includes("\0") || indexName.includes("/") || indexName.includes("\\") || indexName.includes("..")) {
|
|
4393
4540
|
throw new NarsilError(
|
|
4394
|
-
ErrorCodes.
|
|
4395
|
-
|
|
4396
|
-
{
|
|
4541
|
+
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4542
|
+
"Index name contains forbidden characters (/, \\, .., or null bytes)",
|
|
4543
|
+
{ indexName: truncateForDisplay(indexName) }
|
|
4397
4544
|
);
|
|
4398
4545
|
}
|
|
4399
|
-
|
|
4400
|
-
|
|
4401
|
-
|
|
4402
|
-
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
);
|
|
4406
|
-
}
|
|
4546
|
+
if (!INDEX_NAME_PATTERN.test(indexName)) {
|
|
4547
|
+
throw new NarsilError(
|
|
4548
|
+
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4549
|
+
"Index name must start with an alphanumeric character and contain only alphanumeric characters, hyphens, underscores, or dots",
|
|
4550
|
+
{ indexName: truncateForDisplay(indexName) }
|
|
4551
|
+
);
|
|
4407
4552
|
}
|
|
4408
|
-
return vector;
|
|
4409
4553
|
}
|
|
4410
|
-
function
|
|
4411
|
-
|
|
4412
|
-
const segments = path.split(".");
|
|
4413
|
-
let current = document;
|
|
4414
|
-
for (const segment of segments) {
|
|
4415
|
-
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
4416
|
-
current = current[segment];
|
|
4417
|
-
}
|
|
4418
|
-
return current;
|
|
4554
|
+
function isValidInteger(value) {
|
|
4555
|
+
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
4419
4556
|
}
|
|
4420
|
-
function
|
|
4421
|
-
|
|
4422
|
-
document[path] = value;
|
|
4423
|
-
return;
|
|
4424
|
-
}
|
|
4425
|
-
const segments = path.split(".");
|
|
4426
|
-
let current = document;
|
|
4427
|
-
for (let i = 0; i < segments.length - 1; i++) {
|
|
4428
|
-
let next = current[segments[i]];
|
|
4429
|
-
if (next === null || next === void 0 || typeof next !== "object") {
|
|
4430
|
-
next = {};
|
|
4431
|
-
current[segments[i]] = next;
|
|
4432
|
-
}
|
|
4433
|
-
current = next;
|
|
4434
|
-
}
|
|
4435
|
-
current[segments[segments.length - 1]] = value;
|
|
4557
|
+
function indexConfigKey(indexName) {
|
|
4558
|
+
return `${INDEX_CONFIG_PREFIX}${indexName}${INDEX_CONFIG_SUFFIX}`;
|
|
4436
4559
|
}
|
|
4437
|
-
function
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4560
|
+
function isRecord2(value) {
|
|
4561
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
4562
|
+
}
|
|
4563
|
+
function validateDecodedMetadata(decoded, indexName) {
|
|
4564
|
+
if (!isRecord2(decoded)) {
|
|
4565
|
+
throw new NarsilError(
|
|
4566
|
+
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4567
|
+
`Index metadata for '${indexName}' is not an object`,
|
|
4568
|
+
{ indexName }
|
|
4569
|
+
);
|
|
4441
4570
|
}
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
}
|
|
4449
|
-
delete current[segments[segments.length - 1]];
|
|
4450
|
-
}
|
|
4451
|
-
function collectSourceText(document, sourceFields, targetField, documentIndex) {
|
|
4452
|
-
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
4453
|
-
const parts = [];
|
|
4454
|
-
for (const source of sources) {
|
|
4455
|
-
const value = resolveFieldValue(document, source);
|
|
4456
|
-
if (value === void 0 || value === null || value === "") continue;
|
|
4457
|
-
if (typeof value !== "string") {
|
|
4458
|
-
throw new NarsilError(
|
|
4459
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
4460
|
-
`Embedding source field "${source}" must be a string, got ${typeof value}`,
|
|
4461
|
-
{ field: source, targetField }
|
|
4462
|
-
);
|
|
4463
|
-
}
|
|
4464
|
-
parts.push(value);
|
|
4465
|
-
}
|
|
4466
|
-
if (parts.length === 0) {
|
|
4467
|
-
const details = { field: targetField };
|
|
4468
|
-
if (documentIndex !== void 0) {
|
|
4469
|
-
details.documentIndex = documentIndex;
|
|
4470
|
-
}
|
|
4471
|
-
throw new NarsilError(
|
|
4472
|
-
ErrorCodes.EMBEDDING_NO_SOURCE,
|
|
4473
|
-
`All source fields for embedding target "${targetField}" are missing or empty`,
|
|
4474
|
-
details
|
|
4475
|
-
);
|
|
4476
|
-
}
|
|
4477
|
-
return parts.join("\n");
|
|
4478
|
-
}
|
|
4479
|
-
function wrapAdapterError(err, targetField) {
|
|
4480
|
-
if (err instanceof NarsilError) return err;
|
|
4481
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
4482
|
-
return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
|
|
4483
|
-
field: targetField,
|
|
4484
|
-
cause: message
|
|
4485
|
-
});
|
|
4486
|
-
}
|
|
4487
|
-
async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
|
|
4488
|
-
const embeddedFields = /* @__PURE__ */ new Set();
|
|
4489
|
-
try {
|
|
4490
|
-
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
4491
|
-
const existing = resolveFieldValue(document, targetField);
|
|
4492
|
-
if (existing !== void 0 && existing !== null) continue;
|
|
4493
|
-
const text = collectSourceText(document, sourceFields, targetField);
|
|
4494
|
-
let vector;
|
|
4495
|
-
try {
|
|
4496
|
-
const raw = await adapter.embed(text, "document", signal);
|
|
4497
|
-
vector = validateVector(raw, targetField, adapter.dimensions);
|
|
4498
|
-
} catch (err) {
|
|
4499
|
-
throw wrapAdapterError(err, targetField);
|
|
4500
|
-
}
|
|
4501
|
-
setFieldValue(document, targetField, vector);
|
|
4502
|
-
embeddedFields.add(targetField);
|
|
4503
|
-
}
|
|
4504
|
-
} catch (err) {
|
|
4505
|
-
for (const field of embeddedFields) {
|
|
4506
|
-
deleteFieldValue(document, field);
|
|
4507
|
-
}
|
|
4508
|
-
throw err;
|
|
4509
|
-
}
|
|
4510
|
-
return embeddedFields;
|
|
4511
|
-
}
|
|
4512
|
-
var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([ErrorCodes.EMBEDDING_NO_SOURCE, ErrorCodes.DOC_VALIDATION_FAILED]);
|
|
4513
|
-
function isDocumentSourceError(err) {
|
|
4514
|
-
return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
|
|
4515
|
-
}
|
|
4516
|
-
async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
|
|
4517
|
-
const embedded = /* @__PURE__ */ new Map();
|
|
4518
|
-
const failed = /* @__PURE__ */ new Map();
|
|
4519
|
-
try {
|
|
4520
|
-
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
4521
|
-
const needsEmbedding = [];
|
|
4522
|
-
for (let i = 0; i < documents.length; i++) {
|
|
4523
|
-
if (failed.has(i)) continue;
|
|
4524
|
-
const doc = documents[i];
|
|
4525
|
-
const existing = resolveFieldValue(doc, targetField);
|
|
4526
|
-
if (existing !== void 0 && existing !== null) continue;
|
|
4527
|
-
try {
|
|
4528
|
-
const text = collectSourceText(doc, sourceFields, targetField, i);
|
|
4529
|
-
needsEmbedding.push({ docIndex: i, text });
|
|
4530
|
-
} catch (err) {
|
|
4531
|
-
if (isDocumentSourceError(err)) {
|
|
4532
|
-
failed.set(i, err);
|
|
4533
|
-
} else {
|
|
4534
|
-
throw err;
|
|
4535
|
-
}
|
|
4536
|
-
}
|
|
4537
|
-
}
|
|
4538
|
-
if (needsEmbedding.length === 0) continue;
|
|
4539
|
-
const texts = needsEmbedding.map((entry) => entry.text);
|
|
4540
|
-
let vectors;
|
|
4541
|
-
try {
|
|
4542
|
-
if (adapter.embedBatch) {
|
|
4543
|
-
vectors = await adapter.embedBatch(texts, "document", signal);
|
|
4544
|
-
} else {
|
|
4545
|
-
vectors = [];
|
|
4546
|
-
const concurrency = 8;
|
|
4547
|
-
for (let start = 0; start < texts.length; start += concurrency) {
|
|
4548
|
-
const chunk = texts.slice(start, start + concurrency);
|
|
4549
|
-
const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
|
|
4550
|
-
vectors.push(...chunkResults);
|
|
4551
|
-
}
|
|
4552
|
-
}
|
|
4553
|
-
} catch (err) {
|
|
4554
|
-
throw wrapAdapterError(err, targetField);
|
|
4555
|
-
}
|
|
4556
|
-
if (vectors.length !== needsEmbedding.length) {
|
|
4557
|
-
throw new NarsilError(
|
|
4558
|
-
ErrorCodes.EMBEDDING_FAILED,
|
|
4559
|
-
`Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
|
|
4560
|
-
{ field: targetField, expected: needsEmbedding.length, actual: vectors.length }
|
|
4561
|
-
);
|
|
4562
|
-
}
|
|
4563
|
-
for (let j = 0; j < needsEmbedding.length; j++) {
|
|
4564
|
-
const { docIndex } = needsEmbedding[j];
|
|
4565
|
-
const validated = validateVector(vectors[j], targetField, adapter.dimensions);
|
|
4566
|
-
setFieldValue(documents[docIndex], targetField, validated);
|
|
4567
|
-
let fieldSet = embedded.get(docIndex);
|
|
4568
|
-
if (!fieldSet) {
|
|
4569
|
-
fieldSet = /* @__PURE__ */ new Set();
|
|
4570
|
-
embedded.set(docIndex, fieldSet);
|
|
4571
|
-
}
|
|
4572
|
-
fieldSet.add(targetField);
|
|
4573
|
-
}
|
|
4574
|
-
}
|
|
4575
|
-
} catch (err) {
|
|
4576
|
-
for (const [docIndex, fields] of embedded) {
|
|
4577
|
-
for (const field of fields) {
|
|
4578
|
-
deleteFieldValue(documents[docIndex], field);
|
|
4579
|
-
}
|
|
4580
|
-
}
|
|
4581
|
-
throw err;
|
|
4582
|
-
}
|
|
4583
|
-
return { embedded, failed };
|
|
4584
|
-
}
|
|
4585
|
-
var INDEX_CONFIG_PREFIX = "_narsil/index/";
|
|
4586
|
-
var INDEX_CONFIG_SUFFIX = "/config";
|
|
4587
|
-
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
4588
|
-
var MAX_INDEX_NAME_LENGTH = 255;
|
|
4589
|
-
var MAX_PARTITION_COUNT = 65536;
|
|
4590
|
-
var MAX_REPLICATION_FACTOR = 255;
|
|
4591
|
-
function truncateForDisplay(value) {
|
|
4592
|
-
const str = String(value);
|
|
4593
|
-
if (str.length > 100) {
|
|
4594
|
-
return `${str.slice(0, 100)}...`;
|
|
4595
|
-
}
|
|
4596
|
-
return str;
|
|
4597
|
-
}
|
|
4598
|
-
function validateIndexName(indexName) {
|
|
4599
|
-
if (indexName.length === 0 || indexName.length > MAX_INDEX_NAME_LENGTH) {
|
|
4600
|
-
throw new NarsilError(
|
|
4601
|
-
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4602
|
-
`Index name must be between 1 and ${MAX_INDEX_NAME_LENGTH} characters`,
|
|
4603
|
-
{ indexName: truncateForDisplay(indexName) }
|
|
4604
|
-
);
|
|
4605
|
-
}
|
|
4606
|
-
if (indexName.includes("\0") || indexName.includes("/") || indexName.includes("\\") || indexName.includes("..")) {
|
|
4607
|
-
throw new NarsilError(
|
|
4608
|
-
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4609
|
-
"Index name contains forbidden characters (/, \\, .., or null bytes)",
|
|
4610
|
-
{ indexName: truncateForDisplay(indexName) }
|
|
4611
|
-
);
|
|
4612
|
-
}
|
|
4613
|
-
if (!INDEX_NAME_PATTERN.test(indexName)) {
|
|
4614
|
-
throw new NarsilError(
|
|
4615
|
-
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4616
|
-
"Index name must start with an alphanumeric character and contain only alphanumeric characters, hyphens, underscores, or dots",
|
|
4617
|
-
{ indexName: truncateForDisplay(indexName) }
|
|
4618
|
-
);
|
|
4619
|
-
}
|
|
4620
|
-
}
|
|
4621
|
-
function isValidInteger(value) {
|
|
4622
|
-
return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
|
|
4623
|
-
}
|
|
4624
|
-
function indexConfigKey(indexName) {
|
|
4625
|
-
return `${INDEX_CONFIG_PREFIX}${indexName}${INDEX_CONFIG_SUFFIX}`;
|
|
4626
|
-
}
|
|
4627
|
-
function isRecord2(value) {
|
|
4628
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
4629
|
-
}
|
|
4630
|
-
function validateDecodedMetadata(decoded, indexName) {
|
|
4631
|
-
if (!isRecord2(decoded)) {
|
|
4632
|
-
throw new NarsilError(
|
|
4633
|
-
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4634
|
-
`Index metadata for '${indexName}' is not an object`,
|
|
4635
|
-
{ indexName }
|
|
4636
|
-
);
|
|
4637
|
-
}
|
|
4638
|
-
if (typeof decoded.indexName !== "string") {
|
|
4639
|
-
throw new NarsilError(
|
|
4640
|
-
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4641
|
-
`Index metadata for '${indexName}' has invalid indexName`,
|
|
4642
|
-
{ indexName, received: truncateForDisplay(decoded.indexName) }
|
|
4643
|
-
);
|
|
4571
|
+
if (typeof decoded.indexName !== "string") {
|
|
4572
|
+
throw new NarsilError(
|
|
4573
|
+
ErrorCodes.CONTROLLER_METADATA_INVALID,
|
|
4574
|
+
`Index metadata for '${indexName}' has invalid indexName`,
|
|
4575
|
+
{ indexName, received: truncateForDisplay(decoded.indexName) }
|
|
4576
|
+
);
|
|
4644
4577
|
}
|
|
4645
4578
|
if (!isValidInteger(decoded.partitionCount) || decoded.partitionCount <= 0 || decoded.partitionCount > MAX_PARTITION_COUNT) {
|
|
4646
4579
|
throw new NarsilError(
|
|
@@ -4743,27 +4676,361 @@ function validateDocId(docId) {
|
|
|
4743
4676
|
if (!docId || docId.length === 0) {
|
|
4744
4677
|
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
|
|
4745
4678
|
}
|
|
4746
|
-
if (docId.length > MAX_DOC_ID_LENGTH) {
|
|
4679
|
+
if (docId.length > MAX_DOC_ID_LENGTH) {
|
|
4680
|
+
throw new NarsilError(
|
|
4681
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
4682
|
+
`Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
|
|
4683
|
+
{
|
|
4684
|
+
docId,
|
|
4685
|
+
length: docId.length
|
|
4686
|
+
}
|
|
4687
|
+
);
|
|
4688
|
+
}
|
|
4689
|
+
if (docId.includes("\0")) {
|
|
4690
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
|
|
4691
|
+
}
|
|
4692
|
+
}
|
|
4693
|
+
function clampLimit(limit) {
|
|
4694
|
+
if (limit === void 0) return DEFAULT_LIMIT;
|
|
4695
|
+
return Math.max(0, Math.min(limit, MAX_LIMIT));
|
|
4696
|
+
}
|
|
4697
|
+
function clampOffset(offset) {
|
|
4698
|
+
if (offset === void 0) return DEFAULT_OFFSET;
|
|
4699
|
+
return Math.max(0, Math.min(offset, MAX_OFFSET));
|
|
4700
|
+
}
|
|
4701
|
+
|
|
4702
|
+
// src/engine/index-lifecycle.ts
|
|
4703
|
+
async function createEngineIndex(core, config, name, indexConfig) {
|
|
4704
|
+
core.guardShutdown();
|
|
4705
|
+
validateIndexName2(name);
|
|
4706
|
+
if (core.indexRegistry.has(name)) {
|
|
4707
|
+
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
|
|
4708
|
+
}
|
|
4709
|
+
validateSchema(indexConfig.schema);
|
|
4710
|
+
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
4711
|
+
let resolvedEmbeddingAdapter = null;
|
|
4712
|
+
let embeddingAdapterName = null;
|
|
4713
|
+
if (indexConfig.embedding) {
|
|
4714
|
+
let configuredAdapter = indexConfig.embedding.adapter;
|
|
4715
|
+
if (typeof configuredAdapter === "string") {
|
|
4716
|
+
embeddingAdapterName = configuredAdapter;
|
|
4717
|
+
const registered = core.embeddingAdapters.get(configuredAdapter);
|
|
4718
|
+
if (!registered) {
|
|
4719
|
+
throw new NarsilError(
|
|
4720
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4721
|
+
`Embedding adapter "${configuredAdapter}" is not registered on this engine`,
|
|
4722
|
+
{ adapter: configuredAdapter, available: [...core.embeddingAdapters.keys()] }
|
|
4723
|
+
);
|
|
4724
|
+
}
|
|
4725
|
+
configuredAdapter = registered;
|
|
4726
|
+
}
|
|
4727
|
+
resolvedEmbeddingAdapter = validateEmbeddingConfig(
|
|
4728
|
+
{ fields: indexConfig.embedding.fields, adapter: configuredAdapter },
|
|
4729
|
+
indexConfig.schema,
|
|
4730
|
+
config?.embedding
|
|
4731
|
+
);
|
|
4732
|
+
}
|
|
4733
|
+
if (indexConfig.required && indexConfig.required.length > 0) {
|
|
4734
|
+
validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
|
|
4735
|
+
}
|
|
4736
|
+
const language = getLanguage(indexConfig.language ?? "english");
|
|
4737
|
+
core.executor.createIndex(name, indexConfig, language);
|
|
4738
|
+
core.indexRegistry.set(name, {
|
|
4739
|
+
config: indexConfig,
|
|
4740
|
+
language,
|
|
4741
|
+
embeddingAdapter: resolvedEmbeddingAdapter,
|
|
4742
|
+
embeddingAdapterName,
|
|
4743
|
+
vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
|
|
4744
|
+
});
|
|
4745
|
+
if (core.durability) {
|
|
4746
|
+
await core.durability.manager.persistMetadata(name);
|
|
4747
|
+
}
|
|
4748
|
+
await core.pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
|
|
4749
|
+
}
|
|
4750
|
+
async function dropEngineIndex(core, name) {
|
|
4751
|
+
core.guardShutdown();
|
|
4752
|
+
const entry = core.requireIndex(name);
|
|
4753
|
+
core.executor.dropIndex(name);
|
|
4754
|
+
core.indexRegistry.delete(name);
|
|
4755
|
+
if (core.durability) {
|
|
4756
|
+
await core.durability.manager.removeIndex(name);
|
|
4757
|
+
}
|
|
4758
|
+
await core.pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
|
|
4759
|
+
}
|
|
4760
|
+
function registerEngineEmbeddingAdapter(core, name, adapter) {
|
|
4761
|
+
core.guardShutdown();
|
|
4762
|
+
validateRegisteredAdapter(name, adapter);
|
|
4763
|
+
const affected = [];
|
|
4764
|
+
for (const [indexName, entry] of core.indexRegistry) {
|
|
4765
|
+
if (entry.embeddingAdapterName !== name || !entry.config.embedding) continue;
|
|
4766
|
+
try {
|
|
4767
|
+
validateEmbeddingConfig({ fields: entry.config.embedding.fields, adapter }, entry.config.schema, void 0);
|
|
4768
|
+
} catch (err) {
|
|
4769
|
+
if (err instanceof NarsilError) {
|
|
4770
|
+
throw new NarsilError(
|
|
4771
|
+
err.code,
|
|
4772
|
+
`Cannot bind embedding adapter "${name}" to index "${indexName}": ${err.message}`,
|
|
4773
|
+
{
|
|
4774
|
+
indexName,
|
|
4775
|
+
adapter: name
|
|
4776
|
+
}
|
|
4777
|
+
);
|
|
4778
|
+
}
|
|
4779
|
+
throw err;
|
|
4780
|
+
}
|
|
4781
|
+
affected.push(entry);
|
|
4782
|
+
}
|
|
4783
|
+
core.embeddingAdapters.set(name, adapter);
|
|
4784
|
+
for (const entry of affected) {
|
|
4785
|
+
entry.embeddingAdapter = adapter;
|
|
4786
|
+
}
|
|
4787
|
+
}
|
|
4788
|
+
|
|
4789
|
+
// src/engine/lifecycle.ts
|
|
4790
|
+
async function shutdownEngine(core) {
|
|
4791
|
+
const { executor, durability, indexRegistry, eventHandlers, orchestrator } = core;
|
|
4792
|
+
for (const [name] of indexRegistry) {
|
|
4793
|
+
const manager = executor.getManager(name);
|
|
4794
|
+
if (manager) {
|
|
4795
|
+
for (const [, vectorIndex] of manager.getVectorIndexes()) {
|
|
4796
|
+
vectorIndex.dispose();
|
|
4797
|
+
}
|
|
4798
|
+
}
|
|
4799
|
+
}
|
|
4800
|
+
if (durability) {
|
|
4801
|
+
await durability.manager.shutdown();
|
|
4802
|
+
}
|
|
4803
|
+
const adaptersToShutdown = /* @__PURE__ */ new Set();
|
|
4804
|
+
for (const [, entry] of indexRegistry) {
|
|
4805
|
+
if (entry.embeddingAdapter?.shutdown) {
|
|
4806
|
+
adaptersToShutdown.add(entry.embeddingAdapter);
|
|
4807
|
+
}
|
|
4808
|
+
}
|
|
4809
|
+
for (const adapter of adaptersToShutdown) {
|
|
4810
|
+
try {
|
|
4811
|
+
await adapter.shutdown?.();
|
|
4812
|
+
} catch {
|
|
4813
|
+
}
|
|
4814
|
+
}
|
|
4815
|
+
await orchestrator.shutdown();
|
|
4816
|
+
await executor.shutdown();
|
|
4817
|
+
eventHandlers.clear();
|
|
4818
|
+
indexRegistry.clear();
|
|
4819
|
+
}
|
|
4820
|
+
|
|
4821
|
+
// src/engine/embed.ts
|
|
4822
|
+
function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterName) {
|
|
4823
|
+
for (const targetField of Object.keys(embeddingConfig.fields)) {
|
|
4824
|
+
const existing = resolveFieldValue(document, targetField);
|
|
4825
|
+
if (existing === void 0 || existing === null) {
|
|
4826
|
+
throw new NarsilError(
|
|
4827
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4828
|
+
adapterName ? `The index embeds "${targetField}" via adapter "${adapterName}", which is not registered on this engine. Register the adapter or supply the vector on the document.` : `The index embeds "${targetField}" automatically but no embedding adapter is bound. Bind an adapter or supply the vector on the document.`,
|
|
4829
|
+
adapterName ? { field: targetField, adapter: adapterName } : { field: targetField }
|
|
4830
|
+
);
|
|
4831
|
+
}
|
|
4832
|
+
}
|
|
4833
|
+
}
|
|
4834
|
+
function validateVector(vector, targetField, expectedDimensions) {
|
|
4835
|
+
if (!(vector instanceof Float32Array)) {
|
|
4836
|
+
throw new NarsilError(
|
|
4837
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
4838
|
+
`Adapter returned ${typeof vector} for field "${targetField}", expected Float32Array`,
|
|
4839
|
+
{ field: targetField, returnedType: typeof vector }
|
|
4840
|
+
);
|
|
4841
|
+
}
|
|
4842
|
+
if (vector.length !== expectedDimensions) {
|
|
4843
|
+
throw new NarsilError(
|
|
4844
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
4845
|
+
`Adapter returned ${vector.length}-dimensional vector for field "${targetField}", expected ${expectedDimensions}`,
|
|
4846
|
+
{ field: targetField, expected: expectedDimensions, actual: vector.length }
|
|
4847
|
+
);
|
|
4848
|
+
}
|
|
4849
|
+
for (let i = 0; i < vector.length; i++) {
|
|
4850
|
+
if (Number.isNaN(vector[i])) {
|
|
4851
|
+
throw new NarsilError(
|
|
4852
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
4853
|
+
`Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
|
|
4854
|
+
{ field: targetField, index: i }
|
|
4855
|
+
);
|
|
4856
|
+
}
|
|
4857
|
+
}
|
|
4858
|
+
return vector;
|
|
4859
|
+
}
|
|
4860
|
+
function resolveFieldValue(document, path) {
|
|
4861
|
+
if (!path.includes(".")) return document[path];
|
|
4862
|
+
const segments = path.split(".");
|
|
4863
|
+
let current = document;
|
|
4864
|
+
for (const segment of segments) {
|
|
4865
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
4866
|
+
current = current[segment];
|
|
4867
|
+
}
|
|
4868
|
+
return current;
|
|
4869
|
+
}
|
|
4870
|
+
function setFieldValue(document, path, value) {
|
|
4871
|
+
if (!path.includes(".")) {
|
|
4872
|
+
document[path] = value;
|
|
4873
|
+
return;
|
|
4874
|
+
}
|
|
4875
|
+
const segments = path.split(".");
|
|
4876
|
+
let current = document;
|
|
4877
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
4878
|
+
let next = current[segments[i]];
|
|
4879
|
+
if (next === null || next === void 0 || typeof next !== "object") {
|
|
4880
|
+
next = {};
|
|
4881
|
+
current[segments[i]] = next;
|
|
4882
|
+
}
|
|
4883
|
+
current = next;
|
|
4884
|
+
}
|
|
4885
|
+
current[segments[segments.length - 1]] = value;
|
|
4886
|
+
}
|
|
4887
|
+
function deleteFieldValue(document, path) {
|
|
4888
|
+
if (!path.includes(".")) {
|
|
4889
|
+
delete document[path];
|
|
4890
|
+
return;
|
|
4891
|
+
}
|
|
4892
|
+
const segments = path.split(".");
|
|
4893
|
+
let current = document;
|
|
4894
|
+
for (let i = 0; i < segments.length - 1; i++) {
|
|
4895
|
+
const next = current[segments[i]];
|
|
4896
|
+
if (next === null || next === void 0 || typeof next !== "object") return;
|
|
4897
|
+
current = next;
|
|
4898
|
+
}
|
|
4899
|
+
delete current[segments[segments.length - 1]];
|
|
4900
|
+
}
|
|
4901
|
+
function collectSourceText(document, sourceFields, targetField, documentIndex) {
|
|
4902
|
+
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
4903
|
+
const parts = [];
|
|
4904
|
+
for (const source of sources) {
|
|
4905
|
+
const value = resolveFieldValue(document, source);
|
|
4906
|
+
if (value === void 0 || value === null || value === "") continue;
|
|
4907
|
+
if (typeof value !== "string") {
|
|
4908
|
+
throw new NarsilError(
|
|
4909
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
4910
|
+
`Embedding source field "${source}" must be a string, got ${typeof value}`,
|
|
4911
|
+
{ field: source, targetField }
|
|
4912
|
+
);
|
|
4913
|
+
}
|
|
4914
|
+
parts.push(value);
|
|
4915
|
+
}
|
|
4916
|
+
if (parts.length === 0) {
|
|
4917
|
+
const details = { field: targetField };
|
|
4918
|
+
if (documentIndex !== void 0) {
|
|
4919
|
+
details.documentIndex = documentIndex;
|
|
4920
|
+
}
|
|
4747
4921
|
throw new NarsilError(
|
|
4748
|
-
ErrorCodes.
|
|
4749
|
-
`
|
|
4750
|
-
|
|
4751
|
-
docId,
|
|
4752
|
-
length: docId.length
|
|
4753
|
-
}
|
|
4922
|
+
ErrorCodes.EMBEDDING_NO_SOURCE,
|
|
4923
|
+
`All source fields for embedding target "${targetField}" are missing or empty`,
|
|
4924
|
+
details
|
|
4754
4925
|
);
|
|
4755
4926
|
}
|
|
4756
|
-
|
|
4757
|
-
|
|
4927
|
+
return parts.join("\n");
|
|
4928
|
+
}
|
|
4929
|
+
function wrapAdapterError(err, targetField) {
|
|
4930
|
+
if (err instanceof NarsilError) return err;
|
|
4931
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4932
|
+
return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
|
|
4933
|
+
field: targetField,
|
|
4934
|
+
cause: message
|
|
4935
|
+
});
|
|
4936
|
+
}
|
|
4937
|
+
async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
|
|
4938
|
+
const embeddedFields = /* @__PURE__ */ new Set();
|
|
4939
|
+
try {
|
|
4940
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
4941
|
+
const existing = resolveFieldValue(document, targetField);
|
|
4942
|
+
if (existing !== void 0 && existing !== null) continue;
|
|
4943
|
+
const text = collectSourceText(document, sourceFields, targetField);
|
|
4944
|
+
let vector;
|
|
4945
|
+
try {
|
|
4946
|
+
const raw = await adapter.embed(text, "document", signal);
|
|
4947
|
+
vector = validateVector(raw, targetField, adapter.dimensions);
|
|
4948
|
+
} catch (err) {
|
|
4949
|
+
throw wrapAdapterError(err, targetField);
|
|
4950
|
+
}
|
|
4951
|
+
setFieldValue(document, targetField, vector);
|
|
4952
|
+
embeddedFields.add(targetField);
|
|
4953
|
+
}
|
|
4954
|
+
} catch (err) {
|
|
4955
|
+
for (const field of embeddedFields) {
|
|
4956
|
+
deleteFieldValue(document, field);
|
|
4957
|
+
}
|
|
4958
|
+
throw err;
|
|
4758
4959
|
}
|
|
4960
|
+
return embeddedFields;
|
|
4759
4961
|
}
|
|
4760
|
-
|
|
4761
|
-
|
|
4762
|
-
return
|
|
4962
|
+
var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([ErrorCodes.EMBEDDING_NO_SOURCE, ErrorCodes.DOC_VALIDATION_FAILED]);
|
|
4963
|
+
function isDocumentSourceError(err) {
|
|
4964
|
+
return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
|
|
4763
4965
|
}
|
|
4764
|
-
function
|
|
4765
|
-
|
|
4766
|
-
|
|
4966
|
+
async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
|
|
4967
|
+
const embedded = /* @__PURE__ */ new Map();
|
|
4968
|
+
const failed = /* @__PURE__ */ new Map();
|
|
4969
|
+
try {
|
|
4970
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
4971
|
+
const needsEmbedding = [];
|
|
4972
|
+
for (let i = 0; i < documents.length; i++) {
|
|
4973
|
+
if (failed.has(i)) continue;
|
|
4974
|
+
const doc = documents[i];
|
|
4975
|
+
const existing = resolveFieldValue(doc, targetField);
|
|
4976
|
+
if (existing !== void 0 && existing !== null) continue;
|
|
4977
|
+
try {
|
|
4978
|
+
const text = collectSourceText(doc, sourceFields, targetField, i);
|
|
4979
|
+
needsEmbedding.push({ docIndex: i, text });
|
|
4980
|
+
} catch (err) {
|
|
4981
|
+
if (isDocumentSourceError(err)) {
|
|
4982
|
+
failed.set(i, err);
|
|
4983
|
+
} else {
|
|
4984
|
+
throw err;
|
|
4985
|
+
}
|
|
4986
|
+
}
|
|
4987
|
+
}
|
|
4988
|
+
if (needsEmbedding.length === 0) continue;
|
|
4989
|
+
const texts = needsEmbedding.map((entry) => entry.text);
|
|
4990
|
+
let vectors;
|
|
4991
|
+
try {
|
|
4992
|
+
if (adapter.embedBatch) {
|
|
4993
|
+
vectors = await adapter.embedBatch(texts, "document", signal);
|
|
4994
|
+
} else {
|
|
4995
|
+
vectors = [];
|
|
4996
|
+
const concurrency = 8;
|
|
4997
|
+
for (let start = 0; start < texts.length; start += concurrency) {
|
|
4998
|
+
const chunk = texts.slice(start, start + concurrency);
|
|
4999
|
+
const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
|
|
5000
|
+
vectors.push(...chunkResults);
|
|
5001
|
+
}
|
|
5002
|
+
}
|
|
5003
|
+
} catch (err) {
|
|
5004
|
+
throw wrapAdapterError(err, targetField);
|
|
5005
|
+
}
|
|
5006
|
+
if (vectors.length !== needsEmbedding.length) {
|
|
5007
|
+
throw new NarsilError(
|
|
5008
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
5009
|
+
`Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
|
|
5010
|
+
{ field: targetField, expected: needsEmbedding.length, actual: vectors.length }
|
|
5011
|
+
);
|
|
5012
|
+
}
|
|
5013
|
+
for (let j = 0; j < needsEmbedding.length; j++) {
|
|
5014
|
+
const { docIndex } = needsEmbedding[j];
|
|
5015
|
+
const validated = validateVector(vectors[j], targetField, adapter.dimensions);
|
|
5016
|
+
setFieldValue(documents[docIndex], targetField, validated);
|
|
5017
|
+
let fieldSet = embedded.get(docIndex);
|
|
5018
|
+
if (!fieldSet) {
|
|
5019
|
+
fieldSet = /* @__PURE__ */ new Set();
|
|
5020
|
+
embedded.set(docIndex, fieldSet);
|
|
5021
|
+
}
|
|
5022
|
+
fieldSet.add(targetField);
|
|
5023
|
+
}
|
|
5024
|
+
}
|
|
5025
|
+
} catch (err) {
|
|
5026
|
+
for (const [docIndex, fields] of embedded) {
|
|
5027
|
+
for (const field of fields) {
|
|
5028
|
+
deleteFieldValue(documents[docIndex], field);
|
|
5029
|
+
}
|
|
5030
|
+
}
|
|
5031
|
+
throw err;
|
|
5032
|
+
}
|
|
5033
|
+
return { embedded, failed };
|
|
4767
5034
|
}
|
|
4768
5035
|
|
|
4769
5036
|
// src/engine/mutations/durable-rollback.ts
|
|
@@ -4827,13 +5094,21 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
4827
5094
|
if (entry.config.required && entry.config.required.length > 0) {
|
|
4828
5095
|
validateRequiredFields(document, entry.config.required);
|
|
4829
5096
|
}
|
|
4830
|
-
if (entry.
|
|
4831
|
-
|
|
4832
|
-
|
|
4833
|
-
|
|
4834
|
-
|
|
4835
|
-
|
|
4836
|
-
|
|
5097
|
+
if (entry.config.embedding) {
|
|
5098
|
+
if (entry.embeddingAdapter) {
|
|
5099
|
+
await embedDocumentFields(
|
|
5100
|
+
document,
|
|
5101
|
+
entry.config.embedding,
|
|
5102
|
+
entry.embeddingAdapter,
|
|
5103
|
+
ctx.abortController.signal
|
|
5104
|
+
);
|
|
5105
|
+
} else {
|
|
5106
|
+
assertDocumentCarriesMappedVectors(
|
|
5107
|
+
document,
|
|
5108
|
+
entry.config.embedding,
|
|
5109
|
+
entry.embeddingAdapterName
|
|
5110
|
+
);
|
|
5111
|
+
}
|
|
4837
5112
|
}
|
|
4838
5113
|
const insertManager = ctx.requireManager(indexName);
|
|
4839
5114
|
const insertVecIndexes = insertManager.getVectorIndexes();
|
|
@@ -4913,6 +5188,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
4913
5188
|
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
4914
5189
|
const hasRequired = entry.config.required && entry.config.required.length > 0;
|
|
4915
5190
|
const hasEmbedding = entry.embeddingAdapter && entry.config.embedding;
|
|
5191
|
+
const embeddingUnbound = !entry.embeddingAdapter && entry.config.embedding;
|
|
4916
5192
|
const batchManager = ctx.requireManager(indexName);
|
|
4917
5193
|
const batchVecIndexes = batchManager.getVectorIndexes();
|
|
4918
5194
|
const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
@@ -4928,7 +5204,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
4928
5204
|
} catch (err) {
|
|
4929
5205
|
chunkFailedIndexes.add(i);
|
|
4930
5206
|
failed.push({
|
|
4931
|
-
docId: "",
|
|
5207
|
+
docId: providedDocId(documents[i]) ?? "",
|
|
4932
5208
|
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err))
|
|
4933
5209
|
});
|
|
4934
5210
|
}
|
|
@@ -4953,16 +5229,33 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
4953
5229
|
for (const [sliceIndex, error] of embedResult.failed) {
|
|
4954
5230
|
const originalIdx = embeddableOriginalIndexes[sliceIndex];
|
|
4955
5231
|
chunkFailedIndexes.add(originalIdx);
|
|
4956
|
-
failed.push({ docId: "", error });
|
|
5232
|
+
failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error });
|
|
4957
5233
|
}
|
|
4958
5234
|
} catch (err) {
|
|
4959
5235
|
const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
4960
5236
|
for (const originalIdx of embeddableOriginalIndexes) {
|
|
4961
5237
|
chunkFailedIndexes.add(originalIdx);
|
|
4962
|
-
failed.push({ docId: "", error: embeddingError });
|
|
5238
|
+
failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error: embeddingError });
|
|
4963
5239
|
}
|
|
4964
5240
|
}
|
|
4965
5241
|
}
|
|
5242
|
+
} else if (embeddingUnbound) {
|
|
5243
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
5244
|
+
if (chunkFailedIndexes.has(i)) continue;
|
|
5245
|
+
try {
|
|
5246
|
+
assertDocumentCarriesMappedVectors(
|
|
5247
|
+
documents[i],
|
|
5248
|
+
entry.config.embedding,
|
|
5249
|
+
entry.embeddingAdapterName
|
|
5250
|
+
);
|
|
5251
|
+
} catch (err) {
|
|
5252
|
+
chunkFailedIndexes.add(i);
|
|
5253
|
+
failed.push({
|
|
5254
|
+
docId: providedDocId(documents[i]) ?? "",
|
|
5255
|
+
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
|
|
5256
|
+
});
|
|
5257
|
+
}
|
|
5258
|
+
}
|
|
4966
5259
|
}
|
|
4967
5260
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
4968
5261
|
if (ctx.abortController.signal.aborted) break;
|
|
@@ -5156,13 +5449,21 @@ async function updateDocument(ctx, indexName, docId, document) {
|
|
|
5156
5449
|
if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
|
|
5157
5450
|
return;
|
|
5158
5451
|
}
|
|
5159
|
-
if (entry.
|
|
5160
|
-
|
|
5161
|
-
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
|
-
|
|
5165
|
-
|
|
5452
|
+
if (entry.config.embedding) {
|
|
5453
|
+
if (entry.embeddingAdapter) {
|
|
5454
|
+
await embedDocumentFields(
|
|
5455
|
+
document,
|
|
5456
|
+
entry.config.embedding,
|
|
5457
|
+
entry.embeddingAdapter,
|
|
5458
|
+
ctx.abortController.signal
|
|
5459
|
+
);
|
|
5460
|
+
} else {
|
|
5461
|
+
assertDocumentCarriesMappedVectors(
|
|
5462
|
+
document,
|
|
5463
|
+
entry.config.embedding,
|
|
5464
|
+
entry.embeddingAdapterName
|
|
5465
|
+
);
|
|
5466
|
+
}
|
|
5166
5467
|
}
|
|
5167
5468
|
const updateManager = ctx.requireManager(indexName);
|
|
5168
5469
|
const oldDocument = updateManager.get(docId);
|
|
@@ -6212,7 +6513,7 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
|
6212
6513
|
}
|
|
6213
6514
|
|
|
6214
6515
|
// src/engine/resolve-vector-text.ts
|
|
6215
|
-
async function resolveVectorText(params, embeddingAdapter, signal) {
|
|
6516
|
+
async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
|
|
6216
6517
|
if (!params.vector) return params;
|
|
6217
6518
|
if (params.vector.text === void 0 && params.vector.value === void 0) return params;
|
|
6218
6519
|
if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
|
|
@@ -6222,7 +6523,8 @@ async function resolveVectorText(params, embeddingAdapter, signal) {
|
|
|
6222
6523
|
if (!embeddingAdapter) {
|
|
6223
6524
|
throw new NarsilError(
|
|
6224
6525
|
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6225
|
-
"Vector query with 'text' requires an embedding adapter on the index or instance"
|
|
6526
|
+
embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
|
|
6527
|
+
embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
|
|
6226
6528
|
);
|
|
6227
6529
|
}
|
|
6228
6530
|
if (typeof params.vector.text !== "string") {
|
|
@@ -6304,7 +6606,13 @@ async function restoreFromSnapshot(indexName, data, executor, indexRegistry, get
|
|
|
6304
6606
|
const indexConfig = { schema, language: envelope.language };
|
|
6305
6607
|
executor.createIndex(indexName, indexConfig, language);
|
|
6306
6608
|
const vectorFieldPaths = getVectorFieldPaths2(schema);
|
|
6307
|
-
indexRegistry.set(indexName, {
|
|
6609
|
+
indexRegistry.set(indexName, {
|
|
6610
|
+
config: indexConfig,
|
|
6611
|
+
language,
|
|
6612
|
+
embeddingAdapter: null,
|
|
6613
|
+
embeddingAdapterName: null,
|
|
6614
|
+
vectorFieldPaths
|
|
6615
|
+
});
|
|
6308
6616
|
try {
|
|
6309
6617
|
const manager = requireManager(indexName);
|
|
6310
6618
|
while (manager.partitionCount < envelope.partitions.length) {
|
|
@@ -6435,97 +6743,6 @@ function readProcessMemory() {
|
|
|
6435
6743
|
}
|
|
6436
6744
|
}
|
|
6437
6745
|
|
|
6438
|
-
// src/schema/embedding-validator.ts
|
|
6439
|
-
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
6440
|
-
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
6441
|
-
const fieldEntries = Object.keys(embeddingConfig.fields);
|
|
6442
|
-
if (fieldEntries.length === 0) {
|
|
6443
|
-
throw new NarsilError(
|
|
6444
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6445
|
-
"Embedding config must define at least one field mapping"
|
|
6446
|
-
);
|
|
6447
|
-
}
|
|
6448
|
-
const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
|
|
6449
|
-
if (!resolvedAdapter) {
|
|
6450
|
-
throw new NarsilError(
|
|
6451
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6452
|
-
"Embedding config requires an adapter (set on the index or the instance)"
|
|
6453
|
-
);
|
|
6454
|
-
}
|
|
6455
|
-
if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
|
|
6456
|
-
throw new NarsilError(
|
|
6457
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6458
|
-
`Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
|
|
6459
|
-
);
|
|
6460
|
-
}
|
|
6461
|
-
const flatFields = flattenSchema(schema);
|
|
6462
|
-
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
6463
|
-
const targetType = flatFields[targetField];
|
|
6464
|
-
if (targetType === void 0) {
|
|
6465
|
-
throw new NarsilError(
|
|
6466
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6467
|
-
`Embedding target field "${targetField}" does not exist in the schema`,
|
|
6468
|
-
{ field: targetField }
|
|
6469
|
-
);
|
|
6470
|
-
}
|
|
6471
|
-
const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
|
|
6472
|
-
if (!vectorMatch) {
|
|
6473
|
-
throw new NarsilError(
|
|
6474
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6475
|
-
`Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
|
|
6476
|
-
{ field: targetField, type: targetType }
|
|
6477
|
-
);
|
|
6478
|
-
}
|
|
6479
|
-
const schemaDimension = Number.parseInt(vectorMatch[1], 10);
|
|
6480
|
-
if (schemaDimension !== resolvedAdapter.dimensions) {
|
|
6481
|
-
throw new NarsilError(
|
|
6482
|
-
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
6483
|
-
`Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
|
|
6484
|
-
{ field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
|
|
6485
|
-
);
|
|
6486
|
-
}
|
|
6487
|
-
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
6488
|
-
if (sources.length === 0) {
|
|
6489
|
-
throw new NarsilError(
|
|
6490
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6491
|
-
`Embedding target field "${targetField}" must have at least one source field`,
|
|
6492
|
-
{ field: targetField }
|
|
6493
|
-
);
|
|
6494
|
-
}
|
|
6495
|
-
for (const source of sources) {
|
|
6496
|
-
const sourceType = flatFields[source];
|
|
6497
|
-
if (sourceType === void 0) {
|
|
6498
|
-
throw new NarsilError(
|
|
6499
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6500
|
-
`Embedding source field "${source}" does not exist in the schema`,
|
|
6501
|
-
{ field: source, targetField }
|
|
6502
|
-
);
|
|
6503
|
-
}
|
|
6504
|
-
if (sourceType !== "string") {
|
|
6505
|
-
throw new NarsilError(
|
|
6506
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6507
|
-
`Embedding source field "${source}" must be a string type, got "${sourceType}"`,
|
|
6508
|
-
{ field: source, type: sourceType, targetField }
|
|
6509
|
-
);
|
|
6510
|
-
}
|
|
6511
|
-
}
|
|
6512
|
-
}
|
|
6513
|
-
return resolvedAdapter;
|
|
6514
|
-
}
|
|
6515
|
-
function validateRequiredFieldsInSchema(required, schema) {
|
|
6516
|
-
if (required.length === 0) return;
|
|
6517
|
-
const flatFields = flattenSchema(schema);
|
|
6518
|
-
for (const field of required) {
|
|
6519
|
-
if (flatFields[field] === void 0) {
|
|
6520
|
-
throw new NarsilError(
|
|
6521
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
6522
|
-
`Required field "${field}" does not exist in the schema`,
|
|
6523
|
-
{ field }
|
|
6524
|
-
);
|
|
6525
|
-
}
|
|
6526
|
-
}
|
|
6527
|
-
}
|
|
6528
|
-
|
|
6529
6746
|
// src/narsil.ts
|
|
6530
6747
|
async function createNarsil(config) {
|
|
6531
6748
|
const core = createEngineCore(config);
|
|
@@ -6552,43 +6769,14 @@ function createNarsilFromCore(core, config) {
|
|
|
6552
6769
|
rebalanceCtx
|
|
6553
6770
|
} = core;
|
|
6554
6771
|
const narsil = {
|
|
6555
|
-
|
|
6556
|
-
|
|
6557
|
-
validateIndexName2(name);
|
|
6558
|
-
if (indexRegistry.has(name)) {
|
|
6559
|
-
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
|
|
6560
|
-
}
|
|
6561
|
-
validateSchema(indexConfig.schema);
|
|
6562
|
-
let resolvedEmbeddingAdapter = null;
|
|
6563
|
-
if (indexConfig.embedding) {
|
|
6564
|
-
resolvedEmbeddingAdapter = validateEmbeddingConfig(indexConfig.embedding, indexConfig.schema, config?.embedding);
|
|
6565
|
-
}
|
|
6566
|
-
if (indexConfig.required && indexConfig.required.length > 0) {
|
|
6567
|
-
validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
|
|
6568
|
-
}
|
|
6569
|
-
const language = getLanguage(indexConfig.language ?? "english");
|
|
6570
|
-
executor.createIndex(name, indexConfig, language);
|
|
6571
|
-
const vectorFieldPaths = getVectorFieldPaths(indexConfig.schema);
|
|
6572
|
-
indexRegistry.set(name, {
|
|
6573
|
-
config: indexConfig,
|
|
6574
|
-
language,
|
|
6575
|
-
embeddingAdapter: resolvedEmbeddingAdapter,
|
|
6576
|
-
vectorFieldPaths
|
|
6577
|
-
});
|
|
6578
|
-
if (durability) {
|
|
6579
|
-
await durability.manager.persistMetadata(name);
|
|
6580
|
-
}
|
|
6581
|
-
await pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
|
|
6772
|
+
createIndex(name, indexConfig) {
|
|
6773
|
+
return createEngineIndex(core, config, name, indexConfig);
|
|
6582
6774
|
},
|
|
6583
|
-
|
|
6584
|
-
|
|
6585
|
-
|
|
6586
|
-
|
|
6587
|
-
|
|
6588
|
-
if (durability) {
|
|
6589
|
-
await durability.manager.removeIndex(name);
|
|
6590
|
-
}
|
|
6591
|
-
await pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
|
|
6775
|
+
registerEmbeddingAdapter(name, adapter) {
|
|
6776
|
+
registerEngineEmbeddingAdapter(core, name, adapter);
|
|
6777
|
+
},
|
|
6778
|
+
dropIndex(name) {
|
|
6779
|
+
return dropEngineIndex(core, name);
|
|
6592
6780
|
},
|
|
6593
6781
|
listIndexes() {
|
|
6594
6782
|
const infos = [];
|
|
@@ -6667,7 +6855,12 @@ function createNarsilFromCore(core, config) {
|
|
|
6667
6855
|
guardShutdown();
|
|
6668
6856
|
const entry = requireIndex(indexName);
|
|
6669
6857
|
const manager = requireManager(indexName);
|
|
6670
|
-
const resolvedParams = await resolveVectorText(
|
|
6858
|
+
const resolvedParams = await resolveVectorText(
|
|
6859
|
+
params,
|
|
6860
|
+
entry.embeddingAdapter,
|
|
6861
|
+
abortController.signal,
|
|
6862
|
+
entry.embeddingAdapterName
|
|
6863
|
+
);
|
|
6671
6864
|
await pluginRegistry.runHook("beforeSearch", { indexName, params: resolvedParams });
|
|
6672
6865
|
const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
|
|
6673
6866
|
const result = await executeQuery(resolvedParams, {
|
|
@@ -6692,7 +6885,12 @@ function createNarsilFromCore(core, config) {
|
|
|
6692
6885
|
guardShutdown();
|
|
6693
6886
|
const entry = requireIndex(indexName);
|
|
6694
6887
|
const manager = requireManager(indexName);
|
|
6695
|
-
const resolvedParams = await resolveVectorText(
|
|
6888
|
+
const resolvedParams = await resolveVectorText(
|
|
6889
|
+
params,
|
|
6890
|
+
entry.embeddingAdapter,
|
|
6891
|
+
abortController.signal,
|
|
6892
|
+
entry.embeddingAdapterName
|
|
6893
|
+
);
|
|
6696
6894
|
const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
|
|
6697
6895
|
return executePreflight(resolvedParams, {
|
|
6698
6896
|
manager,
|