@delali/narsil 0.1.8 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, validateSchema, createPartitionManager, createVectorIndex, deserializePayloadV2, tokenize, fanOutQuery, validateRequiredFields, flattenSchema, createPartitionIndex, fnv1a, detectRuntime, kWayMerge } from './chunk-FS7XLIUS.mjs';
1
+ import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, createPartitionManager, createVectorIndex, validateSchema, deserializePayloadV2, tokenize, fanOutQuery, validateRequiredFields, validateVectorPromotion, createPartitionIndex, flattenSchema, fnv1a, detectRuntime, kWayMerge } from './chunk-VWF7RKSF.mjs';
2
2
  import { crc32, crc32Init, crc32Update, crc32Final } from './chunk-KMMJMDUA.mjs';
3
3
  import { ErrorCodes, NarsilError } from './chunk-ALB35PRQ.mjs';
4
4
  import { generateId, createRequestId, isValidWorkerResponse } from './chunk-HRABJLPB.mjs';
@@ -132,6 +132,123 @@ function createPluginRegistry() {
132
132
  };
133
133
  }
134
134
 
135
+ // src/schema/embedding-validator.ts
136
+ var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
137
+ function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
138
+ const fieldEntries = Object.keys(embeddingConfig.fields);
139
+ if (fieldEntries.length === 0) {
140
+ throw new NarsilError(
141
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
142
+ "Embedding config must define at least one field mapping"
143
+ );
144
+ }
145
+ if (typeof embeddingConfig.adapter === "string") {
146
+ throw new NarsilError(
147
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
148
+ `Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
149
+ { adapter: embeddingConfig.adapter }
150
+ );
151
+ }
152
+ const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
153
+ if (!resolvedAdapter) {
154
+ throw new NarsilError(
155
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
156
+ "Embedding config requires an adapter (set on the index or the instance)"
157
+ );
158
+ }
159
+ if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
160
+ throw new NarsilError(
161
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
162
+ `Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
163
+ );
164
+ }
165
+ const flatFields = flattenSchema(schema);
166
+ for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
167
+ const targetType = flatFields[targetField];
168
+ if (targetType === void 0) {
169
+ throw new NarsilError(
170
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
171
+ `Embedding target field "${targetField}" does not exist in the schema`,
172
+ { field: targetField }
173
+ );
174
+ }
175
+ const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
176
+ if (!vectorMatch) {
177
+ throw new NarsilError(
178
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
179
+ `Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
180
+ { field: targetField, type: targetType }
181
+ );
182
+ }
183
+ const schemaDimension = Number.parseInt(vectorMatch[1], 10);
184
+ if (schemaDimension !== resolvedAdapter.dimensions) {
185
+ throw new NarsilError(
186
+ ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
187
+ `Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
188
+ { field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
189
+ );
190
+ }
191
+ const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
192
+ if (sources.length === 0) {
193
+ throw new NarsilError(
194
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
195
+ `Embedding target field "${targetField}" must have at least one source field`,
196
+ { field: targetField }
197
+ );
198
+ }
199
+ for (const source of sources) {
200
+ const sourceType = flatFields[source];
201
+ if (sourceType === void 0) {
202
+ throw new NarsilError(
203
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
204
+ `Embedding source field "${source}" does not exist in the schema`,
205
+ { field: source, targetField }
206
+ );
207
+ }
208
+ if (sourceType !== "string") {
209
+ throw new NarsilError(
210
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
211
+ `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
212
+ { field: source, type: sourceType, targetField }
213
+ );
214
+ }
215
+ }
216
+ }
217
+ return resolvedAdapter;
218
+ }
219
+ function validateRegisteredAdapter(name, adapter) {
220
+ if (typeof name !== "string" || name.trim().length === 0) {
221
+ throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
222
+ }
223
+ if (typeof adapter?.embed !== "function") {
224
+ throw new NarsilError(
225
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
226
+ `Embedding adapter "${name}" must provide an embed function`,
227
+ { adapter: name }
228
+ );
229
+ }
230
+ if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
231
+ throw new NarsilError(
232
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
233
+ `Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
234
+ { adapter: name, dimensions: adapter.dimensions }
235
+ );
236
+ }
237
+ }
238
+ function validateRequiredFieldsInSchema(required, schema) {
239
+ if (required.length === 0) return;
240
+ const flatFields = flattenSchema(schema);
241
+ for (const field of required) {
242
+ if (flatFields[field] === void 0) {
243
+ throw new NarsilError(
244
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
245
+ `Required field "${field}" does not exist in the schema`,
246
+ { field }
247
+ );
248
+ }
249
+ }
250
+ }
251
+
135
252
  // src/workers/promoter.ts
136
253
  var DEFAULT_PER_INDEX_THRESHOLD = 1e4;
137
254
  var DEFAULT_TOTAL_THRESHOLD = 5e4;
@@ -739,6 +856,9 @@ function metadataToWire(meta) {
739
856
  if (meta.vectorFields) {
740
857
  wire.vector_fields = meta.vectorFields;
741
858
  }
859
+ if (meta.embedding) {
860
+ wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
861
+ }
742
862
  return wire;
743
863
  }
744
864
  function wireToMetadata(raw) {
@@ -754,6 +874,9 @@ function wireToMetadata(raw) {
754
874
  if (raw.vector_fields) {
755
875
  meta.vectorFields = raw.vector_fields;
756
876
  }
877
+ if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
878
+ meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
879
+ }
757
880
  return meta;
758
881
  }
759
882
  function serializeMetadata(meta) {
@@ -2115,11 +2238,15 @@ function unflattenSchema(flat) {
2115
2238
  return root;
2116
2239
  }
2117
2240
  function reconstructSchemaFromMetadata(metadata) {
2118
- return {
2241
+ const config = {
2119
2242
  schema: unflattenSchema(metadata.schema),
2120
2243
  language: metadata.language,
2121
2244
  bm25: { k1: metadata.bm25Params.k1, b: metadata.bm25Params.b }
2122
2245
  };
2246
+ if (metadata.embedding) {
2247
+ config.embedding = metadata.embedding.adapter !== void 0 ? { adapter: metadata.embedding.adapter, fields: metadata.embedding.fields } : { fields: metadata.embedding.fields };
2248
+ }
2249
+ return config;
2123
2250
  }
2124
2251
 
2125
2252
  // src/persistence/durability/segment/build-segment.ts
@@ -3693,7 +3820,7 @@ function buildMetadata(indexName, hooks) {
3693
3820
  if (config === void 0 || manager === void 0) {
3694
3821
  return void 0;
3695
3822
  }
3696
- return {
3823
+ const metadata = {
3697
3824
  indexName,
3698
3825
  schema: config.schema,
3699
3826
  language: config.language,
@@ -3702,6 +3829,10 @@ function buildMetadata(indexName, hooks) {
3702
3829
  createdAt: Date.now(),
3703
3830
  engineVersion: VERSION
3704
3831
  };
3832
+ if (config.embedding) {
3833
+ metadata.embedding = config.embedding;
3834
+ }
3835
+ return metadata;
3705
3836
  }
3706
3837
  function createDurabilityIntegration(tier, hooks) {
3707
3838
  const managerHooks = {
@@ -4211,11 +4342,16 @@ function createDurabilityFromConfig(config, wiring) {
4211
4342
  if (entry === void 0) {
4212
4343
  return void 0;
4213
4344
  }
4345
+ const embedding = entry.config.embedding ? {
4346
+ fields: entry.config.embedding.fields,
4347
+ ...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
4348
+ } : void 0;
4214
4349
  return {
4215
4350
  schema: flattenSchema(entry.config.schema),
4216
4351
  language: entry.language.name,
4217
4352
  k1: entry.config.bm25?.k1 ?? 1.2,
4218
- b: entry.config.bm25?.b ?? 0.75
4353
+ b: entry.config.bm25?.b ?? 0.75,
4354
+ ...embedding !== void 0 ? { embedding } : {}
4219
4355
  };
4220
4356
  },
4221
4357
  createIndexFromMetadata: wiring.createIndexFromMetadata,
@@ -4234,6 +4370,13 @@ function createEngineCore(config) {
4234
4370
  }
4235
4371
  const idGenerator = config?.idGenerator ?? generateId;
4236
4372
  const indexRegistry = /* @__PURE__ */ new Map();
4373
+ const embeddingAdapters = /* @__PURE__ */ new Map();
4374
+ if (config?.embeddingAdapters) {
4375
+ for (const [name, adapter] of Object.entries(config.embeddingAdapters)) {
4376
+ validateRegisteredAdapter(name, adapter);
4377
+ embeddingAdapters.set(name, adapter);
4378
+ }
4379
+ }
4237
4380
  const eventHandlers = /* @__PURE__ */ new Map();
4238
4381
  const shutdownState = { isShutdown: false };
4239
4382
  const abortController = new AbortController();
@@ -4283,11 +4426,35 @@ function createEngineCore(config) {
4283
4426
  }
4284
4427
  const indexConfig = reconstructSchemaFromMetadata(metadata);
4285
4428
  const language = getLanguage(indexConfig.language ?? "english");
4429
+ const adapterName = metadata.embedding?.adapter ?? null;
4430
+ let embeddingAdapter = null;
4431
+ if (metadata.embedding) {
4432
+ const candidate = adapterName !== null ? embeddingAdapters.get(adapterName) ?? null : config?.embedding ?? null;
4433
+ if (candidate) {
4434
+ try {
4435
+ validateEmbeddingConfig(
4436
+ { fields: metadata.embedding.fields, adapter: candidate },
4437
+ indexConfig.schema,
4438
+ void 0
4439
+ );
4440
+ } catch (err) {
4441
+ if (err instanceof NarsilError) {
4442
+ throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
4443
+ indexName: metadata.indexName,
4444
+ adapter: adapterName ?? void 0
4445
+ });
4446
+ }
4447
+ throw err;
4448
+ }
4449
+ embeddingAdapter = candidate;
4450
+ }
4451
+ }
4286
4452
  executor.createIndex(metadata.indexName, indexConfig, language);
4287
4453
  indexRegistry.set(metadata.indexName, {
4288
4454
  config: indexConfig,
4289
4455
  language,
4290
- embeddingAdapter: null,
4456
+ embeddingAdapter,
4457
+ embeddingAdapterName: adapterName,
4291
4458
  vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
4292
4459
  });
4293
4460
  }
@@ -4330,6 +4497,7 @@ function createEngineCore(config) {
4330
4497
  durability,
4331
4498
  idGenerator,
4332
4499
  indexRegistry,
4500
+ embeddingAdapters,
4333
4501
  eventHandlers,
4334
4502
  shutdownState,
4335
4503
  abortController,
@@ -4347,300 +4515,65 @@ function createEngineCore(config) {
4347
4515
  rebalanceCtx
4348
4516
  };
4349
4517
  }
4350
-
4351
- // src/engine/lifecycle.ts
4352
- async function shutdownEngine(core) {
4353
- const { executor, durability, indexRegistry, eventHandlers, orchestrator } = core;
4354
- for (const [name] of indexRegistry) {
4355
- const manager = executor.getManager(name);
4356
- if (manager) {
4357
- for (const [, vectorIndex] of manager.getVectorIndexes()) {
4358
- vectorIndex.dispose();
4359
- }
4360
- }
4361
- }
4362
- if (durability) {
4363
- await durability.manager.shutdown();
4364
- }
4365
- const adaptersToShutdown = /* @__PURE__ */ new Set();
4366
- for (const [, entry] of indexRegistry) {
4367
- if (entry.embeddingAdapter?.shutdown) {
4368
- adaptersToShutdown.add(entry.embeddingAdapter);
4369
- }
4370
- }
4371
- for (const adapter of adaptersToShutdown) {
4372
- try {
4373
- await adapter.shutdown?.();
4374
- } catch {
4375
- }
4518
+ var INDEX_CONFIG_PREFIX = "_narsil/index/";
4519
+ var INDEX_CONFIG_SUFFIX = "/config";
4520
+ var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
4521
+ var MAX_INDEX_NAME_LENGTH = 255;
4522
+ var MAX_PARTITION_COUNT = 65536;
4523
+ var MAX_REPLICATION_FACTOR = 255;
4524
+ function truncateForDisplay(value) {
4525
+ const str = String(value);
4526
+ if (str.length > 100) {
4527
+ return `${str.slice(0, 100)}...`;
4376
4528
  }
4377
- await orchestrator.shutdown();
4378
- await executor.shutdown();
4379
- eventHandlers.clear();
4380
- indexRegistry.clear();
4529
+ return str;
4381
4530
  }
4382
-
4383
- // src/engine/embed.ts
4384
- function validateVector(vector, targetField, expectedDimensions) {
4385
- if (!(vector instanceof Float32Array)) {
4531
+ function validateIndexName(indexName) {
4532
+ if (indexName.length === 0 || indexName.length > MAX_INDEX_NAME_LENGTH) {
4386
4533
  throw new NarsilError(
4387
- ErrorCodes.EMBEDDING_FAILED,
4388
- `Adapter returned ${typeof vector} for field "${targetField}", expected Float32Array`,
4389
- { field: targetField, returnedType: typeof vector }
4534
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
4535
+ `Index name must be between 1 and ${MAX_INDEX_NAME_LENGTH} characters`,
4536
+ { indexName: truncateForDisplay(indexName) }
4390
4537
  );
4391
4538
  }
4392
- if (vector.length !== expectedDimensions) {
4539
+ if (indexName.includes("\0") || indexName.includes("/") || indexName.includes("\\") || indexName.includes("..")) {
4393
4540
  throw new NarsilError(
4394
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
4395
- `Adapter returned ${vector.length}-dimensional vector for field "${targetField}", expected ${expectedDimensions}`,
4396
- { field: targetField, expected: expectedDimensions, actual: vector.length }
4541
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
4542
+ "Index name contains forbidden characters (/, \\, .., or null bytes)",
4543
+ { indexName: truncateForDisplay(indexName) }
4397
4544
  );
4398
4545
  }
4399
- for (let i = 0; i < vector.length; i++) {
4400
- if (Number.isNaN(vector[i])) {
4401
- throw new NarsilError(
4402
- ErrorCodes.EMBEDDING_FAILED,
4403
- `Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
4404
- { field: targetField, index: i }
4405
- );
4406
- }
4546
+ if (!INDEX_NAME_PATTERN.test(indexName)) {
4547
+ throw new NarsilError(
4548
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
4549
+ "Index name must start with an alphanumeric character and contain only alphanumeric characters, hyphens, underscores, or dots",
4550
+ { indexName: truncateForDisplay(indexName) }
4551
+ );
4407
4552
  }
4408
- return vector;
4409
4553
  }
4410
- function resolveFieldValue(document, path) {
4411
- if (!path.includes(".")) return document[path];
4412
- const segments = path.split(".");
4413
- let current = document;
4414
- for (const segment of segments) {
4415
- if (current === null || current === void 0 || typeof current !== "object") return void 0;
4416
- current = current[segment];
4417
- }
4418
- return current;
4554
+ function isValidInteger(value) {
4555
+ return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
4419
4556
  }
4420
- function setFieldValue(document, path, value) {
4421
- if (!path.includes(".")) {
4422
- document[path] = value;
4423
- return;
4424
- }
4425
- const segments = path.split(".");
4426
- let current = document;
4427
- for (let i = 0; i < segments.length - 1; i++) {
4428
- let next = current[segments[i]];
4429
- if (next === null || next === void 0 || typeof next !== "object") {
4430
- next = {};
4431
- current[segments[i]] = next;
4432
- }
4433
- current = next;
4434
- }
4435
- current[segments[segments.length - 1]] = value;
4557
+ function indexConfigKey(indexName) {
4558
+ return `${INDEX_CONFIG_PREFIX}${indexName}${INDEX_CONFIG_SUFFIX}`;
4436
4559
  }
4437
- function deleteFieldValue(document, path) {
4438
- if (!path.includes(".")) {
4439
- delete document[path];
4440
- return;
4560
+ function isRecord2(value) {
4561
+ return value !== null && typeof value === "object" && !Array.isArray(value);
4562
+ }
4563
+ function validateDecodedMetadata(decoded, indexName) {
4564
+ if (!isRecord2(decoded)) {
4565
+ throw new NarsilError(
4566
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
4567
+ `Index metadata for '${indexName}' is not an object`,
4568
+ { indexName }
4569
+ );
4441
4570
  }
4442
- const segments = path.split(".");
4443
- let current = document;
4444
- for (let i = 0; i < segments.length - 1; i++) {
4445
- const next = current[segments[i]];
4446
- if (next === null || next === void 0 || typeof next !== "object") return;
4447
- current = next;
4448
- }
4449
- delete current[segments[segments.length - 1]];
4450
- }
4451
- function collectSourceText(document, sourceFields, targetField, documentIndex) {
4452
- const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
4453
- const parts = [];
4454
- for (const source of sources) {
4455
- const value = resolveFieldValue(document, source);
4456
- if (value === void 0 || value === null || value === "") continue;
4457
- if (typeof value !== "string") {
4458
- throw new NarsilError(
4459
- ErrorCodes.DOC_VALIDATION_FAILED,
4460
- `Embedding source field "${source}" must be a string, got ${typeof value}`,
4461
- { field: source, targetField }
4462
- );
4463
- }
4464
- parts.push(value);
4465
- }
4466
- if (parts.length === 0) {
4467
- const details = { field: targetField };
4468
- if (documentIndex !== void 0) {
4469
- details.documentIndex = documentIndex;
4470
- }
4471
- throw new NarsilError(
4472
- ErrorCodes.EMBEDDING_NO_SOURCE,
4473
- `All source fields for embedding target "${targetField}" are missing or empty`,
4474
- details
4475
- );
4476
- }
4477
- return parts.join("\n");
4478
- }
4479
- function wrapAdapterError(err, targetField) {
4480
- if (err instanceof NarsilError) return err;
4481
- const message = err instanceof Error ? err.message : String(err);
4482
- return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
4483
- field: targetField,
4484
- cause: message
4485
- });
4486
- }
4487
- async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
4488
- const embeddedFields = /* @__PURE__ */ new Set();
4489
- try {
4490
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
4491
- const existing = resolveFieldValue(document, targetField);
4492
- if (existing !== void 0 && existing !== null) continue;
4493
- const text = collectSourceText(document, sourceFields, targetField);
4494
- let vector;
4495
- try {
4496
- const raw = await adapter.embed(text, "document", signal);
4497
- vector = validateVector(raw, targetField, adapter.dimensions);
4498
- } catch (err) {
4499
- throw wrapAdapterError(err, targetField);
4500
- }
4501
- setFieldValue(document, targetField, vector);
4502
- embeddedFields.add(targetField);
4503
- }
4504
- } catch (err) {
4505
- for (const field of embeddedFields) {
4506
- deleteFieldValue(document, field);
4507
- }
4508
- throw err;
4509
- }
4510
- return embeddedFields;
4511
- }
4512
- var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([ErrorCodes.EMBEDDING_NO_SOURCE, ErrorCodes.DOC_VALIDATION_FAILED]);
4513
- function isDocumentSourceError(err) {
4514
- return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
4515
- }
4516
- async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
4517
- const embedded = /* @__PURE__ */ new Map();
4518
- const failed = /* @__PURE__ */ new Map();
4519
- try {
4520
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
4521
- const needsEmbedding = [];
4522
- for (let i = 0; i < documents.length; i++) {
4523
- if (failed.has(i)) continue;
4524
- const doc = documents[i];
4525
- const existing = resolveFieldValue(doc, targetField);
4526
- if (existing !== void 0 && existing !== null) continue;
4527
- try {
4528
- const text = collectSourceText(doc, sourceFields, targetField, i);
4529
- needsEmbedding.push({ docIndex: i, text });
4530
- } catch (err) {
4531
- if (isDocumentSourceError(err)) {
4532
- failed.set(i, err);
4533
- } else {
4534
- throw err;
4535
- }
4536
- }
4537
- }
4538
- if (needsEmbedding.length === 0) continue;
4539
- const texts = needsEmbedding.map((entry) => entry.text);
4540
- let vectors;
4541
- try {
4542
- if (adapter.embedBatch) {
4543
- vectors = await adapter.embedBatch(texts, "document", signal);
4544
- } else {
4545
- vectors = [];
4546
- const concurrency = 8;
4547
- for (let start = 0; start < texts.length; start += concurrency) {
4548
- const chunk = texts.slice(start, start + concurrency);
4549
- const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
4550
- vectors.push(...chunkResults);
4551
- }
4552
- }
4553
- } catch (err) {
4554
- throw wrapAdapterError(err, targetField);
4555
- }
4556
- if (vectors.length !== needsEmbedding.length) {
4557
- throw new NarsilError(
4558
- ErrorCodes.EMBEDDING_FAILED,
4559
- `Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
4560
- { field: targetField, expected: needsEmbedding.length, actual: vectors.length }
4561
- );
4562
- }
4563
- for (let j = 0; j < needsEmbedding.length; j++) {
4564
- const { docIndex } = needsEmbedding[j];
4565
- const validated = validateVector(vectors[j], targetField, adapter.dimensions);
4566
- setFieldValue(documents[docIndex], targetField, validated);
4567
- let fieldSet = embedded.get(docIndex);
4568
- if (!fieldSet) {
4569
- fieldSet = /* @__PURE__ */ new Set();
4570
- embedded.set(docIndex, fieldSet);
4571
- }
4572
- fieldSet.add(targetField);
4573
- }
4574
- }
4575
- } catch (err) {
4576
- for (const [docIndex, fields] of embedded) {
4577
- for (const field of fields) {
4578
- deleteFieldValue(documents[docIndex], field);
4579
- }
4580
- }
4581
- throw err;
4582
- }
4583
- return { embedded, failed };
4584
- }
4585
- var INDEX_CONFIG_PREFIX = "_narsil/index/";
4586
- var INDEX_CONFIG_SUFFIX = "/config";
4587
- var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
4588
- var MAX_INDEX_NAME_LENGTH = 255;
4589
- var MAX_PARTITION_COUNT = 65536;
4590
- var MAX_REPLICATION_FACTOR = 255;
4591
- function truncateForDisplay(value) {
4592
- const str = String(value);
4593
- if (str.length > 100) {
4594
- return `${str.slice(0, 100)}...`;
4595
- }
4596
- return str;
4597
- }
4598
- function validateIndexName(indexName) {
4599
- if (indexName.length === 0 || indexName.length > MAX_INDEX_NAME_LENGTH) {
4600
- throw new NarsilError(
4601
- ErrorCodes.CONTROLLER_METADATA_INVALID,
4602
- `Index name must be between 1 and ${MAX_INDEX_NAME_LENGTH} characters`,
4603
- { indexName: truncateForDisplay(indexName) }
4604
- );
4605
- }
4606
- if (indexName.includes("\0") || indexName.includes("/") || indexName.includes("\\") || indexName.includes("..")) {
4607
- throw new NarsilError(
4608
- ErrorCodes.CONTROLLER_METADATA_INVALID,
4609
- "Index name contains forbidden characters (/, \\, .., or null bytes)",
4610
- { indexName: truncateForDisplay(indexName) }
4611
- );
4612
- }
4613
- if (!INDEX_NAME_PATTERN.test(indexName)) {
4614
- throw new NarsilError(
4615
- ErrorCodes.CONTROLLER_METADATA_INVALID,
4616
- "Index name must start with an alphanumeric character and contain only alphanumeric characters, hyphens, underscores, or dots",
4617
- { indexName: truncateForDisplay(indexName) }
4618
- );
4619
- }
4620
- }
4621
- function isValidInteger(value) {
4622
- return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
4623
- }
4624
- function indexConfigKey(indexName) {
4625
- return `${INDEX_CONFIG_PREFIX}${indexName}${INDEX_CONFIG_SUFFIX}`;
4626
- }
4627
- function isRecord2(value) {
4628
- return value !== null && typeof value === "object" && !Array.isArray(value);
4629
- }
4630
- function validateDecodedMetadata(decoded, indexName) {
4631
- if (!isRecord2(decoded)) {
4632
- throw new NarsilError(
4633
- ErrorCodes.CONTROLLER_METADATA_INVALID,
4634
- `Index metadata for '${indexName}' is not an object`,
4635
- { indexName }
4636
- );
4637
- }
4638
- if (typeof decoded.indexName !== "string") {
4639
- throw new NarsilError(
4640
- ErrorCodes.CONTROLLER_METADATA_INVALID,
4641
- `Index metadata for '${indexName}' has invalid indexName`,
4642
- { indexName, received: truncateForDisplay(decoded.indexName) }
4643
- );
4571
+ if (typeof decoded.indexName !== "string") {
4572
+ throw new NarsilError(
4573
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
4574
+ `Index metadata for '${indexName}' has invalid indexName`,
4575
+ { indexName, received: truncateForDisplay(decoded.indexName) }
4576
+ );
4644
4577
  }
4645
4578
  if (!isValidInteger(decoded.partitionCount) || decoded.partitionCount <= 0 || decoded.partitionCount > MAX_PARTITION_COUNT) {
4646
4579
  throw new NarsilError(
@@ -4743,27 +4676,361 @@ function validateDocId(docId) {
4743
4676
  if (!docId || docId.length === 0) {
4744
4677
  throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
4745
4678
  }
4746
- if (docId.length > MAX_DOC_ID_LENGTH) {
4679
+ if (docId.length > MAX_DOC_ID_LENGTH) {
4680
+ throw new NarsilError(
4681
+ ErrorCodes.DOC_VALIDATION_FAILED,
4682
+ `Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
4683
+ {
4684
+ docId,
4685
+ length: docId.length
4686
+ }
4687
+ );
4688
+ }
4689
+ if (docId.includes("\0")) {
4690
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
4691
+ }
4692
+ }
4693
+ function clampLimit(limit) {
4694
+ if (limit === void 0) return DEFAULT_LIMIT;
4695
+ return Math.max(0, Math.min(limit, MAX_LIMIT));
4696
+ }
4697
+ function clampOffset(offset) {
4698
+ if (offset === void 0) return DEFAULT_OFFSET;
4699
+ return Math.max(0, Math.min(offset, MAX_OFFSET));
4700
+ }
4701
+
4702
+ // src/engine/index-lifecycle.ts
4703
+ async function createEngineIndex(core, config, name, indexConfig) {
4704
+ core.guardShutdown();
4705
+ validateIndexName2(name);
4706
+ if (core.indexRegistry.has(name)) {
4707
+ throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
4708
+ }
4709
+ validateSchema(indexConfig.schema);
4710
+ validateVectorPromotion(indexConfig.vectorPromotion);
4711
+ let resolvedEmbeddingAdapter = null;
4712
+ let embeddingAdapterName = null;
4713
+ if (indexConfig.embedding) {
4714
+ let configuredAdapter = indexConfig.embedding.adapter;
4715
+ if (typeof configuredAdapter === "string") {
4716
+ embeddingAdapterName = configuredAdapter;
4717
+ const registered = core.embeddingAdapters.get(configuredAdapter);
4718
+ if (!registered) {
4719
+ throw new NarsilError(
4720
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4721
+ `Embedding adapter "${configuredAdapter}" is not registered on this engine`,
4722
+ { adapter: configuredAdapter, available: [...core.embeddingAdapters.keys()] }
4723
+ );
4724
+ }
4725
+ configuredAdapter = registered;
4726
+ }
4727
+ resolvedEmbeddingAdapter = validateEmbeddingConfig(
4728
+ { fields: indexConfig.embedding.fields, adapter: configuredAdapter },
4729
+ indexConfig.schema,
4730
+ config?.embedding
4731
+ );
4732
+ }
4733
+ if (indexConfig.required && indexConfig.required.length > 0) {
4734
+ validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
4735
+ }
4736
+ const language = getLanguage(indexConfig.language ?? "english");
4737
+ core.executor.createIndex(name, indexConfig, language);
4738
+ core.indexRegistry.set(name, {
4739
+ config: indexConfig,
4740
+ language,
4741
+ embeddingAdapter: resolvedEmbeddingAdapter,
4742
+ embeddingAdapterName,
4743
+ vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
4744
+ });
4745
+ if (core.durability) {
4746
+ await core.durability.manager.persistMetadata(name);
4747
+ }
4748
+ await core.pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
4749
+ }
4750
+ async function dropEngineIndex(core, name) {
4751
+ core.guardShutdown();
4752
+ const entry = core.requireIndex(name);
4753
+ core.executor.dropIndex(name);
4754
+ core.indexRegistry.delete(name);
4755
+ if (core.durability) {
4756
+ await core.durability.manager.removeIndex(name);
4757
+ }
4758
+ await core.pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
4759
+ }
4760
+ function registerEngineEmbeddingAdapter(core, name, adapter) {
4761
+ core.guardShutdown();
4762
+ validateRegisteredAdapter(name, adapter);
4763
+ const affected = [];
4764
+ for (const [indexName, entry] of core.indexRegistry) {
4765
+ if (entry.embeddingAdapterName !== name || !entry.config.embedding) continue;
4766
+ try {
4767
+ validateEmbeddingConfig({ fields: entry.config.embedding.fields, adapter }, entry.config.schema, void 0);
4768
+ } catch (err) {
4769
+ if (err instanceof NarsilError) {
4770
+ throw new NarsilError(
4771
+ err.code,
4772
+ `Cannot bind embedding adapter "${name}" to index "${indexName}": ${err.message}`,
4773
+ {
4774
+ indexName,
4775
+ adapter: name
4776
+ }
4777
+ );
4778
+ }
4779
+ throw err;
4780
+ }
4781
+ affected.push(entry);
4782
+ }
4783
+ core.embeddingAdapters.set(name, adapter);
4784
+ for (const entry of affected) {
4785
+ entry.embeddingAdapter = adapter;
4786
+ }
4787
+ }
4788
+
4789
+ // src/engine/lifecycle.ts
4790
+ async function shutdownEngine(core) {
4791
+ const { executor, durability, indexRegistry, eventHandlers, orchestrator } = core;
4792
+ for (const [name] of indexRegistry) {
4793
+ const manager = executor.getManager(name);
4794
+ if (manager) {
4795
+ for (const [, vectorIndex] of manager.getVectorIndexes()) {
4796
+ vectorIndex.dispose();
4797
+ }
4798
+ }
4799
+ }
4800
+ if (durability) {
4801
+ await durability.manager.shutdown();
4802
+ }
4803
+ const adaptersToShutdown = /* @__PURE__ */ new Set();
4804
+ for (const [, entry] of indexRegistry) {
4805
+ if (entry.embeddingAdapter?.shutdown) {
4806
+ adaptersToShutdown.add(entry.embeddingAdapter);
4807
+ }
4808
+ }
4809
+ for (const adapter of adaptersToShutdown) {
4810
+ try {
4811
+ await adapter.shutdown?.();
4812
+ } catch {
4813
+ }
4814
+ }
4815
+ await orchestrator.shutdown();
4816
+ await executor.shutdown();
4817
+ eventHandlers.clear();
4818
+ indexRegistry.clear();
4819
+ }
4820
+
4821
+ // src/engine/embed.ts
4822
+ function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterName) {
4823
+ for (const targetField of Object.keys(embeddingConfig.fields)) {
4824
+ const existing = resolveFieldValue(document, targetField);
4825
+ if (existing === void 0 || existing === null) {
4826
+ throw new NarsilError(
4827
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4828
+ adapterName ? `The index embeds "${targetField}" via adapter "${adapterName}", which is not registered on this engine. Register the adapter or supply the vector on the document.` : `The index embeds "${targetField}" automatically but no embedding adapter is bound. Bind an adapter or supply the vector on the document.`,
4829
+ adapterName ? { field: targetField, adapter: adapterName } : { field: targetField }
4830
+ );
4831
+ }
4832
+ }
4833
+ }
4834
+ function validateVector(vector, targetField, expectedDimensions) {
4835
+ if (!(vector instanceof Float32Array)) {
4836
+ throw new NarsilError(
4837
+ ErrorCodes.EMBEDDING_FAILED,
4838
+ `Adapter returned ${typeof vector} for field "${targetField}", expected Float32Array`,
4839
+ { field: targetField, returnedType: typeof vector }
4840
+ );
4841
+ }
4842
+ if (vector.length !== expectedDimensions) {
4843
+ throw new NarsilError(
4844
+ ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
4845
+ `Adapter returned ${vector.length}-dimensional vector for field "${targetField}", expected ${expectedDimensions}`,
4846
+ { field: targetField, expected: expectedDimensions, actual: vector.length }
4847
+ );
4848
+ }
4849
+ for (let i = 0; i < vector.length; i++) {
4850
+ if (Number.isNaN(vector[i])) {
4851
+ throw new NarsilError(
4852
+ ErrorCodes.EMBEDDING_FAILED,
4853
+ `Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
4854
+ { field: targetField, index: i }
4855
+ );
4856
+ }
4857
+ }
4858
+ return vector;
4859
+ }
4860
+ function resolveFieldValue(document, path) {
4861
+ if (!path.includes(".")) return document[path];
4862
+ const segments = path.split(".");
4863
+ let current = document;
4864
+ for (const segment of segments) {
4865
+ if (current === null || current === void 0 || typeof current !== "object") return void 0;
4866
+ current = current[segment];
4867
+ }
4868
+ return current;
4869
+ }
4870
+ function setFieldValue(document, path, value) {
4871
+ if (!path.includes(".")) {
4872
+ document[path] = value;
4873
+ return;
4874
+ }
4875
+ const segments = path.split(".");
4876
+ let current = document;
4877
+ for (let i = 0; i < segments.length - 1; i++) {
4878
+ let next = current[segments[i]];
4879
+ if (next === null || next === void 0 || typeof next !== "object") {
4880
+ next = {};
4881
+ current[segments[i]] = next;
4882
+ }
4883
+ current = next;
4884
+ }
4885
+ current[segments[segments.length - 1]] = value;
4886
+ }
4887
+ function deleteFieldValue(document, path) {
4888
+ if (!path.includes(".")) {
4889
+ delete document[path];
4890
+ return;
4891
+ }
4892
+ const segments = path.split(".");
4893
+ let current = document;
4894
+ for (let i = 0; i < segments.length - 1; i++) {
4895
+ const next = current[segments[i]];
4896
+ if (next === null || next === void 0 || typeof next !== "object") return;
4897
+ current = next;
4898
+ }
4899
+ delete current[segments[segments.length - 1]];
4900
+ }
4901
+ function collectSourceText(document, sourceFields, targetField, documentIndex) {
4902
+ const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
4903
+ const parts = [];
4904
+ for (const source of sources) {
4905
+ const value = resolveFieldValue(document, source);
4906
+ if (value === void 0 || value === null || value === "") continue;
4907
+ if (typeof value !== "string") {
4908
+ throw new NarsilError(
4909
+ ErrorCodes.DOC_VALIDATION_FAILED,
4910
+ `Embedding source field "${source}" must be a string, got ${typeof value}`,
4911
+ { field: source, targetField }
4912
+ );
4913
+ }
4914
+ parts.push(value);
4915
+ }
4916
+ if (parts.length === 0) {
4917
+ const details = { field: targetField };
4918
+ if (documentIndex !== void 0) {
4919
+ details.documentIndex = documentIndex;
4920
+ }
4747
4921
  throw new NarsilError(
4748
- ErrorCodes.DOC_VALIDATION_FAILED,
4749
- `Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
4750
- {
4751
- docId,
4752
- length: docId.length
4753
- }
4922
+ ErrorCodes.EMBEDDING_NO_SOURCE,
4923
+ `All source fields for embedding target "${targetField}" are missing or empty`,
4924
+ details
4754
4925
  );
4755
4926
  }
4756
- if (docId.includes("\0")) {
4757
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
4927
+ return parts.join("\n");
4928
+ }
4929
+ function wrapAdapterError(err, targetField) {
4930
+ if (err instanceof NarsilError) return err;
4931
+ const message = err instanceof Error ? err.message : String(err);
4932
+ return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
4933
+ field: targetField,
4934
+ cause: message
4935
+ });
4936
+ }
4937
+ async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
4938
+ const embeddedFields = /* @__PURE__ */ new Set();
4939
+ try {
4940
+ for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
4941
+ const existing = resolveFieldValue(document, targetField);
4942
+ if (existing !== void 0 && existing !== null) continue;
4943
+ const text = collectSourceText(document, sourceFields, targetField);
4944
+ let vector;
4945
+ try {
4946
+ const raw = await adapter.embed(text, "document", signal);
4947
+ vector = validateVector(raw, targetField, adapter.dimensions);
4948
+ } catch (err) {
4949
+ throw wrapAdapterError(err, targetField);
4950
+ }
4951
+ setFieldValue(document, targetField, vector);
4952
+ embeddedFields.add(targetField);
4953
+ }
4954
+ } catch (err) {
4955
+ for (const field of embeddedFields) {
4956
+ deleteFieldValue(document, field);
4957
+ }
4958
+ throw err;
4758
4959
  }
4960
+ return embeddedFields;
4759
4961
  }
4760
- function clampLimit(limit) {
4761
- if (limit === void 0) return DEFAULT_LIMIT;
4762
- return Math.max(0, Math.min(limit, MAX_LIMIT));
4962
+ var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([ErrorCodes.EMBEDDING_NO_SOURCE, ErrorCodes.DOC_VALIDATION_FAILED]);
4963
+ function isDocumentSourceError(err) {
4964
+ return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
4763
4965
  }
4764
- function clampOffset(offset) {
4765
- if (offset === void 0) return DEFAULT_OFFSET;
4766
- return Math.max(0, Math.min(offset, MAX_OFFSET));
4966
+ async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
4967
+ const embedded = /* @__PURE__ */ new Map();
4968
+ const failed = /* @__PURE__ */ new Map();
4969
+ try {
4970
+ for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
4971
+ const needsEmbedding = [];
4972
+ for (let i = 0; i < documents.length; i++) {
4973
+ if (failed.has(i)) continue;
4974
+ const doc = documents[i];
4975
+ const existing = resolveFieldValue(doc, targetField);
4976
+ if (existing !== void 0 && existing !== null) continue;
4977
+ try {
4978
+ const text = collectSourceText(doc, sourceFields, targetField, i);
4979
+ needsEmbedding.push({ docIndex: i, text });
4980
+ } catch (err) {
4981
+ if (isDocumentSourceError(err)) {
4982
+ failed.set(i, err);
4983
+ } else {
4984
+ throw err;
4985
+ }
4986
+ }
4987
+ }
4988
+ if (needsEmbedding.length === 0) continue;
4989
+ const texts = needsEmbedding.map((entry) => entry.text);
4990
+ let vectors;
4991
+ try {
4992
+ if (adapter.embedBatch) {
4993
+ vectors = await adapter.embedBatch(texts, "document", signal);
4994
+ } else {
4995
+ vectors = [];
4996
+ const concurrency = 8;
4997
+ for (let start = 0; start < texts.length; start += concurrency) {
4998
+ const chunk = texts.slice(start, start + concurrency);
4999
+ const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
5000
+ vectors.push(...chunkResults);
5001
+ }
5002
+ }
5003
+ } catch (err) {
5004
+ throw wrapAdapterError(err, targetField);
5005
+ }
5006
+ if (vectors.length !== needsEmbedding.length) {
5007
+ throw new NarsilError(
5008
+ ErrorCodes.EMBEDDING_FAILED,
5009
+ `Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
5010
+ { field: targetField, expected: needsEmbedding.length, actual: vectors.length }
5011
+ );
5012
+ }
5013
+ for (let j = 0; j < needsEmbedding.length; j++) {
5014
+ const { docIndex } = needsEmbedding[j];
5015
+ const validated = validateVector(vectors[j], targetField, adapter.dimensions);
5016
+ setFieldValue(documents[docIndex], targetField, validated);
5017
+ let fieldSet = embedded.get(docIndex);
5018
+ if (!fieldSet) {
5019
+ fieldSet = /* @__PURE__ */ new Set();
5020
+ embedded.set(docIndex, fieldSet);
5021
+ }
5022
+ fieldSet.add(targetField);
5023
+ }
5024
+ }
5025
+ } catch (err) {
5026
+ for (const [docIndex, fields] of embedded) {
5027
+ for (const field of fields) {
5028
+ deleteFieldValue(documents[docIndex], field);
5029
+ }
5030
+ }
5031
+ throw err;
5032
+ }
5033
+ return { embedded, failed };
4767
5034
  }
4768
5035
 
4769
5036
  // src/engine/mutations/durable-rollback.ts
@@ -4827,13 +5094,21 @@ async function insertDocument(ctx, indexName, document, docId, options) {
4827
5094
  if (entry.config.required && entry.config.required.length > 0) {
4828
5095
  validateRequiredFields(document, entry.config.required);
4829
5096
  }
4830
- if (entry.embeddingAdapter && entry.config.embedding) {
4831
- await embedDocumentFields(
4832
- document,
4833
- entry.config.embedding,
4834
- entry.embeddingAdapter,
4835
- ctx.abortController.signal
4836
- );
5097
+ if (entry.config.embedding) {
5098
+ if (entry.embeddingAdapter) {
5099
+ await embedDocumentFields(
5100
+ document,
5101
+ entry.config.embedding,
5102
+ entry.embeddingAdapter,
5103
+ ctx.abortController.signal
5104
+ );
5105
+ } else {
5106
+ assertDocumentCarriesMappedVectors(
5107
+ document,
5108
+ entry.config.embedding,
5109
+ entry.embeddingAdapterName
5110
+ );
5111
+ }
4837
5112
  }
4838
5113
  const insertManager = ctx.requireManager(indexName);
4839
5114
  const insertVecIndexes = insertManager.getVectorIndexes();
@@ -4913,6 +5188,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
4913
5188
  const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
4914
5189
  const hasRequired = entry.config.required && entry.config.required.length > 0;
4915
5190
  const hasEmbedding = entry.embeddingAdapter && entry.config.embedding;
5191
+ const embeddingUnbound = !entry.embeddingAdapter && entry.config.embedding;
4916
5192
  const batchManager = ctx.requireManager(indexName);
4917
5193
  const batchVecIndexes = batchManager.getVectorIndexes();
4918
5194
  const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
@@ -4928,7 +5204,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
4928
5204
  } catch (err) {
4929
5205
  chunkFailedIndexes.add(i);
4930
5206
  failed.push({
4931
- docId: "",
5207
+ docId: providedDocId(documents[i]) ?? "",
4932
5208
  error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err))
4933
5209
  });
4934
5210
  }
@@ -4953,16 +5229,33 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
4953
5229
  for (const [sliceIndex, error] of embedResult.failed) {
4954
5230
  const originalIdx = embeddableOriginalIndexes[sliceIndex];
4955
5231
  chunkFailedIndexes.add(originalIdx);
4956
- failed.push({ docId: "", error });
5232
+ failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error });
4957
5233
  }
4958
5234
  } catch (err) {
4959
5235
  const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
4960
5236
  for (const originalIdx of embeddableOriginalIndexes) {
4961
5237
  chunkFailedIndexes.add(originalIdx);
4962
- failed.push({ docId: "", error: embeddingError });
5238
+ failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error: embeddingError });
4963
5239
  }
4964
5240
  }
4965
5241
  }
5242
+ } else if (embeddingUnbound) {
5243
+ for (let i = chunkStart; i < chunkEnd; i++) {
5244
+ if (chunkFailedIndexes.has(i)) continue;
5245
+ try {
5246
+ assertDocumentCarriesMappedVectors(
5247
+ documents[i],
5248
+ entry.config.embedding,
5249
+ entry.embeddingAdapterName
5250
+ );
5251
+ } catch (err) {
5252
+ chunkFailedIndexes.add(i);
5253
+ failed.push({
5254
+ docId: providedDocId(documents[i]) ?? "",
5255
+ error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
5256
+ });
5257
+ }
5258
+ }
4966
5259
  }
4967
5260
  for (let i = chunkStart; i < chunkEnd; i++) {
4968
5261
  if (ctx.abortController.signal.aborted) break;
@@ -5156,13 +5449,21 @@ async function updateDocument(ctx, indexName, docId, document) {
5156
5449
  if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
5157
5450
  return;
5158
5451
  }
5159
- if (entry.embeddingAdapter && entry.config.embedding) {
5160
- await embedDocumentFields(
5161
- document,
5162
- entry.config.embedding,
5163
- entry.embeddingAdapter,
5164
- ctx.abortController.signal
5165
- );
5452
+ if (entry.config.embedding) {
5453
+ if (entry.embeddingAdapter) {
5454
+ await embedDocumentFields(
5455
+ document,
5456
+ entry.config.embedding,
5457
+ entry.embeddingAdapter,
5458
+ ctx.abortController.signal
5459
+ );
5460
+ } else {
5461
+ assertDocumentCarriesMappedVectors(
5462
+ document,
5463
+ entry.config.embedding,
5464
+ entry.embeddingAdapterName
5465
+ );
5466
+ }
5166
5467
  }
5167
5468
  const updateManager = ctx.requireManager(indexName);
5168
5469
  const oldDocument = updateManager.get(docId);
@@ -6212,7 +6513,7 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
6212
6513
  }
6213
6514
 
6214
6515
  // src/engine/resolve-vector-text.ts
6215
- async function resolveVectorText(params, embeddingAdapter, signal) {
6516
+ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
6216
6517
  if (!params.vector) return params;
6217
6518
  if (params.vector.text === void 0 && params.vector.value === void 0) return params;
6218
6519
  if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
@@ -6222,7 +6523,8 @@ async function resolveVectorText(params, embeddingAdapter, signal) {
6222
6523
  if (!embeddingAdapter) {
6223
6524
  throw new NarsilError(
6224
6525
  ErrorCodes.EMBEDDING_CONFIG_INVALID,
6225
- "Vector query with 'text' requires an embedding adapter on the index or instance"
6526
+ embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
6527
+ embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
6226
6528
  );
6227
6529
  }
6228
6530
  if (typeof params.vector.text !== "string") {
@@ -6304,7 +6606,13 @@ async function restoreFromSnapshot(indexName, data, executor, indexRegistry, get
6304
6606
  const indexConfig = { schema, language: envelope.language };
6305
6607
  executor.createIndex(indexName, indexConfig, language);
6306
6608
  const vectorFieldPaths = getVectorFieldPaths2(schema);
6307
- indexRegistry.set(indexName, { config: indexConfig, language, embeddingAdapter: null, vectorFieldPaths });
6609
+ indexRegistry.set(indexName, {
6610
+ config: indexConfig,
6611
+ language,
6612
+ embeddingAdapter: null,
6613
+ embeddingAdapterName: null,
6614
+ vectorFieldPaths
6615
+ });
6308
6616
  try {
6309
6617
  const manager = requireManager(indexName);
6310
6618
  while (manager.partitionCount < envelope.partitions.length) {
@@ -6435,97 +6743,6 @@ function readProcessMemory() {
6435
6743
  }
6436
6744
  }
6437
6745
 
6438
- // src/schema/embedding-validator.ts
6439
- var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
6440
- function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
6441
- const fieldEntries = Object.keys(embeddingConfig.fields);
6442
- if (fieldEntries.length === 0) {
6443
- throw new NarsilError(
6444
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6445
- "Embedding config must define at least one field mapping"
6446
- );
6447
- }
6448
- const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
6449
- if (!resolvedAdapter) {
6450
- throw new NarsilError(
6451
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6452
- "Embedding config requires an adapter (set on the index or the instance)"
6453
- );
6454
- }
6455
- if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
6456
- throw new NarsilError(
6457
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6458
- `Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
6459
- );
6460
- }
6461
- const flatFields = flattenSchema(schema);
6462
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
6463
- const targetType = flatFields[targetField];
6464
- if (targetType === void 0) {
6465
- throw new NarsilError(
6466
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6467
- `Embedding target field "${targetField}" does not exist in the schema`,
6468
- { field: targetField }
6469
- );
6470
- }
6471
- const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
6472
- if (!vectorMatch) {
6473
- throw new NarsilError(
6474
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6475
- `Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
6476
- { field: targetField, type: targetType }
6477
- );
6478
- }
6479
- const schemaDimension = Number.parseInt(vectorMatch[1], 10);
6480
- if (schemaDimension !== resolvedAdapter.dimensions) {
6481
- throw new NarsilError(
6482
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
6483
- `Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
6484
- { field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
6485
- );
6486
- }
6487
- const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
6488
- if (sources.length === 0) {
6489
- throw new NarsilError(
6490
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6491
- `Embedding target field "${targetField}" must have at least one source field`,
6492
- { field: targetField }
6493
- );
6494
- }
6495
- for (const source of sources) {
6496
- const sourceType = flatFields[source];
6497
- if (sourceType === void 0) {
6498
- throw new NarsilError(
6499
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6500
- `Embedding source field "${source}" does not exist in the schema`,
6501
- { field: source, targetField }
6502
- );
6503
- }
6504
- if (sourceType !== "string") {
6505
- throw new NarsilError(
6506
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6507
- `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
6508
- { field: source, type: sourceType, targetField }
6509
- );
6510
- }
6511
- }
6512
- }
6513
- return resolvedAdapter;
6514
- }
6515
- function validateRequiredFieldsInSchema(required, schema) {
6516
- if (required.length === 0) return;
6517
- const flatFields = flattenSchema(schema);
6518
- for (const field of required) {
6519
- if (flatFields[field] === void 0) {
6520
- throw new NarsilError(
6521
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6522
- `Required field "${field}" does not exist in the schema`,
6523
- { field }
6524
- );
6525
- }
6526
- }
6527
- }
6528
-
6529
6746
  // src/narsil.ts
6530
6747
  async function createNarsil(config) {
6531
6748
  const core = createEngineCore(config);
@@ -6552,43 +6769,14 @@ function createNarsilFromCore(core, config) {
6552
6769
  rebalanceCtx
6553
6770
  } = core;
6554
6771
  const narsil = {
6555
- async createIndex(name, indexConfig) {
6556
- guardShutdown();
6557
- validateIndexName2(name);
6558
- if (indexRegistry.has(name)) {
6559
- throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
6560
- }
6561
- validateSchema(indexConfig.schema);
6562
- let resolvedEmbeddingAdapter = null;
6563
- if (indexConfig.embedding) {
6564
- resolvedEmbeddingAdapter = validateEmbeddingConfig(indexConfig.embedding, indexConfig.schema, config?.embedding);
6565
- }
6566
- if (indexConfig.required && indexConfig.required.length > 0) {
6567
- validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
6568
- }
6569
- const language = getLanguage(indexConfig.language ?? "english");
6570
- executor.createIndex(name, indexConfig, language);
6571
- const vectorFieldPaths = getVectorFieldPaths(indexConfig.schema);
6572
- indexRegistry.set(name, {
6573
- config: indexConfig,
6574
- language,
6575
- embeddingAdapter: resolvedEmbeddingAdapter,
6576
- vectorFieldPaths
6577
- });
6578
- if (durability) {
6579
- await durability.manager.persistMetadata(name);
6580
- }
6581
- await pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
6772
+ createIndex(name, indexConfig) {
6773
+ return createEngineIndex(core, config, name, indexConfig);
6582
6774
  },
6583
- async dropIndex(name) {
6584
- guardShutdown();
6585
- const entry = requireIndex(name);
6586
- executor.dropIndex(name);
6587
- indexRegistry.delete(name);
6588
- if (durability) {
6589
- await durability.manager.removeIndex(name);
6590
- }
6591
- await pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
6775
+ registerEmbeddingAdapter(name, adapter) {
6776
+ registerEngineEmbeddingAdapter(core, name, adapter);
6777
+ },
6778
+ dropIndex(name) {
6779
+ return dropEngineIndex(core, name);
6592
6780
  },
6593
6781
  listIndexes() {
6594
6782
  const infos = [];
@@ -6667,7 +6855,12 @@ function createNarsilFromCore(core, config) {
6667
6855
  guardShutdown();
6668
6856
  const entry = requireIndex(indexName);
6669
6857
  const manager = requireManager(indexName);
6670
- const resolvedParams = await resolveVectorText(params, entry.embeddingAdapter, abortController.signal);
6858
+ const resolvedParams = await resolveVectorText(
6859
+ params,
6860
+ entry.embeddingAdapter,
6861
+ abortController.signal,
6862
+ entry.embeddingAdapterName
6863
+ );
6671
6864
  await pluginRegistry.runHook("beforeSearch", { indexName, params: resolvedParams });
6672
6865
  const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
6673
6866
  const result = await executeQuery(resolvedParams, {
@@ -6692,7 +6885,12 @@ function createNarsilFromCore(core, config) {
6692
6885
  guardShutdown();
6693
6886
  const entry = requireIndex(indexName);
6694
6887
  const manager = requireManager(indexName);
6695
- const resolvedParams = await resolveVectorText(params, entry.embeddingAdapter, abortController.signal);
6888
+ const resolvedParams = await resolveVectorText(
6889
+ params,
6890
+ entry.embeddingAdapter,
6891
+ abortController.signal,
6892
+ entry.embeddingAdapterName
6893
+ );
6696
6894
  const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
6697
6895
  return executePreflight(resolvedParams, {
6698
6896
  manager,