@delali/narsil 0.1.8 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, validateSchema, createPartitionManager, createVectorIndex, deserializePayloadV2, tokenize, fanOutQuery, validateRequiredFields, flattenSchema, createPartitionIndex, fnv1a, detectRuntime, kWayMerge } from './chunk-FS7XLIUS.mjs';
1
+ import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, createPartitionManager, createVectorIndex, validateSchema, deserializePayloadV2, tokenize, fanOutQuery, validateRequiredFields, validateVectorPromotion, createPartitionIndex, flattenSchema, fnv1a, detectRuntime, isNodeMainThread, spawnNodeWorker, kWayMerge } from './chunk-Q4OVNESD.mjs';
2
2
  import { crc32, crc32Init, crc32Update, crc32Final } from './chunk-KMMJMDUA.mjs';
3
3
  import { ErrorCodes, NarsilError } from './chunk-ALB35PRQ.mjs';
4
4
  import { generateId, createRequestId, isValidWorkerResponse } from './chunk-HRABJLPB.mjs';
@@ -132,6 +132,123 @@ function createPluginRegistry() {
132
132
  };
133
133
  }
134
134
 
135
+ // src/schema/embedding-validator.ts
136
+ var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
137
+ function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
138
+ const fieldEntries = Object.keys(embeddingConfig.fields);
139
+ if (fieldEntries.length === 0) {
140
+ throw new NarsilError(
141
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
142
+ "Embedding config must define at least one field mapping"
143
+ );
144
+ }
145
+ if (typeof embeddingConfig.adapter === "string") {
146
+ throw new NarsilError(
147
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
148
+ `Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
149
+ { adapter: embeddingConfig.adapter }
150
+ );
151
+ }
152
+ const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
153
+ if (!resolvedAdapter) {
154
+ throw new NarsilError(
155
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
156
+ "Embedding config requires an adapter (set on the index or the instance)"
157
+ );
158
+ }
159
+ if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
160
+ throw new NarsilError(
161
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
162
+ `Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
163
+ );
164
+ }
165
+ const flatFields = flattenSchema(schema);
166
+ for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
167
+ const targetType = flatFields[targetField];
168
+ if (targetType === void 0) {
169
+ throw new NarsilError(
170
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
171
+ `Embedding target field "${targetField}" does not exist in the schema`,
172
+ { field: targetField }
173
+ );
174
+ }
175
+ const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
176
+ if (!vectorMatch) {
177
+ throw new NarsilError(
178
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
179
+ `Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
180
+ { field: targetField, type: targetType }
181
+ );
182
+ }
183
+ const schemaDimension = Number.parseInt(vectorMatch[1], 10);
184
+ if (schemaDimension !== resolvedAdapter.dimensions) {
185
+ throw new NarsilError(
186
+ ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
187
+ `Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
188
+ { field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
189
+ );
190
+ }
191
+ const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
192
+ if (sources.length === 0) {
193
+ throw new NarsilError(
194
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
195
+ `Embedding target field "${targetField}" must have at least one source field`,
196
+ { field: targetField }
197
+ );
198
+ }
199
+ for (const source of sources) {
200
+ const sourceType = flatFields[source];
201
+ if (sourceType === void 0) {
202
+ throw new NarsilError(
203
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
204
+ `Embedding source field "${source}" does not exist in the schema`,
205
+ { field: source, targetField }
206
+ );
207
+ }
208
+ if (sourceType !== "string") {
209
+ throw new NarsilError(
210
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
211
+ `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
212
+ { field: source, type: sourceType, targetField }
213
+ );
214
+ }
215
+ }
216
+ }
217
+ return resolvedAdapter;
218
+ }
219
+ function validateRegisteredAdapter(name, adapter) {
220
+ if (typeof name !== "string" || name.trim().length === 0) {
221
+ throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
222
+ }
223
+ if (typeof adapter?.embed !== "function") {
224
+ throw new NarsilError(
225
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
226
+ `Embedding adapter "${name}" must provide an embed function`,
227
+ { adapter: name }
228
+ );
229
+ }
230
+ if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
231
+ throw new NarsilError(
232
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
233
+ `Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
234
+ { adapter: name, dimensions: adapter.dimensions }
235
+ );
236
+ }
237
+ }
238
+ function validateRequiredFieldsInSchema(required, schema) {
239
+ if (required.length === 0) return;
240
+ const flatFields = flattenSchema(schema);
241
+ for (const field of required) {
242
+ if (flatFields[field] === void 0) {
243
+ throw new NarsilError(
244
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
245
+ `Required field "${field}" does not exist in the schema`,
246
+ { field }
247
+ );
248
+ }
249
+ }
250
+ }
251
+
135
252
  // src/workers/promoter.ts
136
253
  var DEFAULT_PER_INDEX_THRESHOLD = 1e4;
137
254
  var DEFAULT_TOTAL_THRESHOLD = 5e4;
@@ -170,7 +287,7 @@ function createExecutionPromoter(config) {
170
287
  return { check, markPromoted, isPromoted };
171
288
  }
172
289
 
173
- // src/persistence/durability/durable-filesystem.ts
290
+ // src/persistence/durability/node-fs.ts
174
291
  var cachedFs = null;
175
292
  var cachedPath = null;
176
293
  async function getFs() {
@@ -185,6 +302,8 @@ async function getPath() {
185
302
  }
186
303
  return cachedPath;
187
304
  }
305
+
306
+ // src/persistence/durability/durable-filesystem.ts
188
307
  function validateRelativeKey(key) {
189
308
  if (key.length === 0) {
190
309
  throw new NarsilError(ErrorCodes.PERSISTENCE_SAVE_FAILED, "Invalid key: empty");
@@ -452,12 +571,7 @@ var mainThreadResolved = false;
452
571
  var onMainThread = true;
453
572
  async function isOnMainThread() {
454
573
  if (!mainThreadResolved) {
455
- try {
456
- const workerThreads = await import('worker_threads');
457
- onMainThread = workerThreads.isMainThread;
458
- } catch {
459
- onMainThread = true;
460
- }
574
+ onMainThread = await isNodeMainThread();
461
575
  mainThreadResolved = true;
462
576
  }
463
577
  return onMainThread;
@@ -472,8 +586,7 @@ function resolveWorkerEntryPoint() {
472
586
  }
473
587
  async function spawnWorker() {
474
588
  try {
475
- const workerThreads = await import('worker_threads');
476
- return new workerThreads.Worker(new URL(resolveWorkerEntryPoint()));
589
+ return await spawnNodeWorker(new URL(resolveWorkerEntryPoint()));
477
590
  } catch {
478
591
  return null;
479
592
  }
@@ -739,6 +852,9 @@ function metadataToWire(meta) {
739
852
  if (meta.vectorFields) {
740
853
  wire.vector_fields = meta.vectorFields;
741
854
  }
855
+ if (meta.embedding) {
856
+ wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
857
+ }
742
858
  return wire;
743
859
  }
744
860
  function wireToMetadata(raw) {
@@ -754,6 +870,9 @@ function wireToMetadata(raw) {
754
870
  if (raw.vector_fields) {
755
871
  meta.vectorFields = raw.vector_fields;
756
872
  }
873
+ if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
874
+ meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
875
+ }
757
876
  return meta;
758
877
  }
759
878
  function serializeMetadata(meta) {
@@ -1146,8 +1265,10 @@ function resolveWorkerEntryPoint2() {
1146
1265
  }
1147
1266
  async function spawnWorker2() {
1148
1267
  try {
1149
- const workerThreads = await import('worker_threads');
1150
- const worker = new workerThreads.Worker(new URL(resolveWorkerEntryPoint2()));
1268
+ const worker = await spawnNodeWorker(new URL(resolveWorkerEntryPoint2()));
1269
+ if (worker === null) {
1270
+ return null;
1271
+ }
1151
1272
  if (typeof worker.unref === "function") {
1152
1273
  worker.unref();
1153
1274
  }
@@ -2115,11 +2236,15 @@ function unflattenSchema(flat) {
2115
2236
  return root;
2116
2237
  }
2117
2238
  function reconstructSchemaFromMetadata(metadata) {
2118
- return {
2239
+ const config = {
2119
2240
  schema: unflattenSchema(metadata.schema),
2120
2241
  language: metadata.language,
2121
2242
  bm25: { k1: metadata.bm25Params.k1, b: metadata.bm25Params.b }
2122
2243
  };
2244
+ if (metadata.embedding) {
2245
+ config.embedding = metadata.embedding.adapter !== void 0 ? { adapter: metadata.embedding.adapter, fields: metadata.embedding.fields } : { fields: metadata.embedding.fields };
2246
+ }
2247
+ return config;
2123
2248
  }
2124
2249
 
2125
2250
  // src/persistence/durability/segment/build-segment.ts
@@ -3693,7 +3818,7 @@ function buildMetadata(indexName, hooks) {
3693
3818
  if (config === void 0 || manager === void 0) {
3694
3819
  return void 0;
3695
3820
  }
3696
- return {
3821
+ const metadata = {
3697
3822
  indexName,
3698
3823
  schema: config.schema,
3699
3824
  language: config.language,
@@ -3702,6 +3827,10 @@ function buildMetadata(indexName, hooks) {
3702
3827
  createdAt: Date.now(),
3703
3828
  engineVersion: VERSION
3704
3829
  };
3830
+ if (config.embedding) {
3831
+ metadata.embedding = config.embedding;
3832
+ }
3833
+ return metadata;
3705
3834
  }
3706
3835
  function createDurabilityIntegration(tier, hooks) {
3707
3836
  const managerHooks = {
@@ -4211,11 +4340,16 @@ function createDurabilityFromConfig(config, wiring) {
4211
4340
  if (entry === void 0) {
4212
4341
  return void 0;
4213
4342
  }
4343
+ const embedding = entry.config.embedding ? {
4344
+ fields: entry.config.embedding.fields,
4345
+ ...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
4346
+ } : void 0;
4214
4347
  return {
4215
4348
  schema: flattenSchema(entry.config.schema),
4216
4349
  language: entry.language.name,
4217
4350
  k1: entry.config.bm25?.k1 ?? 1.2,
4218
- b: entry.config.bm25?.b ?? 0.75
4351
+ b: entry.config.bm25?.b ?? 0.75,
4352
+ ...embedding !== void 0 ? { embedding } : {}
4219
4353
  };
4220
4354
  },
4221
4355
  createIndexFromMetadata: wiring.createIndexFromMetadata,
@@ -4234,6 +4368,13 @@ function createEngineCore(config) {
4234
4368
  }
4235
4369
  const idGenerator = config?.idGenerator ?? generateId;
4236
4370
  const indexRegistry = /* @__PURE__ */ new Map();
4371
+ const embeddingAdapters = /* @__PURE__ */ new Map();
4372
+ if (config?.embeddingAdapters) {
4373
+ for (const [name, adapter] of Object.entries(config.embeddingAdapters)) {
4374
+ validateRegisteredAdapter(name, adapter);
4375
+ embeddingAdapters.set(name, adapter);
4376
+ }
4377
+ }
4237
4378
  const eventHandlers = /* @__PURE__ */ new Map();
4238
4379
  const shutdownState = { isShutdown: false };
4239
4380
  const abortController = new AbortController();
@@ -4283,11 +4424,35 @@ function createEngineCore(config) {
4283
4424
  }
4284
4425
  const indexConfig = reconstructSchemaFromMetadata(metadata);
4285
4426
  const language = getLanguage(indexConfig.language ?? "english");
4427
+ const adapterName = metadata.embedding?.adapter ?? null;
4428
+ let embeddingAdapter = null;
4429
+ if (metadata.embedding) {
4430
+ const candidate = adapterName !== null ? embeddingAdapters.get(adapterName) ?? null : config?.embedding ?? null;
4431
+ if (candidate) {
4432
+ try {
4433
+ validateEmbeddingConfig(
4434
+ { fields: metadata.embedding.fields, adapter: candidate },
4435
+ indexConfig.schema,
4436
+ void 0
4437
+ );
4438
+ } catch (err) {
4439
+ if (err instanceof NarsilError) {
4440
+ throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
4441
+ indexName: metadata.indexName,
4442
+ adapter: adapterName ?? void 0
4443
+ });
4444
+ }
4445
+ throw err;
4446
+ }
4447
+ embeddingAdapter = candidate;
4448
+ }
4449
+ }
4286
4450
  executor.createIndex(metadata.indexName, indexConfig, language);
4287
4451
  indexRegistry.set(metadata.indexName, {
4288
4452
  config: indexConfig,
4289
4453
  language,
4290
- embeddingAdapter: null,
4454
+ embeddingAdapter,
4455
+ embeddingAdapterName: adapterName,
4291
4456
  vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
4292
4457
  });
4293
4458
  }
@@ -4330,6 +4495,7 @@ function createEngineCore(config) {
4330
4495
  durability,
4331
4496
  idGenerator,
4332
4497
  indexRegistry,
4498
+ embeddingAdapters,
4333
4499
  eventHandlers,
4334
4500
  shutdownState,
4335
4501
  abortController,
@@ -4347,285 +4513,50 @@ function createEngineCore(config) {
4347
4513
  rebalanceCtx
4348
4514
  };
4349
4515
  }
4350
-
4351
- // src/engine/lifecycle.ts
4352
- async function shutdownEngine(core) {
4353
- const { executor, durability, indexRegistry, eventHandlers, orchestrator } = core;
4354
- for (const [name] of indexRegistry) {
4355
- const manager = executor.getManager(name);
4356
- if (manager) {
4357
- for (const [, vectorIndex] of manager.getVectorIndexes()) {
4358
- vectorIndex.dispose();
4359
- }
4360
- }
4361
- }
4362
- if (durability) {
4363
- await durability.manager.shutdown();
4364
- }
4365
- const adaptersToShutdown = /* @__PURE__ */ new Set();
4366
- for (const [, entry] of indexRegistry) {
4367
- if (entry.embeddingAdapter?.shutdown) {
4368
- adaptersToShutdown.add(entry.embeddingAdapter);
4369
- }
4370
- }
4371
- for (const adapter of adaptersToShutdown) {
4372
- try {
4373
- await adapter.shutdown?.();
4374
- } catch {
4375
- }
4516
+ var INDEX_CONFIG_PREFIX = "_narsil/index/";
4517
+ var INDEX_CONFIG_SUFFIX = "/config";
4518
+ var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
4519
+ var MAX_INDEX_NAME_LENGTH = 255;
4520
+ var MAX_PARTITION_COUNT = 65536;
4521
+ var MAX_REPLICATION_FACTOR = 255;
4522
+ function truncateForDisplay(value) {
4523
+ const str = String(value);
4524
+ if (str.length > 100) {
4525
+ return `${str.slice(0, 100)}...`;
4376
4526
  }
4377
- await orchestrator.shutdown();
4378
- await executor.shutdown();
4379
- eventHandlers.clear();
4380
- indexRegistry.clear();
4527
+ return str;
4381
4528
  }
4382
-
4383
- // src/engine/embed.ts
4384
- function validateVector(vector, targetField, expectedDimensions) {
4385
- if (!(vector instanceof Float32Array)) {
4529
+ function validateIndexName(indexName) {
4530
+ if (indexName.length === 0 || indexName.length > MAX_INDEX_NAME_LENGTH) {
4386
4531
  throw new NarsilError(
4387
- ErrorCodes.EMBEDDING_FAILED,
4388
- `Adapter returned ${typeof vector} for field "${targetField}", expected Float32Array`,
4389
- { field: targetField, returnedType: typeof vector }
4532
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
4533
+ `Index name must be between 1 and ${MAX_INDEX_NAME_LENGTH} characters`,
4534
+ { indexName: truncateForDisplay(indexName) }
4390
4535
  );
4391
4536
  }
4392
- if (vector.length !== expectedDimensions) {
4537
+ if (indexName.includes("\0") || indexName.includes("/") || indexName.includes("\\") || indexName.includes("..")) {
4393
4538
  throw new NarsilError(
4394
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
4395
- `Adapter returned ${vector.length}-dimensional vector for field "${targetField}", expected ${expectedDimensions}`,
4396
- { field: targetField, expected: expectedDimensions, actual: vector.length }
4539
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
4540
+ "Index name contains forbidden characters (/, \\, .., or null bytes)",
4541
+ { indexName: truncateForDisplay(indexName) }
4397
4542
  );
4398
4543
  }
4399
- for (let i = 0; i < vector.length; i++) {
4400
- if (Number.isNaN(vector[i])) {
4401
- throw new NarsilError(
4402
- ErrorCodes.EMBEDDING_FAILED,
4403
- `Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
4404
- { field: targetField, index: i }
4405
- );
4406
- }
4544
+ if (!INDEX_NAME_PATTERN.test(indexName)) {
4545
+ throw new NarsilError(
4546
+ ErrorCodes.CONTROLLER_METADATA_INVALID,
4547
+ "Index name must start with an alphanumeric character and contain only alphanumeric characters, hyphens, underscores, or dots",
4548
+ { indexName: truncateForDisplay(indexName) }
4549
+ );
4407
4550
  }
4408
- return vector;
4409
4551
  }
4410
- function resolveFieldValue(document, path) {
4411
- if (!path.includes(".")) return document[path];
4412
- const segments = path.split(".");
4413
- let current = document;
4414
- for (const segment of segments) {
4415
- if (current === null || current === void 0 || typeof current !== "object") return void 0;
4416
- current = current[segment];
4417
- }
4418
- return current;
4552
+ function isValidInteger(value) {
4553
+ return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
4419
4554
  }
4420
- function setFieldValue(document, path, value) {
4421
- if (!path.includes(".")) {
4422
- document[path] = value;
4423
- return;
4424
- }
4425
- const segments = path.split(".");
4426
- let current = document;
4427
- for (let i = 0; i < segments.length - 1; i++) {
4428
- let next = current[segments[i]];
4429
- if (next === null || next === void 0 || typeof next !== "object") {
4430
- next = {};
4431
- current[segments[i]] = next;
4432
- }
4433
- current = next;
4434
- }
4435
- current[segments[segments.length - 1]] = value;
4555
+ function indexConfigKey(indexName) {
4556
+ return `${INDEX_CONFIG_PREFIX}${indexName}${INDEX_CONFIG_SUFFIX}`;
4436
4557
  }
4437
- function deleteFieldValue(document, path) {
4438
- if (!path.includes(".")) {
4439
- delete document[path];
4440
- return;
4441
- }
4442
- const segments = path.split(".");
4443
- let current = document;
4444
- for (let i = 0; i < segments.length - 1; i++) {
4445
- const next = current[segments[i]];
4446
- if (next === null || next === void 0 || typeof next !== "object") return;
4447
- current = next;
4448
- }
4449
- delete current[segments[segments.length - 1]];
4450
- }
4451
- function collectSourceText(document, sourceFields, targetField, documentIndex) {
4452
- const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
4453
- const parts = [];
4454
- for (const source of sources) {
4455
- const value = resolveFieldValue(document, source);
4456
- if (value === void 0 || value === null || value === "") continue;
4457
- if (typeof value !== "string") {
4458
- throw new NarsilError(
4459
- ErrorCodes.DOC_VALIDATION_FAILED,
4460
- `Embedding source field "${source}" must be a string, got ${typeof value}`,
4461
- { field: source, targetField }
4462
- );
4463
- }
4464
- parts.push(value);
4465
- }
4466
- if (parts.length === 0) {
4467
- const details = { field: targetField };
4468
- if (documentIndex !== void 0) {
4469
- details.documentIndex = documentIndex;
4470
- }
4471
- throw new NarsilError(
4472
- ErrorCodes.EMBEDDING_NO_SOURCE,
4473
- `All source fields for embedding target "${targetField}" are missing or empty`,
4474
- details
4475
- );
4476
- }
4477
- return parts.join("\n");
4478
- }
4479
- function wrapAdapterError(err, targetField) {
4480
- if (err instanceof NarsilError) return err;
4481
- const message = err instanceof Error ? err.message : String(err);
4482
- return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
4483
- field: targetField,
4484
- cause: message
4485
- });
4486
- }
4487
- async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
4488
- const embeddedFields = /* @__PURE__ */ new Set();
4489
- try {
4490
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
4491
- const existing = resolveFieldValue(document, targetField);
4492
- if (existing !== void 0 && existing !== null) continue;
4493
- const text = collectSourceText(document, sourceFields, targetField);
4494
- let vector;
4495
- try {
4496
- const raw = await adapter.embed(text, "document", signal);
4497
- vector = validateVector(raw, targetField, adapter.dimensions);
4498
- } catch (err) {
4499
- throw wrapAdapterError(err, targetField);
4500
- }
4501
- setFieldValue(document, targetField, vector);
4502
- embeddedFields.add(targetField);
4503
- }
4504
- } catch (err) {
4505
- for (const field of embeddedFields) {
4506
- deleteFieldValue(document, field);
4507
- }
4508
- throw err;
4509
- }
4510
- return embeddedFields;
4511
- }
4512
- var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([ErrorCodes.EMBEDDING_NO_SOURCE, ErrorCodes.DOC_VALIDATION_FAILED]);
4513
- function isDocumentSourceError(err) {
4514
- return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
4515
- }
4516
- async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
4517
- const embedded = /* @__PURE__ */ new Map();
4518
- const failed = /* @__PURE__ */ new Map();
4519
- try {
4520
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
4521
- const needsEmbedding = [];
4522
- for (let i = 0; i < documents.length; i++) {
4523
- if (failed.has(i)) continue;
4524
- const doc = documents[i];
4525
- const existing = resolveFieldValue(doc, targetField);
4526
- if (existing !== void 0 && existing !== null) continue;
4527
- try {
4528
- const text = collectSourceText(doc, sourceFields, targetField, i);
4529
- needsEmbedding.push({ docIndex: i, text });
4530
- } catch (err) {
4531
- if (isDocumentSourceError(err)) {
4532
- failed.set(i, err);
4533
- } else {
4534
- throw err;
4535
- }
4536
- }
4537
- }
4538
- if (needsEmbedding.length === 0) continue;
4539
- const texts = needsEmbedding.map((entry) => entry.text);
4540
- let vectors;
4541
- try {
4542
- if (adapter.embedBatch) {
4543
- vectors = await adapter.embedBatch(texts, "document", signal);
4544
- } else {
4545
- vectors = [];
4546
- const concurrency = 8;
4547
- for (let start = 0; start < texts.length; start += concurrency) {
4548
- const chunk = texts.slice(start, start + concurrency);
4549
- const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
4550
- vectors.push(...chunkResults);
4551
- }
4552
- }
4553
- } catch (err) {
4554
- throw wrapAdapterError(err, targetField);
4555
- }
4556
- if (vectors.length !== needsEmbedding.length) {
4557
- throw new NarsilError(
4558
- ErrorCodes.EMBEDDING_FAILED,
4559
- `Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
4560
- { field: targetField, expected: needsEmbedding.length, actual: vectors.length }
4561
- );
4562
- }
4563
- for (let j = 0; j < needsEmbedding.length; j++) {
4564
- const { docIndex } = needsEmbedding[j];
4565
- const validated = validateVector(vectors[j], targetField, adapter.dimensions);
4566
- setFieldValue(documents[docIndex], targetField, validated);
4567
- let fieldSet = embedded.get(docIndex);
4568
- if (!fieldSet) {
4569
- fieldSet = /* @__PURE__ */ new Set();
4570
- embedded.set(docIndex, fieldSet);
4571
- }
4572
- fieldSet.add(targetField);
4573
- }
4574
- }
4575
- } catch (err) {
4576
- for (const [docIndex, fields] of embedded) {
4577
- for (const field of fields) {
4578
- deleteFieldValue(documents[docIndex], field);
4579
- }
4580
- }
4581
- throw err;
4582
- }
4583
- return { embedded, failed };
4584
- }
4585
- var INDEX_CONFIG_PREFIX = "_narsil/index/";
4586
- var INDEX_CONFIG_SUFFIX = "/config";
4587
- var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
4588
- var MAX_INDEX_NAME_LENGTH = 255;
4589
- var MAX_PARTITION_COUNT = 65536;
4590
- var MAX_REPLICATION_FACTOR = 255;
4591
- function truncateForDisplay(value) {
4592
- const str = String(value);
4593
- if (str.length > 100) {
4594
- return `${str.slice(0, 100)}...`;
4595
- }
4596
- return str;
4597
- }
4598
- function validateIndexName(indexName) {
4599
- if (indexName.length === 0 || indexName.length > MAX_INDEX_NAME_LENGTH) {
4600
- throw new NarsilError(
4601
- ErrorCodes.CONTROLLER_METADATA_INVALID,
4602
- `Index name must be between 1 and ${MAX_INDEX_NAME_LENGTH} characters`,
4603
- { indexName: truncateForDisplay(indexName) }
4604
- );
4605
- }
4606
- if (indexName.includes("\0") || indexName.includes("/") || indexName.includes("\\") || indexName.includes("..")) {
4607
- throw new NarsilError(
4608
- ErrorCodes.CONTROLLER_METADATA_INVALID,
4609
- "Index name contains forbidden characters (/, \\, .., or null bytes)",
4610
- { indexName: truncateForDisplay(indexName) }
4611
- );
4612
- }
4613
- if (!INDEX_NAME_PATTERN.test(indexName)) {
4614
- throw new NarsilError(
4615
- ErrorCodes.CONTROLLER_METADATA_INVALID,
4616
- "Index name must start with an alphanumeric character and contain only alphanumeric characters, hyphens, underscores, or dots",
4617
- { indexName: truncateForDisplay(indexName) }
4618
- );
4619
- }
4620
- }
4621
- function isValidInteger(value) {
4622
- return typeof value === "number" && Number.isFinite(value) && Number.isInteger(value);
4623
- }
4624
- function indexConfigKey(indexName) {
4625
- return `${INDEX_CONFIG_PREFIX}${indexName}${INDEX_CONFIG_SUFFIX}`;
4626
- }
4627
- function isRecord2(value) {
4628
- return value !== null && typeof value === "object" && !Array.isArray(value);
4558
+ function isRecord2(value) {
4559
+ return value !== null && typeof value === "object" && !Array.isArray(value);
4629
4560
  }
4630
4561
  function validateDecodedMetadata(decoded, indexName) {
4631
4562
  if (!isRecord2(decoded)) {
@@ -4743,27 +4674,361 @@ function validateDocId(docId) {
4743
4674
  if (!docId || docId.length === 0) {
4744
4675
  throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
4745
4676
  }
4746
- if (docId.length > MAX_DOC_ID_LENGTH) {
4677
+ if (docId.length > MAX_DOC_ID_LENGTH) {
4678
+ throw new NarsilError(
4679
+ ErrorCodes.DOC_VALIDATION_FAILED,
4680
+ `Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
4681
+ {
4682
+ docId,
4683
+ length: docId.length
4684
+ }
4685
+ );
4686
+ }
4687
+ if (docId.includes("\0")) {
4688
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
4689
+ }
4690
+ }
4691
+ function clampLimit(limit) {
4692
+ if (limit === void 0) return DEFAULT_LIMIT;
4693
+ return Math.max(0, Math.min(limit, MAX_LIMIT));
4694
+ }
4695
+ function clampOffset(offset) {
4696
+ if (offset === void 0) return DEFAULT_OFFSET;
4697
+ return Math.max(0, Math.min(offset, MAX_OFFSET));
4698
+ }
4699
+
4700
+ // src/engine/index-lifecycle.ts
4701
+ async function createEngineIndex(core, config, name, indexConfig) {
4702
+ core.guardShutdown();
4703
+ validateIndexName2(name);
4704
+ if (core.indexRegistry.has(name)) {
4705
+ throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
4706
+ }
4707
+ validateSchema(indexConfig.schema);
4708
+ validateVectorPromotion(indexConfig.vectorPromotion);
4709
+ let resolvedEmbeddingAdapter = null;
4710
+ let embeddingAdapterName = null;
4711
+ if (indexConfig.embedding) {
4712
+ let configuredAdapter = indexConfig.embedding.adapter;
4713
+ if (typeof configuredAdapter === "string") {
4714
+ embeddingAdapterName = configuredAdapter;
4715
+ const registered = core.embeddingAdapters.get(configuredAdapter);
4716
+ if (!registered) {
4717
+ throw new NarsilError(
4718
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4719
+ `Embedding adapter "${configuredAdapter}" is not registered on this engine`,
4720
+ { adapter: configuredAdapter, available: [...core.embeddingAdapters.keys()] }
4721
+ );
4722
+ }
4723
+ configuredAdapter = registered;
4724
+ }
4725
+ resolvedEmbeddingAdapter = validateEmbeddingConfig(
4726
+ { fields: indexConfig.embedding.fields, adapter: configuredAdapter },
4727
+ indexConfig.schema,
4728
+ config?.embedding
4729
+ );
4730
+ }
4731
+ if (indexConfig.required && indexConfig.required.length > 0) {
4732
+ validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
4733
+ }
4734
+ const language = getLanguage(indexConfig.language ?? "english");
4735
+ core.executor.createIndex(name, indexConfig, language);
4736
+ core.indexRegistry.set(name, {
4737
+ config: indexConfig,
4738
+ language,
4739
+ embeddingAdapter: resolvedEmbeddingAdapter,
4740
+ embeddingAdapterName,
4741
+ vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
4742
+ });
4743
+ if (core.durability) {
4744
+ await core.durability.manager.persistMetadata(name);
4745
+ }
4746
+ await core.pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
4747
+ }
4748
+ async function dropEngineIndex(core, name) {
4749
+ core.guardShutdown();
4750
+ const entry = core.requireIndex(name);
4751
+ core.executor.dropIndex(name);
4752
+ core.indexRegistry.delete(name);
4753
+ if (core.durability) {
4754
+ await core.durability.manager.removeIndex(name);
4755
+ }
4756
+ await core.pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
4757
+ }
4758
+ function registerEngineEmbeddingAdapter(core, name, adapter) {
4759
+ core.guardShutdown();
4760
+ validateRegisteredAdapter(name, adapter);
4761
+ const affected = [];
4762
+ for (const [indexName, entry] of core.indexRegistry) {
4763
+ if (entry.embeddingAdapterName !== name || !entry.config.embedding) continue;
4764
+ try {
4765
+ validateEmbeddingConfig({ fields: entry.config.embedding.fields, adapter }, entry.config.schema, void 0);
4766
+ } catch (err) {
4767
+ if (err instanceof NarsilError) {
4768
+ throw new NarsilError(
4769
+ err.code,
4770
+ `Cannot bind embedding adapter "${name}" to index "${indexName}": ${err.message}`,
4771
+ {
4772
+ indexName,
4773
+ adapter: name
4774
+ }
4775
+ );
4776
+ }
4777
+ throw err;
4778
+ }
4779
+ affected.push(entry);
4780
+ }
4781
+ core.embeddingAdapters.set(name, adapter);
4782
+ for (const entry of affected) {
4783
+ entry.embeddingAdapter = adapter;
4784
+ }
4785
+ }
4786
+
4787
+ // src/engine/lifecycle.ts
4788
+ async function shutdownEngine(core) {
4789
+ const { executor, durability, indexRegistry, eventHandlers, orchestrator } = core;
4790
+ for (const [name] of indexRegistry) {
4791
+ const manager = executor.getManager(name);
4792
+ if (manager) {
4793
+ for (const [, vectorIndex] of manager.getVectorIndexes()) {
4794
+ vectorIndex.dispose();
4795
+ }
4796
+ }
4797
+ }
4798
+ if (durability) {
4799
+ await durability.manager.shutdown();
4800
+ }
4801
+ const adaptersToShutdown = /* @__PURE__ */ new Set();
4802
+ for (const [, entry] of indexRegistry) {
4803
+ if (entry.embeddingAdapter?.shutdown) {
4804
+ adaptersToShutdown.add(entry.embeddingAdapter);
4805
+ }
4806
+ }
4807
+ for (const adapter of adaptersToShutdown) {
4808
+ try {
4809
+ await adapter.shutdown?.();
4810
+ } catch {
4811
+ }
4812
+ }
4813
+ await orchestrator.shutdown();
4814
+ await executor.shutdown();
4815
+ eventHandlers.clear();
4816
+ indexRegistry.clear();
4817
+ }
4818
+
4819
+ // src/engine/embed.ts
4820
+ function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterName) {
4821
+ for (const targetField of Object.keys(embeddingConfig.fields)) {
4822
+ const existing = resolveFieldValue(document, targetField);
4823
+ if (existing === void 0 || existing === null) {
4824
+ throw new NarsilError(
4825
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4826
+ adapterName ? `The index embeds "${targetField}" via adapter "${adapterName}", which is not registered on this engine. Register the adapter or supply the vector on the document.` : `The index embeds "${targetField}" automatically but no embedding adapter is bound. Bind an adapter or supply the vector on the document.`,
4827
+ adapterName ? { field: targetField, adapter: adapterName } : { field: targetField }
4828
+ );
4829
+ }
4830
+ }
4831
+ }
4832
+ function validateVector(vector, targetField, expectedDimensions) {
4833
+ if (!(vector instanceof Float32Array)) {
4834
+ throw new NarsilError(
4835
+ ErrorCodes.EMBEDDING_FAILED,
4836
+ `Adapter returned ${typeof vector} for field "${targetField}", expected Float32Array`,
4837
+ { field: targetField, returnedType: typeof vector }
4838
+ );
4839
+ }
4840
+ if (vector.length !== expectedDimensions) {
4841
+ throw new NarsilError(
4842
+ ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
4843
+ `Adapter returned ${vector.length}-dimensional vector for field "${targetField}", expected ${expectedDimensions}`,
4844
+ { field: targetField, expected: expectedDimensions, actual: vector.length }
4845
+ );
4846
+ }
4847
+ for (let i = 0; i < vector.length; i++) {
4848
+ if (Number.isNaN(vector[i])) {
4849
+ throw new NarsilError(
4850
+ ErrorCodes.EMBEDDING_FAILED,
4851
+ `Adapter returned vector with NaN at index ${i} for field "${targetField}"`,
4852
+ { field: targetField, index: i }
4853
+ );
4854
+ }
4855
+ }
4856
+ return vector;
4857
+ }
4858
+ function resolveFieldValue(document, path) {
4859
+ if (!path.includes(".")) return document[path];
4860
+ const segments = path.split(".");
4861
+ let current = document;
4862
+ for (const segment of segments) {
4863
+ if (current === null || current === void 0 || typeof current !== "object") return void 0;
4864
+ current = current[segment];
4865
+ }
4866
+ return current;
4867
+ }
4868
+ function setFieldValue(document, path, value) {
4869
+ if (!path.includes(".")) {
4870
+ document[path] = value;
4871
+ return;
4872
+ }
4873
+ const segments = path.split(".");
4874
+ let current = document;
4875
+ for (let i = 0; i < segments.length - 1; i++) {
4876
+ let next = current[segments[i]];
4877
+ if (next === null || next === void 0 || typeof next !== "object") {
4878
+ next = {};
4879
+ current[segments[i]] = next;
4880
+ }
4881
+ current = next;
4882
+ }
4883
+ current[segments[segments.length - 1]] = value;
4884
+ }
4885
+ function deleteFieldValue(document, path) {
4886
+ if (!path.includes(".")) {
4887
+ delete document[path];
4888
+ return;
4889
+ }
4890
+ const segments = path.split(".");
4891
+ let current = document;
4892
+ for (let i = 0; i < segments.length - 1; i++) {
4893
+ const next = current[segments[i]];
4894
+ if (next === null || next === void 0 || typeof next !== "object") return;
4895
+ current = next;
4896
+ }
4897
+ delete current[segments[segments.length - 1]];
4898
+ }
4899
+ function collectSourceText(document, sourceFields, targetField, documentIndex) {
4900
+ const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
4901
+ const parts = [];
4902
+ for (const source of sources) {
4903
+ const value = resolveFieldValue(document, source);
4904
+ if (value === void 0 || value === null || value === "") continue;
4905
+ if (typeof value !== "string") {
4906
+ throw new NarsilError(
4907
+ ErrorCodes.DOC_VALIDATION_FAILED,
4908
+ `Embedding source field "${source}" must be a string, got ${typeof value}`,
4909
+ { field: source, targetField }
4910
+ );
4911
+ }
4912
+ parts.push(value);
4913
+ }
4914
+ if (parts.length === 0) {
4915
+ const details = { field: targetField };
4916
+ if (documentIndex !== void 0) {
4917
+ details.documentIndex = documentIndex;
4918
+ }
4747
4919
  throw new NarsilError(
4748
- ErrorCodes.DOC_VALIDATION_FAILED,
4749
- `Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
4750
- {
4751
- docId,
4752
- length: docId.length
4753
- }
4920
+ ErrorCodes.EMBEDDING_NO_SOURCE,
4921
+ `All source fields for embedding target "${targetField}" are missing or empty`,
4922
+ details
4754
4923
  );
4755
4924
  }
4756
- if (docId.includes("\0")) {
4757
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
4925
+ return parts.join("\n");
4926
+ }
4927
+ function wrapAdapterError(err, targetField) {
4928
+ if (err instanceof NarsilError) return err;
4929
+ const message = err instanceof Error ? err.message : String(err);
4930
+ return new NarsilError(ErrorCodes.EMBEDDING_FAILED, `Embedding failed for field "${targetField}": ${message}`, {
4931
+ field: targetField,
4932
+ cause: message
4933
+ });
4934
+ }
4935
+ async function embedDocumentFields(document, embeddingConfig, adapter, signal) {
4936
+ const embeddedFields = /* @__PURE__ */ new Set();
4937
+ try {
4938
+ for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
4939
+ const existing = resolveFieldValue(document, targetField);
4940
+ if (existing !== void 0 && existing !== null) continue;
4941
+ const text = collectSourceText(document, sourceFields, targetField);
4942
+ let vector;
4943
+ try {
4944
+ const raw = await adapter.embed(text, "document", signal);
4945
+ vector = validateVector(raw, targetField, adapter.dimensions);
4946
+ } catch (err) {
4947
+ throw wrapAdapterError(err, targetField);
4948
+ }
4949
+ setFieldValue(document, targetField, vector);
4950
+ embeddedFields.add(targetField);
4951
+ }
4952
+ } catch (err) {
4953
+ for (const field of embeddedFields) {
4954
+ deleteFieldValue(document, field);
4955
+ }
4956
+ throw err;
4758
4957
  }
4958
+ return embeddedFields;
4759
4959
  }
4760
- function clampLimit(limit) {
4761
- if (limit === void 0) return DEFAULT_LIMIT;
4762
- return Math.max(0, Math.min(limit, MAX_LIMIT));
4960
+ var SOURCE_FAILURE_CODES = /* @__PURE__ */ new Set([ErrorCodes.EMBEDDING_NO_SOURCE, ErrorCodes.DOC_VALIDATION_FAILED]);
4961
+ function isDocumentSourceError(err) {
4962
+ return err instanceof NarsilError && SOURCE_FAILURE_CODES.has(err.code);
4763
4963
  }
4764
- function clampOffset(offset) {
4765
- if (offset === void 0) return DEFAULT_OFFSET;
4766
- return Math.max(0, Math.min(offset, MAX_OFFSET));
4964
+ async function embedBatchDocumentFields(documents, embeddingConfig, adapter, signal) {
4965
+ const embedded = /* @__PURE__ */ new Map();
4966
+ const failed = /* @__PURE__ */ new Map();
4967
+ try {
4968
+ for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
4969
+ const needsEmbedding = [];
4970
+ for (let i = 0; i < documents.length; i++) {
4971
+ if (failed.has(i)) continue;
4972
+ const doc = documents[i];
4973
+ const existing = resolveFieldValue(doc, targetField);
4974
+ if (existing !== void 0 && existing !== null) continue;
4975
+ try {
4976
+ const text = collectSourceText(doc, sourceFields, targetField, i);
4977
+ needsEmbedding.push({ docIndex: i, text });
4978
+ } catch (err) {
4979
+ if (isDocumentSourceError(err)) {
4980
+ failed.set(i, err);
4981
+ } else {
4982
+ throw err;
4983
+ }
4984
+ }
4985
+ }
4986
+ if (needsEmbedding.length === 0) continue;
4987
+ const texts = needsEmbedding.map((entry) => entry.text);
4988
+ let vectors;
4989
+ try {
4990
+ if (adapter.embedBatch) {
4991
+ vectors = await adapter.embedBatch(texts, "document", signal);
4992
+ } else {
4993
+ vectors = [];
4994
+ const concurrency = 8;
4995
+ for (let start = 0; start < texts.length; start += concurrency) {
4996
+ const chunk = texts.slice(start, start + concurrency);
4997
+ const chunkResults = await Promise.all(chunk.map((t) => adapter.embed(t, "document", signal)));
4998
+ vectors.push(...chunkResults);
4999
+ }
5000
+ }
5001
+ } catch (err) {
5002
+ throw wrapAdapterError(err, targetField);
5003
+ }
5004
+ if (vectors.length !== needsEmbedding.length) {
5005
+ throw new NarsilError(
5006
+ ErrorCodes.EMBEDDING_FAILED,
5007
+ `Adapter returned ${vectors.length} vectors for ${needsEmbedding.length} inputs on field "${targetField}"`,
5008
+ { field: targetField, expected: needsEmbedding.length, actual: vectors.length }
5009
+ );
5010
+ }
5011
+ for (let j = 0; j < needsEmbedding.length; j++) {
5012
+ const { docIndex } = needsEmbedding[j];
5013
+ const validated = validateVector(vectors[j], targetField, adapter.dimensions);
5014
+ setFieldValue(documents[docIndex], targetField, validated);
5015
+ let fieldSet = embedded.get(docIndex);
5016
+ if (!fieldSet) {
5017
+ fieldSet = /* @__PURE__ */ new Set();
5018
+ embedded.set(docIndex, fieldSet);
5019
+ }
5020
+ fieldSet.add(targetField);
5021
+ }
5022
+ }
5023
+ } catch (err) {
5024
+ for (const [docIndex, fields] of embedded) {
5025
+ for (const field of fields) {
5026
+ deleteFieldValue(documents[docIndex], field);
5027
+ }
5028
+ }
5029
+ throw err;
5030
+ }
5031
+ return { embedded, failed };
4767
5032
  }
4768
5033
 
4769
5034
  // src/engine/mutations/durable-rollback.ts
@@ -4827,13 +5092,21 @@ async function insertDocument(ctx, indexName, document, docId, options) {
4827
5092
  if (entry.config.required && entry.config.required.length > 0) {
4828
5093
  validateRequiredFields(document, entry.config.required);
4829
5094
  }
4830
- if (entry.embeddingAdapter && entry.config.embedding) {
4831
- await embedDocumentFields(
4832
- document,
4833
- entry.config.embedding,
4834
- entry.embeddingAdapter,
4835
- ctx.abortController.signal
4836
- );
5095
+ if (entry.config.embedding) {
5096
+ if (entry.embeddingAdapter) {
5097
+ await embedDocumentFields(
5098
+ document,
5099
+ entry.config.embedding,
5100
+ entry.embeddingAdapter,
5101
+ ctx.abortController.signal
5102
+ );
5103
+ } else {
5104
+ assertDocumentCarriesMappedVectors(
5105
+ document,
5106
+ entry.config.embedding,
5107
+ entry.embeddingAdapterName
5108
+ );
5109
+ }
4837
5110
  }
4838
5111
  const insertManager = ctx.requireManager(indexName);
4839
5112
  const insertVecIndexes = insertManager.getVectorIndexes();
@@ -4913,6 +5186,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
4913
5186
  const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
4914
5187
  const hasRequired = entry.config.required && entry.config.required.length > 0;
4915
5188
  const hasEmbedding = entry.embeddingAdapter && entry.config.embedding;
5189
+ const embeddingUnbound = !entry.embeddingAdapter && entry.config.embedding;
4916
5190
  const batchManager = ctx.requireManager(indexName);
4917
5191
  const batchVecIndexes = batchManager.getVectorIndexes();
4918
5192
  const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
@@ -4928,7 +5202,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
4928
5202
  } catch (err) {
4929
5203
  chunkFailedIndexes.add(i);
4930
5204
  failed.push({
4931
- docId: "",
5205
+ docId: providedDocId(documents[i]) ?? "",
4932
5206
  error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err))
4933
5207
  });
4934
5208
  }
@@ -4953,16 +5227,33 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
4953
5227
  for (const [sliceIndex, error] of embedResult.failed) {
4954
5228
  const originalIdx = embeddableOriginalIndexes[sliceIndex];
4955
5229
  chunkFailedIndexes.add(originalIdx);
4956
- failed.push({ docId: "", error });
5230
+ failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error });
4957
5231
  }
4958
5232
  } catch (err) {
4959
5233
  const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
4960
5234
  for (const originalIdx of embeddableOriginalIndexes) {
4961
5235
  chunkFailedIndexes.add(originalIdx);
4962
- failed.push({ docId: "", error: embeddingError });
5236
+ failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error: embeddingError });
4963
5237
  }
4964
5238
  }
4965
5239
  }
5240
+ } else if (embeddingUnbound) {
5241
+ for (let i = chunkStart; i < chunkEnd; i++) {
5242
+ if (chunkFailedIndexes.has(i)) continue;
5243
+ try {
5244
+ assertDocumentCarriesMappedVectors(
5245
+ documents[i],
5246
+ entry.config.embedding,
5247
+ entry.embeddingAdapterName
5248
+ );
5249
+ } catch (err) {
5250
+ chunkFailedIndexes.add(i);
5251
+ failed.push({
5252
+ docId: providedDocId(documents[i]) ?? "",
5253
+ error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
5254
+ });
5255
+ }
5256
+ }
4966
5257
  }
4967
5258
  for (let i = chunkStart; i < chunkEnd; i++) {
4968
5259
  if (ctx.abortController.signal.aborted) break;
@@ -5156,13 +5447,21 @@ async function updateDocument(ctx, indexName, docId, document) {
5156
5447
  if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
5157
5448
  return;
5158
5449
  }
5159
- if (entry.embeddingAdapter && entry.config.embedding) {
5160
- await embedDocumentFields(
5161
- document,
5162
- entry.config.embedding,
5163
- entry.embeddingAdapter,
5164
- ctx.abortController.signal
5165
- );
5450
+ if (entry.config.embedding) {
5451
+ if (entry.embeddingAdapter) {
5452
+ await embedDocumentFields(
5453
+ document,
5454
+ entry.config.embedding,
5455
+ entry.embeddingAdapter,
5456
+ ctx.abortController.signal
5457
+ );
5458
+ } else {
5459
+ assertDocumentCarriesMappedVectors(
5460
+ document,
5461
+ entry.config.embedding,
5462
+ entry.embeddingAdapterName
5463
+ );
5464
+ }
5166
5465
  }
5167
5466
  const updateManager = ctx.requireManager(indexName);
5168
5467
  const oldDocument = updateManager.get(docId);
@@ -6212,7 +6511,7 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
6212
6511
  }
6213
6512
 
6214
6513
  // src/engine/resolve-vector-text.ts
6215
- async function resolveVectorText(params, embeddingAdapter, signal) {
6514
+ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
6216
6515
  if (!params.vector) return params;
6217
6516
  if (params.vector.text === void 0 && params.vector.value === void 0) return params;
6218
6517
  if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
@@ -6222,7 +6521,8 @@ async function resolveVectorText(params, embeddingAdapter, signal) {
6222
6521
  if (!embeddingAdapter) {
6223
6522
  throw new NarsilError(
6224
6523
  ErrorCodes.EMBEDDING_CONFIG_INVALID,
6225
- "Vector query with 'text' requires an embedding adapter on the index or instance"
6524
+ embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
6525
+ embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
6226
6526
  );
6227
6527
  }
6228
6528
  if (typeof params.vector.text !== "string") {
@@ -6304,7 +6604,13 @@ async function restoreFromSnapshot(indexName, data, executor, indexRegistry, get
6304
6604
  const indexConfig = { schema, language: envelope.language };
6305
6605
  executor.createIndex(indexName, indexConfig, language);
6306
6606
  const vectorFieldPaths = getVectorFieldPaths2(schema);
6307
- indexRegistry.set(indexName, { config: indexConfig, language, embeddingAdapter: null, vectorFieldPaths });
6607
+ indexRegistry.set(indexName, {
6608
+ config: indexConfig,
6609
+ language,
6610
+ embeddingAdapter: null,
6611
+ embeddingAdapterName: null,
6612
+ vectorFieldPaths
6613
+ });
6308
6614
  try {
6309
6615
  const manager = requireManager(indexName);
6310
6616
  while (manager.partitionCount < envelope.partitions.length) {
@@ -6435,97 +6741,6 @@ function readProcessMemory() {
6435
6741
  }
6436
6742
  }
6437
6743
 
6438
- // src/schema/embedding-validator.ts
6439
- var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
6440
- function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
6441
- const fieldEntries = Object.keys(embeddingConfig.fields);
6442
- if (fieldEntries.length === 0) {
6443
- throw new NarsilError(
6444
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6445
- "Embedding config must define at least one field mapping"
6446
- );
6447
- }
6448
- const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
6449
- if (!resolvedAdapter) {
6450
- throw new NarsilError(
6451
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6452
- "Embedding config requires an adapter (set on the index or the instance)"
6453
- );
6454
- }
6455
- if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
6456
- throw new NarsilError(
6457
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6458
- `Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
6459
- );
6460
- }
6461
- const flatFields = flattenSchema(schema);
6462
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
6463
- const targetType = flatFields[targetField];
6464
- if (targetType === void 0) {
6465
- throw new NarsilError(
6466
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6467
- `Embedding target field "${targetField}" does not exist in the schema`,
6468
- { field: targetField }
6469
- );
6470
- }
6471
- const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
6472
- if (!vectorMatch) {
6473
- throw new NarsilError(
6474
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6475
- `Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
6476
- { field: targetField, type: targetType }
6477
- );
6478
- }
6479
- const schemaDimension = Number.parseInt(vectorMatch[1], 10);
6480
- if (schemaDimension !== resolvedAdapter.dimensions) {
6481
- throw new NarsilError(
6482
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
6483
- `Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
6484
- { field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
6485
- );
6486
- }
6487
- const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
6488
- if (sources.length === 0) {
6489
- throw new NarsilError(
6490
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6491
- `Embedding target field "${targetField}" must have at least one source field`,
6492
- { field: targetField }
6493
- );
6494
- }
6495
- for (const source of sources) {
6496
- const sourceType = flatFields[source];
6497
- if (sourceType === void 0) {
6498
- throw new NarsilError(
6499
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6500
- `Embedding source field "${source}" does not exist in the schema`,
6501
- { field: source, targetField }
6502
- );
6503
- }
6504
- if (sourceType !== "string") {
6505
- throw new NarsilError(
6506
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6507
- `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
6508
- { field: source, type: sourceType, targetField }
6509
- );
6510
- }
6511
- }
6512
- }
6513
- return resolvedAdapter;
6514
- }
6515
- function validateRequiredFieldsInSchema(required, schema) {
6516
- if (required.length === 0) return;
6517
- const flatFields = flattenSchema(schema);
6518
- for (const field of required) {
6519
- if (flatFields[field] === void 0) {
6520
- throw new NarsilError(
6521
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
6522
- `Required field "${field}" does not exist in the schema`,
6523
- { field }
6524
- );
6525
- }
6526
- }
6527
- }
6528
-
6529
6744
  // src/narsil.ts
6530
6745
  async function createNarsil(config) {
6531
6746
  const core = createEngineCore(config);
@@ -6552,43 +6767,14 @@ function createNarsilFromCore(core, config) {
6552
6767
  rebalanceCtx
6553
6768
  } = core;
6554
6769
  const narsil = {
6555
- async createIndex(name, indexConfig) {
6556
- guardShutdown();
6557
- validateIndexName2(name);
6558
- if (indexRegistry.has(name)) {
6559
- throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
6560
- }
6561
- validateSchema(indexConfig.schema);
6562
- let resolvedEmbeddingAdapter = null;
6563
- if (indexConfig.embedding) {
6564
- resolvedEmbeddingAdapter = validateEmbeddingConfig(indexConfig.embedding, indexConfig.schema, config?.embedding);
6565
- }
6566
- if (indexConfig.required && indexConfig.required.length > 0) {
6567
- validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
6568
- }
6569
- const language = getLanguage(indexConfig.language ?? "english");
6570
- executor.createIndex(name, indexConfig, language);
6571
- const vectorFieldPaths = getVectorFieldPaths(indexConfig.schema);
6572
- indexRegistry.set(name, {
6573
- config: indexConfig,
6574
- language,
6575
- embeddingAdapter: resolvedEmbeddingAdapter,
6576
- vectorFieldPaths
6577
- });
6578
- if (durability) {
6579
- await durability.manager.persistMetadata(name);
6580
- }
6581
- await pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
6770
+ createIndex(name, indexConfig) {
6771
+ return createEngineIndex(core, config, name, indexConfig);
6582
6772
  },
6583
- async dropIndex(name) {
6584
- guardShutdown();
6585
- const entry = requireIndex(name);
6586
- executor.dropIndex(name);
6587
- indexRegistry.delete(name);
6588
- if (durability) {
6589
- await durability.manager.removeIndex(name);
6590
- }
6591
- await pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
6773
+ registerEmbeddingAdapter(name, adapter) {
6774
+ registerEngineEmbeddingAdapter(core, name, adapter);
6775
+ },
6776
+ dropIndex(name) {
6777
+ return dropEngineIndex(core, name);
6592
6778
  },
6593
6779
  listIndexes() {
6594
6780
  const infos = [];
@@ -6667,7 +6853,12 @@ function createNarsilFromCore(core, config) {
6667
6853
  guardShutdown();
6668
6854
  const entry = requireIndex(indexName);
6669
6855
  const manager = requireManager(indexName);
6670
- const resolvedParams = await resolveVectorText(params, entry.embeddingAdapter, abortController.signal);
6856
+ const resolvedParams = await resolveVectorText(
6857
+ params,
6858
+ entry.embeddingAdapter,
6859
+ abortController.signal,
6860
+ entry.embeddingAdapterName
6861
+ );
6671
6862
  await pluginRegistry.runHook("beforeSearch", { indexName, params: resolvedParams });
6672
6863
  const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
6673
6864
  const result = await executeQuery(resolvedParams, {
@@ -6692,7 +6883,12 @@ function createNarsilFromCore(core, config) {
6692
6883
  guardShutdown();
6693
6884
  const entry = requireIndex(indexName);
6694
6885
  const manager = requireManager(indexName);
6695
- const resolvedParams = await resolveVectorText(params, entry.embeddingAdapter, abortController.signal);
6886
+ const resolvedParams = await resolveVectorText(
6887
+ params,
6888
+ entry.embeddingAdapter,
6889
+ abortController.signal,
6890
+ entry.embeddingAdapterName
6891
+ );
6696
6892
  const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
6697
6893
  return executePreflight(resolvedParams, {
6698
6894
  manager,