@delali/narsil 0.1.8 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1049,6 +1049,38 @@ function extractVectorFieldsFromSchema(schema) {
1049
1049
  }
1050
1050
  return result;
1051
1051
  }
1052
+
1053
+ // src/schema/validator/vector-promotion.ts
1054
+ function fail(message, details) {
1055
+ throw new NarsilError(ErrorCodes.CONFIG_INVALID, message, details);
1056
+ }
1057
+ function isPositiveInteger(value) {
1058
+ return Number.isInteger(value) && value >= 1;
1059
+ }
1060
+ function validateVectorPromotion(config) {
1061
+ if (config === void 0) return;
1062
+ if (config.threshold !== void 0 && !isPositiveInteger(config.threshold)) {
1063
+ fail("vectorPromotion.threshold must be a positive integer", { threshold: config.threshold });
1064
+ }
1065
+ const quantization = config.quantization;
1066
+ if (quantization !== void 0 && quantization !== "sq8" && quantization !== "none") {
1067
+ fail("vectorPromotion.quantization must be 'sq8' or 'none'", { quantization });
1068
+ }
1069
+ const hnsw = config.hnswConfig;
1070
+ if (hnsw === void 0) return;
1071
+ if (hnsw.m !== void 0 && !isPositiveInteger(hnsw.m)) {
1072
+ fail("vectorPromotion.hnswConfig.m must be a positive integer", { m: hnsw.m });
1073
+ }
1074
+ if (hnsw.efConstruction !== void 0 && !isPositiveInteger(hnsw.efConstruction)) {
1075
+ fail("vectorPromotion.hnswConfig.efConstruction must be a positive integer", {
1076
+ efConstruction: hnsw.efConstruction
1077
+ });
1078
+ }
1079
+ const metric = hnsw.metric;
1080
+ if (metric !== void 0 && metric !== "cosine" && metric !== "dotProduct" && metric !== "euclidean") {
1081
+ fail("vectorPromotion.hnswConfig.metric must be 'cosine', 'dotProduct', or 'euclidean'", { metric });
1082
+ }
1083
+ }
1052
1084
  function encodeRawPayloadV2(wire) {
1053
1085
  return encode(wire);
1054
1086
  }
@@ -3992,6 +4024,123 @@ function createPluginRegistry() {
3992
4024
  };
3993
4025
  }
3994
4026
 
4027
+ // src/schema/embedding-validator.ts
4028
+ var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
4029
+ function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
4030
+ const fieldEntries = Object.keys(embeddingConfig.fields);
4031
+ if (fieldEntries.length === 0) {
4032
+ throw new NarsilError(
4033
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4034
+ "Embedding config must define at least one field mapping"
4035
+ );
4036
+ }
4037
+ if (typeof embeddingConfig.adapter === "string") {
4038
+ throw new NarsilError(
4039
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4040
+ `Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
4041
+ { adapter: embeddingConfig.adapter }
4042
+ );
4043
+ }
4044
+ const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
4045
+ if (!resolvedAdapter) {
4046
+ throw new NarsilError(
4047
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4048
+ "Embedding config requires an adapter (set on the index or the instance)"
4049
+ );
4050
+ }
4051
+ if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
4052
+ throw new NarsilError(
4053
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4054
+ `Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
4055
+ );
4056
+ }
4057
+ const flatFields = flattenSchema(schema);
4058
+ for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
4059
+ const targetType = flatFields[targetField];
4060
+ if (targetType === void 0) {
4061
+ throw new NarsilError(
4062
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4063
+ `Embedding target field "${targetField}" does not exist in the schema`,
4064
+ { field: targetField }
4065
+ );
4066
+ }
4067
+ const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
4068
+ if (!vectorMatch) {
4069
+ throw new NarsilError(
4070
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4071
+ `Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
4072
+ { field: targetField, type: targetType }
4073
+ );
4074
+ }
4075
+ const schemaDimension = Number.parseInt(vectorMatch[1], 10);
4076
+ if (schemaDimension !== resolvedAdapter.dimensions) {
4077
+ throw new NarsilError(
4078
+ ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
4079
+ `Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
4080
+ { field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
4081
+ );
4082
+ }
4083
+ const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
4084
+ if (sources.length === 0) {
4085
+ throw new NarsilError(
4086
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4087
+ `Embedding target field "${targetField}" must have at least one source field`,
4088
+ { field: targetField }
4089
+ );
4090
+ }
4091
+ for (const source of sources) {
4092
+ const sourceType = flatFields[source];
4093
+ if (sourceType === void 0) {
4094
+ throw new NarsilError(
4095
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4096
+ `Embedding source field "${source}" does not exist in the schema`,
4097
+ { field: source, targetField }
4098
+ );
4099
+ }
4100
+ if (sourceType !== "string") {
4101
+ throw new NarsilError(
4102
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4103
+ `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
4104
+ { field: source, type: sourceType, targetField }
4105
+ );
4106
+ }
4107
+ }
4108
+ }
4109
+ return resolvedAdapter;
4110
+ }
4111
+ function validateRegisteredAdapter(name, adapter) {
4112
+ if (typeof name !== "string" || name.trim().length === 0) {
4113
+ throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
4114
+ }
4115
+ if (typeof adapter?.embed !== "function") {
4116
+ throw new NarsilError(
4117
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4118
+ `Embedding adapter "${name}" must provide an embed function`,
4119
+ { adapter: name }
4120
+ );
4121
+ }
4122
+ if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
4123
+ throw new NarsilError(
4124
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4125
+ `Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
4126
+ { adapter: name, dimensions: adapter.dimensions }
4127
+ );
4128
+ }
4129
+ }
4130
+ function validateRequiredFieldsInSchema(required, schema) {
4131
+ if (required.length === 0) return;
4132
+ const flatFields = flattenSchema(schema);
4133
+ for (const field of required) {
4134
+ if (flatFields[field] === void 0) {
4135
+ throw new NarsilError(
4136
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
4137
+ `Required field "${field}" does not exist in the schema`,
4138
+ { field }
4139
+ );
4140
+ }
4141
+ }
4142
+ }
4143
+
3995
4144
  // src/search/facets.ts
3996
4145
  function mergeFacets(partitionFacets) {
3997
4146
  const merged = /* @__PURE__ */ new Map();
@@ -7580,6 +7729,9 @@ function metadataToWire(meta) {
7580
7729
  if (meta.vectorFields) {
7581
7730
  wire.vector_fields = meta.vectorFields;
7582
7731
  }
7732
+ if (meta.embedding) {
7733
+ wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
7734
+ }
7583
7735
  return wire;
7584
7736
  }
7585
7737
  function wireToMetadata(raw) {
@@ -7595,6 +7747,9 @@ function wireToMetadata(raw) {
7595
7747
  if (raw.vector_fields) {
7596
7748
  meta.vectorFields = raw.vector_fields;
7597
7749
  }
7750
+ if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
7751
+ meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
7752
+ }
7598
7753
  return meta;
7599
7754
  }
7600
7755
  function serializeMetadata(meta) {
@@ -8947,11 +9102,15 @@ function unflattenSchema(flat) {
8947
9102
  return root;
8948
9103
  }
8949
9104
  function reconstructSchemaFromMetadata(metadata) {
8950
- return {
9105
+ const config = {
8951
9106
  schema: unflattenSchema(metadata.schema),
8952
9107
  language: metadata.language,
8953
9108
  bm25: { k1: metadata.bm25Params.k1, b: metadata.bm25Params.b }
8954
9109
  };
9110
+ if (metadata.embedding) {
9111
+ config.embedding = metadata.embedding.adapter !== void 0 ? { adapter: metadata.embedding.adapter, fields: metadata.embedding.fields } : { fields: metadata.embedding.fields };
9112
+ }
9113
+ return config;
8955
9114
  }
8956
9115
 
8957
9116
  // src/persistence/durability/segment/build-segment.ts
@@ -10399,7 +10558,7 @@ function buildMetadata(indexName, hooks) {
10399
10558
  if (config === void 0 || manager === void 0) {
10400
10559
  return void 0;
10401
10560
  }
10402
- return {
10561
+ const metadata = {
10403
10562
  indexName,
10404
10563
  schema: config.schema,
10405
10564
  language: config.language,
@@ -10408,6 +10567,10 @@ function buildMetadata(indexName, hooks) {
10408
10567
  createdAt: Date.now(),
10409
10568
  engineVersion: VERSION
10410
10569
  };
10570
+ if (config.embedding) {
10571
+ metadata.embedding = config.embedding;
10572
+ }
10573
+ return metadata;
10411
10574
  }
10412
10575
  function createDurabilityIntegration(tier, hooks) {
10413
10576
  const managerHooks = {
@@ -10928,11 +11091,16 @@ function createDurabilityFromConfig(config, wiring) {
10928
11091
  if (entry === void 0) {
10929
11092
  return void 0;
10930
11093
  }
11094
+ const embedding = entry.config.embedding ? {
11095
+ fields: entry.config.embedding.fields,
11096
+ ...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
11097
+ } : void 0;
10931
11098
  return {
10932
11099
  schema: flattenSchema(entry.config.schema),
10933
11100
  language: entry.language.name,
10934
11101
  k1: entry.config.bm25?.k1 ?? 1.2,
10935
- b: entry.config.bm25?.b ?? 0.75
11102
+ b: entry.config.bm25?.b ?? 0.75,
11103
+ ...embedding !== void 0 ? { embedding } : {}
10936
11104
  };
10937
11105
  },
10938
11106
  createIndexFromMetadata: wiring.createIndexFromMetadata,
@@ -10951,6 +11119,13 @@ function createEngineCore(config) {
10951
11119
  }
10952
11120
  const idGenerator = config?.idGenerator ?? generateId;
10953
11121
  const indexRegistry = /* @__PURE__ */ new Map();
11122
+ const embeddingAdapters = /* @__PURE__ */ new Map();
11123
+ if (config?.embeddingAdapters) {
11124
+ for (const [name, adapter] of Object.entries(config.embeddingAdapters)) {
11125
+ validateRegisteredAdapter(name, adapter);
11126
+ embeddingAdapters.set(name, adapter);
11127
+ }
11128
+ }
10954
11129
  const eventHandlers = /* @__PURE__ */ new Map();
10955
11130
  const shutdownState = { isShutdown: false };
10956
11131
  const abortController = new AbortController();
@@ -11000,11 +11175,35 @@ function createEngineCore(config) {
11000
11175
  }
11001
11176
  const indexConfig = reconstructSchemaFromMetadata(metadata);
11002
11177
  const language = getLanguage(indexConfig.language ?? "english");
11178
+ const adapterName = metadata.embedding?.adapter ?? null;
11179
+ let embeddingAdapter = null;
11180
+ if (metadata.embedding) {
11181
+ const candidate = adapterName !== null ? embeddingAdapters.get(adapterName) ?? null : config?.embedding ?? null;
11182
+ if (candidate) {
11183
+ try {
11184
+ validateEmbeddingConfig(
11185
+ { fields: metadata.embedding.fields, adapter: candidate },
11186
+ indexConfig.schema,
11187
+ void 0
11188
+ );
11189
+ } catch (err) {
11190
+ if (err instanceof NarsilError) {
11191
+ throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
11192
+ indexName: metadata.indexName,
11193
+ adapter: adapterName ?? void 0
11194
+ });
11195
+ }
11196
+ throw err;
11197
+ }
11198
+ embeddingAdapter = candidate;
11199
+ }
11200
+ }
11003
11201
  executor.createIndex(metadata.indexName, indexConfig, language);
11004
11202
  indexRegistry.set(metadata.indexName, {
11005
11203
  config: indexConfig,
11006
11204
  language,
11007
- embeddingAdapter: null,
11205
+ embeddingAdapter,
11206
+ embeddingAdapterName: adapterName,
11008
11207
  vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
11009
11208
  });
11010
11209
  }
@@ -11047,6 +11246,7 @@ function createEngineCore(config) {
11047
11246
  durability,
11048
11247
  idGenerator,
11049
11248
  indexRegistry,
11249
+ embeddingAdapters,
11050
11250
  eventHandlers,
11051
11251
  shutdownState,
11052
11252
  abortController,
@@ -11064,6 +11264,160 @@ function createEngineCore(config) {
11064
11264
  rebalanceCtx
11065
11265
  };
11066
11266
  }
11267
+ var MAX_INDEX_NAME_LENGTH = 255;
11268
+
11269
+ // src/engine/validation.ts
11270
+ var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
11271
+ var MAX_DOC_ID_LENGTH = 512;
11272
+ var BATCH_CHUNK_SIZE = 1e3;
11273
+ var MAX_LIMIT = 1e4;
11274
+ var MAX_OFFSET = 1e5;
11275
+ var DEFAULT_LIMIT = 10;
11276
+ var DEFAULT_OFFSET = 0;
11277
+ function now() {
11278
+ if (typeof performance !== "undefined" && typeof performance.now === "function") {
11279
+ return performance.now();
11280
+ }
11281
+ return Date.now();
11282
+ }
11283
+ function validateIndexName(name) {
11284
+ if (!name || name.length === 0) {
11285
+ throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
11286
+ }
11287
+ if (name.length > MAX_INDEX_NAME_LENGTH) {
11288
+ throw new NarsilError(
11289
+ ErrorCodes.INDEX_NOT_FOUND,
11290
+ `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
11291
+ {
11292
+ indexName: name,
11293
+ length: name.length
11294
+ }
11295
+ );
11296
+ }
11297
+ if (!INDEX_NAME_PATTERN.test(name)) {
11298
+ throw new NarsilError(
11299
+ ErrorCodes.INDEX_NOT_FOUND,
11300
+ `Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
11301
+ { indexName: name }
11302
+ );
11303
+ }
11304
+ if (name.includes("..")) {
11305
+ throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
11306
+ }
11307
+ }
11308
+ function validateDocId(docId) {
11309
+ if (!docId || docId.length === 0) {
11310
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
11311
+ }
11312
+ if (docId.length > MAX_DOC_ID_LENGTH) {
11313
+ throw new NarsilError(
11314
+ ErrorCodes.DOC_VALIDATION_FAILED,
11315
+ `Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
11316
+ {
11317
+ docId,
11318
+ length: docId.length
11319
+ }
11320
+ );
11321
+ }
11322
+ if (docId.includes("\0")) {
11323
+ throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
11324
+ }
11325
+ }
11326
+ function clampLimit(limit) {
11327
+ if (limit === void 0) return DEFAULT_LIMIT;
11328
+ return Math.max(0, Math.min(limit, MAX_LIMIT));
11329
+ }
11330
+ function clampOffset(offset) {
11331
+ if (offset === void 0) return DEFAULT_OFFSET;
11332
+ return Math.max(0, Math.min(offset, MAX_OFFSET));
11333
+ }
11334
+
11335
+ // src/engine/index-lifecycle.ts
11336
+ async function createEngineIndex(core, config, name, indexConfig) {
11337
+ core.guardShutdown();
11338
+ validateIndexName(name);
11339
+ if (core.indexRegistry.has(name)) {
11340
+ throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
11341
+ }
11342
+ validateSchema(indexConfig.schema);
11343
+ validateVectorPromotion(indexConfig.vectorPromotion);
11344
+ let resolvedEmbeddingAdapter = null;
11345
+ let embeddingAdapterName = null;
11346
+ if (indexConfig.embedding) {
11347
+ let configuredAdapter = indexConfig.embedding.adapter;
11348
+ if (typeof configuredAdapter === "string") {
11349
+ embeddingAdapterName = configuredAdapter;
11350
+ const registered = core.embeddingAdapters.get(configuredAdapter);
11351
+ if (!registered) {
11352
+ throw new NarsilError(
11353
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
11354
+ `Embedding adapter "${configuredAdapter}" is not registered on this engine`,
11355
+ { adapter: configuredAdapter, available: [...core.embeddingAdapters.keys()] }
11356
+ );
11357
+ }
11358
+ configuredAdapter = registered;
11359
+ }
11360
+ resolvedEmbeddingAdapter = validateEmbeddingConfig(
11361
+ { fields: indexConfig.embedding.fields, adapter: configuredAdapter },
11362
+ indexConfig.schema,
11363
+ config?.embedding
11364
+ );
11365
+ }
11366
+ if (indexConfig.required && indexConfig.required.length > 0) {
11367
+ validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
11368
+ }
11369
+ const language = getLanguage(indexConfig.language ?? "english");
11370
+ core.executor.createIndex(name, indexConfig, language);
11371
+ core.indexRegistry.set(name, {
11372
+ config: indexConfig,
11373
+ language,
11374
+ embeddingAdapter: resolvedEmbeddingAdapter,
11375
+ embeddingAdapterName,
11376
+ vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
11377
+ });
11378
+ if (core.durability) {
11379
+ await core.durability.manager.persistMetadata(name);
11380
+ }
11381
+ await core.pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
11382
+ }
11383
+ async function dropEngineIndex(core, name) {
11384
+ core.guardShutdown();
11385
+ const entry = core.requireIndex(name);
11386
+ core.executor.dropIndex(name);
11387
+ core.indexRegistry.delete(name);
11388
+ if (core.durability) {
11389
+ await core.durability.manager.removeIndex(name);
11390
+ }
11391
+ await core.pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
11392
+ }
11393
+ function registerEngineEmbeddingAdapter(core, name, adapter) {
11394
+ core.guardShutdown();
11395
+ validateRegisteredAdapter(name, adapter);
11396
+ const affected = [];
11397
+ for (const [indexName, entry] of core.indexRegistry) {
11398
+ if (entry.embeddingAdapterName !== name || !entry.config.embedding) continue;
11399
+ try {
11400
+ validateEmbeddingConfig({ fields: entry.config.embedding.fields, adapter }, entry.config.schema, void 0);
11401
+ } catch (err) {
11402
+ if (err instanceof NarsilError) {
11403
+ throw new NarsilError(
11404
+ err.code,
11405
+ `Cannot bind embedding adapter "${name}" to index "${indexName}": ${err.message}`,
11406
+ {
11407
+ indexName,
11408
+ adapter: name
11409
+ }
11410
+ );
11411
+ }
11412
+ throw err;
11413
+ }
11414
+ affected.push(entry);
11415
+ }
11416
+ core.embeddingAdapters.set(name, adapter);
11417
+ for (const entry of affected) {
11418
+ entry.embeddingAdapter = adapter;
11419
+ }
11420
+ }
11067
11421
 
11068
11422
  // src/engine/lifecycle.ts
11069
11423
  async function shutdownEngine(core) {
@@ -11098,6 +11452,18 @@ async function shutdownEngine(core) {
11098
11452
  }
11099
11453
 
11100
11454
  // src/engine/embed.ts
11455
+ function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterName) {
11456
+ for (const targetField of Object.keys(embeddingConfig.fields)) {
11457
+ const existing = resolveFieldValue(document, targetField);
11458
+ if (existing === void 0 || existing === null) {
11459
+ throw new NarsilError(
11460
+ ErrorCodes.EMBEDDING_CONFIG_INVALID,
11461
+ adapterName ? `The index embeds "${targetField}" via adapter "${adapterName}", which is not registered on this engine. Register the adapter or supply the vector on the document.` : `The index embeds "${targetField}" automatically but no embedding adapter is bound. Bind an adapter or supply the vector on the document.`,
11462
+ adapterName ? { field: targetField, adapter: adapterName } : { field: targetField }
11463
+ );
11464
+ }
11465
+ }
11466
+ }
11101
11467
  function validateVector2(vector, targetField, expectedDimensions) {
11102
11468
  if (!(vector instanceof Float32Array)) {
11103
11469
  throw new NarsilError(
@@ -11299,73 +11665,6 @@ async function embedBatchDocumentFields(documents, embeddingConfig, adapter, sig
11299
11665
  }
11300
11666
  return { embedded, failed };
11301
11667
  }
11302
- var MAX_INDEX_NAME_LENGTH = 255;
11303
-
11304
- // src/engine/validation.ts
11305
- var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
11306
- var MAX_DOC_ID_LENGTH = 512;
11307
- var BATCH_CHUNK_SIZE = 1e3;
11308
- var MAX_LIMIT = 1e4;
11309
- var MAX_OFFSET = 1e5;
11310
- var DEFAULT_LIMIT = 10;
11311
- var DEFAULT_OFFSET = 0;
11312
- function now() {
11313
- if (typeof performance !== "undefined" && typeof performance.now === "function") {
11314
- return performance.now();
11315
- }
11316
- return Date.now();
11317
- }
11318
- function validateIndexName(name) {
11319
- if (!name || name.length === 0) {
11320
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
11321
- }
11322
- if (name.length > MAX_INDEX_NAME_LENGTH) {
11323
- throw new NarsilError(
11324
- ErrorCodes.INDEX_NOT_FOUND,
11325
- `Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
11326
- {
11327
- indexName: name,
11328
- length: name.length
11329
- }
11330
- );
11331
- }
11332
- if (!INDEX_NAME_PATTERN.test(name)) {
11333
- throw new NarsilError(
11334
- ErrorCodes.INDEX_NOT_FOUND,
11335
- `Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
11336
- { indexName: name }
11337
- );
11338
- }
11339
- if (name.includes("..")) {
11340
- throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
11341
- }
11342
- }
11343
- function validateDocId(docId) {
11344
- if (!docId || docId.length === 0) {
11345
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
11346
- }
11347
- if (docId.length > MAX_DOC_ID_LENGTH) {
11348
- throw new NarsilError(
11349
- ErrorCodes.DOC_VALIDATION_FAILED,
11350
- `Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
11351
- {
11352
- docId,
11353
- length: docId.length
11354
- }
11355
- );
11356
- }
11357
- if (docId.includes("\0")) {
11358
- throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
11359
- }
11360
- }
11361
- function clampLimit(limit) {
11362
- if (limit === void 0) return DEFAULT_LIMIT;
11363
- return Math.max(0, Math.min(limit, MAX_LIMIT));
11364
- }
11365
- function clampOffset(offset) {
11366
- if (offset === void 0) return DEFAULT_OFFSET;
11367
- return Math.max(0, Math.min(offset, MAX_OFFSET));
11368
- }
11369
11668
 
11370
11669
  // src/engine/mutations/durable-rollback.ts
11371
11670
  function rollbackFailed(docId, originalError, rollbackError) {
@@ -11428,13 +11727,21 @@ async function insertDocument(ctx, indexName, document, docId, options) {
11428
11727
  if (entry.config.required && entry.config.required.length > 0) {
11429
11728
  validateRequiredFields(document, entry.config.required);
11430
11729
  }
11431
- if (entry.embeddingAdapter && entry.config.embedding) {
11432
- await embedDocumentFields(
11433
- document,
11434
- entry.config.embedding,
11435
- entry.embeddingAdapter,
11436
- ctx.abortController.signal
11437
- );
11730
+ if (entry.config.embedding) {
11731
+ if (entry.embeddingAdapter) {
11732
+ await embedDocumentFields(
11733
+ document,
11734
+ entry.config.embedding,
11735
+ entry.embeddingAdapter,
11736
+ ctx.abortController.signal
11737
+ );
11738
+ } else {
11739
+ assertDocumentCarriesMappedVectors(
11740
+ document,
11741
+ entry.config.embedding,
11742
+ entry.embeddingAdapterName
11743
+ );
11744
+ }
11438
11745
  }
11439
11746
  const insertManager = ctx.requireManager(indexName);
11440
11747
  const insertVecIndexes = insertManager.getVectorIndexes();
@@ -11514,6 +11821,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
11514
11821
  const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
11515
11822
  const hasRequired = entry.config.required && entry.config.required.length > 0;
11516
11823
  const hasEmbedding = entry.embeddingAdapter && entry.config.embedding;
11824
+ const embeddingUnbound = !entry.embeddingAdapter && entry.config.embedding;
11517
11825
  const batchManager = ctx.requireManager(indexName);
11518
11826
  const batchVecIndexes = batchManager.getVectorIndexes();
11519
11827
  const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
@@ -11529,7 +11837,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
11529
11837
  } catch (err) {
11530
11838
  chunkFailedIndexes.add(i);
11531
11839
  failed.push({
11532
- docId: "",
11840
+ docId: providedDocId(documents[i]) ?? "",
11533
11841
  error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err))
11534
11842
  });
11535
11843
  }
@@ -11554,16 +11862,33 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
11554
11862
  for (const [sliceIndex, error] of embedResult.failed) {
11555
11863
  const originalIdx = embeddableOriginalIndexes[sliceIndex];
11556
11864
  chunkFailedIndexes.add(originalIdx);
11557
- failed.push({ docId: "", error });
11865
+ failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error });
11558
11866
  }
11559
11867
  } catch (err) {
11560
11868
  const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
11561
11869
  for (const originalIdx of embeddableOriginalIndexes) {
11562
11870
  chunkFailedIndexes.add(originalIdx);
11563
- failed.push({ docId: "", error: embeddingError });
11871
+ failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error: embeddingError });
11564
11872
  }
11565
11873
  }
11566
11874
  }
11875
+ } else if (embeddingUnbound) {
11876
+ for (let i = chunkStart; i < chunkEnd; i++) {
11877
+ if (chunkFailedIndexes.has(i)) continue;
11878
+ try {
11879
+ assertDocumentCarriesMappedVectors(
11880
+ documents[i],
11881
+ entry.config.embedding,
11882
+ entry.embeddingAdapterName
11883
+ );
11884
+ } catch (err) {
11885
+ chunkFailedIndexes.add(i);
11886
+ failed.push({
11887
+ docId: providedDocId(documents[i]) ?? "",
11888
+ error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
11889
+ });
11890
+ }
11891
+ }
11567
11892
  }
11568
11893
  for (let i = chunkStart; i < chunkEnd; i++) {
11569
11894
  if (ctx.abortController.signal.aborted) break;
@@ -11757,13 +12082,21 @@ async function updateDocument(ctx, indexName, docId, document) {
11757
12082
  if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
11758
12083
  return;
11759
12084
  }
11760
- if (entry.embeddingAdapter && entry.config.embedding) {
11761
- await embedDocumentFields(
11762
- document,
11763
- entry.config.embedding,
11764
- entry.embeddingAdapter,
11765
- ctx.abortController.signal
11766
- );
12085
+ if (entry.config.embedding) {
12086
+ if (entry.embeddingAdapter) {
12087
+ await embedDocumentFields(
12088
+ document,
12089
+ entry.config.embedding,
12090
+ entry.embeddingAdapter,
12091
+ ctx.abortController.signal
12092
+ );
12093
+ } else {
12094
+ assertDocumentCarriesMappedVectors(
12095
+ document,
12096
+ entry.config.embedding,
12097
+ entry.embeddingAdapterName
12098
+ );
12099
+ }
11767
12100
  }
11768
12101
  const updateManager = ctx.requireManager(indexName);
11769
12102
  const oldDocument = updateManager.get(docId);
@@ -12813,7 +13146,7 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
12813
13146
  }
12814
13147
 
12815
13148
  // src/engine/resolve-vector-text.ts
12816
- async function resolveVectorText(params, embeddingAdapter, signal) {
13149
+ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
12817
13150
  if (!params.vector) return params;
12818
13151
  if (params.vector.text === void 0 && params.vector.value === void 0) return params;
12819
13152
  if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
@@ -12823,7 +13156,8 @@ async function resolveVectorText(params, embeddingAdapter, signal) {
12823
13156
  if (!embeddingAdapter) {
12824
13157
  throw new NarsilError(
12825
13158
  ErrorCodes.EMBEDDING_CONFIG_INVALID,
12826
- "Vector query with 'text' requires an embedding adapter on the index or instance"
13159
+ embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
13160
+ embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
12827
13161
  );
12828
13162
  }
12829
13163
  if (typeof params.vector.text !== "string") {
@@ -12905,7 +13239,13 @@ async function restoreFromSnapshot(indexName, data, executor, indexRegistry, get
12905
13239
  const indexConfig = { schema, language: envelope.language };
12906
13240
  executor.createIndex(indexName, indexConfig, language);
12907
13241
  const vectorFieldPaths = getVectorFieldPaths2(schema);
12908
- indexRegistry.set(indexName, { config: indexConfig, language, embeddingAdapter: null, vectorFieldPaths });
13242
+ indexRegistry.set(indexName, {
13243
+ config: indexConfig,
13244
+ language,
13245
+ embeddingAdapter: null,
13246
+ embeddingAdapterName: null,
13247
+ vectorFieldPaths
13248
+ });
12909
13249
  try {
12910
13250
  const manager = requireManager(indexName);
12911
13251
  while (manager.partitionCount < envelope.partitions.length) {
@@ -13036,97 +13376,6 @@ function readProcessMemory() {
13036
13376
  }
13037
13377
  }
13038
13378
 
13039
- // src/schema/embedding-validator.ts
13040
- var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
13041
- function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
13042
- const fieldEntries = Object.keys(embeddingConfig.fields);
13043
- if (fieldEntries.length === 0) {
13044
- throw new NarsilError(
13045
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
13046
- "Embedding config must define at least one field mapping"
13047
- );
13048
- }
13049
- const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
13050
- if (!resolvedAdapter) {
13051
- throw new NarsilError(
13052
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
13053
- "Embedding config requires an adapter (set on the index or the instance)"
13054
- );
13055
- }
13056
- if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
13057
- throw new NarsilError(
13058
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
13059
- `Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
13060
- );
13061
- }
13062
- const flatFields = flattenSchema(schema);
13063
- for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
13064
- const targetType = flatFields[targetField];
13065
- if (targetType === void 0) {
13066
- throw new NarsilError(
13067
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
13068
- `Embedding target field "${targetField}" does not exist in the schema`,
13069
- { field: targetField }
13070
- );
13071
- }
13072
- const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
13073
- if (!vectorMatch) {
13074
- throw new NarsilError(
13075
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
13076
- `Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
13077
- { field: targetField, type: targetType }
13078
- );
13079
- }
13080
- const schemaDimension = Number.parseInt(vectorMatch[1], 10);
13081
- if (schemaDimension !== resolvedAdapter.dimensions) {
13082
- throw new NarsilError(
13083
- ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
13084
- `Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
13085
- { field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
13086
- );
13087
- }
13088
- const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
13089
- if (sources.length === 0) {
13090
- throw new NarsilError(
13091
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
13092
- `Embedding target field "${targetField}" must have at least one source field`,
13093
- { field: targetField }
13094
- );
13095
- }
13096
- for (const source of sources) {
13097
- const sourceType = flatFields[source];
13098
- if (sourceType === void 0) {
13099
- throw new NarsilError(
13100
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
13101
- `Embedding source field "${source}" does not exist in the schema`,
13102
- { field: source, targetField }
13103
- );
13104
- }
13105
- if (sourceType !== "string") {
13106
- throw new NarsilError(
13107
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
13108
- `Embedding source field "${source}" must be a string type, got "${sourceType}"`,
13109
- { field: source, type: sourceType, targetField }
13110
- );
13111
- }
13112
- }
13113
- }
13114
- return resolvedAdapter;
13115
- }
13116
- function validateRequiredFieldsInSchema(required, schema) {
13117
- if (required.length === 0) return;
13118
- const flatFields = flattenSchema(schema);
13119
- for (const field of required) {
13120
- if (flatFields[field] === void 0) {
13121
- throw new NarsilError(
13122
- ErrorCodes.EMBEDDING_CONFIG_INVALID,
13123
- `Required field "${field}" does not exist in the schema`,
13124
- { field }
13125
- );
13126
- }
13127
- }
13128
- }
13129
-
13130
13379
  // src/narsil.ts
13131
13380
  async function createNarsil(config) {
13132
13381
  const core = createEngineCore(config);
@@ -13153,43 +13402,14 @@ function createNarsilFromCore(core, config) {
13153
13402
  rebalanceCtx
13154
13403
  } = core;
13155
13404
  const narsil = {
13156
- async createIndex(name, indexConfig) {
13157
- guardShutdown();
13158
- validateIndexName(name);
13159
- if (indexRegistry.has(name)) {
13160
- throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
13161
- }
13162
- validateSchema(indexConfig.schema);
13163
- let resolvedEmbeddingAdapter = null;
13164
- if (indexConfig.embedding) {
13165
- resolvedEmbeddingAdapter = validateEmbeddingConfig(indexConfig.embedding, indexConfig.schema, config?.embedding);
13166
- }
13167
- if (indexConfig.required && indexConfig.required.length > 0) {
13168
- validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
13169
- }
13170
- const language = getLanguage(indexConfig.language ?? "english");
13171
- executor.createIndex(name, indexConfig, language);
13172
- const vectorFieldPaths = getVectorFieldPaths(indexConfig.schema);
13173
- indexRegistry.set(name, {
13174
- config: indexConfig,
13175
- language,
13176
- embeddingAdapter: resolvedEmbeddingAdapter,
13177
- vectorFieldPaths
13178
- });
13179
- if (durability) {
13180
- await durability.manager.persistMetadata(name);
13181
- }
13182
- await pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
13405
+ createIndex(name, indexConfig) {
13406
+ return createEngineIndex(core, config, name, indexConfig);
13183
13407
  },
13184
- async dropIndex(name) {
13185
- guardShutdown();
13186
- const entry = requireIndex(name);
13187
- executor.dropIndex(name);
13188
- indexRegistry.delete(name);
13189
- if (durability) {
13190
- await durability.manager.removeIndex(name);
13191
- }
13192
- await pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
13408
+ registerEmbeddingAdapter(name, adapter) {
13409
+ registerEngineEmbeddingAdapter(core, name, adapter);
13410
+ },
13411
+ dropIndex(name) {
13412
+ return dropEngineIndex(core, name);
13193
13413
  },
13194
13414
  listIndexes() {
13195
13415
  const infos = [];
@@ -13268,7 +13488,12 @@ function createNarsilFromCore(core, config) {
13268
13488
  guardShutdown();
13269
13489
  const entry = requireIndex(indexName);
13270
13490
  const manager = requireManager(indexName);
13271
- const resolvedParams = await resolveVectorText(params, entry.embeddingAdapter, abortController.signal);
13491
+ const resolvedParams = await resolveVectorText(
13492
+ params,
13493
+ entry.embeddingAdapter,
13494
+ abortController.signal,
13495
+ entry.embeddingAdapterName
13496
+ );
13272
13497
  await pluginRegistry.runHook("beforeSearch", { indexName, params: resolvedParams });
13273
13498
  const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
13274
13499
  const result = await executeQuery(resolvedParams, {
@@ -13293,7 +13518,12 @@ function createNarsilFromCore(core, config) {
13293
13518
  guardShutdown();
13294
13519
  const entry = requireIndex(indexName);
13295
13520
  const manager = requireManager(indexName);
13296
- const resolvedParams = await resolveVectorText(params, entry.embeddingAdapter, abortController.signal);
13521
+ const resolvedParams = await resolveVectorText(
13522
+ params,
13523
+ entry.embeddingAdapter,
13524
+ abortController.signal,
13525
+ entry.embeddingAdapterName
13526
+ );
13297
13527
  const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
13298
13528
  return executePreflight(resolvedParams, {
13299
13529
  manager,