@delali/narsil 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +777 -130
- package/dist/{chunk-GQEHCZG7.mjs → chunk-SVDEEMO6.mjs} +648 -450
- package/dist/{chunk-FS7XLIUS.mjs → chunk-VWF7RKSF.mjs} +33 -1
- package/dist/{config-cQLlYR3U.d.ts → config-DX--2Owg.d.ts} +3 -1
- package/dist/{direct-executor-FX56DYD3.mjs → direct-executor-KC5O5D2R.mjs} +1 -1
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +2 -2
- package/dist/embeddings/openai.mjs +5 -0
- package/dist/index.browser.mjs +450 -220
- package/dist/index.d.ts +3 -3
- package/dist/index.mjs +2 -2
- package/dist/{narsil-ChTdkwPu.d.ts → narsil-B3jKyXCr.d.ts} +6 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +2 -2
- package/dist/{schema-BVaw7nqz.d.ts → schema-C3VhNvfw.d.ts} +3 -1
- package/dist/server.d.ts +3 -3
- package/dist/server.mjs +11 -12
- package/dist/{types-Bt7c4hDF.d.ts → types-CTVVm4GG.d.ts} +1 -1
- package/dist/workers/entry.mjs +1 -1
- package/package.json +2 -1
package/dist/index.browser.mjs
CHANGED
|
@@ -1049,6 +1049,38 @@ function extractVectorFieldsFromSchema(schema) {
|
|
|
1049
1049
|
}
|
|
1050
1050
|
return result;
|
|
1051
1051
|
}
|
|
1052
|
+
|
|
1053
|
+
// src/schema/validator/vector-promotion.ts
|
|
1054
|
+
function fail(message, details) {
|
|
1055
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, message, details);
|
|
1056
|
+
}
|
|
1057
|
+
function isPositiveInteger(value) {
|
|
1058
|
+
return Number.isInteger(value) && value >= 1;
|
|
1059
|
+
}
|
|
1060
|
+
function validateVectorPromotion(config) {
|
|
1061
|
+
if (config === void 0) return;
|
|
1062
|
+
if (config.threshold !== void 0 && !isPositiveInteger(config.threshold)) {
|
|
1063
|
+
fail("vectorPromotion.threshold must be a positive integer", { threshold: config.threshold });
|
|
1064
|
+
}
|
|
1065
|
+
const quantization = config.quantization;
|
|
1066
|
+
if (quantization !== void 0 && quantization !== "sq8" && quantization !== "none") {
|
|
1067
|
+
fail("vectorPromotion.quantization must be 'sq8' or 'none'", { quantization });
|
|
1068
|
+
}
|
|
1069
|
+
const hnsw = config.hnswConfig;
|
|
1070
|
+
if (hnsw === void 0) return;
|
|
1071
|
+
if (hnsw.m !== void 0 && !isPositiveInteger(hnsw.m)) {
|
|
1072
|
+
fail("vectorPromotion.hnswConfig.m must be a positive integer", { m: hnsw.m });
|
|
1073
|
+
}
|
|
1074
|
+
if (hnsw.efConstruction !== void 0 && !isPositiveInteger(hnsw.efConstruction)) {
|
|
1075
|
+
fail("vectorPromotion.hnswConfig.efConstruction must be a positive integer", {
|
|
1076
|
+
efConstruction: hnsw.efConstruction
|
|
1077
|
+
});
|
|
1078
|
+
}
|
|
1079
|
+
const metric = hnsw.metric;
|
|
1080
|
+
if (metric !== void 0 && metric !== "cosine" && metric !== "dotProduct" && metric !== "euclidean") {
|
|
1081
|
+
fail("vectorPromotion.hnswConfig.metric must be 'cosine', 'dotProduct', or 'euclidean'", { metric });
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
1052
1084
|
function encodeRawPayloadV2(wire) {
|
|
1053
1085
|
return encode(wire);
|
|
1054
1086
|
}
|
|
@@ -3992,6 +4024,123 @@ function createPluginRegistry() {
|
|
|
3992
4024
|
};
|
|
3993
4025
|
}
|
|
3994
4026
|
|
|
4027
|
+
// src/schema/embedding-validator.ts
|
|
4028
|
+
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
4029
|
+
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
4030
|
+
const fieldEntries = Object.keys(embeddingConfig.fields);
|
|
4031
|
+
if (fieldEntries.length === 0) {
|
|
4032
|
+
throw new NarsilError(
|
|
4033
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4034
|
+
"Embedding config must define at least one field mapping"
|
|
4035
|
+
);
|
|
4036
|
+
}
|
|
4037
|
+
if (typeof embeddingConfig.adapter === "string") {
|
|
4038
|
+
throw new NarsilError(
|
|
4039
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4040
|
+
`Embedding adapter "${embeddingConfig.adapter}" must be resolved against the engine registry before validation`,
|
|
4041
|
+
{ adapter: embeddingConfig.adapter }
|
|
4042
|
+
);
|
|
4043
|
+
}
|
|
4044
|
+
const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
|
|
4045
|
+
if (!resolvedAdapter) {
|
|
4046
|
+
throw new NarsilError(
|
|
4047
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4048
|
+
"Embedding config requires an adapter (set on the index or the instance)"
|
|
4049
|
+
);
|
|
4050
|
+
}
|
|
4051
|
+
if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
|
|
4052
|
+
throw new NarsilError(
|
|
4053
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4054
|
+
`Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
|
|
4055
|
+
);
|
|
4056
|
+
}
|
|
4057
|
+
const flatFields = flattenSchema(schema);
|
|
4058
|
+
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
4059
|
+
const targetType = flatFields[targetField];
|
|
4060
|
+
if (targetType === void 0) {
|
|
4061
|
+
throw new NarsilError(
|
|
4062
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4063
|
+
`Embedding target field "${targetField}" does not exist in the schema`,
|
|
4064
|
+
{ field: targetField }
|
|
4065
|
+
);
|
|
4066
|
+
}
|
|
4067
|
+
const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
|
|
4068
|
+
if (!vectorMatch) {
|
|
4069
|
+
throw new NarsilError(
|
|
4070
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4071
|
+
`Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
|
|
4072
|
+
{ field: targetField, type: targetType }
|
|
4073
|
+
);
|
|
4074
|
+
}
|
|
4075
|
+
const schemaDimension = Number.parseInt(vectorMatch[1], 10);
|
|
4076
|
+
if (schemaDimension !== resolvedAdapter.dimensions) {
|
|
4077
|
+
throw new NarsilError(
|
|
4078
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
4079
|
+
`Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
|
|
4080
|
+
{ field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
|
|
4081
|
+
);
|
|
4082
|
+
}
|
|
4083
|
+
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
4084
|
+
if (sources.length === 0) {
|
|
4085
|
+
throw new NarsilError(
|
|
4086
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4087
|
+
`Embedding target field "${targetField}" must have at least one source field`,
|
|
4088
|
+
{ field: targetField }
|
|
4089
|
+
);
|
|
4090
|
+
}
|
|
4091
|
+
for (const source of sources) {
|
|
4092
|
+
const sourceType = flatFields[source];
|
|
4093
|
+
if (sourceType === void 0) {
|
|
4094
|
+
throw new NarsilError(
|
|
4095
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4096
|
+
`Embedding source field "${source}" does not exist in the schema`,
|
|
4097
|
+
{ field: source, targetField }
|
|
4098
|
+
);
|
|
4099
|
+
}
|
|
4100
|
+
if (sourceType !== "string") {
|
|
4101
|
+
throw new NarsilError(
|
|
4102
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4103
|
+
`Embedding source field "${source}" must be a string type, got "${sourceType}"`,
|
|
4104
|
+
{ field: source, type: sourceType, targetField }
|
|
4105
|
+
);
|
|
4106
|
+
}
|
|
4107
|
+
}
|
|
4108
|
+
}
|
|
4109
|
+
return resolvedAdapter;
|
|
4110
|
+
}
|
|
4111
|
+
function validateRegisteredAdapter(name, adapter) {
|
|
4112
|
+
if (typeof name !== "string" || name.trim().length === 0) {
|
|
4113
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Embedding adapter names must be non-empty strings");
|
|
4114
|
+
}
|
|
4115
|
+
if (typeof adapter?.embed !== "function") {
|
|
4116
|
+
throw new NarsilError(
|
|
4117
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4118
|
+
`Embedding adapter "${name}" must provide an embed function`,
|
|
4119
|
+
{ adapter: name }
|
|
4120
|
+
);
|
|
4121
|
+
}
|
|
4122
|
+
if (typeof adapter.dimensions !== "number" || !Number.isInteger(adapter.dimensions) || adapter.dimensions <= 0) {
|
|
4123
|
+
throw new NarsilError(
|
|
4124
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4125
|
+
`Embedding adapter "${name}" dimensions must be a positive integer, got ${adapter.dimensions}`,
|
|
4126
|
+
{ adapter: name, dimensions: adapter.dimensions }
|
|
4127
|
+
);
|
|
4128
|
+
}
|
|
4129
|
+
}
|
|
4130
|
+
function validateRequiredFieldsInSchema(required, schema) {
|
|
4131
|
+
if (required.length === 0) return;
|
|
4132
|
+
const flatFields = flattenSchema(schema);
|
|
4133
|
+
for (const field of required) {
|
|
4134
|
+
if (flatFields[field] === void 0) {
|
|
4135
|
+
throw new NarsilError(
|
|
4136
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
4137
|
+
`Required field "${field}" does not exist in the schema`,
|
|
4138
|
+
{ field }
|
|
4139
|
+
);
|
|
4140
|
+
}
|
|
4141
|
+
}
|
|
4142
|
+
}
|
|
4143
|
+
|
|
3995
4144
|
// src/search/facets.ts
|
|
3996
4145
|
function mergeFacets(partitionFacets) {
|
|
3997
4146
|
const merged = /* @__PURE__ */ new Map();
|
|
@@ -7580,6 +7729,9 @@ function metadataToWire(meta) {
|
|
|
7580
7729
|
if (meta.vectorFields) {
|
|
7581
7730
|
wire.vector_fields = meta.vectorFields;
|
|
7582
7731
|
}
|
|
7732
|
+
if (meta.embedding) {
|
|
7733
|
+
wire.embedding = meta.embedding.adapter !== void 0 ? { adapter: meta.embedding.adapter, fields: meta.embedding.fields } : { fields: meta.embedding.fields };
|
|
7734
|
+
}
|
|
7583
7735
|
return wire;
|
|
7584
7736
|
}
|
|
7585
7737
|
function wireToMetadata(raw) {
|
|
@@ -7595,6 +7747,9 @@ function wireToMetadata(raw) {
|
|
|
7595
7747
|
if (raw.vector_fields) {
|
|
7596
7748
|
meta.vectorFields = raw.vector_fields;
|
|
7597
7749
|
}
|
|
7750
|
+
if (raw.embedding && typeof raw.embedding === "object" && typeof raw.embedding.fields === "object") {
|
|
7751
|
+
meta.embedding = typeof raw.embedding.adapter === "string" ? { adapter: raw.embedding.adapter, fields: raw.embedding.fields } : { fields: raw.embedding.fields };
|
|
7752
|
+
}
|
|
7598
7753
|
return meta;
|
|
7599
7754
|
}
|
|
7600
7755
|
function serializeMetadata(meta) {
|
|
@@ -8947,11 +9102,15 @@ function unflattenSchema(flat) {
|
|
|
8947
9102
|
return root;
|
|
8948
9103
|
}
|
|
8949
9104
|
function reconstructSchemaFromMetadata(metadata) {
|
|
8950
|
-
|
|
9105
|
+
const config = {
|
|
8951
9106
|
schema: unflattenSchema(metadata.schema),
|
|
8952
9107
|
language: metadata.language,
|
|
8953
9108
|
bm25: { k1: metadata.bm25Params.k1, b: metadata.bm25Params.b }
|
|
8954
9109
|
};
|
|
9110
|
+
if (metadata.embedding) {
|
|
9111
|
+
config.embedding = metadata.embedding.adapter !== void 0 ? { adapter: metadata.embedding.adapter, fields: metadata.embedding.fields } : { fields: metadata.embedding.fields };
|
|
9112
|
+
}
|
|
9113
|
+
return config;
|
|
8955
9114
|
}
|
|
8956
9115
|
|
|
8957
9116
|
// src/persistence/durability/segment/build-segment.ts
|
|
@@ -10399,7 +10558,7 @@ function buildMetadata(indexName, hooks) {
|
|
|
10399
10558
|
if (config === void 0 || manager === void 0) {
|
|
10400
10559
|
return void 0;
|
|
10401
10560
|
}
|
|
10402
|
-
|
|
10561
|
+
const metadata = {
|
|
10403
10562
|
indexName,
|
|
10404
10563
|
schema: config.schema,
|
|
10405
10564
|
language: config.language,
|
|
@@ -10408,6 +10567,10 @@ function buildMetadata(indexName, hooks) {
|
|
|
10408
10567
|
createdAt: Date.now(),
|
|
10409
10568
|
engineVersion: VERSION
|
|
10410
10569
|
};
|
|
10570
|
+
if (config.embedding) {
|
|
10571
|
+
metadata.embedding = config.embedding;
|
|
10572
|
+
}
|
|
10573
|
+
return metadata;
|
|
10411
10574
|
}
|
|
10412
10575
|
function createDurabilityIntegration(tier, hooks) {
|
|
10413
10576
|
const managerHooks = {
|
|
@@ -10928,11 +11091,16 @@ function createDurabilityFromConfig(config, wiring) {
|
|
|
10928
11091
|
if (entry === void 0) {
|
|
10929
11092
|
return void 0;
|
|
10930
11093
|
}
|
|
11094
|
+
const embedding = entry.config.embedding ? {
|
|
11095
|
+
fields: entry.config.embedding.fields,
|
|
11096
|
+
...entry.embeddingAdapterName !== null ? { adapter: entry.embeddingAdapterName } : {}
|
|
11097
|
+
} : void 0;
|
|
10931
11098
|
return {
|
|
10932
11099
|
schema: flattenSchema(entry.config.schema),
|
|
10933
11100
|
language: entry.language.name,
|
|
10934
11101
|
k1: entry.config.bm25?.k1 ?? 1.2,
|
|
10935
|
-
b: entry.config.bm25?.b ?? 0.75
|
|
11102
|
+
b: entry.config.bm25?.b ?? 0.75,
|
|
11103
|
+
...embedding !== void 0 ? { embedding } : {}
|
|
10936
11104
|
};
|
|
10937
11105
|
},
|
|
10938
11106
|
createIndexFromMetadata: wiring.createIndexFromMetadata,
|
|
@@ -10951,6 +11119,13 @@ function createEngineCore(config) {
|
|
|
10951
11119
|
}
|
|
10952
11120
|
const idGenerator = config?.idGenerator ?? generateId;
|
|
10953
11121
|
const indexRegistry = /* @__PURE__ */ new Map();
|
|
11122
|
+
const embeddingAdapters = /* @__PURE__ */ new Map();
|
|
11123
|
+
if (config?.embeddingAdapters) {
|
|
11124
|
+
for (const [name, adapter] of Object.entries(config.embeddingAdapters)) {
|
|
11125
|
+
validateRegisteredAdapter(name, adapter);
|
|
11126
|
+
embeddingAdapters.set(name, adapter);
|
|
11127
|
+
}
|
|
11128
|
+
}
|
|
10954
11129
|
const eventHandlers = /* @__PURE__ */ new Map();
|
|
10955
11130
|
const shutdownState = { isShutdown: false };
|
|
10956
11131
|
const abortController = new AbortController();
|
|
@@ -11000,11 +11175,35 @@ function createEngineCore(config) {
|
|
|
11000
11175
|
}
|
|
11001
11176
|
const indexConfig = reconstructSchemaFromMetadata(metadata);
|
|
11002
11177
|
const language = getLanguage(indexConfig.language ?? "english");
|
|
11178
|
+
const adapterName = metadata.embedding?.adapter ?? null;
|
|
11179
|
+
let embeddingAdapter = null;
|
|
11180
|
+
if (metadata.embedding) {
|
|
11181
|
+
const candidate = adapterName !== null ? embeddingAdapters.get(adapterName) ?? null : config?.embedding ?? null;
|
|
11182
|
+
if (candidate) {
|
|
11183
|
+
try {
|
|
11184
|
+
validateEmbeddingConfig(
|
|
11185
|
+
{ fields: metadata.embedding.fields, adapter: candidate },
|
|
11186
|
+
indexConfig.schema,
|
|
11187
|
+
void 0
|
|
11188
|
+
);
|
|
11189
|
+
} catch (err) {
|
|
11190
|
+
if (err instanceof NarsilError) {
|
|
11191
|
+
throw new NarsilError(err.code, `Recovery of index "${metadata.indexName}" failed: ${err.message}`, {
|
|
11192
|
+
indexName: metadata.indexName,
|
|
11193
|
+
adapter: adapterName ?? void 0
|
|
11194
|
+
});
|
|
11195
|
+
}
|
|
11196
|
+
throw err;
|
|
11197
|
+
}
|
|
11198
|
+
embeddingAdapter = candidate;
|
|
11199
|
+
}
|
|
11200
|
+
}
|
|
11003
11201
|
executor.createIndex(metadata.indexName, indexConfig, language);
|
|
11004
11202
|
indexRegistry.set(metadata.indexName, {
|
|
11005
11203
|
config: indexConfig,
|
|
11006
11204
|
language,
|
|
11007
|
-
embeddingAdapter
|
|
11205
|
+
embeddingAdapter,
|
|
11206
|
+
embeddingAdapterName: adapterName,
|
|
11008
11207
|
vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
|
|
11009
11208
|
});
|
|
11010
11209
|
}
|
|
@@ -11047,6 +11246,7 @@ function createEngineCore(config) {
|
|
|
11047
11246
|
durability,
|
|
11048
11247
|
idGenerator,
|
|
11049
11248
|
indexRegistry,
|
|
11249
|
+
embeddingAdapters,
|
|
11050
11250
|
eventHandlers,
|
|
11051
11251
|
shutdownState,
|
|
11052
11252
|
abortController,
|
|
@@ -11064,6 +11264,160 @@ function createEngineCore(config) {
|
|
|
11064
11264
|
rebalanceCtx
|
|
11065
11265
|
};
|
|
11066
11266
|
}
|
|
11267
|
+
var MAX_INDEX_NAME_LENGTH = 255;
|
|
11268
|
+
|
|
11269
|
+
// src/engine/validation.ts
|
|
11270
|
+
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
11271
|
+
var MAX_DOC_ID_LENGTH = 512;
|
|
11272
|
+
var BATCH_CHUNK_SIZE = 1e3;
|
|
11273
|
+
var MAX_LIMIT = 1e4;
|
|
11274
|
+
var MAX_OFFSET = 1e5;
|
|
11275
|
+
var DEFAULT_LIMIT = 10;
|
|
11276
|
+
var DEFAULT_OFFSET = 0;
|
|
11277
|
+
function now() {
|
|
11278
|
+
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
11279
|
+
return performance.now();
|
|
11280
|
+
}
|
|
11281
|
+
return Date.now();
|
|
11282
|
+
}
|
|
11283
|
+
function validateIndexName(name) {
|
|
11284
|
+
if (!name || name.length === 0) {
|
|
11285
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
|
|
11286
|
+
}
|
|
11287
|
+
if (name.length > MAX_INDEX_NAME_LENGTH) {
|
|
11288
|
+
throw new NarsilError(
|
|
11289
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
11290
|
+
`Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
|
|
11291
|
+
{
|
|
11292
|
+
indexName: name,
|
|
11293
|
+
length: name.length
|
|
11294
|
+
}
|
|
11295
|
+
);
|
|
11296
|
+
}
|
|
11297
|
+
if (!INDEX_NAME_PATTERN.test(name)) {
|
|
11298
|
+
throw new NarsilError(
|
|
11299
|
+
ErrorCodes.INDEX_NOT_FOUND,
|
|
11300
|
+
`Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
|
|
11301
|
+
{ indexName: name }
|
|
11302
|
+
);
|
|
11303
|
+
}
|
|
11304
|
+
if (name.includes("..")) {
|
|
11305
|
+
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
|
|
11306
|
+
}
|
|
11307
|
+
}
|
|
11308
|
+
function validateDocId(docId) {
|
|
11309
|
+
if (!docId || docId.length === 0) {
|
|
11310
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
|
|
11311
|
+
}
|
|
11312
|
+
if (docId.length > MAX_DOC_ID_LENGTH) {
|
|
11313
|
+
throw new NarsilError(
|
|
11314
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
11315
|
+
`Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
|
|
11316
|
+
{
|
|
11317
|
+
docId,
|
|
11318
|
+
length: docId.length
|
|
11319
|
+
}
|
|
11320
|
+
);
|
|
11321
|
+
}
|
|
11322
|
+
if (docId.includes("\0")) {
|
|
11323
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
|
|
11324
|
+
}
|
|
11325
|
+
}
|
|
11326
|
+
function clampLimit(limit) {
|
|
11327
|
+
if (limit === void 0) return DEFAULT_LIMIT;
|
|
11328
|
+
return Math.max(0, Math.min(limit, MAX_LIMIT));
|
|
11329
|
+
}
|
|
11330
|
+
function clampOffset(offset) {
|
|
11331
|
+
if (offset === void 0) return DEFAULT_OFFSET;
|
|
11332
|
+
return Math.max(0, Math.min(offset, MAX_OFFSET));
|
|
11333
|
+
}
|
|
11334
|
+
|
|
11335
|
+
// src/engine/index-lifecycle.ts
|
|
11336
|
+
async function createEngineIndex(core, config, name, indexConfig) {
|
|
11337
|
+
core.guardShutdown();
|
|
11338
|
+
validateIndexName(name);
|
|
11339
|
+
if (core.indexRegistry.has(name)) {
|
|
11340
|
+
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
|
|
11341
|
+
}
|
|
11342
|
+
validateSchema(indexConfig.schema);
|
|
11343
|
+
validateVectorPromotion(indexConfig.vectorPromotion);
|
|
11344
|
+
let resolvedEmbeddingAdapter = null;
|
|
11345
|
+
let embeddingAdapterName = null;
|
|
11346
|
+
if (indexConfig.embedding) {
|
|
11347
|
+
let configuredAdapter = indexConfig.embedding.adapter;
|
|
11348
|
+
if (typeof configuredAdapter === "string") {
|
|
11349
|
+
embeddingAdapterName = configuredAdapter;
|
|
11350
|
+
const registered = core.embeddingAdapters.get(configuredAdapter);
|
|
11351
|
+
if (!registered) {
|
|
11352
|
+
throw new NarsilError(
|
|
11353
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
11354
|
+
`Embedding adapter "${configuredAdapter}" is not registered on this engine`,
|
|
11355
|
+
{ adapter: configuredAdapter, available: [...core.embeddingAdapters.keys()] }
|
|
11356
|
+
);
|
|
11357
|
+
}
|
|
11358
|
+
configuredAdapter = registered;
|
|
11359
|
+
}
|
|
11360
|
+
resolvedEmbeddingAdapter = validateEmbeddingConfig(
|
|
11361
|
+
{ fields: indexConfig.embedding.fields, adapter: configuredAdapter },
|
|
11362
|
+
indexConfig.schema,
|
|
11363
|
+
config?.embedding
|
|
11364
|
+
);
|
|
11365
|
+
}
|
|
11366
|
+
if (indexConfig.required && indexConfig.required.length > 0) {
|
|
11367
|
+
validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
|
|
11368
|
+
}
|
|
11369
|
+
const language = getLanguage(indexConfig.language ?? "english");
|
|
11370
|
+
core.executor.createIndex(name, indexConfig, language);
|
|
11371
|
+
core.indexRegistry.set(name, {
|
|
11372
|
+
config: indexConfig,
|
|
11373
|
+
language,
|
|
11374
|
+
embeddingAdapter: resolvedEmbeddingAdapter,
|
|
11375
|
+
embeddingAdapterName,
|
|
11376
|
+
vectorFieldPaths: getVectorFieldPaths(indexConfig.schema)
|
|
11377
|
+
});
|
|
11378
|
+
if (core.durability) {
|
|
11379
|
+
await core.durability.manager.persistMetadata(name);
|
|
11380
|
+
}
|
|
11381
|
+
await core.pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
|
|
11382
|
+
}
|
|
11383
|
+
async function dropEngineIndex(core, name) {
|
|
11384
|
+
core.guardShutdown();
|
|
11385
|
+
const entry = core.requireIndex(name);
|
|
11386
|
+
core.executor.dropIndex(name);
|
|
11387
|
+
core.indexRegistry.delete(name);
|
|
11388
|
+
if (core.durability) {
|
|
11389
|
+
await core.durability.manager.removeIndex(name);
|
|
11390
|
+
}
|
|
11391
|
+
await core.pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
|
|
11392
|
+
}
|
|
11393
|
+
function registerEngineEmbeddingAdapter(core, name, adapter) {
|
|
11394
|
+
core.guardShutdown();
|
|
11395
|
+
validateRegisteredAdapter(name, adapter);
|
|
11396
|
+
const affected = [];
|
|
11397
|
+
for (const [indexName, entry] of core.indexRegistry) {
|
|
11398
|
+
if (entry.embeddingAdapterName !== name || !entry.config.embedding) continue;
|
|
11399
|
+
try {
|
|
11400
|
+
validateEmbeddingConfig({ fields: entry.config.embedding.fields, adapter }, entry.config.schema, void 0);
|
|
11401
|
+
} catch (err) {
|
|
11402
|
+
if (err instanceof NarsilError) {
|
|
11403
|
+
throw new NarsilError(
|
|
11404
|
+
err.code,
|
|
11405
|
+
`Cannot bind embedding adapter "${name}" to index "${indexName}": ${err.message}`,
|
|
11406
|
+
{
|
|
11407
|
+
indexName,
|
|
11408
|
+
adapter: name
|
|
11409
|
+
}
|
|
11410
|
+
);
|
|
11411
|
+
}
|
|
11412
|
+
throw err;
|
|
11413
|
+
}
|
|
11414
|
+
affected.push(entry);
|
|
11415
|
+
}
|
|
11416
|
+
core.embeddingAdapters.set(name, adapter);
|
|
11417
|
+
for (const entry of affected) {
|
|
11418
|
+
entry.embeddingAdapter = adapter;
|
|
11419
|
+
}
|
|
11420
|
+
}
|
|
11067
11421
|
|
|
11068
11422
|
// src/engine/lifecycle.ts
|
|
11069
11423
|
async function shutdownEngine(core) {
|
|
@@ -11098,6 +11452,18 @@ async function shutdownEngine(core) {
|
|
|
11098
11452
|
}
|
|
11099
11453
|
|
|
11100
11454
|
// src/engine/embed.ts
|
|
11455
|
+
function assertDocumentCarriesMappedVectors(document, embeddingConfig, adapterName) {
|
|
11456
|
+
for (const targetField of Object.keys(embeddingConfig.fields)) {
|
|
11457
|
+
const existing = resolveFieldValue(document, targetField);
|
|
11458
|
+
if (existing === void 0 || existing === null) {
|
|
11459
|
+
throw new NarsilError(
|
|
11460
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
11461
|
+
adapterName ? `The index embeds "${targetField}" via adapter "${adapterName}", which is not registered on this engine. Register the adapter or supply the vector on the document.` : `The index embeds "${targetField}" automatically but no embedding adapter is bound. Bind an adapter or supply the vector on the document.`,
|
|
11462
|
+
adapterName ? { field: targetField, adapter: adapterName } : { field: targetField }
|
|
11463
|
+
);
|
|
11464
|
+
}
|
|
11465
|
+
}
|
|
11466
|
+
}
|
|
11101
11467
|
function validateVector2(vector, targetField, expectedDimensions) {
|
|
11102
11468
|
if (!(vector instanceof Float32Array)) {
|
|
11103
11469
|
throw new NarsilError(
|
|
@@ -11299,73 +11665,6 @@ async function embedBatchDocumentFields(documents, embeddingConfig, adapter, sig
|
|
|
11299
11665
|
}
|
|
11300
11666
|
return { embedded, failed };
|
|
11301
11667
|
}
|
|
11302
|
-
var MAX_INDEX_NAME_LENGTH = 255;
|
|
11303
|
-
|
|
11304
|
-
// src/engine/validation.ts
|
|
11305
|
-
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
11306
|
-
var MAX_DOC_ID_LENGTH = 512;
|
|
11307
|
-
var BATCH_CHUNK_SIZE = 1e3;
|
|
11308
|
-
var MAX_LIMIT = 1e4;
|
|
11309
|
-
var MAX_OFFSET = 1e5;
|
|
11310
|
-
var DEFAULT_LIMIT = 10;
|
|
11311
|
-
var DEFAULT_OFFSET = 0;
|
|
11312
|
-
function now() {
|
|
11313
|
-
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
11314
|
-
return performance.now();
|
|
11315
|
-
}
|
|
11316
|
-
return Date.now();
|
|
11317
|
-
}
|
|
11318
|
-
function validateIndexName(name) {
|
|
11319
|
-
if (!name || name.length === 0) {
|
|
11320
|
-
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, "Index name must not be empty", { indexName: name });
|
|
11321
|
-
}
|
|
11322
|
-
if (name.length > MAX_INDEX_NAME_LENGTH) {
|
|
11323
|
-
throw new NarsilError(
|
|
11324
|
-
ErrorCodes.INDEX_NOT_FOUND,
|
|
11325
|
-
`Index name must not exceed ${MAX_INDEX_NAME_LENGTH} characters`,
|
|
11326
|
-
{
|
|
11327
|
-
indexName: name,
|
|
11328
|
-
length: name.length
|
|
11329
|
-
}
|
|
11330
|
-
);
|
|
11331
|
-
}
|
|
11332
|
-
if (!INDEX_NAME_PATTERN.test(name)) {
|
|
11333
|
-
throw new NarsilError(
|
|
11334
|
-
ErrorCodes.INDEX_NOT_FOUND,
|
|
11335
|
-
`Index name "${name}" contains invalid characters; use alphanumeric, dots, hyphens, and underscores only`,
|
|
11336
|
-
{ indexName: name }
|
|
11337
|
-
);
|
|
11338
|
-
}
|
|
11339
|
-
if (name.includes("..")) {
|
|
11340
|
-
throw new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index name "${name}" must not contain ".."`, { indexName: name });
|
|
11341
|
-
}
|
|
11342
|
-
}
|
|
11343
|
-
function validateDocId(docId) {
|
|
11344
|
-
if (!docId || docId.length === 0) {
|
|
11345
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not be empty", { docId });
|
|
11346
|
-
}
|
|
11347
|
-
if (docId.length > MAX_DOC_ID_LENGTH) {
|
|
11348
|
-
throw new NarsilError(
|
|
11349
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
11350
|
-
`Document ID must not exceed ${MAX_DOC_ID_LENGTH} characters`,
|
|
11351
|
-
{
|
|
11352
|
-
docId,
|
|
11353
|
-
length: docId.length
|
|
11354
|
-
}
|
|
11355
|
-
);
|
|
11356
|
-
}
|
|
11357
|
-
if (docId.includes("\0")) {
|
|
11358
|
-
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Document ID must not contain null bytes", { docId });
|
|
11359
|
-
}
|
|
11360
|
-
}
|
|
11361
|
-
function clampLimit(limit) {
|
|
11362
|
-
if (limit === void 0) return DEFAULT_LIMIT;
|
|
11363
|
-
return Math.max(0, Math.min(limit, MAX_LIMIT));
|
|
11364
|
-
}
|
|
11365
|
-
function clampOffset(offset) {
|
|
11366
|
-
if (offset === void 0) return DEFAULT_OFFSET;
|
|
11367
|
-
return Math.max(0, Math.min(offset, MAX_OFFSET));
|
|
11368
|
-
}
|
|
11369
11668
|
|
|
11370
11669
|
// src/engine/mutations/durable-rollback.ts
|
|
11371
11670
|
function rollbackFailed(docId, originalError, rollbackError) {
|
|
@@ -11428,13 +11727,21 @@ async function insertDocument(ctx, indexName, document, docId, options) {
|
|
|
11428
11727
|
if (entry.config.required && entry.config.required.length > 0) {
|
|
11429
11728
|
validateRequiredFields(document, entry.config.required);
|
|
11430
11729
|
}
|
|
11431
|
-
if (entry.
|
|
11432
|
-
|
|
11433
|
-
|
|
11434
|
-
|
|
11435
|
-
|
|
11436
|
-
|
|
11437
|
-
|
|
11730
|
+
if (entry.config.embedding) {
|
|
11731
|
+
if (entry.embeddingAdapter) {
|
|
11732
|
+
await embedDocumentFields(
|
|
11733
|
+
document,
|
|
11734
|
+
entry.config.embedding,
|
|
11735
|
+
entry.embeddingAdapter,
|
|
11736
|
+
ctx.abortController.signal
|
|
11737
|
+
);
|
|
11738
|
+
} else {
|
|
11739
|
+
assertDocumentCarriesMappedVectors(
|
|
11740
|
+
document,
|
|
11741
|
+
entry.config.embedding,
|
|
11742
|
+
entry.embeddingAdapterName
|
|
11743
|
+
);
|
|
11744
|
+
}
|
|
11438
11745
|
}
|
|
11439
11746
|
const insertManager = ctx.requireManager(indexName);
|
|
11440
11747
|
const insertVecIndexes = insertManager.getVectorIndexes();
|
|
@@ -11514,6 +11821,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
11514
11821
|
const hasAfterHook = ctx.pluginRegistry.hasHooks("afterInsert");
|
|
11515
11822
|
const hasRequired = entry.config.required && entry.config.required.length > 0;
|
|
11516
11823
|
const hasEmbedding = entry.embeddingAdapter && entry.config.embedding;
|
|
11824
|
+
const embeddingUnbound = !entry.embeddingAdapter && entry.config.embedding;
|
|
11517
11825
|
const batchManager = ctx.requireManager(indexName);
|
|
11518
11826
|
const batchVecIndexes = batchManager.getVectorIndexes();
|
|
11519
11827
|
const batchVectorFieldPaths = batchVecIndexes.size > 0 ? entry.vectorFieldPaths : /* @__PURE__ */ new Set();
|
|
@@ -11529,7 +11837,7 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
11529
11837
|
} catch (err) {
|
|
11530
11838
|
chunkFailedIndexes.add(i);
|
|
11531
11839
|
failed.push({
|
|
11532
|
-
docId: "",
|
|
11840
|
+
docId: providedDocId(documents[i]) ?? "",
|
|
11533
11841
|
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, String(err))
|
|
11534
11842
|
});
|
|
11535
11843
|
}
|
|
@@ -11554,16 +11862,33 @@ async function insertDocumentBatch(ctx, indexName, documents, options) {
|
|
|
11554
11862
|
for (const [sliceIndex, error] of embedResult.failed) {
|
|
11555
11863
|
const originalIdx = embeddableOriginalIndexes[sliceIndex];
|
|
11556
11864
|
chunkFailedIndexes.add(originalIdx);
|
|
11557
|
-
failed.push({ docId: "", error });
|
|
11865
|
+
failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error });
|
|
11558
11866
|
}
|
|
11559
11867
|
} catch (err) {
|
|
11560
11868
|
const embeddingError = err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err));
|
|
11561
11869
|
for (const originalIdx of embeddableOriginalIndexes) {
|
|
11562
11870
|
chunkFailedIndexes.add(originalIdx);
|
|
11563
|
-
failed.push({ docId: "", error: embeddingError });
|
|
11871
|
+
failed.push({ docId: providedDocId(documents[originalIdx]) ?? "", error: embeddingError });
|
|
11564
11872
|
}
|
|
11565
11873
|
}
|
|
11566
11874
|
}
|
|
11875
|
+
} else if (embeddingUnbound) {
|
|
11876
|
+
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
11877
|
+
if (chunkFailedIndexes.has(i)) continue;
|
|
11878
|
+
try {
|
|
11879
|
+
assertDocumentCarriesMappedVectors(
|
|
11880
|
+
documents[i],
|
|
11881
|
+
entry.config.embedding,
|
|
11882
|
+
entry.embeddingAdapterName
|
|
11883
|
+
);
|
|
11884
|
+
} catch (err) {
|
|
11885
|
+
chunkFailedIndexes.add(i);
|
|
11886
|
+
failed.push({
|
|
11887
|
+
docId: providedDocId(documents[i]) ?? "",
|
|
11888
|
+
error: err instanceof NarsilError ? err : new NarsilError(ErrorCodes.EMBEDDING_FAILED, String(err))
|
|
11889
|
+
});
|
|
11890
|
+
}
|
|
11891
|
+
}
|
|
11567
11892
|
}
|
|
11568
11893
|
for (let i = chunkStart; i < chunkEnd; i++) {
|
|
11569
11894
|
if (ctx.abortController.signal.aborted) break;
|
|
@@ -11757,13 +12082,21 @@ async function updateDocument(ctx, indexName, docId, document) {
|
|
|
11757
12082
|
if (ctx.bufferIfRebalancing(indexName, { action: "update", docId, document, indexName })) {
|
|
11758
12083
|
return;
|
|
11759
12084
|
}
|
|
11760
|
-
if (entry.
|
|
11761
|
-
|
|
11762
|
-
|
|
11763
|
-
|
|
11764
|
-
|
|
11765
|
-
|
|
11766
|
-
|
|
12085
|
+
if (entry.config.embedding) {
|
|
12086
|
+
if (entry.embeddingAdapter) {
|
|
12087
|
+
await embedDocumentFields(
|
|
12088
|
+
document,
|
|
12089
|
+
entry.config.embedding,
|
|
12090
|
+
entry.embeddingAdapter,
|
|
12091
|
+
ctx.abortController.signal
|
|
12092
|
+
);
|
|
12093
|
+
} else {
|
|
12094
|
+
assertDocumentCarriesMappedVectors(
|
|
12095
|
+
document,
|
|
12096
|
+
entry.config.embedding,
|
|
12097
|
+
entry.embeddingAdapterName
|
|
12098
|
+
);
|
|
12099
|
+
}
|
|
11767
12100
|
}
|
|
11768
12101
|
const updateManager = ctx.requireManager(indexName);
|
|
11769
12102
|
const oldDocument = updateManager.get(docId);
|
|
@@ -12813,7 +13146,7 @@ async function executeRebalance(manager, indexName, targetPartitionCount, ctx) {
|
|
|
12813
13146
|
}
|
|
12814
13147
|
|
|
12815
13148
|
// src/engine/resolve-vector-text.ts
|
|
12816
|
-
async function resolveVectorText(params, embeddingAdapter, signal) {
|
|
13149
|
+
async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
|
|
12817
13150
|
if (!params.vector) return params;
|
|
12818
13151
|
if (params.vector.text === void 0 && params.vector.value === void 0) return params;
|
|
12819
13152
|
if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
|
|
@@ -12823,7 +13156,8 @@ async function resolveVectorText(params, embeddingAdapter, signal) {
|
|
|
12823
13156
|
if (!embeddingAdapter) {
|
|
12824
13157
|
throw new NarsilError(
|
|
12825
13158
|
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
12826
|
-
"Vector query with 'text' requires an embedding adapter on the index or instance"
|
|
13159
|
+
embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
|
|
13160
|
+
embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
|
|
12827
13161
|
);
|
|
12828
13162
|
}
|
|
12829
13163
|
if (typeof params.vector.text !== "string") {
|
|
@@ -12905,7 +13239,13 @@ async function restoreFromSnapshot(indexName, data, executor, indexRegistry, get
|
|
|
12905
13239
|
const indexConfig = { schema, language: envelope.language };
|
|
12906
13240
|
executor.createIndex(indexName, indexConfig, language);
|
|
12907
13241
|
const vectorFieldPaths = getVectorFieldPaths2(schema);
|
|
12908
|
-
indexRegistry.set(indexName, {
|
|
13242
|
+
indexRegistry.set(indexName, {
|
|
13243
|
+
config: indexConfig,
|
|
13244
|
+
language,
|
|
13245
|
+
embeddingAdapter: null,
|
|
13246
|
+
embeddingAdapterName: null,
|
|
13247
|
+
vectorFieldPaths
|
|
13248
|
+
});
|
|
12909
13249
|
try {
|
|
12910
13250
|
const manager = requireManager(indexName);
|
|
12911
13251
|
while (manager.partitionCount < envelope.partitions.length) {
|
|
@@ -13036,97 +13376,6 @@ function readProcessMemory() {
|
|
|
13036
13376
|
}
|
|
13037
13377
|
}
|
|
13038
13378
|
|
|
13039
|
-
// src/schema/embedding-validator.ts
|
|
13040
|
-
var VECTOR_DIMENSION_PATTERN = /^vector\[(\d+)]$/;
|
|
13041
|
-
function validateEmbeddingConfig(embeddingConfig, schema, instanceAdapter) {
|
|
13042
|
-
const fieldEntries = Object.keys(embeddingConfig.fields);
|
|
13043
|
-
if (fieldEntries.length === 0) {
|
|
13044
|
-
throw new NarsilError(
|
|
13045
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
13046
|
-
"Embedding config must define at least one field mapping"
|
|
13047
|
-
);
|
|
13048
|
-
}
|
|
13049
|
-
const resolvedAdapter = embeddingConfig.adapter ?? instanceAdapter;
|
|
13050
|
-
if (!resolvedAdapter) {
|
|
13051
|
-
throw new NarsilError(
|
|
13052
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
13053
|
-
"Embedding config requires an adapter (set on the index or the instance)"
|
|
13054
|
-
);
|
|
13055
|
-
}
|
|
13056
|
-
if (typeof resolvedAdapter.dimensions !== "number" || !Number.isInteger(resolvedAdapter.dimensions) || resolvedAdapter.dimensions <= 0) {
|
|
13057
|
-
throw new NarsilError(
|
|
13058
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
13059
|
-
`Adapter dimensions must be a positive integer, got ${resolvedAdapter.dimensions}`
|
|
13060
|
-
);
|
|
13061
|
-
}
|
|
13062
|
-
const flatFields = flattenSchema(schema);
|
|
13063
|
-
for (const [targetField, sourceFields] of Object.entries(embeddingConfig.fields)) {
|
|
13064
|
-
const targetType = flatFields[targetField];
|
|
13065
|
-
if (targetType === void 0) {
|
|
13066
|
-
throw new NarsilError(
|
|
13067
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
13068
|
-
`Embedding target field "${targetField}" does not exist in the schema`,
|
|
13069
|
-
{ field: targetField }
|
|
13070
|
-
);
|
|
13071
|
-
}
|
|
13072
|
-
const vectorMatch = VECTOR_DIMENSION_PATTERN.exec(targetType);
|
|
13073
|
-
if (!vectorMatch) {
|
|
13074
|
-
throw new NarsilError(
|
|
13075
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
13076
|
-
`Embedding target field "${targetField}" must be a vector type, got "${targetType}"`,
|
|
13077
|
-
{ field: targetField, type: targetType }
|
|
13078
|
-
);
|
|
13079
|
-
}
|
|
13080
|
-
const schemaDimension = Number.parseInt(vectorMatch[1], 10);
|
|
13081
|
-
if (schemaDimension !== resolvedAdapter.dimensions) {
|
|
13082
|
-
throw new NarsilError(
|
|
13083
|
-
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
13084
|
-
`Embedding target field "${targetField}" has dimension ${schemaDimension} but the adapter produces ${resolvedAdapter.dimensions}-dimensional vectors`,
|
|
13085
|
-
{ field: targetField, expected: schemaDimension, actual: resolvedAdapter.dimensions }
|
|
13086
|
-
);
|
|
13087
|
-
}
|
|
13088
|
-
const sources = Array.isArray(sourceFields) ? sourceFields : [sourceFields];
|
|
13089
|
-
if (sources.length === 0) {
|
|
13090
|
-
throw new NarsilError(
|
|
13091
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
13092
|
-
`Embedding target field "${targetField}" must have at least one source field`,
|
|
13093
|
-
{ field: targetField }
|
|
13094
|
-
);
|
|
13095
|
-
}
|
|
13096
|
-
for (const source of sources) {
|
|
13097
|
-
const sourceType = flatFields[source];
|
|
13098
|
-
if (sourceType === void 0) {
|
|
13099
|
-
throw new NarsilError(
|
|
13100
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
13101
|
-
`Embedding source field "${source}" does not exist in the schema`,
|
|
13102
|
-
{ field: source, targetField }
|
|
13103
|
-
);
|
|
13104
|
-
}
|
|
13105
|
-
if (sourceType !== "string") {
|
|
13106
|
-
throw new NarsilError(
|
|
13107
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
13108
|
-
`Embedding source field "${source}" must be a string type, got "${sourceType}"`,
|
|
13109
|
-
{ field: source, type: sourceType, targetField }
|
|
13110
|
-
);
|
|
13111
|
-
}
|
|
13112
|
-
}
|
|
13113
|
-
}
|
|
13114
|
-
return resolvedAdapter;
|
|
13115
|
-
}
|
|
13116
|
-
function validateRequiredFieldsInSchema(required, schema) {
|
|
13117
|
-
if (required.length === 0) return;
|
|
13118
|
-
const flatFields = flattenSchema(schema);
|
|
13119
|
-
for (const field of required) {
|
|
13120
|
-
if (flatFields[field] === void 0) {
|
|
13121
|
-
throw new NarsilError(
|
|
13122
|
-
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
13123
|
-
`Required field "${field}" does not exist in the schema`,
|
|
13124
|
-
{ field }
|
|
13125
|
-
);
|
|
13126
|
-
}
|
|
13127
|
-
}
|
|
13128
|
-
}
|
|
13129
|
-
|
|
13130
13379
|
// src/narsil.ts
|
|
13131
13380
|
async function createNarsil(config) {
|
|
13132
13381
|
const core = createEngineCore(config);
|
|
@@ -13153,43 +13402,14 @@ function createNarsilFromCore(core, config) {
|
|
|
13153
13402
|
rebalanceCtx
|
|
13154
13403
|
} = core;
|
|
13155
13404
|
const narsil = {
|
|
13156
|
-
|
|
13157
|
-
|
|
13158
|
-
validateIndexName(name);
|
|
13159
|
-
if (indexRegistry.has(name)) {
|
|
13160
|
-
throw new NarsilError(ErrorCodes.INDEX_ALREADY_EXISTS, `Index "${name}" already exists`, { indexName: name });
|
|
13161
|
-
}
|
|
13162
|
-
validateSchema(indexConfig.schema);
|
|
13163
|
-
let resolvedEmbeddingAdapter = null;
|
|
13164
|
-
if (indexConfig.embedding) {
|
|
13165
|
-
resolvedEmbeddingAdapter = validateEmbeddingConfig(indexConfig.embedding, indexConfig.schema, config?.embedding);
|
|
13166
|
-
}
|
|
13167
|
-
if (indexConfig.required && indexConfig.required.length > 0) {
|
|
13168
|
-
validateRequiredFieldsInSchema(indexConfig.required, indexConfig.schema);
|
|
13169
|
-
}
|
|
13170
|
-
const language = getLanguage(indexConfig.language ?? "english");
|
|
13171
|
-
executor.createIndex(name, indexConfig, language);
|
|
13172
|
-
const vectorFieldPaths = getVectorFieldPaths(indexConfig.schema);
|
|
13173
|
-
indexRegistry.set(name, {
|
|
13174
|
-
config: indexConfig,
|
|
13175
|
-
language,
|
|
13176
|
-
embeddingAdapter: resolvedEmbeddingAdapter,
|
|
13177
|
-
vectorFieldPaths
|
|
13178
|
-
});
|
|
13179
|
-
if (durability) {
|
|
13180
|
-
await durability.manager.persistMetadata(name);
|
|
13181
|
-
}
|
|
13182
|
-
await pluginRegistry.runHook("onIndexCreate", { indexName: name, config: indexConfig });
|
|
13405
|
+
createIndex(name, indexConfig) {
|
|
13406
|
+
return createEngineIndex(core, config, name, indexConfig);
|
|
13183
13407
|
},
|
|
13184
|
-
|
|
13185
|
-
|
|
13186
|
-
|
|
13187
|
-
|
|
13188
|
-
|
|
13189
|
-
if (durability) {
|
|
13190
|
-
await durability.manager.removeIndex(name);
|
|
13191
|
-
}
|
|
13192
|
-
await pluginRegistry.runHook("onIndexDrop", { indexName: name, config: entry.config });
|
|
13408
|
+
registerEmbeddingAdapter(name, adapter) {
|
|
13409
|
+
registerEngineEmbeddingAdapter(core, name, adapter);
|
|
13410
|
+
},
|
|
13411
|
+
dropIndex(name) {
|
|
13412
|
+
return dropEngineIndex(core, name);
|
|
13193
13413
|
},
|
|
13194
13414
|
listIndexes() {
|
|
13195
13415
|
const infos = [];
|
|
@@ -13268,7 +13488,12 @@ function createNarsilFromCore(core, config) {
|
|
|
13268
13488
|
guardShutdown();
|
|
13269
13489
|
const entry = requireIndex(indexName);
|
|
13270
13490
|
const manager = requireManager(indexName);
|
|
13271
|
-
const resolvedParams = await resolveVectorText(
|
|
13491
|
+
const resolvedParams = await resolveVectorText(
|
|
13492
|
+
params,
|
|
13493
|
+
entry.embeddingAdapter,
|
|
13494
|
+
abortController.signal,
|
|
13495
|
+
entry.embeddingAdapterName
|
|
13496
|
+
);
|
|
13272
13497
|
await pluginRegistry.runHook("beforeSearch", { indexName, params: resolvedParams });
|
|
13273
13498
|
const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
|
|
13274
13499
|
const result = await executeQuery(resolvedParams, {
|
|
@@ -13293,7 +13518,12 @@ function createNarsilFromCore(core, config) {
|
|
|
13293
13518
|
guardShutdown();
|
|
13294
13519
|
const entry = requireIndex(indexName);
|
|
13295
13520
|
const manager = requireManager(indexName);
|
|
13296
|
-
const resolvedParams = await resolveVectorText(
|
|
13521
|
+
const resolvedParams = await resolveVectorText(
|
|
13522
|
+
params,
|
|
13523
|
+
entry.embeddingAdapter,
|
|
13524
|
+
abortController.signal,
|
|
13525
|
+
entry.embeddingAdapterName
|
|
13526
|
+
);
|
|
13297
13527
|
const workerSearch = orchestrator.isPromoted() ? orchestrator.searchViaWorker.bind(orchestrator) : void 0;
|
|
13298
13528
|
return executePreflight(resolvedParams, {
|
|
13299
13529
|
manager,
|