@h-ai/vecdb 1.0.0-beta.17 → 1.0.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +12 -3
- package/README.ja.md +12 -3
- package/README.md +12 -3
- package/dist/index.d.ts +3 -2
- package/dist/index.js +482 -146
- package/dist/index.js.map +1 -1
- package/package.json +9 -10
package/dist/index.js
CHANGED
|
@@ -181,13 +181,17 @@ var ChromaConfigSchema = z.object({
|
|
|
181
181
|
} catch {
|
|
182
182
|
return false;
|
|
183
183
|
}
|
|
184
|
-
}, {
|
|
184
|
+
}, {
|
|
185
|
+
error: () => vecdbM("vecdb_chromaEndpointInvalid")
|
|
186
|
+
}).optional(),
|
|
185
187
|
/** 主机(默认 localhost) */
|
|
186
188
|
host: z.string().default("localhost"),
|
|
187
189
|
/** 端口(默认 8000) */
|
|
188
190
|
port: z.number().int().min(1).max(65535).default(8e3),
|
|
189
191
|
/** 可选 Bearer token,发送 Authorization 请求头;用于采用此协议的自托管服务/网关,不是 Chroma Cloud 配置。空串表示未配置。 */
|
|
190
|
-
apiKey: z.string().regex(/^[\x21-\x7E]*$/, {
|
|
192
|
+
apiKey: z.string().regex(/^[\x21-\x7E]*$/, {
|
|
193
|
+
error: () => vecdbM("vecdb_chromaTokenInvalid")
|
|
194
|
+
}).optional(),
|
|
191
195
|
/** 拉起本地服务的可执行命令(默认 'chroma',可指定绝对路径) */
|
|
192
196
|
serverCommand: z.string().default("chroma"),
|
|
193
197
|
/** 拉起本地服务的就绪等待超时(毫秒,默认 30000) */
|
|
@@ -219,6 +223,13 @@ var VecdbErrorInfo = {
|
|
|
219
223
|
SERIALIZATION_FAILED: "014:500"
|
|
220
224
|
};
|
|
221
225
|
var HaiVecdbError = core.error.buildHaiErrorsDef("vecdb", VecdbErrorInfo);
|
|
226
|
+
function vectorScoreFromDistance(distance, metric) {
|
|
227
|
+
if (metric === "cosine")
|
|
228
|
+
return Math.max(0, Math.min(1, 1 - distance / 2));
|
|
229
|
+
if (metric === "dot")
|
|
230
|
+
return 1 / (1 + Math.exp(distance));
|
|
231
|
+
return 1 / (1 + Math.max(0, distance));
|
|
232
|
+
}
|
|
222
233
|
function stringifyOperationPayload(payload, maxLength) {
|
|
223
234
|
if (maxLength <= 0) {
|
|
224
235
|
return "";
|
|
@@ -253,13 +264,21 @@ function logVecdbOperation(ctx, category, operation, payload) {
|
|
|
253
264
|
function errorMsgFromCode(def, errorStr) {
|
|
254
265
|
switch (def) {
|
|
255
266
|
case HaiVecdbError.DELETE_FAILED:
|
|
256
|
-
return vecdbM("vecdb_deleteFailed", {
|
|
267
|
+
return vecdbM("vecdb_deleteFailed", {
|
|
268
|
+
params: { error: errorStr }
|
|
269
|
+
});
|
|
257
270
|
case HaiVecdbError.INSERT_FAILED:
|
|
258
|
-
return vecdbM("vecdb_insertFailed", {
|
|
271
|
+
return vecdbM("vecdb_insertFailed", {
|
|
272
|
+
params: { error: errorStr }
|
|
273
|
+
});
|
|
259
274
|
case HaiVecdbError.UPDATE_FAILED:
|
|
260
|
-
return vecdbM("vecdb_updateFailed", {
|
|
275
|
+
return vecdbM("vecdb_updateFailed", {
|
|
276
|
+
params: { error: errorStr }
|
|
277
|
+
});
|
|
261
278
|
default:
|
|
262
|
-
return vecdbM("vecdb_queryFailed", {
|
|
279
|
+
return vecdbM("vecdb_queryFailed", {
|
|
280
|
+
params: { error: errorStr }
|
|
281
|
+
});
|
|
263
282
|
}
|
|
264
283
|
}
|
|
265
284
|
async function wrapOp(ctx, fn, errorDef, errorLabel, errorMeta) {
|
|
@@ -279,7 +298,11 @@ function createBaseCollectionOps(ctx, driver) {
|
|
|
279
298
|
if (!Number.isInteger(options.dimension) || options.dimension <= 0) {
|
|
280
299
|
return err(
|
|
281
300
|
HaiVecdbError.DIMENSION_MISMATCH,
|
|
282
|
-
vecdbM("vecdb_invalidDimension", {
|
|
301
|
+
vecdbM("vecdb_invalidDimension", {
|
|
302
|
+
params: {
|
|
303
|
+
dimension: String(options.dimension)
|
|
304
|
+
}
|
|
305
|
+
})
|
|
283
306
|
);
|
|
284
307
|
}
|
|
285
308
|
logVecdbOperation(ctx, "write", "collection.create", { name, options });
|
|
@@ -416,9 +439,7 @@ function fromChromaSpace(space) {
|
|
|
416
439
|
return "cosine";
|
|
417
440
|
}
|
|
418
441
|
function distanceToScore(distance, metric) {
|
|
419
|
-
|
|
420
|
-
return Math.max(0, Math.min(1, 1 - distance));
|
|
421
|
-
return 1 / (1 + Math.max(0, distance));
|
|
442
|
+
return vectorScoreFromDistance(metric === "dot" ? distance - 1 : distance, metric);
|
|
422
443
|
}
|
|
423
444
|
function buildWhere(filter) {
|
|
424
445
|
if (!filter)
|
|
@@ -426,7 +447,9 @@ function buildWhere(filter) {
|
|
|
426
447
|
const entries = Object.entries(filter);
|
|
427
448
|
if (entries.length === 0)
|
|
428
449
|
return void 0;
|
|
429
|
-
const where = entries.length === 1 ? { [entries[0][0]]: entries[0][1] } : {
|
|
450
|
+
const where = entries.length === 1 ? { [entries[0][0]]: entries[0][1] } : {
|
|
451
|
+
$and: entries.map(([key, value]) => ({ [key]: value }))
|
|
452
|
+
};
|
|
430
453
|
return where;
|
|
431
454
|
}
|
|
432
455
|
function createChromaProvider() {
|
|
@@ -445,19 +468,33 @@ function createChromaProvider() {
|
|
|
445
468
|
if (!isChromaModule(mod)) {
|
|
446
469
|
const error = new Error("chromadb driver module does not export ChromaClient");
|
|
447
470
|
logger.error("Failed to load Chroma driver", { error: error.message });
|
|
448
|
-
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", {
|
|
471
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", {
|
|
472
|
+
params: { driver: "chromadb" }
|
|
473
|
+
}), error);
|
|
449
474
|
}
|
|
450
475
|
return ok(mod);
|
|
451
476
|
} catch (error) {
|
|
452
477
|
if (isMissingModuleError(error)) {
|
|
453
|
-
logger.error("Failed to load Chroma driver", {
|
|
454
|
-
|
|
478
|
+
logger.error("Failed to load Chroma driver", {
|
|
479
|
+
error: error instanceof Error ? error.message : String(error)
|
|
480
|
+
});
|
|
481
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", {
|
|
482
|
+
params: { driver: "chromadb" }
|
|
483
|
+
}), error);
|
|
455
484
|
}
|
|
456
485
|
throw error;
|
|
457
486
|
}
|
|
458
487
|
}
|
|
459
488
|
async function startEmbeddedServer(chromaConfig) {
|
|
460
|
-
const args = [
|
|
489
|
+
const args = [
|
|
490
|
+
"run",
|
|
491
|
+
"--path",
|
|
492
|
+
chromaConfig.path ?? "./data/chroma",
|
|
493
|
+
"--host",
|
|
494
|
+
chromaConfig.host,
|
|
495
|
+
"--port",
|
|
496
|
+
String(chromaConfig.port)
|
|
497
|
+
];
|
|
461
498
|
logger.info("Starting embedded Chroma server", { command: chromaConfig.serverCommand, host: chromaConfig.host, port: chromaConfig.port });
|
|
462
499
|
spawnFailed = false;
|
|
463
500
|
const proc = spawn(chromaConfig.serverCommand, args, { stdio: "ignore" });
|
|
@@ -536,7 +573,16 @@ function createChromaProvider() {
|
|
|
536
573
|
logger.info("Chroma connection closed");
|
|
537
574
|
return ok(void 0);
|
|
538
575
|
} catch (cause) {
|
|
539
|
-
return err({
|
|
576
|
+
return err({
|
|
577
|
+
...HaiVecdbError.CONNECTION_FAILED,
|
|
578
|
+
message: vecdbM("vecdb_closeFailed", {
|
|
579
|
+
params: {
|
|
580
|
+
error: cause instanceof Error ? cause.message : String(cause)
|
|
581
|
+
}
|
|
582
|
+
}),
|
|
583
|
+
cause,
|
|
584
|
+
ext: { cleanupPending: true }
|
|
585
|
+
});
|
|
540
586
|
}
|
|
541
587
|
})().finally(() => {
|
|
542
588
|
closing = null;
|
|
@@ -570,21 +616,36 @@ function createChromaProvider() {
|
|
|
570
616
|
return null;
|
|
571
617
|
return err(
|
|
572
618
|
HaiVecdbError.DIMENSION_MISMATCH,
|
|
573
|
-
vecdbM("vecdb_dimensionMismatch", {
|
|
619
|
+
vecdbM("vecdb_dimensionMismatch", {
|
|
620
|
+
params: {
|
|
621
|
+
expected: String(expected),
|
|
622
|
+
actual: String(actual)
|
|
623
|
+
}
|
|
624
|
+
})
|
|
574
625
|
);
|
|
575
626
|
}
|
|
576
|
-
const ctx = {
|
|
627
|
+
const ctx = {
|
|
628
|
+
isConnected: () => client !== null,
|
|
629
|
+
logger,
|
|
630
|
+
operationLog: () => config?.operationLog
|
|
631
|
+
};
|
|
577
632
|
const collectionDriver = {
|
|
578
633
|
async create(name, options) {
|
|
579
634
|
logger.debug("Creating collection", { name, dimension: options.dimension, metric: options.metric });
|
|
580
635
|
const existing = await client.listCollections();
|
|
581
636
|
const exists = existing.some((item) => item.name === name);
|
|
582
|
-
if (exists)
|
|
583
|
-
return err(HaiVecdbError.COLLECTION_ALREADY_EXISTS, vecdbM("vecdb_collectionAlreadyExists", {
|
|
637
|
+
if (exists) {
|
|
638
|
+
return err(HaiVecdbError.COLLECTION_ALREADY_EXISTS, vecdbM("vecdb_collectionAlreadyExists", {
|
|
639
|
+
params: { name }
|
|
640
|
+
}));
|
|
641
|
+
}
|
|
584
642
|
const metric = options.metric ?? config?.metric ?? "cosine";
|
|
585
643
|
await client.getOrCreateCollection({
|
|
586
644
|
name,
|
|
587
|
-
metadata: {
|
|
645
|
+
metadata: {
|
|
646
|
+
[SPACE_METADATA_KEY]: toChromaSpace(metric),
|
|
647
|
+
[DIMENSION_METADATA_KEY]: options.dimension
|
|
648
|
+
},
|
|
588
649
|
embeddingFunction: NOOP_EMBEDDING_FUNCTION
|
|
589
650
|
});
|
|
590
651
|
logger.info("Collection created", { name, dimension: options.dimension, metric });
|
|
@@ -594,8 +655,11 @@ function createChromaProvider() {
|
|
|
594
655
|
logger.debug("Dropping collection", { name });
|
|
595
656
|
const existing = await client.listCollections();
|
|
596
657
|
const exists = existing.some((item) => item.name === name);
|
|
597
|
-
if (!exists)
|
|
598
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
658
|
+
if (!exists) {
|
|
659
|
+
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
660
|
+
params: { name }
|
|
661
|
+
}));
|
|
662
|
+
}
|
|
599
663
|
await client.deleteCollection({ name });
|
|
600
664
|
logger.info("Collection dropped", { name });
|
|
601
665
|
return ok(void 0);
|
|
@@ -608,8 +672,11 @@ function createChromaProvider() {
|
|
|
608
672
|
async info(name) {
|
|
609
673
|
logger.debug("Getting collection info", { name });
|
|
610
674
|
const existing = await client.listCollections();
|
|
611
|
-
if (!existing.some((item) => item.name === name))
|
|
612
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
675
|
+
if (!existing.some((item) => item.name === name)) {
|
|
676
|
+
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
677
|
+
params: { name }
|
|
678
|
+
}));
|
|
679
|
+
}
|
|
613
680
|
const collection = await getCollection(name);
|
|
614
681
|
const count = await collection.count();
|
|
615
682
|
return ok({
|
|
@@ -679,7 +746,11 @@ function createChromaProvider() {
|
|
|
679
746
|
if (validation)
|
|
680
747
|
return validation;
|
|
681
748
|
const metric = fromChromaSpace(target.metadata?.[SPACE_METADATA_KEY]);
|
|
682
|
-
const response = await target.query({
|
|
749
|
+
const response = await target.query({
|
|
750
|
+
queryEmbeddings: [vector],
|
|
751
|
+
nResults: topK,
|
|
752
|
+
where: buildWhere(options?.filter)
|
|
753
|
+
});
|
|
683
754
|
const ids = response.ids[0] ?? [];
|
|
684
755
|
const distances = response.distances?.[0] ?? [];
|
|
685
756
|
const metadatas = response.metadatas?.[0] ?? [];
|
|
@@ -708,8 +779,11 @@ function createChromaProvider() {
|
|
|
708
779
|
return {
|
|
709
780
|
name: "chroma",
|
|
710
781
|
async connect(cfg) {
|
|
711
|
-
if (cfg.type !== "chroma")
|
|
712
|
-
return err(HaiVecdbError.UNSUPPORTED_TYPE, vecdbM("vecdb_unsupportedType", {
|
|
782
|
+
if (cfg.type !== "chroma") {
|
|
783
|
+
return err(HaiVecdbError.UNSUPPORTED_TYPE, vecdbM("vecdb_unsupportedType", {
|
|
784
|
+
params: { type: cfg.type }
|
|
785
|
+
}));
|
|
786
|
+
}
|
|
713
787
|
if (serverProcess || client || closing) {
|
|
714
788
|
const cleaned = await closeResources();
|
|
715
789
|
if (!cleaned.success)
|
|
@@ -727,7 +801,11 @@ function createChromaProvider() {
|
|
|
727
801
|
if (embedded)
|
|
728
802
|
await startEmbeddedServer(chromaConfig);
|
|
729
803
|
const endpoint = chromaConfig.url ? new URL(chromaConfig.url) : void 0;
|
|
730
|
-
const clientParams = endpoint ? {
|
|
804
|
+
const clientParams = endpoint ? {
|
|
805
|
+
host: endpoint.hostname,
|
|
806
|
+
port: Number(endpoint.port || (endpoint.protocol === "https:" ? 443 : 80)),
|
|
807
|
+
ssl: endpoint.protocol === "https:"
|
|
808
|
+
} : { host: chromaConfig.host, port: chromaConfig.port, ssl: false };
|
|
731
809
|
if (chromaConfig.apiKey)
|
|
732
810
|
clientParams.headers = { Authorization: `Bearer ${chromaConfig.apiKey}` };
|
|
733
811
|
clientParams.fetchOptions = { signal: startupAbort.signal };
|
|
@@ -739,14 +817,28 @@ function createChromaProvider() {
|
|
|
739
817
|
throw new Error("Chroma child exited before readiness was confirmed");
|
|
740
818
|
client = chromaClient;
|
|
741
819
|
config = chromaConfig;
|
|
742
|
-
logger.info("Chroma connected", {
|
|
820
|
+
logger.info("Chroma connected", {
|
|
821
|
+
embedded,
|
|
822
|
+
host: clientParams.host,
|
|
823
|
+
port: clientParams.port,
|
|
824
|
+
ssl: clientParams.ssl
|
|
825
|
+
});
|
|
743
826
|
return ok(void 0);
|
|
744
827
|
} catch (error) {
|
|
745
828
|
startupAbort.abort();
|
|
746
829
|
const cleanup = await closeResources();
|
|
747
|
-
logger.error("Failed to connect to Chroma", {
|
|
748
|
-
|
|
749
|
-
|
|
830
|
+
logger.error("Failed to connect to Chroma", {
|
|
831
|
+
error: error instanceof Error ? error.message : String(error)
|
|
832
|
+
});
|
|
833
|
+
const failure = err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_connectionFailed", {
|
|
834
|
+
params: {
|
|
835
|
+
error: String(error)
|
|
836
|
+
}
|
|
837
|
+
}), error);
|
|
838
|
+
return !failure.success && !cleanup.success ? err({
|
|
839
|
+
...failure.error,
|
|
840
|
+
ext: { cleanupPending: true, cleanupError: cleanup.error }
|
|
841
|
+
}) : failure;
|
|
750
842
|
} finally {
|
|
751
843
|
clearTimeout(startupTimer);
|
|
752
844
|
}
|
|
@@ -792,19 +884,26 @@ function createLancedbProvider() {
|
|
|
792
884
|
return ok(mod);
|
|
793
885
|
} catch (error) {
|
|
794
886
|
logger2.error("Failed to load @lancedb/lancedb", { error });
|
|
795
|
-
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", {
|
|
887
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", {
|
|
888
|
+
params: { driver: "@lancedb/lancedb" }
|
|
889
|
+
}), error);
|
|
796
890
|
}
|
|
797
891
|
}
|
|
798
892
|
async function withCollection(name, action) {
|
|
799
|
-
if (!(await connection.tableNames()).includes(name))
|
|
800
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
893
|
+
if (!(await connection.tableNames()).includes(name)) {
|
|
894
|
+
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
895
|
+
params: { name }
|
|
896
|
+
}));
|
|
897
|
+
}
|
|
801
898
|
const table = await connection.openTable(name);
|
|
802
899
|
try {
|
|
803
900
|
const schema = await table.schema();
|
|
804
901
|
const vectorType = schema.fields.find((field) => field.name === "vector")?.type;
|
|
805
902
|
const metric = DistanceMetricSchema.safeParse(schema.metadata.get(COLLECTION_METRIC_KEY) ?? null);
|
|
806
903
|
if (!vectorType || !("listSize" in vectorType) || typeof vectorType.listSize !== "number" || !Number.isInteger(vectorType.listSize) || vectorType.listSize <= 0 || !metric.success) {
|
|
807
|
-
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", {
|
|
904
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", {
|
|
905
|
+
params: { name }
|
|
906
|
+
}));
|
|
808
907
|
}
|
|
809
908
|
const primaryKeys = schema.fields.filter((field) => field.metadata.has(PRIMARY_KEY_POSITION) || ["true", "1", "yes"].includes(field.metadata.get(PRIMARY_KEY_FLAG)?.toLowerCase() ?? ""));
|
|
810
909
|
const idField = primaryKeys[0];
|
|
@@ -813,8 +912,15 @@ function createLancedbProvider() {
|
|
|
813
912
|
if (!driver.success)
|
|
814
913
|
return driver;
|
|
815
914
|
const jsonMetadata = schema.fields.find((field) => field.name === "metadata")?.metadata.get("ARROW:extension:name") === "lance.json";
|
|
816
|
-
const inputSchema = jsonMetadata ? schema.assign([
|
|
817
|
-
|
|
915
|
+
const inputSchema = jsonMetadata ? schema.assign([
|
|
916
|
+
driver.data.makeJsonField("metadata", false)
|
|
917
|
+
]) : void 0;
|
|
918
|
+
return await action(table, {
|
|
919
|
+
dimension: vectorType.listSize,
|
|
920
|
+
metric: metric.data,
|
|
921
|
+
atomicWrites,
|
|
922
|
+
inputSchema
|
|
923
|
+
});
|
|
818
924
|
} finally {
|
|
819
925
|
table.close();
|
|
820
926
|
}
|
|
@@ -823,7 +929,10 @@ function createLancedbProvider() {
|
|
|
823
929
|
return err(
|
|
824
930
|
HaiVecdbError.DIMENSION_MISMATCH,
|
|
825
931
|
vecdbM("vecdb_dimensionMismatch", {
|
|
826
|
-
params: {
|
|
932
|
+
params: {
|
|
933
|
+
expected: String(expected),
|
|
934
|
+
actual: String(actual)
|
|
935
|
+
}
|
|
827
936
|
})
|
|
828
937
|
);
|
|
829
938
|
}
|
|
@@ -850,29 +959,49 @@ function createLancedbProvider() {
|
|
|
850
959
|
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_duplicateDocumentIds"));
|
|
851
960
|
return null;
|
|
852
961
|
}
|
|
853
|
-
const ctx = {
|
|
962
|
+
const ctx = {
|
|
963
|
+
isConnected: () => connection !== null,
|
|
964
|
+
logger: logger2,
|
|
965
|
+
operationLog: () => config?.operationLog
|
|
966
|
+
};
|
|
854
967
|
const collectionDriver = {
|
|
855
968
|
async create(name, options) {
|
|
856
969
|
logger2.debug("Creating collection", { name, dimension: options.dimension, metric: options.metric });
|
|
857
970
|
const tableNames = await connection.tableNames();
|
|
858
971
|
if (tableNames.includes(name)) {
|
|
859
|
-
return err(HaiVecdbError.COLLECTION_ALREADY_EXISTS, vecdbM("vecdb_collectionAlreadyExists", {
|
|
972
|
+
return err(HaiVecdbError.COLLECTION_ALREADY_EXISTS, vecdbM("vecdb_collectionAlreadyExists", {
|
|
973
|
+
params: { name }
|
|
974
|
+
}));
|
|
860
975
|
}
|
|
861
976
|
const dimension = options.dimension;
|
|
862
977
|
const parsedMetric = DistanceMetricSchema.safeParse(options.metric ?? config?.metric);
|
|
863
|
-
if (!parsedMetric.success)
|
|
864
|
-
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", {
|
|
978
|
+
if (!parsedMetric.success) {
|
|
979
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", {
|
|
980
|
+
params: { name }
|
|
981
|
+
}));
|
|
982
|
+
}
|
|
865
983
|
const metric = parsedMetric.data;
|
|
866
984
|
const driver = loadLancedb();
|
|
867
985
|
if (!driver.success)
|
|
868
986
|
return driver;
|
|
869
|
-
const inferred = driver.data.makeArrowTable([
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
987
|
+
const inferred = driver.data.makeArrowTable([
|
|
988
|
+
{
|
|
989
|
+
id: "",
|
|
990
|
+
vector: Array.from({ length: dimension }).fill(0),
|
|
991
|
+
content: "",
|
|
992
|
+
metadata: "{}"
|
|
993
|
+
}
|
|
994
|
+
], {}, /* @__PURE__ */ new Map([
|
|
995
|
+
[COLLECTION_METRIC_KEY, metric]
|
|
996
|
+
])).schema;
|
|
997
|
+
const schema = inferred.assign(inferred.fields.map((field) => field.name === "id" ? field.clone({
|
|
998
|
+
nullable: false,
|
|
999
|
+
metadata: new Map([
|
|
1000
|
+
...field.metadata,
|
|
1001
|
+
[PRIMARY_KEY_FLAG, "true"],
|
|
1002
|
+
[PRIMARY_KEY_POSITION, "1"]
|
|
1003
|
+
])
|
|
1004
|
+
}) : field.name === "metadata" ? driver.data.makeJsonField("metadata", false) : field));
|
|
876
1005
|
const table = await connection.createEmptyTable(name, schema);
|
|
877
1006
|
table.close();
|
|
878
1007
|
logger2.info("Collection created", { name, dimension, metric });
|
|
@@ -882,7 +1011,9 @@ function createLancedbProvider() {
|
|
|
882
1011
|
logger2.debug("Dropping collection", { name });
|
|
883
1012
|
const tableNames = await connection.tableNames();
|
|
884
1013
|
if (!tableNames.includes(name)) {
|
|
885
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
1014
|
+
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
1015
|
+
params: { name }
|
|
1016
|
+
}));
|
|
886
1017
|
}
|
|
887
1018
|
await connection.dropTable(name);
|
|
888
1019
|
logger2.info("Collection dropped", { name });
|
|
@@ -895,7 +1026,12 @@ function createLancedbProvider() {
|
|
|
895
1026
|
},
|
|
896
1027
|
async info(name) {
|
|
897
1028
|
logger2.debug("Getting collection info", { name });
|
|
898
|
-
return withCollection(name, async (table, meta) => ok({
|
|
1029
|
+
return withCollection(name, async (table, meta) => ok({
|
|
1030
|
+
name,
|
|
1031
|
+
dimension: meta.dimension,
|
|
1032
|
+
metric: meta.metric,
|
|
1033
|
+
count: await table.countRows()
|
|
1034
|
+
}));
|
|
899
1035
|
},
|
|
900
1036
|
async list() {
|
|
901
1037
|
logger2.debug("Listing collections");
|
|
@@ -983,7 +1119,7 @@ function createLancedbProvider() {
|
|
|
983
1119
|
const results = await query.toArray();
|
|
984
1120
|
const searchResults = results.map((row) => {
|
|
985
1121
|
const distance = row._distance ?? 0;
|
|
986
|
-
const score =
|
|
1122
|
+
const score = vectorScoreFromDistance(meta.metric === "dot" ? distance - 1 : distance, meta.metric);
|
|
987
1123
|
return {
|
|
988
1124
|
id: row.id,
|
|
989
1125
|
score,
|
|
@@ -1006,7 +1142,9 @@ function createLancedbProvider() {
|
|
|
1006
1142
|
name: "lancedb",
|
|
1007
1143
|
async connect(cfg) {
|
|
1008
1144
|
if (cfg.type !== "lancedb") {
|
|
1009
|
-
return err(HaiVecdbError.UNSUPPORTED_TYPE, vecdbM("vecdb_unsupportedType", {
|
|
1145
|
+
return err(HaiVecdbError.UNSUPPORTED_TYPE, vecdbM("vecdb_unsupportedType", {
|
|
1146
|
+
params: { type: cfg.type }
|
|
1147
|
+
}));
|
|
1010
1148
|
}
|
|
1011
1149
|
const lanceConfig = cfg;
|
|
1012
1150
|
const loadResult = loadLancedb();
|
|
@@ -1023,7 +1161,11 @@ function createLancedbProvider() {
|
|
|
1023
1161
|
connection = null;
|
|
1024
1162
|
config = null;
|
|
1025
1163
|
logger2.error("Failed to connect to LanceDB", { error });
|
|
1026
|
-
return err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_connectionFailed", {
|
|
1164
|
+
return err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_connectionFailed", {
|
|
1165
|
+
params: {
|
|
1166
|
+
error: String(error)
|
|
1167
|
+
}
|
|
1168
|
+
}), error);
|
|
1027
1169
|
}
|
|
1028
1170
|
},
|
|
1029
1171
|
async close() {
|
|
@@ -1070,16 +1212,7 @@ function createPgvectorProvider() {
|
|
|
1070
1212
|
}
|
|
1071
1213
|
}
|
|
1072
1214
|
function distanceToScore2(distance, metric) {
|
|
1073
|
-
|
|
1074
|
-
case "cosine":
|
|
1075
|
-
return 1 - distance / 2;
|
|
1076
|
-
case "euclidean":
|
|
1077
|
-
return 1 / (1 + distance);
|
|
1078
|
-
case "dot":
|
|
1079
|
-
return -distance;
|
|
1080
|
-
default:
|
|
1081
|
-
return 1 / (1 + distance);
|
|
1082
|
-
}
|
|
1215
|
+
return vectorScoreFromDistance(distance, metric);
|
|
1083
1216
|
}
|
|
1084
1217
|
function isMissingModuleError(error, packageName) {
|
|
1085
1218
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -1091,13 +1224,19 @@ function createPgvectorProvider() {
|
|
|
1091
1224
|
if (!isPgModule(mod)) {
|
|
1092
1225
|
const error = new Error("pg driver module does not export Pool");
|
|
1093
1226
|
logger3.error("Failed to load pg driver", { error: error.message });
|
|
1094
|
-
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", {
|
|
1227
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", {
|
|
1228
|
+
params: { driver: "pg" }
|
|
1229
|
+
}), error);
|
|
1095
1230
|
}
|
|
1096
1231
|
return ok(mod);
|
|
1097
1232
|
} catch (error) {
|
|
1098
1233
|
if (isMissingModuleError(error, "pg")) {
|
|
1099
|
-
logger3.error("Failed to load pg driver", {
|
|
1100
|
-
|
|
1234
|
+
logger3.error("Failed to load pg driver", {
|
|
1235
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1236
|
+
});
|
|
1237
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", {
|
|
1238
|
+
params: { driver: "pg" }
|
|
1239
|
+
}), error);
|
|
1101
1240
|
}
|
|
1102
1241
|
throw error;
|
|
1103
1242
|
}
|
|
@@ -1109,19 +1248,25 @@ function createPgvectorProvider() {
|
|
|
1109
1248
|
[table]
|
|
1110
1249
|
);
|
|
1111
1250
|
if (!existsResult.rows[0].exists) {
|
|
1112
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
1251
|
+
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
1252
|
+
params: { name }
|
|
1253
|
+
}));
|
|
1113
1254
|
}
|
|
1114
1255
|
const dimResult = await pool.query(`
|
|
1115
1256
|
SELECT atttypmod FROM pg_attribute
|
|
1116
1257
|
WHERE attrelid = $1::regclass AND attname = 'vector'
|
|
1117
|
-
`, [
|
|
1258
|
+
`, [
|
|
1259
|
+
quoteIdent(table)
|
|
1260
|
+
]);
|
|
1118
1261
|
return ok(dimResult.rows.length > 0 ? Number(dimResult.rows[0].atttypmod) : 0);
|
|
1119
1262
|
}
|
|
1120
1263
|
async function getCollectionMetric(name) {
|
|
1121
1264
|
const result = await pool.query(`SELECT col_description(a.attrelid, a.attnum) AS definition,
|
|
1122
1265
|
(SELECT opc.opcname FROM pg_index i JOIN pg_opclass opc ON opc.oid = ANY(i.indclass)
|
|
1123
1266
|
WHERE i.indrelid = a.attrelid AND opc.opcname IN ('vector_cosine_ops', 'vector_l2_ops', 'vector_ip_ops') LIMIT 1) AS operator
|
|
1124
|
-
FROM pg_attribute a WHERE a.attrelid = $1::regclass AND a.attname = 'vector'`, [
|
|
1267
|
+
FROM pg_attribute a WHERE a.attrelid = $1::regclass AND a.attname = 'vector'`, [
|
|
1268
|
+
quoteIdent(tableName(name))
|
|
1269
|
+
]);
|
|
1125
1270
|
const row = result.rows[0];
|
|
1126
1271
|
const stored = typeof row?.definition === "string" && row.definition.startsWith("hai.vecdb.metric=") ? row.definition.slice("hai.vecdb.metric=".length) : void 0;
|
|
1127
1272
|
const inherited = row?.operator === "vector_cosine_ops" ? "cosine" : row?.operator === "vector_l2_ops" ? "euclidean" : row?.operator === "vector_ip_ops" ? "dot" : void 0;
|
|
@@ -1138,7 +1283,10 @@ function createPgvectorProvider() {
|
|
|
1138
1283
|
return err(
|
|
1139
1284
|
HaiVecdbError.DIMENSION_MISMATCH,
|
|
1140
1285
|
vecdbM("vecdb_dimensionMismatch", {
|
|
1141
|
-
params: {
|
|
1286
|
+
params: {
|
|
1287
|
+
expected: String(expected),
|
|
1288
|
+
actual: String(actual)
|
|
1289
|
+
}
|
|
1142
1290
|
})
|
|
1143
1291
|
);
|
|
1144
1292
|
}
|
|
@@ -1151,7 +1299,11 @@ function createPgvectorProvider() {
|
|
|
1151
1299
|
}
|
|
1152
1300
|
return null;
|
|
1153
1301
|
}
|
|
1154
|
-
const ctx = {
|
|
1302
|
+
const ctx = {
|
|
1303
|
+
isConnected: () => pool !== null,
|
|
1304
|
+
logger: logger3,
|
|
1305
|
+
operationLog: () => config?.operationLog
|
|
1306
|
+
};
|
|
1155
1307
|
const collectionDriver = {
|
|
1156
1308
|
async create(name, options) {
|
|
1157
1309
|
const table = tableName(name);
|
|
@@ -1164,7 +1316,9 @@ function createPgvectorProvider() {
|
|
|
1164
1316
|
[table]
|
|
1165
1317
|
);
|
|
1166
1318
|
if (checkResult.rows[0].exists) {
|
|
1167
|
-
return err(HaiVecdbError.COLLECTION_ALREADY_EXISTS, vecdbM("vecdb_collectionAlreadyExists", {
|
|
1319
|
+
return err(HaiVecdbError.COLLECTION_ALREADY_EXISTS, vecdbM("vecdb_collectionAlreadyExists", {
|
|
1320
|
+
params: { name }
|
|
1321
|
+
}));
|
|
1168
1322
|
}
|
|
1169
1323
|
const qi = quoteIdent(table);
|
|
1170
1324
|
await pool.query(`
|
|
@@ -1201,7 +1355,9 @@ function createPgvectorProvider() {
|
|
|
1201
1355
|
[table]
|
|
1202
1356
|
);
|
|
1203
1357
|
if (!checkResult.rows[0].exists) {
|
|
1204
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
1358
|
+
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
1359
|
+
params: { name }
|
|
1360
|
+
}));
|
|
1205
1361
|
}
|
|
1206
1362
|
await pool.query(`DROP TABLE ${quoteIdent(table)}`);
|
|
1207
1363
|
logger3.info("Collection dropped", { name });
|
|
@@ -1211,7 +1367,9 @@ function createPgvectorProvider() {
|
|
|
1211
1367
|
logger3.debug("Checking collection exists", { name });
|
|
1212
1368
|
const result = await pool.query(
|
|
1213
1369
|
`SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $1)`,
|
|
1214
|
-
[
|
|
1370
|
+
[
|
|
1371
|
+
tableName(name)
|
|
1372
|
+
]
|
|
1215
1373
|
);
|
|
1216
1374
|
return ok(result.rows[0].exists);
|
|
1217
1375
|
},
|
|
@@ -1376,7 +1534,9 @@ function createPgvectorProvider() {
|
|
|
1376
1534
|
name: "pgvector",
|
|
1377
1535
|
async connect(cfg) {
|
|
1378
1536
|
if (cfg.type !== "pgvector") {
|
|
1379
|
-
return err(HaiVecdbError.UNSUPPORTED_TYPE, vecdbM("vecdb_unsupportedType", {
|
|
1537
|
+
return err(HaiVecdbError.UNSUPPORTED_TYPE, vecdbM("vecdb_unsupportedType", {
|
|
1538
|
+
params: { type: cfg.type }
|
|
1539
|
+
}));
|
|
1380
1540
|
}
|
|
1381
1541
|
const pgConfig = cfg;
|
|
1382
1542
|
try {
|
|
@@ -1398,8 +1558,14 @@ function createPgvectorProvider() {
|
|
|
1398
1558
|
logger3.info("pgvector connected", { host: pgConfig.host, database: pgConfig.database });
|
|
1399
1559
|
return ok(void 0);
|
|
1400
1560
|
} catch (error) {
|
|
1401
|
-
logger3.error("Failed to connect to pgvector", {
|
|
1402
|
-
|
|
1561
|
+
logger3.error("Failed to connect to pgvector", {
|
|
1562
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1563
|
+
});
|
|
1564
|
+
return err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_connectionFailed", {
|
|
1565
|
+
params: {
|
|
1566
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1567
|
+
}
|
|
1568
|
+
}), error);
|
|
1403
1569
|
}
|
|
1404
1570
|
},
|
|
1405
1571
|
async close() {
|
|
@@ -1412,8 +1578,14 @@ function createPgvectorProvider() {
|
|
|
1412
1578
|
logger3.info("pgvector connection closed");
|
|
1413
1579
|
return ok(void 0);
|
|
1414
1580
|
} catch (error) {
|
|
1415
|
-
logger3.error("Failed to close pgvector connection", {
|
|
1416
|
-
|
|
1581
|
+
logger3.error("Failed to close pgvector connection", {
|
|
1582
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1583
|
+
});
|
|
1584
|
+
return err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_closeFailed", {
|
|
1585
|
+
params: {
|
|
1586
|
+
error: String(error)
|
|
1587
|
+
}
|
|
1588
|
+
}), error);
|
|
1417
1589
|
}
|
|
1418
1590
|
},
|
|
1419
1591
|
isConnected() {
|
|
@@ -1423,14 +1595,152 @@ function createPgvectorProvider() {
|
|
|
1423
1595
|
vector: createBaseVectorOps(ctx, vectorDriver)
|
|
1424
1596
|
};
|
|
1425
1597
|
}
|
|
1426
|
-
var
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1598
|
+
var MAX_RESPONSE_BYTES = 16 * 1024 * 1024;
|
|
1599
|
+
var pointSchema = z.object({
|
|
1600
|
+
id: z.union([
|
|
1601
|
+
z.string(),
|
|
1602
|
+
z.number()
|
|
1603
|
+
]),
|
|
1604
|
+
score: z.number().optional(),
|
|
1605
|
+
payload: z.record(z.string(), z.unknown()).nullable().optional(),
|
|
1606
|
+
vector: z.array(z.number()).optional()
|
|
1607
|
+
});
|
|
1608
|
+
var collectionSchema = z.object({
|
|
1609
|
+
config: z.object({
|
|
1610
|
+
params: z.object({
|
|
1611
|
+
vectors: z.unknown()
|
|
1612
|
+
})
|
|
1613
|
+
}),
|
|
1614
|
+
points_count: z.number().nullable().optional()
|
|
1615
|
+
});
|
|
1616
|
+
var QdrantHttpError = class extends Error {
|
|
1617
|
+
constructor(status) {
|
|
1618
|
+
super(`Qdrant HTTP request failed (${status})`);
|
|
1619
|
+
this.status = status;
|
|
1430
1620
|
}
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1621
|
+
status;
|
|
1622
|
+
};
|
|
1623
|
+
var QdrantRestClient = class {
|
|
1624
|
+
constructor(url, apiKey) {
|
|
1625
|
+
this.apiKey = apiKey;
|
|
1626
|
+
this.base = new URL(url);
|
|
1627
|
+
if (!["http:", "https:"].includes(this.base.protocol) || this.base.username || this.base.password)
|
|
1628
|
+
throw new Error("Qdrant URL must use HTTP(S) without embedded credentials");
|
|
1629
|
+
}
|
|
1630
|
+
apiKey;
|
|
1631
|
+
base;
|
|
1632
|
+
async perform(path, method, schema, body) {
|
|
1633
|
+
const url = new URL(this.base);
|
|
1634
|
+
const [pathname, query] = path.split("?");
|
|
1635
|
+
url.pathname = `${url.pathname.replace(/\/$/, "")}${pathname}`;
|
|
1636
|
+
for (const [key, value] of new URLSearchParams(query)) url.searchParams.set(key, value);
|
|
1637
|
+
const response = await fetch(url, {
|
|
1638
|
+
method,
|
|
1639
|
+
redirect: "error",
|
|
1640
|
+
signal: AbortSignal.timeout(3e4),
|
|
1641
|
+
headers: {
|
|
1642
|
+
"content-type": "application/json",
|
|
1643
|
+
...this.apiKey ? { "api-key": this.apiKey } : {}
|
|
1644
|
+
},
|
|
1645
|
+
body: body === void 0 ? void 0 : JSON.stringify(body)
|
|
1646
|
+
});
|
|
1647
|
+
if (!response.ok) {
|
|
1648
|
+
await response.body?.cancel();
|
|
1649
|
+
throw new QdrantHttpError(response.status);
|
|
1650
|
+
}
|
|
1651
|
+
const reader = response.body?.getReader();
|
|
1652
|
+
if (!reader)
|
|
1653
|
+
throw new Error("Qdrant response body is missing");
|
|
1654
|
+
let size = 0;
|
|
1655
|
+
const chunks = [];
|
|
1656
|
+
try {
|
|
1657
|
+
while (true) {
|
|
1658
|
+
const chunk = await reader.read();
|
|
1659
|
+
if (chunk.done)
|
|
1660
|
+
break;
|
|
1661
|
+
size += chunk.value.byteLength;
|
|
1662
|
+
if (size > MAX_RESPONSE_BYTES)
|
|
1663
|
+
throw new Error("Qdrant response exceeds 16 MiB");
|
|
1664
|
+
chunks.push(chunk.value);
|
|
1665
|
+
}
|
|
1666
|
+
} finally {
|
|
1667
|
+
await reader.cancel().catch(() => void 0);
|
|
1668
|
+
reader.releaseLock();
|
|
1669
|
+
}
|
|
1670
|
+
const bytes = new Uint8Array(size);
|
|
1671
|
+
let offset = 0;
|
|
1672
|
+
for (const chunk of chunks) {
|
|
1673
|
+
bytes.set(chunk, offset);
|
|
1674
|
+
offset += chunk.byteLength;
|
|
1675
|
+
}
|
|
1676
|
+
let bodyJson;
|
|
1677
|
+
try {
|
|
1678
|
+
bodyJson = JSON.parse(new TextDecoder().decode(bytes));
|
|
1679
|
+
} catch {
|
|
1680
|
+
throw new Error("Qdrant returned invalid JSON");
|
|
1681
|
+
}
|
|
1682
|
+
const envelope = z.object({
|
|
1683
|
+
status: z.literal("ok"),
|
|
1684
|
+
result: schema
|
|
1685
|
+
}).safeParse(bodyJson);
|
|
1686
|
+
if (!envelope.success)
|
|
1687
|
+
throw new Error("Qdrant returned an invalid response");
|
|
1688
|
+
return envelope.data.result;
|
|
1689
|
+
}
|
|
1690
|
+
async request(path, method, schema, body) {
|
|
1691
|
+
try {
|
|
1692
|
+
return await this.perform(path, method, schema, body);
|
|
1693
|
+
} catch (error) {
|
|
1694
|
+
if (error instanceof QdrantHttpError)
|
|
1695
|
+
throw error;
|
|
1696
|
+
core.logger.child({ module: "vecdb", scope: "qdrant-http" }).error("Qdrant HTTP transport failed", {
|
|
1697
|
+
error: core.sanitize.sanitizeSensitiveFields(error)
|
|
1698
|
+
});
|
|
1699
|
+
throw new Error("Qdrant request failed");
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
getCollections() {
|
|
1703
|
+
return this.request("/collections", "GET", z.object({
|
|
1704
|
+
collections: z.array(z.object({
|
|
1705
|
+
name: z.string()
|
|
1706
|
+
}))
|
|
1707
|
+
}));
|
|
1708
|
+
}
|
|
1709
|
+
getCollection(name) {
|
|
1710
|
+
return this.request(`/collections/${encodeURIComponent(name)}`, "GET", collectionSchema);
|
|
1711
|
+
}
|
|
1712
|
+
createCollection(name, options) {
|
|
1713
|
+
return this.request(`/collections/${encodeURIComponent(name)}`, "PUT", z.boolean(), options);
|
|
1714
|
+
}
|
|
1715
|
+
deleteCollection(name) {
|
|
1716
|
+
return this.request(`/collections/${encodeURIComponent(name)}`, "DELETE", z.boolean());
|
|
1717
|
+
}
|
|
1718
|
+
upsert(name, options) {
|
|
1719
|
+
return this.request(`/collections/${encodeURIComponent(name)}/points?wait=${options.wait}`, "PUT", z.object({
|
|
1720
|
+
status: z.literal("completed")
|
|
1721
|
+
}), { points: options.points });
|
|
1722
|
+
}
|
|
1723
|
+
delete(name, options) {
|
|
1724
|
+
return this.request(`/collections/${encodeURIComponent(name)}/points/delete?wait=${options.wait}`, "POST", z.object({
|
|
1725
|
+
status: z.literal("completed")
|
|
1726
|
+
}), { points: options.points });
|
|
1727
|
+
}
|
|
1728
|
+
query(name, options) {
|
|
1729
|
+
return this.request(`/collections/${encodeURIComponent(name)}/points/query`, "POST", z.object({
|
|
1730
|
+
points: z.array(pointSchema.extend({
|
|
1731
|
+
score: z.number()
|
|
1732
|
+
}))
|
|
1733
|
+
}), options);
|
|
1734
|
+
}
|
|
1735
|
+
count(name, options) {
|
|
1736
|
+
return this.request(`/collections/${encodeURIComponent(name)}/points/count`, "POST", z.object({
|
|
1737
|
+
count: z.number()
|
|
1738
|
+
}), options);
|
|
1739
|
+
}
|
|
1740
|
+
};
|
|
1741
|
+
|
|
1742
|
+
// src/providers/vecdb-provider-qdrant.ts
|
|
1743
|
+
var logger4 = core.logger.child({ module: "vecdb", scope: "qdrant" });
|
|
1434
1744
|
function hashToUuid(id) {
|
|
1435
1745
|
const hash = createHash("sha256").update(id).digest("hex");
|
|
1436
1746
|
return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
|
|
@@ -1448,27 +1758,6 @@ function toQdrantDistance(metric) {
|
|
|
1448
1758
|
function createQdrantProvider() {
|
|
1449
1759
|
let client = null;
|
|
1450
1760
|
let config = null;
|
|
1451
|
-
function isMissingModuleError(error, packageName) {
|
|
1452
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
1453
|
-
return message.includes(`'${packageName}'`) || message.includes(`"${packageName}"`) || message.includes(packageName);
|
|
1454
|
-
}
|
|
1455
|
-
async function loadQdrantClient() {
|
|
1456
|
-
try {
|
|
1457
|
-
const mod = await import('@qdrant/js-client-rest');
|
|
1458
|
-
if (!isQdrantModule(mod)) {
|
|
1459
|
-
const error = new Error("@qdrant/js-client-rest driver module does not export QdrantClient");
|
|
1460
|
-
logger4.error("Failed to load Qdrant driver", { error: error.message });
|
|
1461
|
-
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", { params: { driver: "@qdrant/js-client-rest" } }), error);
|
|
1462
|
-
}
|
|
1463
|
-
return ok(mod);
|
|
1464
|
-
} catch (error) {
|
|
1465
|
-
if (isMissingModuleError(error, "@qdrant/js-client-rest")) {
|
|
1466
|
-
logger4.error("Failed to load Qdrant driver", { error: error instanceof Error ? error.message : String(error) });
|
|
1467
|
-
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", { params: { driver: "@qdrant/js-client-rest" } }), error);
|
|
1468
|
-
}
|
|
1469
|
-
throw error;
|
|
1470
|
-
}
|
|
1471
|
-
}
|
|
1472
1761
|
function isQdrantNotFoundError(error) {
|
|
1473
1762
|
return typeof error === "object" && error !== null && "status" in error && error.status === 404;
|
|
1474
1763
|
}
|
|
@@ -1477,14 +1766,16 @@ function createQdrantProvider() {
|
|
|
1477
1766
|
return ok(await client.getCollection(name));
|
|
1478
1767
|
} catch (error) {
|
|
1479
1768
|
if (isQdrantNotFoundError(error)) {
|
|
1480
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
1769
|
+
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", {
|
|
1770
|
+
params: { name }
|
|
1771
|
+
}));
|
|
1481
1772
|
}
|
|
1482
1773
|
throw error;
|
|
1483
1774
|
}
|
|
1484
1775
|
}
|
|
1485
1776
|
function readVectorDefinition(info) {
|
|
1486
1777
|
const vectors = info.config.params.vectors;
|
|
1487
|
-
if (!vectors || !("size" in vectors) || typeof vectors.size !== "number" || !Number.isSafeInteger(vectors.size) || vectors.size <= 0)
|
|
1778
|
+
if (!vectors || typeof vectors !== "object" || !("size" in vectors) || typeof vectors.size !== "number" || !Number.isSafeInteger(vectors.size) || vectors.size <= 0)
|
|
1488
1779
|
return void 0;
|
|
1489
1780
|
if (!("distance" in vectors))
|
|
1490
1781
|
return void 0;
|
|
@@ -1497,7 +1788,9 @@ function createQdrantProvider() {
|
|
|
1497
1788
|
return infoResult;
|
|
1498
1789
|
}
|
|
1499
1790
|
const definition = readVectorDefinition(infoResult.data);
|
|
1500
|
-
return definition ? ok(definition.dimension) : err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", {
|
|
1791
|
+
return definition ? ok(definition.dimension) : err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", {
|
|
1792
|
+
params: { name }
|
|
1793
|
+
}));
|
|
1501
1794
|
}
|
|
1502
1795
|
function validateVectorDimension(expected, actual) {
|
|
1503
1796
|
if (expected === actual) {
|
|
@@ -1506,18 +1799,27 @@ function createQdrantProvider() {
|
|
|
1506
1799
|
return err(
|
|
1507
1800
|
HaiVecdbError.DIMENSION_MISMATCH,
|
|
1508
1801
|
vecdbM("vecdb_dimensionMismatch", {
|
|
1509
|
-
params: {
|
|
1802
|
+
params: {
|
|
1803
|
+
expected: String(expected),
|
|
1804
|
+
actual: String(actual)
|
|
1805
|
+
}
|
|
1510
1806
|
})
|
|
1511
1807
|
);
|
|
1512
1808
|
}
|
|
1513
|
-
const ctx = {
|
|
1809
|
+
const ctx = {
|
|
1810
|
+
isConnected: () => client !== null,
|
|
1811
|
+
logger: logger4,
|
|
1812
|
+
operationLog: () => config?.operationLog
|
|
1813
|
+
};
|
|
1514
1814
|
const collectionDriver = {
|
|
1515
1815
|
async create(name, options) {
|
|
1516
1816
|
logger4.debug("Creating collection", { name, dimension: options.dimension, metric: options.metric });
|
|
1517
1817
|
const collections = await client.getCollections();
|
|
1518
1818
|
const exists = collections.collections.some((c) => c.name === name);
|
|
1519
1819
|
if (exists) {
|
|
1520
|
-
return err(HaiVecdbError.COLLECTION_ALREADY_EXISTS, vecdbM("vecdb_collectionAlreadyExists", {
|
|
1820
|
+
return err(HaiVecdbError.COLLECTION_ALREADY_EXISTS, vecdbM("vecdb_collectionAlreadyExists", {
|
|
1821
|
+
params: { name }
|
|
1822
|
+
}));
|
|
1521
1823
|
}
|
|
1522
1824
|
const metric = options.metric ?? config?.metric ?? "cosine";
|
|
1523
1825
|
await client.createCollection(name, {
|
|
@@ -1559,8 +1861,11 @@ function createQdrantProvider() {
|
|
|
1559
1861
|
}
|
|
1560
1862
|
const collectionInfo = infoResult.data;
|
|
1561
1863
|
const definition = readVectorDefinition(collectionInfo);
|
|
1562
|
-
if (!definition)
|
|
1563
|
-
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", {
|
|
1864
|
+
if (!definition) {
|
|
1865
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", {
|
|
1866
|
+
params: { name }
|
|
1867
|
+
}));
|
|
1868
|
+
}
|
|
1564
1869
|
return ok({
|
|
1565
1870
|
name,
|
|
1566
1871
|
...definition,
|
|
@@ -1636,11 +1941,16 @@ function createQdrantProvider() {
|
|
|
1636
1941
|
const topK = options?.topK ?? 10;
|
|
1637
1942
|
const minScore = options?.minScore ?? 0;
|
|
1638
1943
|
logger4.debug("Searching vectors", { collection, topK, hasFilter: !!options?.filter });
|
|
1639
|
-
const
|
|
1640
|
-
if (!
|
|
1641
|
-
return
|
|
1944
|
+
const collectionInfo = await getCollectionInfo(collection);
|
|
1945
|
+
if (!collectionInfo.success)
|
|
1946
|
+
return collectionInfo;
|
|
1947
|
+
const definition = readVectorDefinition(collectionInfo.data);
|
|
1948
|
+
if (!definition) {
|
|
1949
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", {
|
|
1950
|
+
params: { name: collection }
|
|
1951
|
+
}));
|
|
1642
1952
|
}
|
|
1643
|
-
const dimensionValidation = validateVectorDimension(
|
|
1953
|
+
const dimensionValidation = validateVectorDimension(definition.dimension, vector.length);
|
|
1644
1954
|
if (dimensionValidation) {
|
|
1645
1955
|
return dimensionValidation;
|
|
1646
1956
|
}
|
|
@@ -1655,7 +1965,6 @@ function createQdrantProvider() {
|
|
|
1655
1965
|
const searchResult = await client.query(collection, {
|
|
1656
1966
|
query: vector,
|
|
1657
1967
|
limit: topK,
|
|
1658
|
-
score_threshold: minScore > 0 ? minScore : void 0,
|
|
1659
1968
|
filter,
|
|
1660
1969
|
with_payload: true
|
|
1661
1970
|
});
|
|
@@ -1664,11 +1973,12 @@ function createQdrantProvider() {
|
|
|
1664
1973
|
const { _id, _content, ...metadata } = payload;
|
|
1665
1974
|
return {
|
|
1666
1975
|
id: typeof _id === "string" ? _id : "",
|
|
1667
|
-
|
|
1976
|
+
// Qdrant 余弦/内积返回相似值,欧氏返回距离;统一后再应用公开阈值。
|
|
1977
|
+
score: vectorScoreFromDistance(definition.metric === "cosine" ? 1 - point.score : definition.metric === "dot" ? -point.score : point.score, definition.metric),
|
|
1668
1978
|
content: typeof _content === "string" && _content.length > 0 ? _content : void 0,
|
|
1669
1979
|
metadata: Object.keys(metadata).length > 0 ? metadata : void 0
|
|
1670
1980
|
};
|
|
1671
|
-
});
|
|
1981
|
+
}).filter((point) => point.score >= minScore);
|
|
1672
1982
|
return ok(results);
|
|
1673
1983
|
},
|
|
1674
1984
|
async count(collection) {
|
|
@@ -1681,27 +1991,27 @@ function createQdrantProvider() {
|
|
|
1681
1991
|
name: "qdrant",
|
|
1682
1992
|
async connect(cfg) {
|
|
1683
1993
|
if (cfg.type !== "qdrant") {
|
|
1684
|
-
return err(HaiVecdbError.UNSUPPORTED_TYPE, vecdbM("vecdb_unsupportedType", {
|
|
1994
|
+
return err(HaiVecdbError.UNSUPPORTED_TYPE, vecdbM("vecdb_unsupportedType", {
|
|
1995
|
+
params: { type: cfg.type }
|
|
1996
|
+
}));
|
|
1685
1997
|
}
|
|
1686
1998
|
const qdrantConfig = cfg;
|
|
1687
1999
|
try {
|
|
1688
|
-
const
|
|
1689
|
-
if (!loadResult.success) {
|
|
1690
|
-
return loadResult;
|
|
1691
|
-
}
|
|
1692
|
-
const { QdrantClient: QdrantClientClass } = loadResult.data;
|
|
1693
|
-
const qdrantClient = new QdrantClientClass({
|
|
1694
|
-
url: qdrantConfig.url,
|
|
1695
|
-
apiKey: qdrantConfig.apiKey
|
|
1696
|
-
});
|
|
2000
|
+
const qdrantClient = new QdrantRestClient(qdrantConfig.url, qdrantConfig.apiKey);
|
|
1697
2001
|
await qdrantClient.getCollections();
|
|
1698
2002
|
client = qdrantClient;
|
|
1699
2003
|
config = qdrantConfig;
|
|
1700
|
-
logger4.info("Qdrant connected", {
|
|
2004
|
+
logger4.info("Qdrant connected", {
|
|
2005
|
+
url: core.sanitize.sanitizeUrl(qdrantConfig.url)
|
|
2006
|
+
});
|
|
1701
2007
|
return ok(void 0);
|
|
1702
2008
|
} catch (error) {
|
|
1703
|
-
logger4.error("Failed to connect to Qdrant", {
|
|
1704
|
-
|
|
2009
|
+
logger4.error("Failed to connect to Qdrant", {
|
|
2010
|
+
error: core.sanitize.sanitizeSensitiveFields(error)
|
|
2011
|
+
});
|
|
2012
|
+
return err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_connectionFailed", {
|
|
2013
|
+
params: { error: "Qdrant connection failed" }
|
|
2014
|
+
}));
|
|
1705
2015
|
}
|
|
1706
2016
|
},
|
|
1707
2017
|
async close() {
|
|
@@ -1736,24 +2046,44 @@ async function closeOwnedProvider() {
|
|
|
1736
2046
|
return ok(void 0);
|
|
1737
2047
|
try {
|
|
1738
2048
|
const result = await pendingProvider.close();
|
|
1739
|
-
if (!result.success)
|
|
1740
|
-
return err({
|
|
2049
|
+
if (!result.success) {
|
|
2050
|
+
return err({
|
|
2051
|
+
...result.error,
|
|
2052
|
+
ext: { ...result.error.ext, cleanupPending: true }
|
|
2053
|
+
});
|
|
2054
|
+
}
|
|
1741
2055
|
pendingProvider = null;
|
|
1742
2056
|
return result;
|
|
1743
2057
|
} catch (cause) {
|
|
1744
|
-
return err({
|
|
2058
|
+
return err({
|
|
2059
|
+
...HaiVecdbError.CONNECTION_FAILED,
|
|
2060
|
+
message: vecdbM("vecdb_closeFailed", {
|
|
2061
|
+
params: {
|
|
2062
|
+
error: cause instanceof Error ? cause.message : String(cause)
|
|
2063
|
+
}
|
|
2064
|
+
}),
|
|
2065
|
+
cause,
|
|
2066
|
+
ext: { cleanupPending: true }
|
|
2067
|
+
});
|
|
1745
2068
|
}
|
|
1746
2069
|
}
|
|
1747
2070
|
async function failInitialization(result) {
|
|
1748
2071
|
const cleanup = await closeOwnedProvider();
|
|
1749
2072
|
if (result.success)
|
|
1750
2073
|
return result;
|
|
1751
|
-
if (!cleanup.success)
|
|
1752
|
-
return err({
|
|
2074
|
+
if (!cleanup.success) {
|
|
2075
|
+
return err({
|
|
2076
|
+
...result.error,
|
|
2077
|
+
ext: { ...result.error.ext, cleanupPending: true, cleanupError: cleanup.error }
|
|
2078
|
+
});
|
|
2079
|
+
}
|
|
1753
2080
|
const ext = { ...result.error.ext };
|
|
1754
2081
|
delete ext.cleanupPending;
|
|
1755
2082
|
delete ext.cleanupError;
|
|
1756
|
-
return err({
|
|
2083
|
+
return err({
|
|
2084
|
+
...result.error,
|
|
2085
|
+
ext: Object.keys(ext).length ? ext : void 0
|
|
2086
|
+
});
|
|
1757
2087
|
}
|
|
1758
2088
|
function createProvider(config) {
|
|
1759
2089
|
switch (config.type) {
|
|
@@ -1812,7 +2142,9 @@ var vecdb = {
|
|
|
1812
2142
|
const parseResult = VecdbConfigSchema.safeParse(config);
|
|
1813
2143
|
if (!parseResult.success) {
|
|
1814
2144
|
logger5.error("Vecdb config validation failed", { error: parseResult.error.message });
|
|
1815
|
-
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_configError", {
|
|
2145
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_configError", {
|
|
2146
|
+
params: { error: parseResult.error.message }
|
|
2147
|
+
}), parseResult.error);
|
|
1816
2148
|
}
|
|
1817
2149
|
const parsed = parseResult.data;
|
|
1818
2150
|
if (core.module.requiresSharedState(process.env) && (parsed.type === "lancedb" || parsed.type === "chroma" && !!parsed.path && !parsed.url))
|
|
@@ -1835,7 +2167,11 @@ var vecdb = {
|
|
|
1835
2167
|
return ok(void 0);
|
|
1836
2168
|
} catch (error) {
|
|
1837
2169
|
logger5.error("Vecdb module initialization failed", { error });
|
|
1838
|
-
return await failInitialization(err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_initFailed", {
|
|
2170
|
+
return await failInitialization(err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_initFailed", {
|
|
2171
|
+
params: {
|
|
2172
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2173
|
+
}
|
|
2174
|
+
}), error));
|
|
1839
2175
|
}
|
|
1840
2176
|
} finally {
|
|
1841
2177
|
initInProgress = false;
|