@h-ai/vecdb 0.1.0-alpha9 → 1.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +186 -0
- package/README.ja.md +186 -0
- package/README.md +103 -12
- package/dist/index.d.ts +372 -133
- package/dist/index.js +1163 -343
- package/dist/index.js.map +1 -1
- package/package.json +21 -16
package/dist/index.js
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import { core,
|
|
2
|
+
import { core, err, ok } from '@h-ai/core';
|
|
3
|
+
import process from 'process';
|
|
4
|
+
import { spawn } from 'child_process';
|
|
5
|
+
import { setTimeout as setTimeout$1 } from 'timers/promises';
|
|
6
|
+
import { createRequire } from 'module';
|
|
3
7
|
import { createHash } from 'crypto';
|
|
4
8
|
|
|
5
9
|
// src/vecdb-config.ts
|
|
@@ -17,6 +21,11 @@ var en_US_default = {
|
|
|
17
21
|
vecdb_connectionFailed: "Failed to connect to vector database: {error}",
|
|
18
22
|
vecdb_collectionNotFound: "Collection not found: {name}",
|
|
19
23
|
vecdb_collectionAlreadyExists: "Collection already exists: {name}",
|
|
24
|
+
vecdb_invalidCollectionSchema: "Collection has no valid persisted vector dimension or distance metric: {name}",
|
|
25
|
+
vecdb_atomicUpsertUnavailable: "LanceDB collection or driver lacks safe atomic merge primary-key support; no writes were performed. Rebuild the collection or use a supported driver",
|
|
26
|
+
vecdb_duplicateDocumentIds: "Vector document IDs must be unique within a batch; no writes were performed",
|
|
27
|
+
vecdb_jsonMetadataRequired: "LanceDB collection requires native JSON metadata; rebuild legacy string metadata collections before writing or filtering",
|
|
28
|
+
vecdb_invalidMetadataFilter: "LanceDB metadata filter must be a plain object of top-level string, finite number, boolean or null values; nested objects, arrays and undefined are unsupported",
|
|
20
29
|
vecdb_dimensionMismatch: "Vector dimension mismatch: expected {expected}, actual {actual}",
|
|
21
30
|
vecdb_insertFailed: "Failed to insert vectors: {error}",
|
|
22
31
|
vecdb_deleteFailed: "Failed to delete vectors: {error}",
|
|
@@ -26,7 +35,43 @@ var en_US_default = {
|
|
|
26
35
|
vecdb_serializationFailed: "Serialization/deserialization failed: {error}",
|
|
27
36
|
vecdb_invalidDimension: "Invalid vector dimension: {dimension}, must be a positive integer",
|
|
28
37
|
vecdb_configPathRequired: "Database storage path is required",
|
|
29
|
-
vecdb_configDatabaseRequired: "Database name is required"
|
|
38
|
+
vecdb_configDatabaseRequired: "Database name is required",
|
|
39
|
+
vecdb_sharedBackendRequired: "Production mode requires a shared backend; local files and memory backends are for development or explicit end-to-end tests.",
|
|
40
|
+
vecdb_chromaEndpointInvalid: "Chroma requires an HTTP(S) root endpoint without credentials, path prefix, query or fragment",
|
|
41
|
+
vecdb_chromaTokenInvalid: "Chroma Bearer token must contain only printable ASCII characters without whitespace"
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
// messages/ja-JP.json
|
|
45
|
+
var ja_JP_default = {
|
|
46
|
+
$schema: "https://inlang.com/schema/inlang-message-format",
|
|
47
|
+
vecdb_notInitialized: "\u30D9\u30AF\u30C8\u30EB\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u304C\u521D\u671F\u5316\u3055\u308C\u3066\u3044\u307E\u305B\u3093\u3002\u5148\u306Bvecdb.init()\u3092\u547C\u3073\u51FA\u3057\u3066\u304F\u3060\u3055\u3044",
|
|
48
|
+
vecdb_initFailed: "\u30D9\u30AF\u30C8\u30EB\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u306E\u521D\u671F\u5316\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
|
|
49
|
+
vecdb_closeFailed: "\u30D9\u30AF\u30C8\u30EB\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u306E\u7D42\u4E86\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
|
|
50
|
+
vecdb_configError: "\u30D9\u30AF\u30C8\u30EB\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u8A2D\u5B9A\u306E\u691C\u8A3C\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
|
|
51
|
+
vecdb_initInProgress: "\u30D9\u30AF\u30C8\u30EB\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u306E\u521D\u671F\u5316\u306F\u65E2\u306B\u5B9F\u884C\u4E2D\u3067\u3059",
|
|
52
|
+
vecdb_unsupportedType: "\u672A\u5BFE\u5FDC\u306E\u30D9\u30AF\u30C8\u30EB\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u306E\u7A2E\u985E\u3067\u3059: {type}",
|
|
53
|
+
vecdb_driverNotFound: "\u30D9\u30AF\u30C8\u30EB\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u306E\u30C9\u30E9\u30A4\u30D0\u30FC\u304C\u30A4\u30F3\u30B9\u30C8\u30FC\u30EB\u3055\u308C\u3066\u3044\u307E\u305B\u3093: {driver}\u3002pnpm add {driver}\u3092\u5B9F\u884C\u3057\u3066\u304F\u3060\u3055\u3044",
|
|
54
|
+
vecdb_connectionFailed: "\u30D9\u30AF\u30C8\u30EB\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u3078\u306E\u63A5\u7D9A\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
|
|
55
|
+
vecdb_collectionNotFound: "\u30B3\u30EC\u30AF\u30B7\u30E7\u30F3\u304C\u898B\u3064\u304B\u308A\u307E\u305B\u3093: {name}",
|
|
56
|
+
vecdb_collectionAlreadyExists: "\u30B3\u30EC\u30AF\u30B7\u30E7\u30F3\u306F\u65E2\u306B\u5B58\u5728\u3057\u307E\u3059: {name}",
|
|
57
|
+
vecdb_invalidCollectionSchema: "\u30B3\u30EC\u30AF\u30B7\u30E7\u30F3\u306B\u6709\u52B9\u306A\u6C38\u7D9A\u5316\u6E08\u307F\u306E\u30D9\u30AF\u30C8\u30EB\u6B21\u5143\u307E\u305F\u306F\u8DDD\u96E2\u30E1\u30C8\u30EA\u30C3\u30AF\u304C\u3042\u308A\u307E\u305B\u3093\uFF1A{name}",
|
|
58
|
+
vecdb_atomicUpsertUnavailable: "LanceDB \u30B3\u30EC\u30AF\u30B7\u30E7\u30F3\u307E\u305F\u306F\u30C9\u30E9\u30A4\u30D0\u30FC\u306B\u5B89\u5168\u306A\u4E3B\u30AD\u30FC\u306B\u3088\u308B\u30A2\u30C8\u30DF\u30C3\u30AF\u306A merge \u6A5F\u80FD\u304C\u3042\u308A\u307E\u305B\u3093\u3002\u66F8\u304D\u8FBC\u307F\u306F\u884C\u3063\u3066\u3044\u307E\u305B\u3093\u3002\u30B3\u30EC\u30AF\u30B7\u30E7\u30F3\u3092\u518D\u4F5C\u6210\u3059\u308B\u304B\u3001\u5BFE\u5FDC\u30C9\u30E9\u30A4\u30D0\u30FC\u3092\u4F7F\u7528\u3057\u3066\u304F\u3060\u3055\u3044",
|
|
59
|
+
vecdb_duplicateDocumentIds: "\u540C\u4E00\u30D0\u30C3\u30C1\u306E\u30D9\u30AF\u30C8\u30EB\u6587\u66F8 ID \u306F\u4E00\u610F\u3067\u3042\u308B\u5FC5\u8981\u304C\u3042\u308A\u307E\u3059\u3002\u66F8\u304D\u8FBC\u307F\u306F\u884C\u3063\u3066\u3044\u307E\u305B\u3093",
|
|
60
|
+
vecdb_jsonMetadataRequired: "LanceDB \u30B3\u30EC\u30AF\u30B7\u30E7\u30F3\u306B\u306F\u30CD\u30A4\u30C6\u30A3\u30D6 JSON \u30E1\u30BF\u30C7\u30FC\u30BF\u304C\u5FC5\u8981\u3067\u3059\u3002\u65E7\u5F62\u5F0F\u306E\u6587\u5B57\u5217\u30E1\u30BF\u30C7\u30FC\u30BF\u3092\u4F7F\u7528\u3059\u308B\u30B3\u30EC\u30AF\u30B7\u30E7\u30F3\u306F\u3001\u66F8\u304D\u8FBC\u307F\u3084\u30D5\u30A3\u30EB\u30BF\u30FC\u51E6\u7406\u306E\u524D\u306B\u518D\u4F5C\u6210\u3057\u3066\u304F\u3060\u3055\u3044",
|
|
61
|
+
vecdb_invalidMetadataFilter: "LanceDB \u30E1\u30BF\u30C7\u30FC\u30BF\u30D5\u30A3\u30EB\u30BF\u30FC\u306F\u901A\u5E38\u306E\u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u3068\u3057\u3001\u6700\u4E0A\u4F4D\u306E\u5024\u306B\u306F\u6587\u5B57\u5217\u3001\u6709\u9650\u6570\u3001\u771F\u507D\u5024\u3001null \u306E\u307F\u6307\u5B9A\u3067\u304D\u307E\u3059\u3002\u5165\u308C\u5B50\u306E\u30AA\u30D6\u30B8\u30A7\u30AF\u30C8\u3001\u914D\u5217\u3001undefined \u306F\u4F7F\u7528\u3067\u304D\u307E\u305B\u3093",
|
|
62
|
+
vecdb_dimensionMismatch: "\u30D9\u30AF\u30C8\u30EB\u306E\u6B21\u5143\u304C\u4E00\u81F4\u3057\u307E\u305B\u3093: \u671F\u5F85\u5024{expected}\u3001\u5B9F\u969B\u306E\u5024{actual}",
|
|
63
|
+
vecdb_insertFailed: "\u30D9\u30AF\u30C8\u30EB\u306E\u633F\u5165\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
|
|
64
|
+
vecdb_deleteFailed: "\u30D9\u30AF\u30C8\u30EB\u306E\u524A\u9664\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
|
|
65
|
+
vecdb_updateFailed: "\u30D9\u30AF\u30C8\u30EB\u306E\u66F4\u65B0\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
|
|
66
|
+
vecdb_queryFailed: "\u30D9\u30AF\u30C8\u30EB\u306E\u691C\u7D22\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
|
|
67
|
+
vecdb_indexBuildFailed: "\u30A4\u30F3\u30C7\u30C3\u30AF\u30B9\u306E\u69CB\u7BC9\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
|
|
68
|
+
vecdb_serializationFailed: "\u30B7\u30EA\u30A2\u30E9\u30A4\u30BA\u307E\u305F\u306F\u30C7\u30B7\u30EA\u30A2\u30E9\u30A4\u30BA\u306B\u5931\u6557\u3057\u307E\u3057\u305F: {error}",
|
|
69
|
+
vecdb_invalidDimension: "\u30D9\u30AF\u30C8\u30EB\u306E\u6B21\u5143\u304C\u7121\u52B9\u3067\u3059: {dimension}\u3002\u6B63\u306E\u6574\u6570\u3092\u6307\u5B9A\u3057\u3066\u304F\u3060\u3055\u3044",
|
|
70
|
+
vecdb_configPathRequired: "\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u306E\u4FDD\u5B58\u30D1\u30B9\u306F\u5FC5\u9808\u3067\u3059",
|
|
71
|
+
vecdb_configDatabaseRequired: "\u30C7\u30FC\u30BF\u30D9\u30FC\u30B9\u540D\u306F\u5FC5\u9808\u3067\u3059",
|
|
72
|
+
vecdb_sharedBackendRequired: "\u672C\u756A\u74B0\u5883\u306B\u306F\u5171\u6709\u30D0\u30C3\u30AF\u30A8\u30F3\u30C9\u304C\u5FC5\u8981\u3067\u3059\u3002\u30ED\u30FC\u30AB\u30EB\u30D5\u30A1\u30A4\u30EB\u3068\u30E1\u30E2\u30EA\u30FC\u30D0\u30C3\u30AF\u30A8\u30F3\u30C9\u306F\u958B\u767A\u74B0\u5883\u307E\u305F\u306F\u660E\u793A\u7684\u306A E2E \u30C6\u30B9\u30C8\u3067\u306E\u307F\u4F7F\u7528\u3067\u304D\u307E\u3059\u3002",
|
|
73
|
+
vecdb_chromaEndpointInvalid: "Chroma \u306B\u306F HTTP(S) \u306E\u30EB\u30FC\u30C8\u30A8\u30F3\u30C9\u30DD\u30A4\u30F3\u30C8\u3092\u6307\u5B9A\u3057\u3066\u304F\u3060\u3055\u3044\u3002\u8A8D\u8A3C\u60C5\u5831\u3001\u30D1\u30B9\u30D7\u30EC\u30D5\u30A3\u30C3\u30AF\u30B9\u3001\u30AF\u30A8\u30EA\u3001\u30D5\u30E9\u30B0\u30E1\u30F3\u30C8\u306F\u4F7F\u7528\u3067\u304D\u307E\u305B\u3093",
|
|
74
|
+
vecdb_chromaTokenInvalid: "Chroma Bearer \u30C8\u30FC\u30AF\u30F3\u306B\u306F\u7A7A\u767D\u3092\u542B\u307E\u306A\u3044\u8868\u793A\u53EF\u80FD\u306A ASCII \u6587\u5B57\u306E\u307F\u4F7F\u7528\u3067\u304D\u307E\u3059"
|
|
30
75
|
};
|
|
31
76
|
|
|
32
77
|
// messages/zh-CN.json
|
|
@@ -42,6 +87,11 @@ var zh_CN_default = {
|
|
|
42
87
|
vecdb_connectionFailed: "\u8FDE\u63A5\u5411\u91CF\u6570\u636E\u5E93\u5931\u8D25: {error}",
|
|
43
88
|
vecdb_collectionNotFound: "\u96C6\u5408\u4E0D\u5B58\u5728: {name}",
|
|
44
89
|
vecdb_collectionAlreadyExists: "\u96C6\u5408\u5DF2\u5B58\u5728: {name}",
|
|
90
|
+
vecdb_invalidCollectionSchema: "\u96C6\u5408\u7F3A\u5C11\u6709\u6548\u7684\u6301\u4E45\u5316\u5411\u91CF\u7EF4\u5EA6\u6216\u8DDD\u79BB\u5EA6\u91CF\u5B9A\u4E49: {name}",
|
|
91
|
+
vecdb_atomicUpsertUnavailable: "LanceDB \u96C6\u5408\u6216\u9A71\u52A8\u7F3A\u5C11\u5B89\u5168\u7684\u539F\u5B50 merge \u4E3B\u952E\u80FD\u529B\uFF0C\u672A\u6267\u884C\u5199\u5165\uFF0C\u8BF7\u91CD\u5EFA\u96C6\u5408\u6216\u4F7F\u7528\u53D7\u652F\u6301\u7684\u9A71\u52A8",
|
|
92
|
+
vecdb_duplicateDocumentIds: "\u540C\u4E00\u6279\u5411\u91CF\u6587\u6863\u7684 ID \u4E0D\u5F97\u91CD\u590D\uFF0C\u672A\u6267\u884C\u5199\u5165",
|
|
93
|
+
vecdb_jsonMetadataRequired: "LanceDB \u96C6\u5408\u987B\u4F7F\u7528\u539F\u751F JSON \u5143\u6570\u636E\uFF0C\u65E7\u5B57\u7B26\u4E32\u5143\u6570\u636E\u96C6\u5408\u9700\u91CD\u5EFA\u540E\u624D\u80FD\u5199\u5165\u6216\u8FC7\u6EE4",
|
|
94
|
+
vecdb_invalidMetadataFilter: "LanceDB \u5143\u6570\u636E\u8FC7\u6EE4\u987B\u4E3A\u666E\u901A\u5BF9\u8C61\uFF0C\u503C\u4EC5\u652F\u6301\u9876\u5C42\u5B57\u7B26\u4E32\u3001\u6709\u9650\u6570\u503C\u3001\u5E03\u5C14\u6216 null\uFF1B\u4E0D\u652F\u6301\u5D4C\u5957\u5BF9\u8C61\u3001\u6570\u7EC4\u548C undefined",
|
|
45
95
|
vecdb_dimensionMismatch: "\u5411\u91CF\u7EF4\u5EA6\u4E0D\u5339\u914D: \u671F\u671B {expected}\uFF0C\u5B9E\u9645 {actual}",
|
|
46
96
|
vecdb_insertFailed: "\u63D2\u5165\u5411\u91CF\u5931\u8D25: {error}",
|
|
47
97
|
vecdb_deleteFailed: "\u5220\u9664\u5411\u91CF\u5931\u8D25: {error}",
|
|
@@ -51,24 +101,37 @@ var zh_CN_default = {
|
|
|
51
101
|
vecdb_serializationFailed: "\u5E8F\u5217\u5316/\u53CD\u5E8F\u5217\u5316\u5931\u8D25: {error}",
|
|
52
102
|
vecdb_invalidDimension: "\u65E0\u6548\u7684\u5411\u91CF\u7EF4\u5EA6: {dimension}\uFF0C\u5FC5\u987B\u4E3A\u6B63\u6574\u6570",
|
|
53
103
|
vecdb_configPathRequired: "\u6570\u636E\u5E93\u5B58\u50A8\u8DEF\u5F84\u4E0D\u80FD\u4E3A\u7A7A",
|
|
54
|
-
vecdb_configDatabaseRequired: "\u6570\u636E\u5E93\u540D\u79F0\u4E0D\u80FD\u4E3A\u7A7A"
|
|
104
|
+
vecdb_configDatabaseRequired: "\u6570\u636E\u5E93\u540D\u79F0\u4E0D\u80FD\u4E3A\u7A7A",
|
|
105
|
+
vecdb_sharedBackendRequired: "\u751F\u4EA7\u6A21\u5F0F\u9700\u8981\u8DE8\u8282\u70B9\u5171\u4EAB\u540E\u7AEF\uFF1B\u672C\u673A\u6587\u4EF6\u6216\u5185\u5B58\u540E\u7AEF\u4EC5\u7528\u4E8E\u5F00\u53D1\u548C\u663E\u5F0F\u7AEF\u5230\u7AEF\u6D4B\u8BD5\u3002",
|
|
106
|
+
vecdb_chromaEndpointInvalid: "Chroma \u4EC5\u652F\u6301 HTTP(S) \u6839\u7AEF\u70B9\uFF0C\u4E0D\u5F97\u5305\u542B\u51ED\u636E\u3001\u8DEF\u5F84\u524D\u7F00\u3001query \u6216 fragment",
|
|
107
|
+
vecdb_chromaTokenInvalid: "Chroma Bearer token \u4EC5\u652F\u6301\u65E0\u7A7A\u767D\u7684\u53EF\u6253\u5370 ASCII \u5B57\u7B26"
|
|
55
108
|
};
|
|
56
109
|
|
|
57
110
|
// src/vecdb-i18n.ts
|
|
58
111
|
var vecdbM = core.i18n.createMessageGetter({
|
|
59
112
|
"zh-CN": zh_CN_default,
|
|
60
|
-
"en-US": en_US_default
|
|
113
|
+
"en-US": en_US_default,
|
|
114
|
+
"ja-JP": ja_JP_default
|
|
61
115
|
});
|
|
62
116
|
|
|
63
117
|
// src/vecdb-config.ts
|
|
64
|
-
var VecdbTypeSchema = z.enum(["lancedb", "pgvector", "qdrant"]);
|
|
65
118
|
var DistanceMetricSchema = z.enum(["cosine", "euclidean", "dot"]).default("cosine");
|
|
119
|
+
var VecdbOperationLogConfigSchema = z.object({
|
|
120
|
+
read: z.boolean().default(false),
|
|
121
|
+
write: z.boolean().default(false),
|
|
122
|
+
maxLength: z.number().int().min(0).default(1e3),
|
|
123
|
+
level: z.enum(["info", "debug", "trace"]).default("debug")
|
|
124
|
+
});
|
|
125
|
+
var OperationLogConfigFieldSchema = {
|
|
126
|
+
operationLog: VecdbOperationLogConfigSchema.optional()
|
|
127
|
+
};
|
|
66
128
|
var LancedbConfigSchema = z.object({
|
|
67
129
|
type: z.literal("lancedb"),
|
|
68
|
-
/**
|
|
130
|
+
/** 持久化本地目录;独立节点的同名路径不会自动共享,分布式部署优先使用服务端后端。 */
|
|
69
131
|
path: z.string().min(1, vecdbM("vecdb_configPathRequired")),
|
|
70
|
-
/**
|
|
71
|
-
metric: DistanceMetricSchema.optional()
|
|
132
|
+
/** 新建集合的默认距离度量;已有集合使用 Arrow schema 中的持久定义。 */
|
|
133
|
+
metric: DistanceMetricSchema.optional(),
|
|
134
|
+
...OperationLogConfigFieldSchema
|
|
72
135
|
});
|
|
73
136
|
var PgvectorIndexTypeSchema = z.enum(["ivfflat", "hnsw"]).default("hnsw");
|
|
74
137
|
var PgvectorConfigSchema = z.object({
|
|
@@ -90,7 +153,8 @@ var PgvectorConfigSchema = z.object({
|
|
|
90
153
|
/** 距离度量(默认 cosine) */
|
|
91
154
|
metric: DistanceMetricSchema.optional(),
|
|
92
155
|
/** 表名前缀(默认 'vec_') */
|
|
93
|
-
tablePrefix: z.string().default("vec_")
|
|
156
|
+
tablePrefix: z.string().default("vec_"),
|
|
157
|
+
...OperationLogConfigFieldSchema
|
|
94
158
|
});
|
|
95
159
|
var QdrantConfigSchema = z.object({
|
|
96
160
|
type: z.literal("qdrant"),
|
|
@@ -99,12 +163,41 @@ var QdrantConfigSchema = z.object({
|
|
|
99
163
|
/** API Key(可选) */
|
|
100
164
|
apiKey: z.string().optional(),
|
|
101
165
|
/** 距离度量(默认 cosine) */
|
|
102
|
-
metric: DistanceMetricSchema.optional()
|
|
166
|
+
metric: DistanceMetricSchema.optional(),
|
|
167
|
+
...OperationLogConfigFieldSchema
|
|
168
|
+
});
|
|
169
|
+
var ChromaConfigSchema = z.object({
|
|
170
|
+
type: z.literal("chroma"),
|
|
171
|
+
/** 本地持久化路径(嵌入式模式:提供且无 url 时自动拉起本地服务) */
|
|
172
|
+
path: z.string().optional(),
|
|
173
|
+
/** Chroma HTTP(S) 根端点;不可包含凭据、路径前缀、query 或 fragment;提供时优先于 host/port。 */
|
|
174
|
+
url: z.string().url().refine((value) => {
|
|
175
|
+
try {
|
|
176
|
+
const endpoint = new URL(value);
|
|
177
|
+
return ["http:", "https:"].includes(endpoint.protocol) && !endpoint.username && !endpoint.password && endpoint.pathname === "/" && !endpoint.search && !endpoint.hash;
|
|
178
|
+
} catch {
|
|
179
|
+
return false;
|
|
180
|
+
}
|
|
181
|
+
}, { error: () => vecdbM("vecdb_chromaEndpointInvalid") }).optional(),
|
|
182
|
+
/** 主机(默认 localhost) */
|
|
183
|
+
host: z.string().default("localhost"),
|
|
184
|
+
/** 端口(默认 8000) */
|
|
185
|
+
port: z.number().int().min(1).max(65535).default(8e3),
|
|
186
|
+
/** 可选 Bearer token,发送 Authorization 请求头;用于采用此协议的自托管服务/网关,不是 Chroma Cloud 配置。空串表示未配置。 */
|
|
187
|
+
apiKey: z.string().regex(/^[\x21-\x7E]*$/, { error: () => vecdbM("vecdb_chromaTokenInvalid") }).optional(),
|
|
188
|
+
/** 拉起本地服务的可执行命令(默认 'chroma',可指定绝对路径) */
|
|
189
|
+
serverCommand: z.string().default("chroma"),
|
|
190
|
+
/** 拉起本地服务的就绪等待超时(毫秒,默认 30000) */
|
|
191
|
+
startupTimeout: z.number().int().positive().default(3e4),
|
|
192
|
+
/** 距离度量(默认 cosine) */
|
|
193
|
+
metric: DistanceMetricSchema.optional(),
|
|
194
|
+
...OperationLogConfigFieldSchema
|
|
103
195
|
});
|
|
104
196
|
var VecdbConfigSchema = z.discriminatedUnion("type", [
|
|
105
197
|
LancedbConfigSchema,
|
|
106
198
|
PgvectorConfigSchema,
|
|
107
|
-
QdrantConfigSchema
|
|
199
|
+
QdrantConfigSchema,
|
|
200
|
+
ChromaConfigSchema
|
|
108
201
|
]);
|
|
109
202
|
var VecdbErrorInfo = {
|
|
110
203
|
CONNECTION_FAILED: "001:500",
|
|
@@ -123,6 +216,37 @@ var VecdbErrorInfo = {
|
|
|
123
216
|
SERIALIZATION_FAILED: "014:500"
|
|
124
217
|
};
|
|
125
218
|
var HaiVecdbError = core.error.buildHaiErrorsDef("vecdb", VecdbErrorInfo);
|
|
219
|
+
function stringifyOperationPayload(payload, maxLength) {
|
|
220
|
+
if (maxLength <= 0) {
|
|
221
|
+
return "";
|
|
222
|
+
}
|
|
223
|
+
try {
|
|
224
|
+
const text = JSON.stringify(payload, (_key, value) => {
|
|
225
|
+
if (typeof value === "bigint") {
|
|
226
|
+
return value.toString();
|
|
227
|
+
}
|
|
228
|
+
if (typeof value === "function") {
|
|
229
|
+
return "[Function]";
|
|
230
|
+
}
|
|
231
|
+
return value;
|
|
232
|
+
});
|
|
233
|
+
return text.length > maxLength ? `${text.slice(0, maxLength)}...` : text;
|
|
234
|
+
} catch {
|
|
235
|
+
const text = String(payload);
|
|
236
|
+
return text.length > maxLength ? `${text.slice(0, maxLength)}...` : text;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
function logVecdbOperation(ctx, category, operation, payload) {
|
|
240
|
+
const config = ctx.operationLog?.();
|
|
241
|
+
if (!config?.[category]) {
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
ctx.logger[config.level]("VecDB operation executed", {
|
|
245
|
+
category,
|
|
246
|
+
operation,
|
|
247
|
+
payload: stringifyOperationPayload(payload, config.maxLength)
|
|
248
|
+
});
|
|
249
|
+
}
|
|
126
250
|
function errorMsgFromCode(def, errorStr) {
|
|
127
251
|
switch (def) {
|
|
128
252
|
case HaiVecdbError.DELETE_FAILED:
|
|
@@ -155,6 +279,7 @@ function createBaseCollectionOps(ctx, driver) {
|
|
|
155
279
|
vecdbM("vecdb_invalidDimension", { params: { dimension: String(options.dimension) } })
|
|
156
280
|
);
|
|
157
281
|
}
|
|
282
|
+
logVecdbOperation(ctx, "write", "collection.create", { name, options });
|
|
158
283
|
return wrapOp(
|
|
159
284
|
ctx,
|
|
160
285
|
() => driver.create(name, options),
|
|
@@ -163,250 +288,715 @@ function createBaseCollectionOps(ctx, driver) {
|
|
|
163
288
|
{ name }
|
|
164
289
|
);
|
|
165
290
|
},
|
|
166
|
-
drop: (name) =>
|
|
167
|
-
ctx,
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
"
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
)
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
291
|
+
drop: (name) => {
|
|
292
|
+
logVecdbOperation(ctx, "write", "collection.drop", { name });
|
|
293
|
+
return wrapOp(
|
|
294
|
+
ctx,
|
|
295
|
+
() => driver.drop(name),
|
|
296
|
+
HaiVecdbError.DELETE_FAILED,
|
|
297
|
+
"Failed to drop collection",
|
|
298
|
+
{ name }
|
|
299
|
+
);
|
|
300
|
+
},
|
|
301
|
+
exists: (name) => {
|
|
302
|
+
logVecdbOperation(ctx, "read", "collection.exists", { name });
|
|
303
|
+
return wrapOp(
|
|
304
|
+
ctx,
|
|
305
|
+
() => driver.exists(name),
|
|
306
|
+
HaiVecdbError.QUERY_FAILED,
|
|
307
|
+
"Failed to check collection",
|
|
308
|
+
{ name }
|
|
309
|
+
);
|
|
310
|
+
},
|
|
311
|
+
info: (name) => {
|
|
312
|
+
logVecdbOperation(ctx, "read", "collection.info", { name });
|
|
313
|
+
return wrapOp(
|
|
314
|
+
ctx,
|
|
315
|
+
() => driver.info(name),
|
|
316
|
+
HaiVecdbError.QUERY_FAILED,
|
|
317
|
+
"Failed to get collection info",
|
|
318
|
+
{ name }
|
|
319
|
+
);
|
|
320
|
+
},
|
|
321
|
+
list: () => {
|
|
322
|
+
logVecdbOperation(ctx, "read", "collection.list", {});
|
|
323
|
+
return wrapOp(
|
|
324
|
+
ctx,
|
|
325
|
+
() => driver.list(),
|
|
326
|
+
HaiVecdbError.QUERY_FAILED,
|
|
327
|
+
"Failed to list collections"
|
|
328
|
+
);
|
|
329
|
+
}
|
|
193
330
|
};
|
|
194
331
|
}
|
|
195
332
|
function createBaseVectorOps(ctx, driver) {
|
|
196
333
|
return {
|
|
197
|
-
insert: (collection, documents) =>
|
|
198
|
-
ctx,
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
"
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
)
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
334
|
+
insert: (collection, documents) => {
|
|
335
|
+
logVecdbOperation(ctx, "write", "vector.insert", { collection, count: documents.length, documents });
|
|
336
|
+
return wrapOp(
|
|
337
|
+
ctx,
|
|
338
|
+
() => documents.length === 0 ? Promise.resolve(ok(void 0)) : driver.insert(collection, documents),
|
|
339
|
+
HaiVecdbError.INSERT_FAILED,
|
|
340
|
+
"Failed to insert vectors",
|
|
341
|
+
{ collection }
|
|
342
|
+
);
|
|
343
|
+
},
|
|
344
|
+
upsert: (collection, documents) => {
|
|
345
|
+
logVecdbOperation(ctx, "write", "vector.upsert", { collection, count: documents.length, documents });
|
|
346
|
+
return wrapOp(
|
|
347
|
+
ctx,
|
|
348
|
+
() => documents.length === 0 ? Promise.resolve(ok(void 0)) : driver.upsert(collection, documents),
|
|
349
|
+
HaiVecdbError.UPDATE_FAILED,
|
|
350
|
+
"Failed to upsert vectors",
|
|
351
|
+
{ collection }
|
|
352
|
+
);
|
|
353
|
+
},
|
|
354
|
+
delete: (collection, ids) => {
|
|
355
|
+
logVecdbOperation(ctx, "write", "vector.delete", { collection, count: ids.length, ids });
|
|
356
|
+
return wrapOp(
|
|
357
|
+
ctx,
|
|
358
|
+
() => ids.length === 0 ? Promise.resolve(ok(void 0)) : driver.delete(collection, ids),
|
|
359
|
+
HaiVecdbError.DELETE_FAILED,
|
|
360
|
+
"Failed to delete vectors",
|
|
361
|
+
{ collection }
|
|
362
|
+
);
|
|
363
|
+
},
|
|
364
|
+
search: (collection, vector, options) => {
|
|
365
|
+
logVecdbOperation(ctx, "read", "vector.search", { collection, vector, options });
|
|
366
|
+
return wrapOp(
|
|
367
|
+
ctx,
|
|
368
|
+
() => driver.search(collection, vector, options),
|
|
369
|
+
HaiVecdbError.QUERY_FAILED,
|
|
370
|
+
"Failed to search vectors",
|
|
371
|
+
{ collection }
|
|
372
|
+
);
|
|
373
|
+
},
|
|
374
|
+
count: (collection) => {
|
|
375
|
+
logVecdbOperation(ctx, "read", "vector.count", { collection });
|
|
376
|
+
return wrapOp(
|
|
377
|
+
ctx,
|
|
378
|
+
() => driver.count(collection),
|
|
379
|
+
HaiVecdbError.QUERY_FAILED,
|
|
380
|
+
"Failed to count vectors",
|
|
381
|
+
{ collection }
|
|
382
|
+
);
|
|
383
|
+
}
|
|
232
384
|
};
|
|
233
385
|
}
|
|
234
386
|
|
|
235
|
-
// src/providers/vecdb-provider-
|
|
236
|
-
var logger = core.logger.child({ module: "vecdb", scope: "
|
|
237
|
-
|
|
238
|
-
|
|
387
|
+
// src/providers/vecdb-provider-chroma.ts
|
|
388
|
+
var logger = core.logger.child({ module: "vecdb", scope: "chroma" });
|
|
389
|
+
var DIMENSION_METADATA_KEY = "hai_dimension";
|
|
390
|
+
var SPACE_METADATA_KEY = "hnsw:space";
|
|
391
|
+
var HEARTBEAT_POLL_INTERVAL = 300;
|
|
392
|
+
function isChromaModule(value) {
|
|
393
|
+
return typeof value === "object" && value !== null && typeof value.ChromaClient === "function";
|
|
239
394
|
}
|
|
240
|
-
|
|
241
|
-
|
|
395
|
+
var NOOP_EMBEDDING_FUNCTION = {
|
|
396
|
+
name: "hai-noop",
|
|
397
|
+
async generate() {
|
|
398
|
+
throw new Error("Chroma embedding function should not be called: vectors are provided explicitly");
|
|
399
|
+
}
|
|
400
|
+
};
|
|
401
|
+
function toChromaSpace(metric) {
|
|
402
|
+
if (metric === "euclidean")
|
|
403
|
+
return "l2";
|
|
404
|
+
if (metric === "dot")
|
|
405
|
+
return "ip";
|
|
406
|
+
return "cosine";
|
|
407
|
+
}
|
|
408
|
+
function fromChromaSpace(space) {
|
|
409
|
+
if (space === "l2")
|
|
410
|
+
return "euclidean";
|
|
411
|
+
if (space === "ip")
|
|
412
|
+
return "dot";
|
|
413
|
+
return "cosine";
|
|
414
|
+
}
|
|
415
|
+
function distanceToScore(distance, metric) {
|
|
416
|
+
if (metric === "cosine")
|
|
417
|
+
return Math.max(0, Math.min(1, 1 - distance));
|
|
418
|
+
return 1 / (1 + Math.max(0, distance));
|
|
419
|
+
}
|
|
420
|
+
function buildWhere(filter) {
|
|
421
|
+
if (!filter)
|
|
422
|
+
return void 0;
|
|
423
|
+
const entries = Object.entries(filter);
|
|
424
|
+
if (entries.length === 0)
|
|
425
|
+
return void 0;
|
|
426
|
+
const where = entries.length === 1 ? { [entries[0][0]]: entries[0][1] } : { $and: entries.map(([key, value]) => ({ [key]: value })) };
|
|
427
|
+
return where;
|
|
428
|
+
}
|
|
429
|
+
function createChromaProvider() {
|
|
430
|
+
let client = null;
|
|
242
431
|
let config = null;
|
|
243
|
-
|
|
244
|
-
|
|
432
|
+
let serverProcess = null;
|
|
433
|
+
let spawnFailed = false;
|
|
434
|
+
let closing = null;
|
|
435
|
+
function isMissingModuleError(error) {
|
|
436
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
437
|
+
return message.includes("chromadb");
|
|
438
|
+
}
|
|
439
|
+
async function loadChromaModule() {
|
|
245
440
|
try {
|
|
246
|
-
const mod = await import('
|
|
441
|
+
const mod = await import('chromadb');
|
|
442
|
+
if (!isChromaModule(mod)) {
|
|
443
|
+
const error = new Error("chromadb driver module does not export ChromaClient");
|
|
444
|
+
logger.error("Failed to load Chroma driver", { error: error.message });
|
|
445
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", { params: { driver: "chromadb" } }), error);
|
|
446
|
+
}
|
|
247
447
|
return ok(mod);
|
|
248
448
|
} catch (error) {
|
|
249
|
-
|
|
250
|
-
|
|
449
|
+
if (isMissingModuleError(error)) {
|
|
450
|
+
logger.error("Failed to load Chroma driver", { error: error instanceof Error ? error.message : String(error) });
|
|
451
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", { params: { driver: "chromadb" } }), error);
|
|
452
|
+
}
|
|
453
|
+
throw error;
|
|
251
454
|
}
|
|
252
455
|
}
|
|
253
|
-
async function
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
456
|
+
async function startEmbeddedServer(chromaConfig) {
|
|
457
|
+
const args = ["run", "--path", chromaConfig.path ?? "./data/chroma", "--host", chromaConfig.host, "--port", String(chromaConfig.port)];
|
|
458
|
+
logger.info("Starting embedded Chroma server", { command: chromaConfig.serverCommand, host: chromaConfig.host, port: chromaConfig.port });
|
|
459
|
+
spawnFailed = false;
|
|
460
|
+
const proc = spawn(chromaConfig.serverCommand, args, { stdio: "ignore" });
|
|
461
|
+
proc.on("error", (error) => {
|
|
462
|
+
spawnFailed = proc.pid === void 0;
|
|
463
|
+
logger.error("Chroma server process error", { error: error.message });
|
|
464
|
+
});
|
|
465
|
+
serverProcess = proc;
|
|
466
|
+
await new Promise((resolve, reject) => {
|
|
467
|
+
proc.once("spawn", resolve);
|
|
468
|
+
proc.once("error", reject);
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
async function stopEmbeddedServer() {
|
|
472
|
+
const owned = serverProcess;
|
|
473
|
+
if (!owned)
|
|
474
|
+
return;
|
|
475
|
+
const hasExited = () => owned.exitCode !== null || owned.signalCode !== null || spawnFailed && owned.pid === void 0;
|
|
476
|
+
if (!hasExited()) {
|
|
477
|
+
const child = owned;
|
|
478
|
+
await new Promise((resolve, reject) => {
|
|
479
|
+
let timer;
|
|
480
|
+
let settled = false;
|
|
481
|
+
function cleanup() {
|
|
482
|
+
clearTimeout(timer);
|
|
483
|
+
child.removeListener("exit", exited);
|
|
484
|
+
child.removeListener("error", failed);
|
|
485
|
+
}
|
|
486
|
+
function exited() {
|
|
487
|
+
if (settled)
|
|
488
|
+
return;
|
|
489
|
+
settled = true;
|
|
490
|
+
cleanup();
|
|
491
|
+
resolve();
|
|
492
|
+
}
|
|
493
|
+
function failed(cause) {
|
|
494
|
+
if (settled)
|
|
495
|
+
return;
|
|
496
|
+
if (hasExited()) {
|
|
497
|
+
exited();
|
|
498
|
+
return;
|
|
499
|
+
}
|
|
500
|
+
settled = true;
|
|
501
|
+
cleanup();
|
|
502
|
+
reject(cause);
|
|
503
|
+
}
|
|
504
|
+
function signal(name) {
|
|
505
|
+
try {
|
|
506
|
+
if (!child.kill(name))
|
|
507
|
+
failed(new Error("Chroma child termination signal was rejected"));
|
|
508
|
+
} catch (cause) {
|
|
509
|
+
failed(cause instanceof Error ? cause : new Error(String(cause)));
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
child.once("exit", exited);
|
|
513
|
+
child.once("error", failed);
|
|
514
|
+
timer = setTimeout(() => {
|
|
515
|
+
timer = setTimeout(() => failed(new Error("Chroma child exit was not confirmed")), 1500);
|
|
516
|
+
signal("SIGKILL");
|
|
517
|
+
}, 1500);
|
|
518
|
+
if (hasExited())
|
|
519
|
+
exited();
|
|
520
|
+
else signal("SIGTERM");
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
if (serverProcess === owned)
|
|
524
|
+
serverProcess = null;
|
|
525
|
+
}
|
|
526
|
+
async function closeResources() {
|
|
527
|
+
if (!closing) {
|
|
528
|
+
client = null;
|
|
529
|
+
config = null;
|
|
530
|
+
closing = (async () => {
|
|
531
|
+
try {
|
|
532
|
+
await stopEmbeddedServer();
|
|
533
|
+
logger.info("Chroma connection closed");
|
|
534
|
+
return ok(void 0);
|
|
535
|
+
} catch (cause) {
|
|
536
|
+
return err({ ...HaiVecdbError.CONNECTION_FAILED, message: vecdbM("vecdb_closeFailed", { params: { error: cause instanceof Error ? cause.message : String(cause) } }), cause, ext: { cleanupPending: true } });
|
|
537
|
+
}
|
|
538
|
+
})().finally(() => {
|
|
539
|
+
closing = null;
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
return await closing;
|
|
543
|
+
}
|
|
544
|
+
async function waitForHeartbeat(chromaClient, timeout) {
|
|
545
|
+
const deadline = Date.now() + timeout;
|
|
546
|
+
let lastError;
|
|
547
|
+
while (Date.now() < deadline) {
|
|
548
|
+
try {
|
|
549
|
+
await chromaClient.heartbeat();
|
|
550
|
+
return;
|
|
551
|
+
} catch (error) {
|
|
552
|
+
lastError = error;
|
|
553
|
+
await setTimeout$1(HEARTBEAT_POLL_INTERVAL);
|
|
260
554
|
}
|
|
261
|
-
return null;
|
|
262
|
-
} catch {
|
|
263
|
-
return null;
|
|
264
555
|
}
|
|
556
|
+
throw new Error(`Chroma heartbeat timed out after ${timeout}ms: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
|
|
557
|
+
}
|
|
558
|
+
async function getCollection(name) {
|
|
559
|
+
return client.getCollection({ name, embeddingFunction: NOOP_EMBEDDING_FUNCTION });
|
|
560
|
+
}
|
|
561
|
+
function readDimension(collection) {
|
|
562
|
+
const value = collection.metadata?.[DIMENSION_METADATA_KEY];
|
|
563
|
+
return typeof value === "number" ? value : 0;
|
|
265
564
|
}
|
|
266
|
-
|
|
565
|
+
function validateDimension(expected, actual) {
|
|
566
|
+
if (expected === 0 || expected === actual)
|
|
567
|
+
return null;
|
|
568
|
+
return err(
|
|
569
|
+
HaiVecdbError.DIMENSION_MISMATCH,
|
|
570
|
+
vecdbM("vecdb_dimensionMismatch", { params: { expected: String(expected), actual: String(actual) } })
|
|
571
|
+
);
|
|
572
|
+
}
|
|
573
|
+
const ctx = { isConnected: () => client !== null, logger, operationLog: () => config?.operationLog };
|
|
267
574
|
const collectionDriver = {
|
|
268
575
|
async create(name, options) {
|
|
269
576
|
logger.debug("Creating collection", { name, dimension: options.dimension, metric: options.metric });
|
|
270
|
-
const
|
|
271
|
-
|
|
577
|
+
const existing = await client.listCollections();
|
|
578
|
+
const exists = existing.some((item) => item.name === name);
|
|
579
|
+
if (exists)
|
|
272
580
|
return err(HaiVecdbError.COLLECTION_ALREADY_EXISTS, vecdbM("vecdb_collectionAlreadyExists", { params: { name } }));
|
|
273
|
-
}
|
|
274
|
-
const dimension = options.dimension;
|
|
275
581
|
const metric = options.metric ?? config?.metric ?? "cosine";
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
};
|
|
282
|
-
const table = await connection.createTable(name, [initRecord]);
|
|
283
|
-
await table.delete('id = "__init__"');
|
|
284
|
-
collectionMetas.set(name, { dimension, metric });
|
|
285
|
-
logger.info("Collection created", { name, dimension, metric });
|
|
582
|
+
await client.getOrCreateCollection({
|
|
583
|
+
name,
|
|
584
|
+
metadata: { [SPACE_METADATA_KEY]: toChromaSpace(metric), [DIMENSION_METADATA_KEY]: options.dimension },
|
|
585
|
+
embeddingFunction: NOOP_EMBEDDING_FUNCTION
|
|
586
|
+
});
|
|
587
|
+
logger.info("Collection created", { name, dimension: options.dimension, metric });
|
|
286
588
|
return ok(void 0);
|
|
287
589
|
},
|
|
288
590
|
async drop(name) {
|
|
289
591
|
logger.debug("Dropping collection", { name });
|
|
290
|
-
const
|
|
291
|
-
|
|
592
|
+
const existing = await client.listCollections();
|
|
593
|
+
const exists = existing.some((item) => item.name === name);
|
|
594
|
+
if (!exists)
|
|
292
595
|
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name } }));
|
|
293
|
-
}
|
|
294
|
-
await connection.dropTable(name);
|
|
295
|
-
collectionMetas.delete(name);
|
|
596
|
+
await client.deleteCollection({ name });
|
|
296
597
|
logger.info("Collection dropped", { name });
|
|
297
598
|
return ok(void 0);
|
|
298
599
|
},
|
|
299
600
|
async exists(name) {
|
|
300
601
|
logger.debug("Checking collection exists", { name });
|
|
301
|
-
const
|
|
302
|
-
return ok(
|
|
602
|
+
const existing = await client.listCollections();
|
|
603
|
+
return ok(existing.some((item) => item.name === name));
|
|
303
604
|
},
|
|
304
605
|
async info(name) {
|
|
305
606
|
logger.debug("Getting collection info", { name });
|
|
306
|
-
const
|
|
307
|
-
if (!
|
|
607
|
+
const existing = await client.listCollections();
|
|
608
|
+
if (!existing.some((item) => item.name === name))
|
|
308
609
|
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name } }));
|
|
309
|
-
|
|
310
|
-
const count = await
|
|
311
|
-
const meta = collectionMetas.get(name);
|
|
610
|
+
const collection = await getCollection(name);
|
|
611
|
+
const count = await collection.count();
|
|
312
612
|
return ok({
|
|
313
613
|
name,
|
|
314
|
-
dimension:
|
|
315
|
-
metric:
|
|
614
|
+
dimension: readDimension(collection),
|
|
615
|
+
metric: fromChromaSpace(collection.metadata?.[SPACE_METADATA_KEY]),
|
|
316
616
|
count
|
|
317
617
|
});
|
|
318
618
|
},
|
|
319
619
|
async list() {
|
|
320
620
|
logger.debug("Listing collections");
|
|
321
|
-
const
|
|
322
|
-
return ok(
|
|
621
|
+
const existing = await client.listCollections();
|
|
622
|
+
return ok(existing.map((item) => item.name));
|
|
323
623
|
}
|
|
324
624
|
};
|
|
625
|
+
function toWriteParams(documents) {
|
|
626
|
+
return {
|
|
627
|
+
ids: documents.map((doc) => doc.id),
|
|
628
|
+
embeddings: documents.map((doc) => doc.vector),
|
|
629
|
+
documents: documents.map((doc) => doc.content ?? ""),
|
|
630
|
+
// 统一 API 的 metadata 更宽;SDK 在 add/upsert 中先执行 validateMetadata,再发送请求。
|
|
631
|
+
// 使用上游 Metadata 类型跨越此边界,避免复制其标量、数组及稀疏向量规则;原生拒绝会由 base 转为 HaiResult。
|
|
632
|
+
metadatas: documents.map((doc) => ({ ...doc.metadata, _content: doc.content ?? "" }))
|
|
633
|
+
};
|
|
634
|
+
}
|
|
325
635
|
const vectorDriver = {
|
|
326
636
|
async insert(collection, documents) {
|
|
327
637
|
logger.debug("Inserting vectors", { collection, count: documents.length });
|
|
328
|
-
const
|
|
329
|
-
|
|
330
|
-
|
|
638
|
+
const target = await getCollection(collection);
|
|
639
|
+
const expected = readDimension(target);
|
|
640
|
+
for (const doc of documents) {
|
|
641
|
+
const validation = validateDimension(expected, doc.vector.length);
|
|
642
|
+
if (validation)
|
|
643
|
+
return validation;
|
|
331
644
|
}
|
|
332
|
-
|
|
333
|
-
id: doc.id,
|
|
334
|
-
vector: doc.vector,
|
|
335
|
-
content: doc.content ?? "",
|
|
336
|
-
metadata: JSON.stringify(doc.metadata ?? {})
|
|
337
|
-
}));
|
|
338
|
-
await table.add(records);
|
|
645
|
+
await target.add(toWriteParams(documents));
|
|
339
646
|
logger.info("Vectors inserted", { collection, count: documents.length });
|
|
340
647
|
return ok(void 0);
|
|
341
648
|
},
|
|
342
649
|
async upsert(collection, documents) {
|
|
343
650
|
logger.debug("Upserting vectors", { collection, count: documents.length });
|
|
344
|
-
const
|
|
345
|
-
|
|
346
|
-
|
|
651
|
+
const target = await getCollection(collection);
|
|
652
|
+
const expected = readDimension(target);
|
|
653
|
+
for (const doc of documents) {
|
|
654
|
+
const validation = validateDimension(expected, doc.vector.length);
|
|
655
|
+
if (validation)
|
|
656
|
+
return validation;
|
|
347
657
|
}
|
|
348
|
-
|
|
349
|
-
id: doc.id,
|
|
350
|
-
vector: doc.vector,
|
|
351
|
-
content: doc.content ?? "",
|
|
352
|
-
metadata: JSON.stringify(doc.metadata ?? {})
|
|
353
|
-
}));
|
|
354
|
-
const ids = documents.map((d) => `"${escapeLanceFilterValue(d.id)}"`).join(", ");
|
|
355
|
-
await table.delete(`id IN (${ids})`);
|
|
356
|
-
await table.add(records);
|
|
658
|
+
await target.upsert(toWriteParams(documents));
|
|
357
659
|
logger.info("Vectors upserted", { collection, count: documents.length });
|
|
358
660
|
return ok(void 0);
|
|
359
661
|
},
|
|
360
662
|
async delete(collection, ids) {
|
|
361
663
|
logger.debug("Deleting vectors", { collection, count: ids.length });
|
|
362
|
-
const
|
|
363
|
-
|
|
364
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name: collection } }));
|
|
365
|
-
}
|
|
366
|
-
const idList = ids.map((id) => `"${escapeLanceFilterValue(id)}"`).join(", ");
|
|
367
|
-
await table.delete(`id IN (${idList})`);
|
|
664
|
+
const target = await getCollection(collection);
|
|
665
|
+
await target.delete({ ids });
|
|
368
666
|
logger.info("Vectors deleted", { collection, count: ids.length });
|
|
369
667
|
return ok(void 0);
|
|
370
668
|
},
|
|
371
669
|
async search(collection, vector, options) {
|
|
372
|
-
logger.debug("Searching vectors", { collection, topK: options?.topK, hasFilter: !!options?.filter });
|
|
373
|
-
const table = await openTable(collection);
|
|
374
|
-
if (!table) {
|
|
375
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name: collection } }));
|
|
376
|
-
}
|
|
377
670
|
const topK = options?.topK ?? 10;
|
|
378
671
|
const minScore = options?.minScore ?? 0;
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
const
|
|
390
|
-
const
|
|
391
|
-
|
|
392
|
-
const score =
|
|
393
|
-
|
|
394
|
-
|
|
672
|
+
logger.debug("Searching vectors", { collection, topK, hasFilter: !!options?.filter });
|
|
673
|
+
const target = await getCollection(collection);
|
|
674
|
+
const expected = readDimension(target);
|
|
675
|
+
const validation = validateDimension(expected, vector.length);
|
|
676
|
+
if (validation)
|
|
677
|
+
return validation;
|
|
678
|
+
const metric = fromChromaSpace(target.metadata?.[SPACE_METADATA_KEY]);
|
|
679
|
+
const response = await target.query({ queryEmbeddings: [vector], nResults: topK, where: buildWhere(options?.filter) });
|
|
680
|
+
const ids = response.ids[0] ?? [];
|
|
681
|
+
const distances = response.distances?.[0] ?? [];
|
|
682
|
+
const metadatas = response.metadatas?.[0] ?? [];
|
|
683
|
+
const results = [];
|
|
684
|
+
for (let i = 0; i < ids.length; i++) {
|
|
685
|
+
const score = distanceToScore(distances[i] ?? 0, metric);
|
|
686
|
+
if (score < minScore)
|
|
687
|
+
continue;
|
|
688
|
+
const metadata = metadatas[i] ?? {};
|
|
689
|
+
const { _content, ...rest } = metadata;
|
|
690
|
+
results.push({
|
|
691
|
+
id: ids[i],
|
|
395
692
|
score,
|
|
396
|
-
content:
|
|
397
|
-
metadata:
|
|
398
|
-
};
|
|
399
|
-
}
|
|
400
|
-
return ok(
|
|
693
|
+
content: typeof _content === "string" && _content.length > 0 ? _content : void 0,
|
|
694
|
+
metadata: Object.keys(rest).length > 0 ? rest : void 0
|
|
695
|
+
});
|
|
696
|
+
}
|
|
697
|
+
return ok(results);
|
|
401
698
|
},
|
|
402
699
|
async count(collection) {
|
|
403
700
|
logger.debug("Counting vectors", { collection });
|
|
404
|
-
const
|
|
405
|
-
|
|
406
|
-
|
|
701
|
+
const target = await getCollection(collection);
|
|
702
|
+
return ok(await target.count());
|
|
703
|
+
}
|
|
704
|
+
};
|
|
705
|
+
return {
|
|
706
|
+
name: "chroma",
|
|
707
|
+
async connect(cfg) {
|
|
708
|
+
if (cfg.type !== "chroma")
|
|
709
|
+
return err(HaiVecdbError.UNSUPPORTED_TYPE, vecdbM("vecdb_unsupportedType", { params: { type: cfg.type } }));
|
|
710
|
+
if (serverProcess || client || closing) {
|
|
711
|
+
const cleaned = await closeResources();
|
|
712
|
+
if (!cleaned.success)
|
|
713
|
+
return cleaned;
|
|
714
|
+
}
|
|
715
|
+
const chromaConfig = cfg;
|
|
716
|
+
const embedded = !!chromaConfig.path && !chromaConfig.url;
|
|
717
|
+
const loadResult = await loadChromaModule();
|
|
718
|
+
if (!loadResult.success)
|
|
719
|
+
return loadResult;
|
|
720
|
+
const startupAbort = new AbortController();
|
|
721
|
+
const startupTimeout = embedded ? chromaConfig.startupTimeout : 5e3;
|
|
722
|
+
const startupTimer = setTimeout(() => startupAbort.abort(), startupTimeout);
|
|
723
|
+
try {
|
|
724
|
+
if (embedded)
|
|
725
|
+
await startEmbeddedServer(chromaConfig);
|
|
726
|
+
const endpoint = chromaConfig.url ? new URL(chromaConfig.url) : void 0;
|
|
727
|
+
const clientParams = endpoint ? { host: endpoint.hostname, port: Number(endpoint.port || (endpoint.protocol === "https:" ? 443 : 80)), ssl: endpoint.protocol === "https:" } : { host: chromaConfig.host, port: chromaConfig.port, ssl: false };
|
|
728
|
+
if (chromaConfig.apiKey)
|
|
729
|
+
clientParams.headers = { Authorization: `Bearer ${chromaConfig.apiKey}` };
|
|
730
|
+
clientParams.fetchOptions = { signal: startupAbort.signal };
|
|
731
|
+
const chromaClient = new loadResult.data.ChromaClient(clientParams);
|
|
732
|
+
await waitForHeartbeat(chromaClient, startupTimeout);
|
|
733
|
+
await chromaClient.countCollections();
|
|
734
|
+
startupAbort.signal.throwIfAborted();
|
|
735
|
+
if (embedded && (spawnFailed || !serverProcess || serverProcess.exitCode !== null || serverProcess.signalCode !== null))
|
|
736
|
+
throw new Error("Chroma child exited before readiness was confirmed");
|
|
737
|
+
client = chromaClient;
|
|
738
|
+
config = chromaConfig;
|
|
739
|
+
logger.info("Chroma connected", { embedded, host: clientParams.host, port: clientParams.port, ssl: clientParams.ssl });
|
|
740
|
+
return ok(void 0);
|
|
741
|
+
} catch (error) {
|
|
742
|
+
startupAbort.abort();
|
|
743
|
+
const cleanup = await closeResources();
|
|
744
|
+
logger.error("Failed to connect to Chroma", { error: error instanceof Error ? error.message : String(error) });
|
|
745
|
+
const failure = err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_connectionFailed", { params: { error: String(error) } }), error);
|
|
746
|
+
return !failure.success && !cleanup.success ? err({ ...failure.error, ext: { cleanupPending: true, cleanupError: cleanup.error } }) : failure;
|
|
747
|
+
} finally {
|
|
748
|
+
clearTimeout(startupTimer);
|
|
749
|
+
}
|
|
750
|
+
},
|
|
751
|
+
async close() {
|
|
752
|
+
return await closeResources();
|
|
753
|
+
},
|
|
754
|
+
isConnected() {
|
|
755
|
+
return client !== null;
|
|
756
|
+
},
|
|
757
|
+
collection: createBaseCollectionOps(ctx, collectionDriver),
|
|
758
|
+
vector: createBaseVectorOps(ctx, vectorDriver)
|
|
759
|
+
};
|
|
760
|
+
}
|
|
761
|
+
var require2 = createRequire(import.meta.url);
|
|
762
|
+
var logger2 = core.logger.child({ module: "vecdb", scope: "lancedb" });
|
|
763
|
+
var COLLECTION_METRIC_KEY = "hai.vecdb.metric";
|
|
764
|
+
var PRIMARY_KEY_FLAG = "lance-schema:unenforced-primary-key";
|
|
765
|
+
var PRIMARY_KEY_POSITION = "lance-schema:unenforced-primary-key:position";
|
|
766
|
+
function escapeLanceSqlValue(value) {
|
|
767
|
+
return value.replace(/'/g, "''");
|
|
768
|
+
}
|
|
769
|
+
function buildMetadataPredicate(filter) {
|
|
770
|
+
if (typeof filter !== "object" || filter === null || Array.isArray(filter) || ![Object.prototype, null].includes(Object.getPrototypeOf(filter)) || Reflect.ownKeys(filter).some((key) => typeof key !== "string")) {
|
|
771
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidMetadataFilter"));
|
|
772
|
+
}
|
|
773
|
+
const predicates = [];
|
|
774
|
+
for (const [key, value] of Object.entries(filter)) {
|
|
775
|
+
if (value !== null && typeof value !== "string" && typeof value !== "boolean" && !(typeof value === "number" && Number.isFinite(value))) {
|
|
776
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidMetadataFilter"));
|
|
777
|
+
}
|
|
778
|
+
const extracted = `json_extract(json_get(metadata, '${escapeLanceSqlValue(key)}'), '$')`;
|
|
779
|
+
predicates.push(typeof value === "number" ? `TRY_CAST(${extracted} AS DOUBLE) = CAST(${value} AS DOUBLE)` : `${extracted} = '${escapeLanceSqlValue(JSON.stringify(value))}'`);
|
|
780
|
+
}
|
|
781
|
+
return ok(predicates.join(" AND "));
|
|
782
|
+
}
|
|
783
|
+
function createLancedbProvider() {
|
|
784
|
+
let connection = null;
|
|
785
|
+
let config = null;
|
|
786
|
+
function loadLancedb() {
|
|
787
|
+
try {
|
|
788
|
+
const mod = require2("@lancedb/lancedb");
|
|
789
|
+
return ok(mod);
|
|
790
|
+
} catch (error) {
|
|
791
|
+
logger2.error("Failed to load @lancedb/lancedb", { error });
|
|
792
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", { params: { driver: "@lancedb/lancedb" } }), error);
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
async function withCollection(name, action) {
|
|
796
|
+
if (!(await connection.tableNames()).includes(name))
|
|
797
|
+
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name } }));
|
|
798
|
+
const table = await connection.openTable(name);
|
|
799
|
+
try {
|
|
800
|
+
const schema = await table.schema();
|
|
801
|
+
const vectorType = schema.fields.find((field) => field.name === "vector")?.type;
|
|
802
|
+
const metric = DistanceMetricSchema.safeParse(schema.metadata.get(COLLECTION_METRIC_KEY) ?? null);
|
|
803
|
+
if (!vectorType || !("listSize" in vectorType) || typeof vectorType.listSize !== "number" || !Number.isInteger(vectorType.listSize) || vectorType.listSize <= 0 || !metric.success) {
|
|
804
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", { params: { name } }));
|
|
805
|
+
}
|
|
806
|
+
const primaryKeys = schema.fields.filter((field) => field.metadata.has(PRIMARY_KEY_POSITION) || ["true", "1", "yes"].includes(field.metadata.get(PRIMARY_KEY_FLAG)?.toLowerCase() ?? ""));
|
|
807
|
+
const idField = primaryKeys[0];
|
|
808
|
+
const atomicWrites = primaryKeys.length === 1 && idField.name === "id" && !idField.nullable && ["true", "1", "yes"].includes(idField.metadata.get(PRIMARY_KEY_FLAG)?.toLowerCase() ?? "") && idField.metadata.get(PRIMARY_KEY_POSITION) === "1" && typeof table.mergeInsert === "function";
|
|
809
|
+
const driver = loadLancedb();
|
|
810
|
+
if (!driver.success)
|
|
811
|
+
return driver;
|
|
812
|
+
const jsonMetadata = schema.fields.find((field) => field.name === "metadata")?.metadata.get("ARROW:extension:name") === "lance.json";
|
|
813
|
+
const inputSchema = jsonMetadata ? schema.assign([driver.data.makeJsonField("metadata", false)]) : void 0;
|
|
814
|
+
return await action(table, { dimension: vectorType.listSize, metric: metric.data, atomicWrites, inputSchema });
|
|
815
|
+
} finally {
|
|
816
|
+
table.close();
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
function createDimensionMismatchResult(expected, actual) {
|
|
820
|
+
return err(
|
|
821
|
+
HaiVecdbError.DIMENSION_MISMATCH,
|
|
822
|
+
vecdbM("vecdb_dimensionMismatch", {
|
|
823
|
+
params: { expected: String(expected), actual: String(actual) }
|
|
824
|
+
})
|
|
825
|
+
);
|
|
826
|
+
}
|
|
827
|
+
function validateVectorDimension(expected, vector) {
|
|
828
|
+
if (vector.length === expected) {
|
|
829
|
+
return null;
|
|
830
|
+
}
|
|
831
|
+
return createDimensionMismatchResult(expected, vector.length);
|
|
832
|
+
}
|
|
833
|
+
function validateDocumentsDimension(expected, documents) {
|
|
834
|
+
for (const document of documents) {
|
|
835
|
+
if (document.vector.length !== expected) {
|
|
836
|
+
return createDimensionMismatchResult(expected, document.vector.length);
|
|
407
837
|
}
|
|
408
|
-
|
|
409
|
-
|
|
838
|
+
}
|
|
839
|
+
return null;
|
|
840
|
+
}
|
|
841
|
+
function validateAtomicWrite(meta, documents) {
|
|
842
|
+
if (!meta.inputSchema)
|
|
843
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_jsonMetadataRequired"));
|
|
844
|
+
if (!meta.atomicWrites)
|
|
845
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_atomicUpsertUnavailable"));
|
|
846
|
+
if (new Set(documents.map((document) => document.id)).size !== documents.length)
|
|
847
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_duplicateDocumentIds"));
|
|
848
|
+
return null;
|
|
849
|
+
}
|
|
850
|
+
const ctx = { isConnected: () => connection !== null, logger: logger2, operationLog: () => config?.operationLog };
|
|
851
|
+
const collectionDriver = {
|
|
852
|
+
async create(name, options) {
|
|
853
|
+
logger2.debug("Creating collection", { name, dimension: options.dimension, metric: options.metric });
|
|
854
|
+
const tableNames = await connection.tableNames();
|
|
855
|
+
if (tableNames.includes(name)) {
|
|
856
|
+
return err(HaiVecdbError.COLLECTION_ALREADY_EXISTS, vecdbM("vecdb_collectionAlreadyExists", { params: { name } }));
|
|
857
|
+
}
|
|
858
|
+
const dimension = options.dimension;
|
|
859
|
+
const parsedMetric = DistanceMetricSchema.safeParse(options.metric ?? config?.metric);
|
|
860
|
+
if (!parsedMetric.success)
|
|
861
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", { params: { name } }));
|
|
862
|
+
const metric = parsedMetric.data;
|
|
863
|
+
const driver = loadLancedb();
|
|
864
|
+
if (!driver.success)
|
|
865
|
+
return driver;
|
|
866
|
+
const inferred = driver.data.makeArrowTable([{
|
|
867
|
+
id: "",
|
|
868
|
+
vector: Array.from({ length: dimension }).fill(0),
|
|
869
|
+
content: "",
|
|
870
|
+
metadata: "{}"
|
|
871
|
+
}], {}, /* @__PURE__ */ new Map([[COLLECTION_METRIC_KEY, metric]])).schema;
|
|
872
|
+
const schema = inferred.assign(inferred.fields.map((field) => field.name === "id" ? field.clone({ nullable: false, metadata: new Map([...field.metadata, [PRIMARY_KEY_FLAG, "true"], [PRIMARY_KEY_POSITION, "1"]]) }) : field.name === "metadata" ? driver.data.makeJsonField("metadata", false) : field));
|
|
873
|
+
const table = await connection.createEmptyTable(name, schema);
|
|
874
|
+
table.close();
|
|
875
|
+
logger2.info("Collection created", { name, dimension, metric });
|
|
876
|
+
return ok(void 0);
|
|
877
|
+
},
|
|
878
|
+
async drop(name) {
|
|
879
|
+
logger2.debug("Dropping collection", { name });
|
|
880
|
+
const tableNames = await connection.tableNames();
|
|
881
|
+
if (!tableNames.includes(name)) {
|
|
882
|
+
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name } }));
|
|
883
|
+
}
|
|
884
|
+
await connection.dropTable(name);
|
|
885
|
+
logger2.info("Collection dropped", { name });
|
|
886
|
+
return ok(void 0);
|
|
887
|
+
},
|
|
888
|
+
async exists(name) {
|
|
889
|
+
logger2.debug("Checking collection exists", { name });
|
|
890
|
+
const tableNames = await connection.tableNames();
|
|
891
|
+
return ok(tableNames.includes(name));
|
|
892
|
+
},
|
|
893
|
+
async info(name) {
|
|
894
|
+
logger2.debug("Getting collection info", { name });
|
|
895
|
+
return withCollection(name, async (table, meta) => ok({ name, dimension: meta.dimension, metric: meta.metric, count: await table.countRows() }));
|
|
896
|
+
},
|
|
897
|
+
async list() {
|
|
898
|
+
logger2.debug("Listing collections");
|
|
899
|
+
const tableNames = await connection.tableNames();
|
|
900
|
+
return ok(tableNames);
|
|
901
|
+
}
|
|
902
|
+
};
|
|
903
|
+
const vectorDriver = {
|
|
904
|
+
async insert(collection, documents) {
|
|
905
|
+
logger2.debug("Inserting vectors", { collection, count: documents.length });
|
|
906
|
+
return withCollection(collection, async (table, meta) => {
|
|
907
|
+
const atomicValidation = validateAtomicWrite(meta, documents);
|
|
908
|
+
if (atomicValidation)
|
|
909
|
+
return atomicValidation;
|
|
910
|
+
const dimensionValidation = validateDocumentsDimension(meta.dimension, documents);
|
|
911
|
+
if (dimensionValidation) {
|
|
912
|
+
return dimensionValidation;
|
|
913
|
+
}
|
|
914
|
+
const records = documents.map((doc) => ({
|
|
915
|
+
id: doc.id,
|
|
916
|
+
vector: doc.vector,
|
|
917
|
+
content: doc.content ?? "",
|
|
918
|
+
metadata: JSON.stringify(doc.metadata ?? {})
|
|
919
|
+
}));
|
|
920
|
+
const driver = loadLancedb();
|
|
921
|
+
if (!driver.success)
|
|
922
|
+
return driver;
|
|
923
|
+
await table.mergeInsert("id").whenNotMatchedInsertAll().execute(driver.data.makeArrowTable(records, { schema: meta.inputSchema }));
|
|
924
|
+
logger2.info("Vectors inserted", { collection, count: documents.length });
|
|
925
|
+
return ok(void 0);
|
|
926
|
+
});
|
|
927
|
+
},
|
|
928
|
+
async upsert(collection, documents) {
|
|
929
|
+
logger2.debug("Upserting vectors", { collection, count: documents.length });
|
|
930
|
+
return withCollection(collection, async (table, meta) => {
|
|
931
|
+
const atomicValidation = validateAtomicWrite(meta, documents);
|
|
932
|
+
if (atomicValidation)
|
|
933
|
+
return atomicValidation;
|
|
934
|
+
const dimensionValidation = validateDocumentsDimension(meta.dimension, documents);
|
|
935
|
+
if (dimensionValidation) {
|
|
936
|
+
return dimensionValidation;
|
|
937
|
+
}
|
|
938
|
+
const records = documents.map((doc) => ({
|
|
939
|
+
id: doc.id,
|
|
940
|
+
vector: doc.vector,
|
|
941
|
+
content: doc.content ?? "",
|
|
942
|
+
metadata: JSON.stringify(doc.metadata ?? {})
|
|
943
|
+
}));
|
|
944
|
+
const driver = loadLancedb();
|
|
945
|
+
if (!driver.success)
|
|
946
|
+
return driver;
|
|
947
|
+
await table.mergeInsert("id").whenMatchedUpdateAll().whenNotMatchedInsertAll().execute(driver.data.makeArrowTable(records, { schema: meta.inputSchema }));
|
|
948
|
+
logger2.info("Vectors upserted", { collection, count: documents.length });
|
|
949
|
+
return ok(void 0);
|
|
950
|
+
});
|
|
951
|
+
},
|
|
952
|
+
async delete(collection, ids) {
|
|
953
|
+
logger2.debug("Deleting vectors", { collection, count: ids.length });
|
|
954
|
+
return withCollection(collection, async (table) => {
|
|
955
|
+
const idList = ids.map((id) => `'${escapeLanceSqlValue(id)}'`).join(", ");
|
|
956
|
+
await table.delete(`id IN (${idList})`);
|
|
957
|
+
logger2.info("Vectors deleted", { collection, count: ids.length });
|
|
958
|
+
return ok(void 0);
|
|
959
|
+
});
|
|
960
|
+
},
|
|
961
|
+
async search(collection, vector, options) {
|
|
962
|
+
logger2.debug("Searching vectors", { collection, topK: options?.topK, hasFilter: !!options?.filter });
|
|
963
|
+
return withCollection(collection, async (table, meta) => {
|
|
964
|
+
const dimensionValidation = validateVectorDimension(meta.dimension, vector);
|
|
965
|
+
if (dimensionValidation) {
|
|
966
|
+
return dimensionValidation;
|
|
967
|
+
}
|
|
968
|
+
const topK = options?.topK ?? 10;
|
|
969
|
+
const minScore = options?.minScore ?? 0;
|
|
970
|
+
let query = table.vectorSearch(vector).distanceType(meta.metric === "euclidean" ? "l2" : meta.metric).limit(topK);
|
|
971
|
+
if (options?.filter !== void 0) {
|
|
972
|
+
if (!meta.inputSchema)
|
|
973
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_jsonMetadataRequired"));
|
|
974
|
+
const predicate = buildMetadataPredicate(options.filter);
|
|
975
|
+
if (!predicate.success)
|
|
976
|
+
return predicate;
|
|
977
|
+
if (predicate.data)
|
|
978
|
+
query = query.where(predicate.data);
|
|
979
|
+
}
|
|
980
|
+
const results = await query.toArray();
|
|
981
|
+
const searchResults = results.map((row) => {
|
|
982
|
+
const distance = row._distance ?? 0;
|
|
983
|
+
const score = meta.metric === "cosine" ? Math.max(0, Math.min(1, 1 - distance / 2)) : meta.metric === "dot" ? 1 / (1 + Math.exp(distance - 1)) : 1 / (1 + Math.max(0, distance));
|
|
984
|
+
return {
|
|
985
|
+
id: row.id,
|
|
986
|
+
score,
|
|
987
|
+
content: row.content || void 0,
|
|
988
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : void 0
|
|
989
|
+
};
|
|
990
|
+
}).filter((r) => r.score >= minScore);
|
|
991
|
+
return ok(searchResults);
|
|
992
|
+
});
|
|
993
|
+
},
|
|
994
|
+
async count(collection) {
|
|
995
|
+
logger2.debug("Counting vectors", { collection });
|
|
996
|
+
return withCollection(collection, async (table) => {
|
|
997
|
+
const count = await table.countRows();
|
|
998
|
+
return ok(count);
|
|
999
|
+
});
|
|
410
1000
|
}
|
|
411
1001
|
};
|
|
412
1002
|
return {
|
|
@@ -416,42 +1006,28 @@ function createLancedbProvider() {
|
|
|
416
1006
|
return err(HaiVecdbError.UNSUPPORTED_TYPE, vecdbM("vecdb_unsupportedType", { params: { type: cfg.type } }));
|
|
417
1007
|
}
|
|
418
1008
|
const lanceConfig = cfg;
|
|
419
|
-
const loadResult =
|
|
1009
|
+
const loadResult = loadLancedb();
|
|
420
1010
|
if (!loadResult.success)
|
|
421
1011
|
return loadResult;
|
|
422
1012
|
const lancedb = loadResult.data;
|
|
423
1013
|
try {
|
|
424
|
-
connection = await lancedb.connect(lanceConfig.path);
|
|
1014
|
+
connection = await lancedb.connect(lanceConfig.path, { readConsistencyInterval: 0 });
|
|
425
1015
|
config = lanceConfig;
|
|
426
|
-
|
|
427
|
-
await Promise.allSettled(tableNames.map(async (name) => {
|
|
428
|
-
try {
|
|
429
|
-
const table = await connection.openTable(name);
|
|
430
|
-
const count = await table.countRows();
|
|
431
|
-
if (count === 0)
|
|
432
|
-
return;
|
|
433
|
-
const rows = await table.search(Array.from({ length: 1 }).fill(0)).limit(1).toArray();
|
|
434
|
-
if (rows.length > 0 && rows[0].vector) {
|
|
435
|
-
collectionMetas.set(name, {
|
|
436
|
-
dimension: rows[0].vector.length,
|
|
437
|
-
metric: lanceConfig.metric ?? "cosine"
|
|
438
|
-
});
|
|
439
|
-
}
|
|
440
|
-
} catch {
|
|
441
|
-
}
|
|
442
|
-
}));
|
|
443
|
-
logger.info("LanceDB connected", { path: lanceConfig.path });
|
|
1016
|
+
logger2.info("LanceDB connected", { path: lanceConfig.path });
|
|
444
1017
|
return ok(void 0);
|
|
445
1018
|
} catch (error) {
|
|
446
|
-
|
|
1019
|
+
connection?.close();
|
|
1020
|
+
connection = null;
|
|
1021
|
+
config = null;
|
|
1022
|
+
logger2.error("Failed to connect to LanceDB", { error });
|
|
447
1023
|
return err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_connectionFailed", { params: { error: String(error) } }), error);
|
|
448
1024
|
}
|
|
449
1025
|
},
|
|
450
1026
|
async close() {
|
|
1027
|
+
connection?.close();
|
|
451
1028
|
connection = null;
|
|
452
1029
|
config = null;
|
|
453
|
-
|
|
454
|
-
logger.info("LanceDB connection closed");
|
|
1030
|
+
logger2.info("LanceDB connection closed");
|
|
455
1031
|
return ok(void 0);
|
|
456
1032
|
},
|
|
457
1033
|
isConnected() {
|
|
@@ -461,7 +1037,14 @@ function createLancedbProvider() {
|
|
|
461
1037
|
vector: createBaseVectorOps(ctx, vectorDriver)
|
|
462
1038
|
};
|
|
463
1039
|
}
|
|
464
|
-
var
|
|
1040
|
+
var logger3 = core.logger.child({ module: "vecdb", scope: "pgvector" });
|
|
1041
|
+
function isPgModule(value) {
|
|
1042
|
+
if (typeof value !== "object" || value === null) {
|
|
1043
|
+
return false;
|
|
1044
|
+
}
|
|
1045
|
+
const maybeModule = value;
|
|
1046
|
+
return typeof maybeModule.Pool === "function";
|
|
1047
|
+
}
|
|
465
1048
|
function createPgvectorProvider() {
|
|
466
1049
|
let pool = null;
|
|
467
1050
|
let config = null;
|
|
@@ -484,7 +1067,7 @@ function createPgvectorProvider() {
|
|
|
484
1067
|
return "<=>";
|
|
485
1068
|
}
|
|
486
1069
|
}
|
|
487
|
-
function
|
|
1070
|
+
function distanceToScore2(distance) {
|
|
488
1071
|
const metric = config?.metric ?? "cosine";
|
|
489
1072
|
switch (metric) {
|
|
490
1073
|
case "cosine":
|
|
@@ -497,14 +1080,70 @@ function createPgvectorProvider() {
|
|
|
497
1080
|
return 1 / (1 + distance);
|
|
498
1081
|
}
|
|
499
1082
|
}
|
|
500
|
-
|
|
1083
|
+
function isMissingModuleError(error, packageName) {
|
|
1084
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1085
|
+
return message.includes(`'${packageName}'`) || message.includes(`"${packageName}"`) || message.includes(packageName);
|
|
1086
|
+
}
|
|
1087
|
+
async function loadPg() {
|
|
1088
|
+
try {
|
|
1089
|
+
const mod = await import('pg');
|
|
1090
|
+
if (!isPgModule(mod)) {
|
|
1091
|
+
const error = new Error("pg driver module does not export Pool");
|
|
1092
|
+
logger3.error("Failed to load pg driver", { error: error.message });
|
|
1093
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", { params: { driver: "pg" } }), error);
|
|
1094
|
+
}
|
|
1095
|
+
return ok(mod);
|
|
1096
|
+
} catch (error) {
|
|
1097
|
+
if (isMissingModuleError(error, "pg")) {
|
|
1098
|
+
logger3.error("Failed to load pg driver", { error: error instanceof Error ? error.message : String(error) });
|
|
1099
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", { params: { driver: "pg" } }), error);
|
|
1100
|
+
}
|
|
1101
|
+
throw error;
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
async function getCollectionDimension(name) {
|
|
1105
|
+
const table = tableName(name);
|
|
1106
|
+
const existsResult = await pool.query(
|
|
1107
|
+
`SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $1)`,
|
|
1108
|
+
[table]
|
|
1109
|
+
);
|
|
1110
|
+
if (!existsResult.rows[0].exists) {
|
|
1111
|
+
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name } }));
|
|
1112
|
+
}
|
|
1113
|
+
const dimResult = await pool.query(`
|
|
1114
|
+
SELECT atttypmod FROM pg_attribute
|
|
1115
|
+
WHERE attrelid = $1::regclass AND attname = 'vector'
|
|
1116
|
+
`, [quoteIdent(table)]);
|
|
1117
|
+
return ok(dimResult.rows.length > 0 ? Number(dimResult.rows[0].atttypmod) : 0);
|
|
1118
|
+
}
|
|
1119
|
+
function validateVectorDimension(expected, actual) {
|
|
1120
|
+
if (expected === actual) {
|
|
1121
|
+
return null;
|
|
1122
|
+
}
|
|
1123
|
+
return err(
|
|
1124
|
+
HaiVecdbError.DIMENSION_MISMATCH,
|
|
1125
|
+
vecdbM("vecdb_dimensionMismatch", {
|
|
1126
|
+
params: { expected: String(expected), actual: String(actual) }
|
|
1127
|
+
})
|
|
1128
|
+
);
|
|
1129
|
+
}
|
|
1130
|
+
function validateDocumentsDimension(expected, documents) {
|
|
1131
|
+
for (const document of documents) {
|
|
1132
|
+
const validation = validateVectorDimension(expected, document.vector.length);
|
|
1133
|
+
if (validation) {
|
|
1134
|
+
return validation;
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
return null;
|
|
1138
|
+
}
|
|
1139
|
+
const ctx = { isConnected: () => pool !== null, logger: logger3, operationLog: () => config?.operationLog };
|
|
501
1140
|
const collectionDriver = {
|
|
502
1141
|
async create(name, options) {
|
|
503
1142
|
const table = tableName(name);
|
|
504
1143
|
const dimension = options.dimension;
|
|
505
1144
|
const metric = options.metric ?? config?.metric ?? "cosine";
|
|
506
1145
|
const indexType = config?.indexType ?? "hnsw";
|
|
507
|
-
|
|
1146
|
+
logger3.debug("Creating collection", { name, dimension, metric, indexType });
|
|
508
1147
|
const checkResult = await pool.query(
|
|
509
1148
|
`SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $1)`,
|
|
510
1149
|
[table]
|
|
@@ -535,12 +1174,12 @@ function createPgvectorProvider() {
|
|
|
535
1174
|
WITH (lists = 100)
|
|
536
1175
|
`);
|
|
537
1176
|
}
|
|
538
|
-
|
|
1177
|
+
logger3.info("Collection created", { name, dimension, metric, indexType });
|
|
539
1178
|
return ok(void 0);
|
|
540
1179
|
},
|
|
541
1180
|
async drop(name) {
|
|
542
1181
|
const table = tableName(name);
|
|
543
|
-
|
|
1182
|
+
logger3.debug("Dropping collection", { name });
|
|
544
1183
|
const checkResult = await pool.query(
|
|
545
1184
|
`SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $1)`,
|
|
546
1185
|
[table]
|
|
@@ -549,11 +1188,11 @@ function createPgvectorProvider() {
|
|
|
549
1188
|
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name } }));
|
|
550
1189
|
}
|
|
551
1190
|
await pool.query(`DROP TABLE ${quoteIdent(table)}`);
|
|
552
|
-
|
|
1191
|
+
logger3.info("Collection dropped", { name });
|
|
553
1192
|
return ok(void 0);
|
|
554
1193
|
},
|
|
555
1194
|
async exists(name) {
|
|
556
|
-
|
|
1195
|
+
logger3.debug("Checking collection exists", { name });
|
|
557
1196
|
const result = await pool.query(
|
|
558
1197
|
`SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = $1)`,
|
|
559
1198
|
[tableName(name)]
|
|
@@ -561,32 +1200,24 @@ function createPgvectorProvider() {
|
|
|
561
1200
|
return ok(result.rows[0].exists);
|
|
562
1201
|
},
|
|
563
1202
|
async info(name) {
|
|
1203
|
+
logger3.debug("Getting collection info", { name });
|
|
564
1204
|
const table = tableName(name);
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
[table]
|
|
569
|
-
);
|
|
570
|
-
if (!existsResult.rows[0].exists) {
|
|
571
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name } }));
|
|
1205
|
+
const dimensionResult = await getCollectionDimension(name);
|
|
1206
|
+
if (!dimensionResult.success) {
|
|
1207
|
+
return dimensionResult;
|
|
572
1208
|
}
|
|
573
1209
|
const countResult = await pool.query(`SELECT COUNT(*) as count FROM ${quoteIdent(table)}`);
|
|
574
1210
|
const count = Number.parseInt(String(countResult.rows[0].count), 10);
|
|
575
|
-
const dimResult = await pool.query(`
|
|
576
|
-
SELECT atttypmod FROM pg_attribute
|
|
577
|
-
WHERE attrelid = $1::regclass AND attname = 'vector'
|
|
578
|
-
`, [quoteIdent(table)]);
|
|
579
|
-
const dimension = dimResult.rows.length > 0 ? Number(dimResult.rows[0].atttypmod) : 0;
|
|
580
1211
|
return ok({
|
|
581
1212
|
name,
|
|
582
|
-
dimension,
|
|
1213
|
+
dimension: dimensionResult.data,
|
|
583
1214
|
metric: config?.metric ?? "cosine",
|
|
584
1215
|
count
|
|
585
1216
|
});
|
|
586
1217
|
},
|
|
587
1218
|
async list() {
|
|
588
1219
|
const prefix = config?.tablePrefix ?? "vec_";
|
|
589
|
-
|
|
1220
|
+
logger3.debug("Listing collections");
|
|
590
1221
|
const result = await pool.query(
|
|
591
1222
|
`SELECT table_name FROM information_schema.tables WHERE table_name LIKE $1 AND table_schema = 'public'`,
|
|
592
1223
|
[`${prefix}%`]
|
|
@@ -600,7 +1231,15 @@ function createPgvectorProvider() {
|
|
|
600
1231
|
const vectorDriver = {
|
|
601
1232
|
async insert(collection, documents) {
|
|
602
1233
|
const table = tableName(collection);
|
|
603
|
-
|
|
1234
|
+
logger3.debug("Inserting vectors", { collection, count: documents.length });
|
|
1235
|
+
const dimensionResult = await getCollectionDimension(collection);
|
|
1236
|
+
if (!dimensionResult.success) {
|
|
1237
|
+
return dimensionResult;
|
|
1238
|
+
}
|
|
1239
|
+
const dimensionValidation = validateDocumentsDimension(dimensionResult.data, documents);
|
|
1240
|
+
if (dimensionValidation) {
|
|
1241
|
+
return dimensionValidation;
|
|
1242
|
+
}
|
|
604
1243
|
const qi = quoteIdent(table);
|
|
605
1244
|
const values = [];
|
|
606
1245
|
const params = [];
|
|
@@ -614,12 +1253,20 @@ function createPgvectorProvider() {
|
|
|
614
1253
|
`INSERT INTO ${qi} (id, vector, content, metadata) VALUES ${values.join(", ")}`,
|
|
615
1254
|
params
|
|
616
1255
|
);
|
|
617
|
-
|
|
1256
|
+
logger3.info("Vectors inserted", { collection, count: documents.length });
|
|
618
1257
|
return ok(void 0);
|
|
619
1258
|
},
|
|
620
1259
|
async upsert(collection, documents) {
|
|
621
1260
|
const table = tableName(collection);
|
|
622
|
-
|
|
1261
|
+
logger3.debug("Upserting vectors", { collection, count: documents.length });
|
|
1262
|
+
const dimensionResult = await getCollectionDimension(collection);
|
|
1263
|
+
if (!dimensionResult.success) {
|
|
1264
|
+
return dimensionResult;
|
|
1265
|
+
}
|
|
1266
|
+
const dimensionValidation = validateDocumentsDimension(dimensionResult.data, documents);
|
|
1267
|
+
if (dimensionValidation) {
|
|
1268
|
+
return dimensionValidation;
|
|
1269
|
+
}
|
|
623
1270
|
const qi = quoteIdent(table);
|
|
624
1271
|
const values = [];
|
|
625
1272
|
const params = [];
|
|
@@ -637,15 +1284,15 @@ function createPgvectorProvider() {
|
|
|
637
1284
|
metadata = EXCLUDED.metadata`,
|
|
638
1285
|
params
|
|
639
1286
|
);
|
|
640
|
-
|
|
1287
|
+
logger3.info("Vectors upserted", { collection, count: documents.length });
|
|
641
1288
|
return ok(void 0);
|
|
642
1289
|
},
|
|
643
1290
|
async delete(collection, ids) {
|
|
644
1291
|
const table = tableName(collection);
|
|
645
|
-
|
|
1292
|
+
logger3.debug("Deleting vectors", { collection, count: ids.length });
|
|
646
1293
|
const placeholders = ids.map((_, i) => `$${i + 1}`).join(", ");
|
|
647
1294
|
await pool.query(`DELETE FROM ${quoteIdent(table)} WHERE id IN (${placeholders})`, ids);
|
|
648
|
-
|
|
1295
|
+
logger3.info("Vectors deleted", { collection, count: ids.length });
|
|
649
1296
|
return ok(void 0);
|
|
650
1297
|
},
|
|
651
1298
|
async search(collection, vector, options) {
|
|
@@ -653,7 +1300,15 @@ function createPgvectorProvider() {
|
|
|
653
1300
|
const topK = options?.topK ?? 10;
|
|
654
1301
|
const minScore = options?.minScore ?? 0;
|
|
655
1302
|
const op = distanceOp();
|
|
656
|
-
|
|
1303
|
+
logger3.debug("Searching vectors", { collection, topK, hasFilter: !!options?.filter });
|
|
1304
|
+
const dimensionResult = await getCollectionDimension(collection);
|
|
1305
|
+
if (!dimensionResult.success) {
|
|
1306
|
+
return dimensionResult;
|
|
1307
|
+
}
|
|
1308
|
+
const dimensionValidation = validateVectorDimension(dimensionResult.data, vector.length);
|
|
1309
|
+
if (dimensionValidation) {
|
|
1310
|
+
return dimensionValidation;
|
|
1311
|
+
}
|
|
657
1312
|
const vectorStr = `[${vector.join(",")}]`;
|
|
658
1313
|
let filterSQL = "";
|
|
659
1314
|
const filterParams = [vectorStr, topK];
|
|
@@ -678,7 +1333,7 @@ function createPgvectorProvider() {
|
|
|
678
1333
|
filterParams
|
|
679
1334
|
);
|
|
680
1335
|
const searchResults = result.rows.map((row) => {
|
|
681
|
-
const score =
|
|
1336
|
+
const score = distanceToScore2(row.distance);
|
|
682
1337
|
return {
|
|
683
1338
|
id: row.id,
|
|
684
1339
|
score,
|
|
@@ -690,7 +1345,7 @@ function createPgvectorProvider() {
|
|
|
690
1345
|
},
|
|
691
1346
|
async count(collection) {
|
|
692
1347
|
const table = tableName(collection);
|
|
693
|
-
|
|
1348
|
+
logger3.debug("Counting vectors", { collection });
|
|
694
1349
|
const result = await pool.query(`SELECT COUNT(*) as count FROM ${quoteIdent(table)}`);
|
|
695
1350
|
return ok(Number.parseInt(String(result.rows[0].count), 10));
|
|
696
1351
|
}
|
|
@@ -703,7 +1358,11 @@ function createPgvectorProvider() {
|
|
|
703
1358
|
}
|
|
704
1359
|
const pgConfig = cfg;
|
|
705
1360
|
try {
|
|
706
|
-
const
|
|
1361
|
+
const loadResult = await loadPg();
|
|
1362
|
+
if (!loadResult.success) {
|
|
1363
|
+
return loadResult;
|
|
1364
|
+
}
|
|
1365
|
+
const { Pool } = loadResult.data;
|
|
707
1366
|
const poolConfig = pgConfig.url ? { connectionString: pgConfig.url } : {
|
|
708
1367
|
host: pgConfig.host,
|
|
709
1368
|
port: pgConfig.port,
|
|
@@ -714,10 +1373,10 @@ function createPgvectorProvider() {
|
|
|
714
1373
|
pool = new Pool(poolConfig);
|
|
715
1374
|
await pool.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
716
1375
|
config = pgConfig;
|
|
717
|
-
|
|
1376
|
+
logger3.info("pgvector connected", { host: pgConfig.host, database: pgConfig.database });
|
|
718
1377
|
return ok(void 0);
|
|
719
1378
|
} catch (error) {
|
|
720
|
-
|
|
1379
|
+
logger3.error("Failed to connect to pgvector", { error: error instanceof Error ? error.message : String(error) });
|
|
721
1380
|
return err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_connectionFailed", { params: { error: error instanceof Error ? error.message : String(error) } }), error);
|
|
722
1381
|
}
|
|
723
1382
|
},
|
|
@@ -728,10 +1387,10 @@ function createPgvectorProvider() {
|
|
|
728
1387
|
}
|
|
729
1388
|
pool = null;
|
|
730
1389
|
config = null;
|
|
731
|
-
|
|
1390
|
+
logger3.info("pgvector connection closed");
|
|
732
1391
|
return ok(void 0);
|
|
733
1392
|
} catch (error) {
|
|
734
|
-
|
|
1393
|
+
logger3.error("Failed to close pgvector connection", { error: error instanceof Error ? error.message : String(error) });
|
|
735
1394
|
return err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_closeFailed", { params: { error: String(error) } }), error);
|
|
736
1395
|
}
|
|
737
1396
|
},
|
|
@@ -742,19 +1401,14 @@ function createPgvectorProvider() {
|
|
|
742
1401
|
vector: createBaseVectorOps(ctx, vectorDriver)
|
|
743
1402
|
};
|
|
744
1403
|
}
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
parsed.username = "***";
|
|
750
|
-
if (parsed.password)
|
|
751
|
-
parsed.password = "***";
|
|
752
|
-
return parsed.toString();
|
|
753
|
-
} catch {
|
|
754
|
-
return "(invalid url)";
|
|
1404
|
+
var logger4 = core.logger.child({ module: "vecdb", scope: "qdrant" });
|
|
1405
|
+
function isQdrantModule(value) {
|
|
1406
|
+
if (typeof value !== "object" || value === null) {
|
|
1407
|
+
return false;
|
|
755
1408
|
}
|
|
1409
|
+
const maybeModule = value;
|
|
1410
|
+
return typeof maybeModule.QdrantClient === "function";
|
|
756
1411
|
}
|
|
757
|
-
var logger3 = core.logger.child({ module: "vecdb", scope: "qdrant" });
|
|
758
1412
|
function hashToUuid(id) {
|
|
759
1413
|
const hash = createHash("sha256").update(id).digest("hex");
|
|
760
1414
|
return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
|
|
@@ -767,17 +1421,77 @@ function toQdrantDistance(metric) {
|
|
|
767
1421
|
return "Euclid";
|
|
768
1422
|
case "dot":
|
|
769
1423
|
return "Dot";
|
|
770
|
-
default:
|
|
771
|
-
return "Cosine";
|
|
772
1424
|
}
|
|
773
1425
|
}
|
|
774
1426
|
function createQdrantProvider() {
|
|
775
1427
|
let client = null;
|
|
776
1428
|
let config = null;
|
|
777
|
-
|
|
1429
|
+
function isMissingModuleError(error, packageName) {
|
|
1430
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1431
|
+
return message.includes(`'${packageName}'`) || message.includes(`"${packageName}"`) || message.includes(packageName);
|
|
1432
|
+
}
|
|
1433
|
+
async function loadQdrantClient() {
|
|
1434
|
+
try {
|
|
1435
|
+
const mod = await import('@qdrant/js-client-rest');
|
|
1436
|
+
if (!isQdrantModule(mod)) {
|
|
1437
|
+
const error = new Error("@qdrant/js-client-rest driver module does not export QdrantClient");
|
|
1438
|
+
logger4.error("Failed to load Qdrant driver", { error: error.message });
|
|
1439
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", { params: { driver: "@qdrant/js-client-rest" } }), error);
|
|
1440
|
+
}
|
|
1441
|
+
return ok(mod);
|
|
1442
|
+
} catch (error) {
|
|
1443
|
+
if (isMissingModuleError(error, "@qdrant/js-client-rest")) {
|
|
1444
|
+
logger4.error("Failed to load Qdrant driver", { error: error instanceof Error ? error.message : String(error) });
|
|
1445
|
+
return err(HaiVecdbError.DRIVER_NOT_FOUND, vecdbM("vecdb_driverNotFound", { params: { driver: "@qdrant/js-client-rest" } }), error);
|
|
1446
|
+
}
|
|
1447
|
+
throw error;
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1450
|
+
function isQdrantNotFoundError(error) {
|
|
1451
|
+
return typeof error === "object" && error !== null && "status" in error && error.status === 404;
|
|
1452
|
+
}
|
|
1453
|
+
async function getCollectionInfo(name) {
|
|
1454
|
+
try {
|
|
1455
|
+
return ok(await client.getCollection(name));
|
|
1456
|
+
} catch (error) {
|
|
1457
|
+
if (isQdrantNotFoundError(error)) {
|
|
1458
|
+
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name } }));
|
|
1459
|
+
}
|
|
1460
|
+
throw error;
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
function readVectorDefinition(info) {
|
|
1464
|
+
const vectors = info.config.params.vectors;
|
|
1465
|
+
if (!vectors || !("size" in vectors) || typeof vectors.size !== "number" || !Number.isSafeInteger(vectors.size) || vectors.size <= 0)
|
|
1466
|
+
return void 0;
|
|
1467
|
+
if (!("distance" in vectors))
|
|
1468
|
+
return void 0;
|
|
1469
|
+
const metric = vectors.distance === "Cosine" ? "cosine" : vectors.distance === "Euclid" ? "euclidean" : vectors.distance === "Dot" ? "dot" : void 0;
|
|
1470
|
+
return metric ? { dimension: vectors.size, metric } : void 0;
|
|
1471
|
+
}
|
|
1472
|
+
async function getCollectionDimension(name) {
|
|
1473
|
+
const infoResult = await getCollectionInfo(name);
|
|
1474
|
+
if (!infoResult.success) {
|
|
1475
|
+
return infoResult;
|
|
1476
|
+
}
|
|
1477
|
+
const definition = readVectorDefinition(infoResult.data);
|
|
1478
|
+
return definition ? ok(definition.dimension) : err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", { params: { name } }));
|
|
1479
|
+
}
|
|
1480
|
+
function validateVectorDimension(expected, actual) {
|
|
1481
|
+
if (expected === actual) {
|
|
1482
|
+
return null;
|
|
1483
|
+
}
|
|
1484
|
+
return err(
|
|
1485
|
+
HaiVecdbError.DIMENSION_MISMATCH,
|
|
1486
|
+
vecdbM("vecdb_dimensionMismatch", {
|
|
1487
|
+
params: { expected: String(expected), actual: String(actual) }
|
|
1488
|
+
})
|
|
1489
|
+
);
|
|
1490
|
+
}
|
|
1491
|
+
const ctx = { isConnected: () => client !== null, logger: logger4, operationLog: () => config?.operationLog };
|
|
778
1492
|
const collectionDriver = {
|
|
779
1493
|
async create(name, options) {
|
|
780
|
-
|
|
1494
|
+
logger4.debug("Creating collection", { name, dimension: options.dimension, metric: options.metric });
|
|
781
1495
|
const collections = await client.getCollections();
|
|
782
1496
|
const exists = collections.collections.some((c) => c.name === name);
|
|
783
1497
|
if (exists) {
|
|
@@ -790,50 +1504,49 @@ function createQdrantProvider() {
|
|
|
790
1504
|
distance: toQdrantDistance(metric)
|
|
791
1505
|
}
|
|
792
1506
|
});
|
|
793
|
-
|
|
1507
|
+
logger4.info("Collection created", { name, dimension: options.dimension, metric });
|
|
794
1508
|
return ok(void 0);
|
|
795
1509
|
},
|
|
796
1510
|
async drop(name) {
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name } }));
|
|
1511
|
+
logger4.debug("Dropping collection", { name });
|
|
1512
|
+
const infoResult = await getCollectionInfo(name);
|
|
1513
|
+
if (!infoResult.success) {
|
|
1514
|
+
return infoResult;
|
|
802
1515
|
}
|
|
803
1516
|
await client.deleteCollection(name);
|
|
804
|
-
|
|
1517
|
+
logger4.info("Collection dropped", { name });
|
|
805
1518
|
return ok(void 0);
|
|
806
1519
|
},
|
|
807
1520
|
async exists(name) {
|
|
808
|
-
|
|
1521
|
+
logger4.debug("Checking collection exists", { name });
|
|
809
1522
|
try {
|
|
810
1523
|
await client.getCollection(name);
|
|
811
1524
|
return ok(true);
|
|
812
|
-
} catch {
|
|
813
|
-
|
|
1525
|
+
} catch (error) {
|
|
1526
|
+
if (isQdrantNotFoundError(error)) {
|
|
1527
|
+
return ok(false);
|
|
1528
|
+
}
|
|
1529
|
+
throw error;
|
|
814
1530
|
}
|
|
815
1531
|
},
|
|
816
1532
|
async info(name) {
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
} catch {
|
|
822
|
-
return err(HaiVecdbError.COLLECTION_NOT_FOUND, vecdbM("vecdb_collectionNotFound", { params: { name } }));
|
|
1533
|
+
logger4.debug("Getting collection info", { name });
|
|
1534
|
+
const infoResult = await getCollectionInfo(name);
|
|
1535
|
+
if (!infoResult.success) {
|
|
1536
|
+
return infoResult;
|
|
823
1537
|
}
|
|
824
|
-
const
|
|
825
|
-
const
|
|
826
|
-
|
|
827
|
-
|
|
1538
|
+
const collectionInfo = infoResult.data;
|
|
1539
|
+
const definition = readVectorDefinition(collectionInfo);
|
|
1540
|
+
if (!definition)
|
|
1541
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_invalidCollectionSchema", { params: { name } }));
|
|
828
1542
|
return ok({
|
|
829
1543
|
name,
|
|
830
|
-
|
|
831
|
-
metric,
|
|
1544
|
+
...definition,
|
|
832
1545
|
count: collectionInfo.points_count ?? 0
|
|
833
1546
|
});
|
|
834
1547
|
},
|
|
835
1548
|
async list() {
|
|
836
|
-
|
|
1549
|
+
logger4.debug("Listing collections");
|
|
837
1550
|
const collections = await client.getCollections();
|
|
838
1551
|
const names = collections.collections.map((c) => c.name);
|
|
839
1552
|
return ok(names);
|
|
@@ -841,7 +1554,17 @@ function createQdrantProvider() {
|
|
|
841
1554
|
};
|
|
842
1555
|
const vectorDriver = {
|
|
843
1556
|
async insert(collection, documents) {
|
|
844
|
-
|
|
1557
|
+
logger4.debug("Inserting vectors", { collection, count: documents.length });
|
|
1558
|
+
const dimensionResult = await getCollectionDimension(collection);
|
|
1559
|
+
if (!dimensionResult.success) {
|
|
1560
|
+
return dimensionResult;
|
|
1561
|
+
}
|
|
1562
|
+
for (const document of documents) {
|
|
1563
|
+
const validation = validateVectorDimension(dimensionResult.data, document.vector.length);
|
|
1564
|
+
if (validation) {
|
|
1565
|
+
return validation;
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
845
1568
|
const points = documents.map((doc) => ({
|
|
846
1569
|
id: hashToUuid(doc.id),
|
|
847
1570
|
vector: doc.vector,
|
|
@@ -851,12 +1574,22 @@ function createQdrantProvider() {
|
|
|
851
1574
|
_content: doc.content ?? ""
|
|
852
1575
|
}
|
|
853
1576
|
}));
|
|
854
|
-
await client.upsert(collection, { points });
|
|
855
|
-
|
|
1577
|
+
await client.upsert(collection, { points, wait: true });
|
|
1578
|
+
logger4.info("Vectors inserted", { collection, count: documents.length });
|
|
856
1579
|
return ok(void 0);
|
|
857
1580
|
},
|
|
858
1581
|
async upsert(collection, documents) {
|
|
859
|
-
|
|
1582
|
+
logger4.debug("Upserting vectors", { collection, count: documents.length });
|
|
1583
|
+
const dimensionResult = await getCollectionDimension(collection);
|
|
1584
|
+
if (!dimensionResult.success) {
|
|
1585
|
+
return dimensionResult;
|
|
1586
|
+
}
|
|
1587
|
+
for (const document of documents) {
|
|
1588
|
+
const validation = validateVectorDimension(dimensionResult.data, document.vector.length);
|
|
1589
|
+
if (validation) {
|
|
1590
|
+
return validation;
|
|
1591
|
+
}
|
|
1592
|
+
}
|
|
860
1593
|
const points = documents.map((doc) => ({
|
|
861
1594
|
id: hashToUuid(doc.id),
|
|
862
1595
|
vector: doc.vector,
|
|
@@ -866,21 +1599,29 @@ function createQdrantProvider() {
|
|
|
866
1599
|
_content: doc.content ?? ""
|
|
867
1600
|
}
|
|
868
1601
|
}));
|
|
869
|
-
await client.upsert(collection, { points });
|
|
870
|
-
|
|
1602
|
+
await client.upsert(collection, { points, wait: true });
|
|
1603
|
+
logger4.info("Vectors upserted", { collection, count: documents.length });
|
|
871
1604
|
return ok(void 0);
|
|
872
1605
|
},
|
|
873
1606
|
async delete(collection, ids) {
|
|
874
|
-
|
|
1607
|
+
logger4.debug("Deleting vectors", { collection, count: ids.length });
|
|
875
1608
|
const uuids = ids.map(hashToUuid);
|
|
876
|
-
await client.delete(collection, { points: uuids });
|
|
877
|
-
|
|
1609
|
+
await client.delete(collection, { points: uuids, wait: true });
|
|
1610
|
+
logger4.info("Vectors deleted", { collection, count: ids.length });
|
|
878
1611
|
return ok(void 0);
|
|
879
1612
|
},
|
|
880
1613
|
async search(collection, vector, options) {
|
|
881
1614
|
const topK = options?.topK ?? 10;
|
|
882
1615
|
const minScore = options?.minScore ?? 0;
|
|
883
|
-
|
|
1616
|
+
logger4.debug("Searching vectors", { collection, topK, hasFilter: !!options?.filter });
|
|
1617
|
+
const dimensionResult = await getCollectionDimension(collection);
|
|
1618
|
+
if (!dimensionResult.success) {
|
|
1619
|
+
return dimensionResult;
|
|
1620
|
+
}
|
|
1621
|
+
const dimensionValidation = validateVectorDimension(dimensionResult.data, vector.length);
|
|
1622
|
+
if (dimensionValidation) {
|
|
1623
|
+
return dimensionValidation;
|
|
1624
|
+
}
|
|
884
1625
|
let filter;
|
|
885
1626
|
if (options?.filter && Object.keys(options.filter).length > 0) {
|
|
886
1627
|
const mustConditions = Object.entries(options.filter).map(([key, value]) => ({
|
|
@@ -889,29 +1630,29 @@ function createQdrantProvider() {
|
|
|
889
1630
|
}));
|
|
890
1631
|
filter = { must: mustConditions };
|
|
891
1632
|
}
|
|
892
|
-
const searchResult = await client.
|
|
893
|
-
vector,
|
|
1633
|
+
const searchResult = await client.query(collection, {
|
|
1634
|
+
query: vector,
|
|
894
1635
|
limit: topK,
|
|
895
1636
|
score_threshold: minScore > 0 ? minScore : void 0,
|
|
896
1637
|
filter,
|
|
897
1638
|
with_payload: true
|
|
898
1639
|
});
|
|
899
|
-
const results = searchResult.map((point) => {
|
|
1640
|
+
const results = searchResult.points.map((point) => {
|
|
900
1641
|
const payload = point.payload ?? {};
|
|
901
1642
|
const { _id, _content, ...metadata } = payload;
|
|
902
1643
|
return {
|
|
903
|
-
id: _id
|
|
1644
|
+
id: typeof _id === "string" ? _id : "",
|
|
904
1645
|
score: point.score,
|
|
905
|
-
content: _content
|
|
1646
|
+
content: typeof _content === "string" && _content.length > 0 ? _content : void 0,
|
|
906
1647
|
metadata: Object.keys(metadata).length > 0 ? metadata : void 0
|
|
907
1648
|
};
|
|
908
1649
|
});
|
|
909
1650
|
return ok(results);
|
|
910
1651
|
},
|
|
911
1652
|
async count(collection) {
|
|
912
|
-
|
|
913
|
-
const
|
|
914
|
-
return ok(
|
|
1653
|
+
logger4.debug("Counting vectors", { collection });
|
|
1654
|
+
const count = await client.count(collection, { exact: true });
|
|
1655
|
+
return ok(count.count);
|
|
915
1656
|
}
|
|
916
1657
|
};
|
|
917
1658
|
return {
|
|
@@ -922,7 +1663,11 @@ function createQdrantProvider() {
|
|
|
922
1663
|
}
|
|
923
1664
|
const qdrantConfig = cfg;
|
|
924
1665
|
try {
|
|
925
|
-
const
|
|
1666
|
+
const loadResult = await loadQdrantClient();
|
|
1667
|
+
if (!loadResult.success) {
|
|
1668
|
+
return loadResult;
|
|
1669
|
+
}
|
|
1670
|
+
const { QdrantClient: QdrantClientClass } = loadResult.data;
|
|
926
1671
|
const qdrantClient = new QdrantClientClass({
|
|
927
1672
|
url: qdrantConfig.url,
|
|
928
1673
|
apiKey: qdrantConfig.apiKey
|
|
@@ -930,17 +1675,17 @@ function createQdrantProvider() {
|
|
|
930
1675
|
await qdrantClient.getCollections();
|
|
931
1676
|
client = qdrantClient;
|
|
932
1677
|
config = qdrantConfig;
|
|
933
|
-
|
|
1678
|
+
logger4.info("Qdrant connected", { url: core.sanitize.sanitizeUrl(qdrantConfig.url) });
|
|
934
1679
|
return ok(void 0);
|
|
935
1680
|
} catch (error) {
|
|
936
|
-
|
|
1681
|
+
logger4.error("Failed to connect to Qdrant", { error: error instanceof Error ? error.message : String(error) });
|
|
937
1682
|
return err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_connectionFailed", { params: { error: String(error) } }), error);
|
|
938
1683
|
}
|
|
939
1684
|
},
|
|
940
1685
|
async close() {
|
|
941
1686
|
client = null;
|
|
942
1687
|
config = null;
|
|
943
|
-
|
|
1688
|
+
logger4.info("Qdrant connection closed");
|
|
944
1689
|
return ok(void 0);
|
|
945
1690
|
},
|
|
946
1691
|
isConnected() {
|
|
@@ -952,10 +1697,42 @@ function createQdrantProvider() {
|
|
|
952
1697
|
}
|
|
953
1698
|
|
|
954
1699
|
// src/vecdb-main.ts
|
|
955
|
-
var
|
|
1700
|
+
var logger5 = core.logger.child({ module: "vecdb", scope: "main" });
|
|
956
1701
|
var currentProvider = null;
|
|
957
1702
|
var currentConfig = null;
|
|
958
1703
|
var initInProgress = false;
|
|
1704
|
+
var pendingProvider = null;
|
|
1705
|
+
var initCompletion = null;
|
|
1706
|
+
var closingPromise = null;
|
|
1707
|
+
async function closeOwnedProvider() {
|
|
1708
|
+
if (currentProvider) {
|
|
1709
|
+
pendingProvider = currentProvider;
|
|
1710
|
+
currentProvider = null;
|
|
1711
|
+
}
|
|
1712
|
+
currentConfig = null;
|
|
1713
|
+
if (!pendingProvider)
|
|
1714
|
+
return ok(void 0);
|
|
1715
|
+
try {
|
|
1716
|
+
const result = await pendingProvider.close();
|
|
1717
|
+
if (!result.success)
|
|
1718
|
+
return err({ ...result.error, ext: { ...result.error.ext, cleanupPending: true } });
|
|
1719
|
+
pendingProvider = null;
|
|
1720
|
+
return result;
|
|
1721
|
+
} catch (cause) {
|
|
1722
|
+
return err({ ...HaiVecdbError.CONNECTION_FAILED, message: vecdbM("vecdb_closeFailed", { params: { error: cause instanceof Error ? cause.message : String(cause) } }), cause, ext: { cleanupPending: true } });
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
async function failInitialization(result) {
|
|
1726
|
+
const cleanup = await closeOwnedProvider();
|
|
1727
|
+
if (result.success)
|
|
1728
|
+
return result;
|
|
1729
|
+
if (!cleanup.success)
|
|
1730
|
+
return err({ ...result.error, ext: { ...result.error.ext, cleanupPending: true, cleanupError: cleanup.error } });
|
|
1731
|
+
const ext = { ...result.error.ext };
|
|
1732
|
+
delete ext.cleanupPending;
|
|
1733
|
+
delete ext.cleanupError;
|
|
1734
|
+
return err({ ...result.error, ext: Object.keys(ext).length ? ext : void 0 });
|
|
1735
|
+
}
|
|
959
1736
|
function createProvider(config) {
|
|
960
1737
|
switch (config.type) {
|
|
961
1738
|
case "lancedb":
|
|
@@ -964,6 +1741,8 @@ function createProvider(config) {
|
|
|
964
1741
|
return createPgvectorProvider();
|
|
965
1742
|
case "qdrant":
|
|
966
1743
|
return createQdrantProvider();
|
|
1744
|
+
case "chroma":
|
|
1745
|
+
return createChromaProvider();
|
|
967
1746
|
}
|
|
968
1747
|
}
|
|
969
1748
|
var notInitialized = core.module.createNotInitializedKit(
|
|
@@ -972,6 +1751,20 @@ var notInitialized = core.module.createNotInitializedKit(
|
|
|
972
1751
|
);
|
|
973
1752
|
var notInitializedCollection = notInitialized.proxy();
|
|
974
1753
|
var notInitializedVector = notInitialized.proxy();
|
|
1754
|
+
var collectionOps = {
|
|
1755
|
+
create: (name, options) => currentProvider && !closingPromise ? currentProvider.collection.create(name, options) : notInitializedCollection.create(name, options),
|
|
1756
|
+
drop: (name) => currentProvider && !closingPromise ? currentProvider.collection.drop(name) : notInitializedCollection.drop(name),
|
|
1757
|
+
exists: (name) => currentProvider && !closingPromise ? currentProvider.collection.exists(name) : notInitializedCollection.exists(name),
|
|
1758
|
+
info: (name) => currentProvider && !closingPromise ? currentProvider.collection.info(name) : notInitializedCollection.info(name),
|
|
1759
|
+
list: () => currentProvider && !closingPromise ? currentProvider.collection.list() : notInitializedCollection.list()
|
|
1760
|
+
};
|
|
1761
|
+
var vectorOps = {
|
|
1762
|
+
insert: (collection, documents) => currentProvider && !closingPromise ? currentProvider.vector.insert(collection, documents) : notInitializedVector.insert(collection, documents),
|
|
1763
|
+
upsert: (collection, documents) => currentProvider && !closingPromise ? currentProvider.vector.upsert(collection, documents) : notInitializedVector.upsert(collection, documents),
|
|
1764
|
+
delete: (collection, ids) => currentProvider && !closingPromise ? currentProvider.vector.delete(collection, ids) : notInitializedVector.delete(collection, ids),
|
|
1765
|
+
search: (collection, vector, options) => currentProvider && !closingPromise ? currentProvider.vector.search(collection, vector, options) : notInitializedVector.search(collection, vector, options),
|
|
1766
|
+
count: (collection) => currentProvider && !closingPromise ? currentProvider.vector.count(collection) : notInitializedVector.count(collection)
|
|
1767
|
+
};
|
|
975
1768
|
var vecdb = {
|
|
976
1769
|
/**
|
|
977
1770
|
* 初始化向量数据库连接
|
|
@@ -980,43 +1773,52 @@ var vecdb = {
|
|
|
980
1773
|
* @returns 初始化结果,失败时包含错误信息
|
|
981
1774
|
*/
|
|
982
1775
|
async init(config) {
|
|
983
|
-
if (initInProgress) {
|
|
984
|
-
|
|
1776
|
+
if (initInProgress || closingPromise) {
|
|
1777
|
+
logger5.warn("Vecdb init already in progress, skipping concurrent call");
|
|
985
1778
|
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_initInProgress"));
|
|
986
1779
|
}
|
|
987
1780
|
initInProgress = true;
|
|
1781
|
+
let resolveInit;
|
|
1782
|
+
initCompletion = new Promise((resolve) => {
|
|
1783
|
+
resolveInit = resolve;
|
|
1784
|
+
});
|
|
988
1785
|
try {
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
logger4.info("Initializing vecdb module");
|
|
1786
|
+
const cleanup = await closeOwnedProvider();
|
|
1787
|
+
if (!cleanup.success)
|
|
1788
|
+
return cleanup;
|
|
1789
|
+
logger5.info("Initializing vecdb module");
|
|
994
1790
|
const parseResult = VecdbConfigSchema.safeParse(config);
|
|
995
1791
|
if (!parseResult.success) {
|
|
996
|
-
|
|
1792
|
+
logger5.error("Vecdb config validation failed", { error: parseResult.error.message });
|
|
997
1793
|
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_configError", { params: { error: parseResult.error.message } }), parseResult.error);
|
|
998
1794
|
}
|
|
999
1795
|
const parsed = parseResult.data;
|
|
1796
|
+
if (core.module.requiresSharedState(process.env) && (parsed.type === "lancedb" || parsed.type === "chroma" && !!parsed.path && !parsed.url))
|
|
1797
|
+
return err(HaiVecdbError.CONFIG_ERROR, vecdbM("vecdb_sharedBackendRequired"));
|
|
1000
1798
|
try {
|
|
1001
1799
|
const provider = createProvider(parsed);
|
|
1800
|
+
pendingProvider = provider;
|
|
1002
1801
|
const connectResult = await provider.connect(parsed);
|
|
1003
1802
|
if (!connectResult.success) {
|
|
1004
|
-
|
|
1803
|
+
logger5.error("Vecdb module initialization failed", {
|
|
1005
1804
|
code: connectResult.error.code,
|
|
1006
1805
|
message: connectResult.error.message
|
|
1007
1806
|
});
|
|
1008
|
-
return connectResult;
|
|
1807
|
+
return await failInitialization(connectResult);
|
|
1009
1808
|
}
|
|
1809
|
+
pendingProvider = null;
|
|
1010
1810
|
currentProvider = provider;
|
|
1011
1811
|
currentConfig = parsed;
|
|
1012
|
-
|
|
1812
|
+
logger5.info("Vecdb module initialized", { type: parsed.type });
|
|
1013
1813
|
return ok(void 0);
|
|
1014
1814
|
} catch (error) {
|
|
1015
|
-
|
|
1016
|
-
return err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_initFailed", { params: { error: error instanceof Error ? error.message : String(error) } }), error);
|
|
1815
|
+
logger5.error("Vecdb module initialization failed", { error });
|
|
1816
|
+
return await failInitialization(err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_initFailed", { params: { error: error instanceof Error ? error.message : String(error) } }), error));
|
|
1017
1817
|
}
|
|
1018
1818
|
} finally {
|
|
1019
1819
|
initInProgress = false;
|
|
1820
|
+
initCompletion = null;
|
|
1821
|
+
resolveInit();
|
|
1020
1822
|
}
|
|
1021
1823
|
},
|
|
1022
1824
|
/**
|
|
@@ -1025,7 +1827,7 @@ var vecdb = {
|
|
|
1025
1827
|
* 未初始化时返回占位对象(所有调用返回 NOT_INITIALIZED)。
|
|
1026
1828
|
*/
|
|
1027
1829
|
get collection() {
|
|
1028
|
-
return
|
|
1830
|
+
return collectionOps;
|
|
1029
1831
|
},
|
|
1030
1832
|
/**
|
|
1031
1833
|
* 获取向量操作接口
|
|
@@ -1033,15 +1835,44 @@ var vecdb = {
|
|
|
1033
1835
|
* 未初始化时返回占位对象(所有调用返回 NOT_INITIALIZED)。
|
|
1034
1836
|
*/
|
|
1035
1837
|
get vector() {
|
|
1036
|
-
return
|
|
1838
|
+
return vectorOps;
|
|
1037
1839
|
},
|
|
1038
|
-
/**
|
|
1840
|
+
/** 获取当前脱敏配置快照(未初始化时为 null) */
|
|
1039
1841
|
get config() {
|
|
1040
|
-
return currentConfig;
|
|
1842
|
+
return currentConfig && !closingPromise ? core.sanitize.sanitizeSensitiveFields(currentConfig) : null;
|
|
1843
|
+
},
|
|
1844
|
+
/** 获取底层后端的原始连接信息(含凭证,未初始化时为 null;仅供同进程内集成使用) */
|
|
1845
|
+
get rawConnection() {
|
|
1846
|
+
if (!currentConfig || closingPromise)
|
|
1847
|
+
return null;
|
|
1848
|
+
if (currentConfig.type === "qdrant")
|
|
1849
|
+
return { type: "qdrant", url: currentConfig.url, apiKey: currentConfig.apiKey };
|
|
1850
|
+
if (currentConfig.type === "pgvector") {
|
|
1851
|
+
return {
|
|
1852
|
+
type: "pgvector",
|
|
1853
|
+
connectionString: currentConfig.url,
|
|
1854
|
+
host: currentConfig.host,
|
|
1855
|
+
port: currentConfig.port,
|
|
1856
|
+
database: currentConfig.database,
|
|
1857
|
+
user: currentConfig.user,
|
|
1858
|
+
password: currentConfig.password
|
|
1859
|
+
};
|
|
1860
|
+
}
|
|
1861
|
+
if (currentConfig.type === "chroma") {
|
|
1862
|
+
return {
|
|
1863
|
+
type: "chroma",
|
|
1864
|
+
url: currentConfig.url,
|
|
1865
|
+
host: currentConfig.host,
|
|
1866
|
+
port: currentConfig.port,
|
|
1867
|
+
apiKey: currentConfig.apiKey,
|
|
1868
|
+
path: currentConfig.path
|
|
1869
|
+
};
|
|
1870
|
+
}
|
|
1871
|
+
return { type: currentConfig.type, path: currentConfig.path };
|
|
1041
1872
|
},
|
|
1042
1873
|
/** 检查是否已初始化 */
|
|
1043
1874
|
get isInitialized() {
|
|
1044
|
-
return currentProvider !== null && currentProvider.isConnected();
|
|
1875
|
+
return currentProvider !== null && !closingPromise && currentProvider.isConnected();
|
|
1045
1876
|
},
|
|
1046
1877
|
/**
|
|
1047
1878
|
* 关闭向量数据库连接
|
|
@@ -1049,30 +1880,19 @@ var vecdb = {
|
|
|
1049
1880
|
* 多次调用安全,未初始化时直接返回。
|
|
1050
1881
|
*/
|
|
1051
1882
|
async close() {
|
|
1052
|
-
if (!
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
if (!closeResult.success) {
|
|
1061
|
-
logger4.error("Vecdb module close failed", { code: closeResult.error.code, message: closeResult.error.message });
|
|
1062
|
-
return closeResult;
|
|
1063
|
-
}
|
|
1064
|
-
logger4.info("Vecdb module closed");
|
|
1065
|
-
return ok(void 0);
|
|
1066
|
-
} catch (error) {
|
|
1067
|
-
logger4.error("Vecdb module close failed", { error });
|
|
1068
|
-
return err(HaiVecdbError.CONNECTION_FAILED, vecdbM("vecdb_closeFailed", { params: { error: error instanceof Error ? error.message : String(error) } }), error);
|
|
1069
|
-
} finally {
|
|
1070
|
-
currentProvider = null;
|
|
1071
|
-
currentConfig = null;
|
|
1883
|
+
if (!closingPromise) {
|
|
1884
|
+
closingPromise = (async () => {
|
|
1885
|
+
if (initCompletion)
|
|
1886
|
+
await initCompletion;
|
|
1887
|
+
return closeOwnedProvider();
|
|
1888
|
+
})().finally(() => {
|
|
1889
|
+
closingPromise = null;
|
|
1890
|
+
});
|
|
1072
1891
|
}
|
|
1892
|
+
return await closingPromise;
|
|
1073
1893
|
}
|
|
1074
1894
|
};
|
|
1075
1895
|
|
|
1076
|
-
export { DistanceMetricSchema, HaiVecdbError,
|
|
1896
|
+
export { DistanceMetricSchema, HaiVecdbError, VecdbConfigSchema, VecdbOperationLogConfigSchema, vecdb };
|
|
1077
1897
|
//# sourceMappingURL=index.js.map
|
|
1078
1898
|
//# sourceMappingURL=index.js.map
|