@vivantel/rag-store-chromadb 1.2.3 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/store.d.ts +3 -1
- package/dist/store.d.ts.map +1 -1
- package/dist/store.js +13 -1
- package/dist/store.js.map +1 -1
- package/package.json +1 -1
package/dist/store.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { VectorDocument, VectorSearchResult, VectorStore, IndexStats, QueryPerfReport } from "@vivantel/rag-core";
|
|
1
|
+
import type { VectorDocument, VectorSearchResult, VectorStore, IndexStats, QueryPerfReport, Logger } from "@vivantel/rag-core";
|
|
2
2
|
export interface ChromaVectorStoreOptions {
|
|
3
3
|
path?: string;
|
|
4
4
|
collectionName?: string;
|
|
@@ -13,7 +13,9 @@ export declare class ChromaVectorStore implements VectorStore {
|
|
|
13
13
|
private readonly apiKey;
|
|
14
14
|
private client;
|
|
15
15
|
private collection;
|
|
16
|
+
private logger;
|
|
16
17
|
constructor(options: ChromaVectorStoreOptions);
|
|
18
|
+
setLogger(logger: Logger): void;
|
|
17
19
|
initialize(): Promise<void>;
|
|
18
20
|
upsert(documents: VectorDocument[]): Promise<void>;
|
|
19
21
|
deleteBySourceFile(sourceFiles: string[]): Promise<void>;
|
package/dist/store.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,kBAAkB,EAClB,WAAW,EACX,UAAU,EACV,eAAe,
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,kBAAkB,EAClB,WAAW,EACX,UAAU,EACV,eAAe,EACf,MAAM,EACP,MAAM,oBAAoB,CAAC;AAK5B,MAAM,WAAW,wBAAwB;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAOD,qBAAa,iBAAkB,YAAW,WAAW;IACnD,QAAQ,CAAC,IAAI,cAAc;IAE3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,UAAU,CAAc;IAChC,OAAO,CAAC,MAAM,CAAuB;gBAEzB,OAAO,EAAE,wBAAwB;IAO7C,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAIzB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAoB3B,MAAM,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAwBlD,kBAAkB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAUxD,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IA8B/C,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC;IAIpC,kBAAkB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAIpE,MAAM,CACV,cAAc,EAAE,MAAM,EAAE,EACxB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,kBAAkB,EAAE,CAAC;CAiCjC"}
|
package/dist/store.js
CHANGED
|
@@ -13,13 +13,18 @@ export class ChromaVectorStore {
|
|
|
13
13
|
apiKey;
|
|
14
14
|
client;
|
|
15
15
|
collection;
|
|
16
|
+
logger = null;
|
|
16
17
|
constructor(options) {
|
|
17
18
|
this.path = options.path ?? "http://localhost:8000";
|
|
18
19
|
this.collectionName = options.collectionName ?? DEFAULT_COLLECTION;
|
|
19
20
|
this.dimensions = options.dimensions ?? DEFAULT_DIMENSIONS;
|
|
20
21
|
this.apiKey = options.apiKey;
|
|
21
22
|
}
|
|
23
|
+
setLogger(logger) {
|
|
24
|
+
this.logger = logger.withTag("chromadb");
|
|
25
|
+
}
|
|
22
26
|
async initialize() {
|
|
27
|
+
this.logger?.info(`Connecting to chromadb at ${this.path}, collection: ${this.collectionName}`);
|
|
23
28
|
this.client = new ChromaClient({
|
|
24
29
|
path: this.path,
|
|
25
30
|
auth: this.apiKey
|
|
@@ -30,12 +35,15 @@ export class ChromaVectorStore {
|
|
|
30
35
|
name: this.collectionName,
|
|
31
36
|
metadata: { "hnsw:space": "cosine" },
|
|
32
37
|
});
|
|
38
|
+
this.logger?.debug(`Collection "${this.collectionName}" ready (cosine, ${this.dimensions}d)`);
|
|
33
39
|
}
|
|
34
40
|
async upsert(documents) {
|
|
41
|
+
this.logger?.verbose(`Upserting ${documents.length} docs into "${this.collectionName}"`);
|
|
35
42
|
for (let i = 0; i < documents.length; i += UPSERT_BATCH_SIZE) {
|
|
36
43
|
const batch = documents.slice(i, i + UPSERT_BATCH_SIZE);
|
|
44
|
+
const ids = batch.map((doc) => doc.id ?? crypto.randomUUID());
|
|
37
45
|
await this.collection.upsert({
|
|
38
|
-
ids
|
|
46
|
+
ids,
|
|
39
47
|
embeddings: batch.map((doc) => doc.embedding),
|
|
40
48
|
documents: batch.map((doc) => doc.content),
|
|
41
49
|
metadatas: batch.map((doc) => ({
|
|
@@ -45,11 +53,13 @@ export class ChromaVectorStore {
|
|
|
45
53
|
...doc.metadata,
|
|
46
54
|
})),
|
|
47
55
|
});
|
|
56
|
+
this.logger?.trace(` Upserted IDs: ${ids.map((id) => id.slice(0, 8)).join(", ")}`);
|
|
48
57
|
}
|
|
49
58
|
}
|
|
50
59
|
async deleteBySourceFile(sourceFiles) {
|
|
51
60
|
if (sourceFiles.length === 0)
|
|
52
61
|
return;
|
|
62
|
+
this.logger?.verbose(`Deleting docs for ${sourceFiles.length} source file(s)`);
|
|
53
63
|
await this.collection.delete({
|
|
54
64
|
where: { source_file: { $in: sourceFiles } },
|
|
55
65
|
});
|
|
@@ -76,6 +86,7 @@ export class ChromaVectorStore {
|
|
|
76
86
|
break;
|
|
77
87
|
offset += SCROLL_PAGE_SIZE;
|
|
78
88
|
}
|
|
89
|
+
this.logger?.verbose(`getCurrentState: ${state.size} source version(s)`);
|
|
79
90
|
return state;
|
|
80
91
|
}
|
|
81
92
|
async getIndexStats() {
|
|
@@ -85,6 +96,7 @@ export class ChromaVectorStore {
|
|
|
85
96
|
return getQueryPerfReport(this.collection, this.dimensions, timeframeHours);
|
|
86
97
|
}
|
|
87
98
|
async search(queryEmbedding, topK) {
|
|
99
|
+
this.logger?.debug(`Search: topK=${topK}`);
|
|
88
100
|
const result = await this.collection.query({
|
|
89
101
|
queryEmbeddings: [queryEmbedding],
|
|
90
102
|
nResults: topK,
|
package/dist/store.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAmB,MAAM,UAAU,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AASrD,MAAM,kBAAkB,GAAG,WAAW,CAAC;AACvC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,UAAU,CAAC;IAEV,IAAI,CAAS;IACb,cAAc,CAAS;IACvB,UAAU,CAAS;IACnB,MAAM,CAAqB;IACpC,MAAM,CAAgB;IACtB,UAAU,CAAc;IACxB,MAAM,GAAkB,IAAI,CAAC;IAErC,YAAY,OAAiC;QAC3C,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,uBAAuB,CAAC;QACpD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,kBAAkB,CAAC;QACnE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;QAC3D,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,SAAS,CAAC,MAAc;QACtB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,MAAM,EAAE,IAAI,CACf,6BAA6B,IAAI,CAAC,IAAI,iBAAiB,IAAI,CAAC,cAAc,EAAE,CAC7E,CAAC;QACF,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,CAAC;YAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,MAAM;gBACf,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE;gBACjD,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC;YACxD,IAAI,EAAE,IAAI,CAAC,cAAc;YACzB,QAAQ,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE;SACrC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB,eAAe,IAAI,CAAC,cAAc,oBAAoB,IAAI,CAAC,UAAU,IAAI,CAC1E,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAA2B;QACtC,IAAI,CAAC,MAAM,EAAE,OAAO,CAClB,aAAa,SAAS,CAAC,MAAM,eAAe,IAAI,CAAC,cAAc,GAAG,CACnE,CAAC;QACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,iBAAiB,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,CAAC;YACxD,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YAC9D,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;gBAC3B,GAAG;gBACH,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC;gBAC7C,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC;gBAC1C,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAC7B,WAAW,EAAE,GAAG,CAAC,UAAU;oBAC3B,WAAW,EAAE,GAAG,CAAC,UAAU;oBAC3B,YAAY,EAAE,GAAG,CAAC,WAAW;oBAC7B,GAAG,GAAG,CAAC,QAAQ;iBAChB,CAAC,CAAC;aACJ,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,EAAE,KAAK,CAChB,mBAAmB,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,WAAqB;QAC5C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACrC,IAAI,CAAC,MAAM,EAAE,OAAO,CAClB,qBAAqB,WAAW,CAAC,MAAM,iBAAiB,CACzD,CAAC;QACF,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAC3B,KAAK,EAAE,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE,EAAE;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;QACxC,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBACvC,OAAO,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC;gBAChC,KAAK,EAAE,gBAAgB;gBACvB,MAAM;aACP,CAAC,CAAC;YAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACpC,IAAI,CAAC,IAAI;oBAAE,SAAS;gBACpB,MAAM,UAAU,GACd,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;gBACjE,MAAM,UAAU,GACd,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;gBACjE,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;oBAC7B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;gBACpC,CAAC;YACH,CAAC;YAED,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,gBAAgB;gBAAE,MAAM;YAChD,MAAM,IAAI,gBAAgB,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,oBAAoB,KAAK,CAAC,IAAI,oBAAoB,CAAC,CAAC;QACzE,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,OAAO,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,cAAsB;QAC7C,OAAO,kBAAkB,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,MAAM,CACV,cAAwB,EACxB,IAAY;QAEZ,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YACzC,eAAe,EAAE,CAAC,cAAc,CAAC;YACjC,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE;gBACP,WAAW,CAAC,SAAS;gBACrB,WAAW,CAAC,SAAS;gBACrB,WAAW,CAAC,SAAS;aACtB;SACF,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE9C,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACzB,EAAE;YACF,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE;YAC3B,QAAQ,EAAE,CAAC,GAAG,EAAE;gBACd,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;oBAAE,OAAO,EAAE,CAAC;gBACjD,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,GAAG,IAAI,EAAE,GACvD,IAA+B,CAAC;gBAClC,KAAK,WAAW,CAAC;gBACjB,KAAK,WAAW,CAAC;gBACjB,KAAK,YAAY,CAAC;gBAClB,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,EAAE;YACJ,UAAU,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SACpC,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|