@vivantel/rag-store-chromadb 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +14 -0
- package/dist/index.js.map +1 -0
- package/dist/query-perf.d.ts +4 -0
- package/dist/query-perf.d.ts.map +1 -0
- package/dist/query-perf.js +65 -0
- package/dist/query-perf.js.map +1 -0
- package/dist/query-perf.test.d.ts +2 -0
- package/dist/query-perf.test.d.ts.map +1 -0
- package/dist/query-perf.test.js +73 -0
- package/dist/query-perf.test.js.map +1 -0
- package/dist/stats.d.ts +4 -0
- package/dist/stats.d.ts.map +1 -0
- package/dist/stats.js +19 -0
- package/dist/stats.js.map +1 -0
- package/dist/stats.test.d.ts +2 -0
- package/dist/stats.test.d.ts.map +1 -0
- package/dist/stats.test.js +41 -0
- package/dist/stats.test.js.map +1 -0
- package/dist/store.d.ts +25 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +118 -0
- package/dist/store.js.map +1 -0
- package/package.json +70 -0
package/README.md
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# @vivantel/rag-store-chromadb
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/@vivantel/rag-store-chromadb)
|
|
4
|
+
|
|
5
|
+
ChromaDB vector store for [`@vivantel/rag-core`](../rag-core/README.md). Works with a locally-running Chroma server or hosted Chroma Cloud.
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install @vivantel/rag-store-chromadb @vivantel/rag-core
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Quick start (JSON config)
|
|
14
|
+
|
|
15
|
+
Local Chroma server:
|
|
16
|
+
|
|
17
|
+
```json
|
|
18
|
+
{
|
|
19
|
+
"vectorStore": {
|
|
20
|
+
"package": "@vivantel/rag-store-chromadb",
|
|
21
|
+
"config": {
|
|
22
|
+
"path": "http://localhost:8000"
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Running Chroma locally
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
docker run -p 8000:8000 chromadb/chroma
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Hosted Chroma
|
|
35
|
+
|
|
36
|
+
Pass your Chroma Cloud URL and API key:
|
|
37
|
+
|
|
38
|
+
```json
|
|
39
|
+
{
|
|
40
|
+
"vectorStore": {
|
|
41
|
+
"package": "@vivantel/rag-store-chromadb",
|
|
42
|
+
"config": {
|
|
43
|
+
"path": "${CHROMA_URL}",
|
|
44
|
+
"apiKey": "${CHROMA_API_KEY}"
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Configuration
|
|
51
|
+
|
|
52
|
+
| Option | Type | Default | Description |
|
|
53
|
+
|--------|------|---------|-------------|
|
|
54
|
+
| `path` | `string` | `"http://localhost:8000"` | Chroma server URL |
|
|
55
|
+
| `apiKey` | `string` | `undefined` | API key for hosted Chroma (token auth) |
|
|
56
|
+
| `collectionName` | `string` | `"documents"` | Collection name |
|
|
57
|
+
| `dimensions` | `number` | `undefined` | Vector size (inferred from first upsert if omitted) |
|
|
58
|
+
|
|
59
|
+
## TypeScript usage
|
|
60
|
+
|
|
61
|
+
```typescript
|
|
62
|
+
import { ChromaVectorStore } from "@vivantel/rag-store-chromadb";
|
|
63
|
+
|
|
64
|
+
const store = new ChromaVectorStore({
|
|
65
|
+
path: "http://localhost:8000",
|
|
66
|
+
collectionName: "my-docs",
|
|
67
|
+
});
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Self-registration
|
|
71
|
+
|
|
72
|
+
This package declares a `"rag-plugin"` field in its `package.json`. Once installed, `rag-update init` discovers it automatically — no manual config required.
|
|
73
|
+
|
|
74
|
+
```jsonc
|
|
75
|
+
// package.json (excerpt)
|
|
76
|
+
"rag-plugin": {
|
|
77
|
+
"type": "vectorStore",
|
|
78
|
+
"label": "ChromaDB (local or hosted)",
|
|
79
|
+
"key": "chromadb",
|
|
80
|
+
"envVars": [],
|
|
81
|
+
"defaultConfig": { "path": "http://localhost:8000" }
|
|
82
|
+
}
|
|
83
|
+
```
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { ChromaVectorStore, type ChromaVectorStoreOptions } from "./store.js";
|
|
2
|
+
import type { VectorStore } from "@vivantel/rag-core";
|
|
3
|
+
/** Factory used by the JSON config loader. */
|
|
4
|
+
export declare function createVectorStore(config: Record<string, unknown>): VectorStore;
|
|
5
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,KAAK,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAE9E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGtD,8CAA8C;AAC9C,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B,WAAW,CAWb"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export { ChromaVectorStore } from "./store.js";
|
|
2
|
+
import { ChromaVectorStore } from "./store.js";
|
|
3
|
+
/** Factory used by the JSON config loader. */
|
|
4
|
+
export function createVectorStore(config) {
|
|
5
|
+
return new ChromaVectorStore({
|
|
6
|
+
path: typeof config.path === "string" ? config.path : undefined,
|
|
7
|
+
collectionName: typeof config.collectionName === "string"
|
|
8
|
+
? config.collectionName
|
|
9
|
+
: undefined,
|
|
10
|
+
dimensions: typeof config.dimensions === "number" ? config.dimensions : undefined,
|
|
11
|
+
apiKey: typeof config.apiKey === "string" ? config.apiKey : undefined,
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAiC,MAAM,YAAY,CAAC;AAG9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAE/C,8CAA8C;AAC9C,MAAM,UAAU,iBAAiB,CAC/B,MAA+B;IAE/B,OAAO,IAAI,iBAAiB,CAAC;QAC3B,IAAI,EAAE,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;QAC/D,cAAc,EACZ,OAAO,MAAM,CAAC,cAAc,KAAK,QAAQ;YACvC,CAAC,CAAC,MAAM,CAAC,cAAc;YACvB,CAAC,CAAC,SAAS;QACf,UAAU,EACR,OAAO,MAAM,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;QACvE,MAAM,EAAE,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;KACtE,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { QueryPerfReport } from "@vivantel/rag-core";
|
|
2
|
+
import type { Collection } from "chromadb";
|
|
3
|
+
export declare function getQueryPerfReport(collection: Collection, dimensions: number, timeframeHours: number): Promise<QueryPerfReport>;
|
|
4
|
+
//# sourceMappingURL=query-perf.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query-perf.d.ts","sourceRoot":"","sources":["../src/query-perf.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAa3C,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,UAAU,EACtB,UAAU,EAAE,MAAM,EAClB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,eAAe,CAAC,CAiE1B"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { IncludeEnum } from "chromadb";
|
|
2
|
+
const SAMPLE_COUNT = 20;
|
|
3
|
+
function percentile(sorted, p) {
|
|
4
|
+
if (sorted.length === 0)
|
|
5
|
+
return -1;
|
|
6
|
+
return (sorted[Math.floor((p / 100) * sorted.length)] ??
|
|
7
|
+
sorted[sorted.length - 1] ??
|
|
8
|
+
0);
|
|
9
|
+
}
|
|
10
|
+
export async function getQueryPerfReport(collection, dimensions, timeframeHours) {
|
|
11
|
+
// Try to fetch real embeddings for representative queries; fall back to zero vectors.
|
|
12
|
+
let sampleEmbeddings = [];
|
|
13
|
+
try {
|
|
14
|
+
const result = await collection.get({
|
|
15
|
+
include: [IncludeEnum.Embeddings],
|
|
16
|
+
limit: SAMPLE_COUNT,
|
|
17
|
+
});
|
|
18
|
+
if (result.embeddings && result.embeddings.length > 0) {
|
|
19
|
+
sampleEmbeddings = result.embeddings.map((e) => Array.from(e));
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
catch {
|
|
23
|
+
// Fall through to synthetic vectors
|
|
24
|
+
}
|
|
25
|
+
if (sampleEmbeddings.length === 0) {
|
|
26
|
+
sampleEmbeddings = Array.from({ length: SAMPLE_COUNT }, () => Array(dimensions).fill(0));
|
|
27
|
+
}
|
|
28
|
+
const latencies = [];
|
|
29
|
+
for (const embedding of sampleEmbeddings) {
|
|
30
|
+
const start = performance.now();
|
|
31
|
+
try {
|
|
32
|
+
await collection.query({
|
|
33
|
+
queryEmbeddings: [embedding],
|
|
34
|
+
nResults: 10,
|
|
35
|
+
include: [],
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
// Empty collection or other transient error — still record timing
|
|
40
|
+
}
|
|
41
|
+
latencies.push(performance.now() - start);
|
|
42
|
+
}
|
|
43
|
+
latencies.sort((a, b) => a - b);
|
|
44
|
+
const p50 = percentile(latencies, 50);
|
|
45
|
+
const p95 = percentile(latencies, 95);
|
|
46
|
+
const p99 = percentile(latencies, 99);
|
|
47
|
+
const slowQueryCount = latencies.filter((ms) => ms > 100).length;
|
|
48
|
+
const suggestedIndexes = [];
|
|
49
|
+
if (p95 > 100) {
|
|
50
|
+
suggestedIndexes.push(`p95 latency (${p95.toFixed(1)} ms) is high. ` +
|
|
51
|
+
`Consider increasing ChromaDB server resources or tuning HNSW ef_search.`);
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
suggestedIndexes.push("Query performance looks healthy.");
|
|
55
|
+
}
|
|
56
|
+
return {
|
|
57
|
+
timeframeHours,
|
|
58
|
+
p50LatencyMs: Math.round(p50 * 10) / 10,
|
|
59
|
+
p95LatencyMs: Math.round(p95 * 10) / 10,
|
|
60
|
+
p99LatencyMs: Math.round(p99 * 10) / 10,
|
|
61
|
+
slowQueryCount,
|
|
62
|
+
suggestedIndexes,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
//# sourceMappingURL=query-perf.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query-perf.js","sourceRoot":"","sources":["../src/query-perf.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAGvC,MAAM,YAAY,GAAG,EAAE,CAAC;AAExB,SAAS,UAAU,CAAC,MAAgB,EAAE,CAAS;IAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,CAAC;IACnC,OAAO,CACL,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACzB,CAAC,CACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,UAAsB,EACtB,UAAkB,EAClB,cAAsB;IAEtB,sFAAsF;IACtF,IAAI,gBAAgB,GAAe,EAAE,CAAC;IAEtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC;YAClC,OAAO,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC;YACjC,KAAK,EAAE,YAAY;SACpB,CAAC,CAAC;QACH,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtD,gBAAgB,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7C,KAAK,CAAC,IAAI,CAAC,CAAsB,CAAC,CACnC,CAAC;QACJ,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,oCAAoC;IACtC,CAAC;IAED,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClC,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,GAAG,EAAE,CAC3D,KAAK,CAAS,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAClC,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAa,EAAE,CAAC;IAE/B,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,UAAU,CAAC,KAAK,CAAC;gBACrB,eAAe,EAAE,CAAC,SAAS,CAAC;gBAC5B,QAAQ,EAAE,EAAE;gBACZ,OAAO,EAAE,EAAQ;aAClB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,kEAAkE;QACpE,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IAC5C,CAAC;IAED,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEhC,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACtC,MAAM,cAAc,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;QACd,gBAAgB,CAAC,IAAI,CACnB,gBAAgB,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB;YAC5C,yEAAyE,CAC5E,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,gBAAgB,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO;QACL,cAAc;QACd,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,EAAE;QACvC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,EAAE;QACvC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,EAAE,CAAC,GAAG,EAAE;QACvC,cAAc;QACd,gBAAgB;KACjB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query-perf.test.d.ts","sourceRoot":"","sources":["../src/query-perf.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from "vitest";
|
|
2
|
+
import { getQueryPerfReport } from "./query-perf.js";
|
|
3
|
+
function makeCollection(overrides = {}) {
|
|
4
|
+
const delay = overrides.queryDelayMs ?? 2;
|
|
5
|
+
return {
|
|
6
|
+
get: overrides.get ?? vi.fn().mockResolvedValue({ embeddings: [], ids: [] }),
|
|
7
|
+
query: overrides.query ??
|
|
8
|
+
vi
|
|
9
|
+
.fn()
|
|
10
|
+
.mockImplementation(() => new Promise((resolve) => setTimeout(() => resolve({ ids: [[]] }), delay))),
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
describe("getQueryPerfReport (ChromaDB)", () => {
|
|
14
|
+
it("returns valid latency percentiles (p50 ≤ p95 ≤ p99)", async () => {
|
|
15
|
+
const report = await getQueryPerfReport(makeCollection({ queryDelayMs: 2 }), 384, 24);
|
|
16
|
+
expect(report.timeframeHours).toBe(24);
|
|
17
|
+
expect(report.p50LatencyMs).toBeGreaterThanOrEqual(0);
|
|
18
|
+
expect(report.p95LatencyMs).toBeGreaterThanOrEqual(report.p50LatencyMs);
|
|
19
|
+
expect(report.p99LatencyMs).toBeGreaterThanOrEqual(report.p95LatencyMs);
|
|
20
|
+
});
|
|
21
|
+
it("timeframeHours is reflected in the report", async () => {
|
|
22
|
+
const report = await getQueryPerfReport(makeCollection(), 384, 48);
|
|
23
|
+
expect(report.timeframeHours).toBe(48);
|
|
24
|
+
});
|
|
25
|
+
it("slowQueryCount is 0 for fast queries (< 100 ms)", async () => {
|
|
26
|
+
const report = await getQueryPerfReport(makeCollection({ queryDelayMs: 1 }), 384, 24);
|
|
27
|
+
expect(report.slowQueryCount).toBe(0);
|
|
28
|
+
});
|
|
29
|
+
it("returns healthy suggestion when p95 ≤ 100 ms", async () => {
|
|
30
|
+
const report = await getQueryPerfReport(makeCollection({ queryDelayMs: 1 }), 384, 24);
|
|
31
|
+
expect(report.suggestedIndexes[0]).toMatch(/healthy/i);
|
|
32
|
+
});
|
|
33
|
+
it("uses real embeddings from collection.get() when available", async () => {
|
|
34
|
+
const fakeEmbeddings = [Array(4).fill(0.1), Array(4).fill(0.2)];
|
|
35
|
+
const querySpy = vi.fn().mockResolvedValue({ ids: [[]] });
|
|
36
|
+
const collection = makeCollection({
|
|
37
|
+
get: vi
|
|
38
|
+
.fn()
|
|
39
|
+
.mockResolvedValue({ embeddings: fakeEmbeddings, ids: ["a", "b"] }),
|
|
40
|
+
query: querySpy,
|
|
41
|
+
});
|
|
42
|
+
await getQueryPerfReport(collection, 4, 24);
|
|
43
|
+
// Exactly 2 real embeddings → exactly 2 query calls
|
|
44
|
+
expect(querySpy).toHaveBeenCalledTimes(2);
|
|
45
|
+
const firstArg = querySpy.mock.calls[0][0];
|
|
46
|
+
expect(firstArg.queryEmbeddings[0]).toEqual(expect.arrayContaining([0.1]));
|
|
47
|
+
});
|
|
48
|
+
it("falls back to 20 synthetic zero-vectors when collection.get() fails", async () => {
|
|
49
|
+
const querySpy = vi.fn().mockResolvedValue({ ids: [[]] });
|
|
50
|
+
const collection = makeCollection({
|
|
51
|
+
get: vi.fn().mockRejectedValue(new Error("not supported")),
|
|
52
|
+
query: querySpy,
|
|
53
|
+
});
|
|
54
|
+
const report = await getQueryPerfReport(collection, 384, 24);
|
|
55
|
+
expect(report.p50LatencyMs).toBeGreaterThanOrEqual(0);
|
|
56
|
+
// 20 synthetic samples (SAMPLE_COUNT)
|
|
57
|
+
expect(querySpy).toHaveBeenCalledTimes(20);
|
|
58
|
+
});
|
|
59
|
+
it("suggests HNSW tuning when p95 > 100 ms", async () => {
|
|
60
|
+
const report = await getQueryPerfReport(makeCollection({ queryDelayMs: 120 }), 384, 24);
|
|
61
|
+
expect(report.suggestedIndexes[0]).toMatch(/HNSW|hnsw/i);
|
|
62
|
+
}, 15_000);
|
|
63
|
+
it("handles query errors gracefully (still returns a complete report)", async () => {
|
|
64
|
+
const collection = makeCollection({
|
|
65
|
+
query: vi.fn().mockRejectedValue(new Error("connection refused")),
|
|
66
|
+
});
|
|
67
|
+
const report = await getQueryPerfReport(collection, 384, 24);
|
|
68
|
+
expect(report.p50LatencyMs).toBeGreaterThanOrEqual(0);
|
|
69
|
+
expect(report.slowQueryCount).toBeGreaterThanOrEqual(0);
|
|
70
|
+
expect(report.suggestedIndexes.length).toBeGreaterThan(0);
|
|
71
|
+
});
|
|
72
|
+
}, 60_000);
|
|
73
|
+
//# sourceMappingURL=query-perf.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"query-perf.test.js","sourceRoot":"","sources":["../src/query-perf.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAGrD,SAAS,cAAc,CACrB,YAIK,EAAE;IAEP,MAAM,KAAK,GAAG,SAAS,CAAC,YAAY,IAAI,CAAC,CAAC;IAE1C,OAAO;QACL,GAAG,EACD,SAAS,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,UAAU,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;QACzE,KAAK,EACH,SAAS,CAAC,KAAK;YACf,EAAE;iBACC,EAAE,EAAE;iBACJ,kBAAkB,CACjB,GAAG,EAAE,CACH,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CACtB,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAChD,CACJ;KACmB,CAAC;AAC7B,CAAC;AAED,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC7C,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,MAAM,GAAG,MAAM,kBAAkB,CACrC,cAAc,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,EACnC,GAAG,EACH,EAAE,CACH,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,sBAAsB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACxE,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,sBAAsB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QACnE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,MAAM,GAAG,MAAM,kBAAkB,CACrC,cAAc,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,EACnC,GAAG,EACH,EAAE,CACH,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,MAAM,GAAG,MAAM,kBAAkB,CACrC,cAAc,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,EACnC,GAAG,EACH,EAAE,CACH,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,MAAM,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,cAAc,CAAC;YAChC,GAAG,EAAE,EAAE;iBACJ,EAAE,EAAE;iBACJ,iBAAiB,CAAC,EAAE,UAAU,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;YACrE,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;QAEH,MAAM,kBAAkB,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAE5C,oDAAoD;QACpD,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAExC,CAAC;QACF,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC7E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACnF,MAAM,QAAQ,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1D,MAAM,UAAU,GAAG,cAAc,CAAC;YAChC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;YAC1D,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,UAAU,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAE7D,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACtD,sCAAsC;QACtC,MAAM,CAAC,QAAQ,CAAC,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,MAAM,GAAG,MAAM,kBAAkB,CACrC,cAAc,CAAC,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,EACrC,GAAG,EACH,EAAE,CACH,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IAC3D,CAAC,EAAE,MAAM,CAAC,CAAC;IAEX,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;QACjF,MAAM,UAAU,GAAG,cAAc,CAAC;YAChC,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;SAClE,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,UAAU,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAE7D,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC,EAAE,MAAM,CAAC,CAAC"}
|
package/dist/stats.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../src/stats.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C,wBAAsB,aAAa,CACjC,UAAU,EAAE,UAAU,GACrB,OAAO,CAAC,UAAU,CAAC,CAoBrB"}
|
package/dist/stats.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export async function getIndexStats(collection) {
|
|
2
|
+
const totalVectors = await collection.count();
|
|
3
|
+
const suggestions = [];
|
|
4
|
+
if (totalVectors === 0) {
|
|
5
|
+
suggestions.push("Collection is empty.");
|
|
6
|
+
}
|
|
7
|
+
else {
|
|
8
|
+
suggestions.push("ChromaDB uses HNSW internally — no manual reindex needed.");
|
|
9
|
+
}
|
|
10
|
+
return {
|
|
11
|
+
totalVectors,
|
|
12
|
+
indexType: "hnsw",
|
|
13
|
+
annRecallAt10: -1,
|
|
14
|
+
indexAgeHours: -1,
|
|
15
|
+
deadTupleFraction: 0,
|
|
16
|
+
suggestions,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=stats.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.js","sourceRoot":"","sources":["../src/stats.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAAsB;IAEtB,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;IAC9C,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;QACvB,WAAW,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAC3C,CAAC;SAAM,CAAC;QACN,WAAW,CAAC,IAAI,CACd,2DAA2D,CAC5D,CAAC;IACJ,CAAC;IAED,OAAO;QACL,YAAY;QACZ,SAAS,EAAE,MAAM;QACjB,aAAa,EAAE,CAAC,CAAC;QACjB,aAAa,EAAE,CAAC,CAAC;QACjB,iBAAiB,EAAE,CAAC;QACpB,WAAW;KACZ,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.test.d.ts","sourceRoot":"","sources":["../src/stats.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from "vitest";
|
|
2
|
+
import { getIndexStats } from "./stats.js";
|
|
3
|
+
function makeCollection(count) {
|
|
4
|
+
return {
|
|
5
|
+
count: vi.fn().mockResolvedValue(count),
|
|
6
|
+
};
|
|
7
|
+
}
|
|
8
|
+
describe("getIndexStats (ChromaDB)", () => {
|
|
9
|
+
it("always returns hnsw indexType", async () => {
|
|
10
|
+
const stats = await getIndexStats(makeCollection(100));
|
|
11
|
+
expect(stats.indexType).toBe("hnsw");
|
|
12
|
+
});
|
|
13
|
+
it("reflects the collection count as totalVectors", async () => {
|
|
14
|
+
const stats = await getIndexStats(makeCollection(4200));
|
|
15
|
+
expect(stats.totalVectors).toBe(4200);
|
|
16
|
+
});
|
|
17
|
+
it("totalVectors is 0 for an empty collection", async () => {
|
|
18
|
+
const stats = await getIndexStats(makeCollection(0));
|
|
19
|
+
expect(stats.totalVectors).toBe(0);
|
|
20
|
+
});
|
|
21
|
+
it("suggestion mentions HNSW for a non-empty collection", async () => {
|
|
22
|
+
const stats = await getIndexStats(makeCollection(50));
|
|
23
|
+
expect(stats.suggestions[0]).toMatch(/HNSW/i);
|
|
24
|
+
expect(stats.suggestions[0]).not.toMatch(/empty/i);
|
|
25
|
+
});
|
|
26
|
+
it("suggestion mentions empty for a zero-count collection", async () => {
|
|
27
|
+
const stats = await getIndexStats(makeCollection(0));
|
|
28
|
+
expect(stats.suggestions[0]).toMatch(/empty/i);
|
|
29
|
+
});
|
|
30
|
+
it("always returns annRecallAt10 = -1, indexAgeHours = -1, deadTupleFraction = 0", async () => {
|
|
31
|
+
const stats = await getIndexStats(makeCollection(10));
|
|
32
|
+
expect(stats.annRecallAt10).toBe(-1);
|
|
33
|
+
expect(stats.indexAgeHours).toBe(-1);
|
|
34
|
+
expect(stats.deadTupleFraction).toBe(0);
|
|
35
|
+
});
|
|
36
|
+
it("returns exactly one suggestion", async () => {
|
|
37
|
+
const stats = await getIndexStats(makeCollection(100));
|
|
38
|
+
expect(stats.suggestions).toHaveLength(1);
|
|
39
|
+
});
|
|
40
|
+
});
|
|
41
|
+
//# sourceMappingURL=stats.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stats.test.js","sourceRoot":"","sources":["../src/stats.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAG3C,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO;QACL,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,KAAK,CAAC;KACf,CAAC;AAC7B,CAAC;AAED,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACxC,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,KAAK,IAAI,EAAE;QAC7D,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9C,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8EAA8E,EAAE,KAAK,IAAI,EAAE;QAC5F,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/store.d.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { VectorDocument, VectorSearchResult, VectorStore, IndexStats, QueryPerfReport } from "@vivantel/rag-core";
|
|
2
|
+
export interface ChromaVectorStoreOptions {
|
|
3
|
+
path?: string;
|
|
4
|
+
collectionName?: string;
|
|
5
|
+
dimensions?: number;
|
|
6
|
+
apiKey?: string;
|
|
7
|
+
}
|
|
8
|
+
export declare class ChromaVectorStore implements VectorStore {
|
|
9
|
+
readonly name = "chromadb";
|
|
10
|
+
private readonly path;
|
|
11
|
+
private readonly collectionName;
|
|
12
|
+
private readonly dimensions;
|
|
13
|
+
private readonly apiKey;
|
|
14
|
+
private client;
|
|
15
|
+
private collection;
|
|
16
|
+
constructor(options: ChromaVectorStoreOptions);
|
|
17
|
+
initialize(): Promise<void>;
|
|
18
|
+
upsert(documents: VectorDocument[]): Promise<void>;
|
|
19
|
+
deleteBySourceFile(sourceFiles: string[]): Promise<void>;
|
|
20
|
+
getCurrentState(): Promise<Map<string, string>>;
|
|
21
|
+
getIndexStats(): Promise<IndexStats>;
|
|
22
|
+
getQueryPerfReport(timeframeHours: number): Promise<QueryPerfReport>;
|
|
23
|
+
search(queryEmbedding: number[], topK: number): Promise<VectorSearchResult[]>;
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,kBAAkB,EAClB,WAAW,EACX,UAAU,EACV,eAAe,EAChB,MAAM,oBAAoB,CAAC;AAK5B,MAAM,WAAW,wBAAwB;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAOD,qBAAa,iBAAkB,YAAW,WAAW;IACnD,QAAQ,CAAC,IAAI,cAAc;IAE3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,UAAU,CAAc;gBAEpB,OAAO,EAAE,wBAAwB;IAOvC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAc3B,MAAM,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBlD,kBAAkB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAOxD,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IA6B/C,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC;IAIpC,kBAAkB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAIpE,MAAM,CACV,cAAc,EAAE,MAAM,EAAE,EACxB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,kBAAkB,EAAE,CAAC;CAgCjC"}
|
package/dist/store.js
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { ChromaClient, IncludeEnum } from "chromadb";
|
|
2
|
+
import { getIndexStats } from "./stats.js";
|
|
3
|
+
import { getQueryPerfReport } from "./query-perf.js";
|
|
4
|
+
const DEFAULT_COLLECTION = "documents";
|
|
5
|
+
const DEFAULT_DIMENSIONS = 1536;
|
|
6
|
+
const UPSERT_BATCH_SIZE = 100;
|
|
7
|
+
const SCROLL_PAGE_SIZE = 1000;
|
|
8
|
+
export class ChromaVectorStore {
|
|
9
|
+
name = "chromadb";
|
|
10
|
+
path;
|
|
11
|
+
collectionName;
|
|
12
|
+
dimensions;
|
|
13
|
+
apiKey;
|
|
14
|
+
client;
|
|
15
|
+
collection;
|
|
16
|
+
constructor(options) {
|
|
17
|
+
this.path = options.path ?? "http://localhost:8000";
|
|
18
|
+
this.collectionName = options.collectionName ?? DEFAULT_COLLECTION;
|
|
19
|
+
this.dimensions = options.dimensions ?? DEFAULT_DIMENSIONS;
|
|
20
|
+
this.apiKey = options.apiKey;
|
|
21
|
+
}
|
|
22
|
+
async initialize() {
|
|
23
|
+
this.client = new ChromaClient({
|
|
24
|
+
path: this.path,
|
|
25
|
+
auth: this.apiKey
|
|
26
|
+
? { provider: "token", credentials: this.apiKey }
|
|
27
|
+
: undefined,
|
|
28
|
+
});
|
|
29
|
+
this.collection = await this.client.getOrCreateCollection({
|
|
30
|
+
name: this.collectionName,
|
|
31
|
+
metadata: { "hnsw:space": "cosine" },
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
async upsert(documents) {
|
|
35
|
+
for (let i = 0; i < documents.length; i += UPSERT_BATCH_SIZE) {
|
|
36
|
+
const batch = documents.slice(i, i + UPSERT_BATCH_SIZE);
|
|
37
|
+
await this.collection.upsert({
|
|
38
|
+
ids: batch.map((doc) => doc.id ?? crypto.randomUUID()),
|
|
39
|
+
embeddings: batch.map((doc) => doc.embedding),
|
|
40
|
+
documents: batch.map((doc) => doc.content),
|
|
41
|
+
metadatas: batch.map((doc) => ({
|
|
42
|
+
source_file: doc.sourceFile,
|
|
43
|
+
commit_hash: doc.commitHash,
|
|
44
|
+
content_hash: doc.contentHash,
|
|
45
|
+
...doc.metadata,
|
|
46
|
+
})),
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
async deleteBySourceFile(sourceFiles) {
|
|
51
|
+
if (sourceFiles.length === 0)
|
|
52
|
+
return;
|
|
53
|
+
await this.collection.delete({
|
|
54
|
+
where: { source_file: { $in: sourceFiles } },
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
async getCurrentState() {
|
|
58
|
+
const state = new Map();
|
|
59
|
+
let offset = 0;
|
|
60
|
+
while (true) {
|
|
61
|
+
const result = await this.collection.get({
|
|
62
|
+
include: [IncludeEnum.Metadatas],
|
|
63
|
+
limit: SCROLL_PAGE_SIZE,
|
|
64
|
+
offset,
|
|
65
|
+
});
|
|
66
|
+
for (const meta of result.metadatas) {
|
|
67
|
+
if (!meta)
|
|
68
|
+
continue;
|
|
69
|
+
const sourceFile = typeof meta.source_file === "string" ? meta.source_file : null;
|
|
70
|
+
const commitHash = typeof meta.commit_hash === "string" ? meta.commit_hash : null;
|
|
71
|
+
if (sourceFile && commitHash) {
|
|
72
|
+
state.set(sourceFile, commitHash);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
if (result.ids.length < SCROLL_PAGE_SIZE)
|
|
76
|
+
break;
|
|
77
|
+
offset += SCROLL_PAGE_SIZE;
|
|
78
|
+
}
|
|
79
|
+
return state;
|
|
80
|
+
}
|
|
81
|
+
async getIndexStats() {
|
|
82
|
+
return getIndexStats(this.collection);
|
|
83
|
+
}
|
|
84
|
+
async getQueryPerfReport(timeframeHours) {
|
|
85
|
+
return getQueryPerfReport(this.collection, this.dimensions, timeframeHours);
|
|
86
|
+
}
|
|
87
|
+
async search(queryEmbedding, topK) {
|
|
88
|
+
const result = await this.collection.query({
|
|
89
|
+
queryEmbeddings: [queryEmbedding],
|
|
90
|
+
nResults: topK,
|
|
91
|
+
include: [
|
|
92
|
+
IncludeEnum.Documents,
|
|
93
|
+
IncludeEnum.Metadatas,
|
|
94
|
+
IncludeEnum.Distances,
|
|
95
|
+
],
|
|
96
|
+
});
|
|
97
|
+
const ids = result.ids[0] ?? [];
|
|
98
|
+
const documents = result.documents[0] ?? [];
|
|
99
|
+
const metadatas = result.metadatas[0] ?? [];
|
|
100
|
+
const distances = result.distances?.[0] ?? [];
|
|
101
|
+
return ids.map((id, i) => ({
|
|
102
|
+
id,
|
|
103
|
+
content: documents[i] ?? "",
|
|
104
|
+
metadata: (() => {
|
|
105
|
+
const meta = metadatas[i];
|
|
106
|
+
if (!meta || typeof meta !== "object")
|
|
107
|
+
return {};
|
|
108
|
+
const { source_file, commit_hash, content_hash, ...rest } = meta;
|
|
109
|
+
void source_file;
|
|
110
|
+
void commit_hash;
|
|
111
|
+
void content_hash;
|
|
112
|
+
return rest;
|
|
113
|
+
})(),
|
|
114
|
+
similarity: 1 - (distances[i] ?? 1),
|
|
115
|
+
}));
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
//# sourceMappingURL=store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAmB,MAAM,UAAU,CAAC;AACtE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AASrD,MAAM,kBAAkB,GAAG,WAAW,CAAC;AACvC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,MAAM,OAAO,iBAAiB;IACnB,IAAI,GAAG,UAAU,CAAC;IAEV,IAAI,CAAS;IACb,cAAc,CAAS;IACvB,UAAU,CAAS;IACnB,MAAM,CAAqB;IACpC,MAAM,CAAgB;IACtB,UAAU,CAAc;IAEhC,YAAY,OAAiC;QAC3C,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,uBAAuB,CAAC;QACpD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,kBAAkB,CAAC;QACnE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;QAC3D,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,CAAC;YAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,MAAM;gBACf,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE;gBACjD,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC;YACxD,IAAI,EAAE,IAAI,CAAC,cAAc;YACzB,QAAQ,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE;SACrC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAA2B;QACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,iBAAiB,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC,CAAC;YACxD,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;gBAC3B,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;gBACtD,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC;gBAC7C,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC;gBAC1C,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;oBAC7B,WAAW,EAAE,GAAG,CAAC,UAAU;oBAC3B,WAAW,EAAE,GAAG,CAAC,UAAU;oBAC3B,YAAY,EAAE,GAAG,CAAC,WAAW;oBAC7B,GAAG,GAAG,CAAC,QAAQ;iBAChB,CAAC,CAAC;aACJ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,WAAqB;QAC5C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACrC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAC3B,KAAK,EAAE,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,WAAW,EAAE,EAAE;SAC7C,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;QACxC,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;gBACvC,OAAO,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC;gBAChC,KAAK,EAAE,gBAAgB;gBACvB,MAAM;aACP,CAAC,CAAC;YAEH,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACpC,IAAI,CAAC,IAAI;oBAAE,SAAS;gBACpB,MAAM,UAAU,GACd,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;gBACjE,MAAM,UAAU,GACd,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;gBACjE,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;oBAC7B,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;gBACpC,CAAC;YACH,CAAC;YAED,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,gBAAgB;gBAAE,MAAM;YAChD,MAAM,IAAI,gBAAgB,CAAC;QAC7B,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,OAAO,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,cAAsB;QAC7C,OAAO,kBAAkB,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,MAAM,CACV,cAAwB,EACxB,IAAY;QAEZ,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YACzC,eAAe,EAAE,CAAC,cAAc,CAAC;YACjC,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE;gBACP,WAAW,CAAC,SAAS;gBACrB,WAAW,CAAC,SAAS;gBACrB,WAAW,CAAC,SAAS;aACtB;SACF,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAE9C,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACzB,EAAE;YACF,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE;YAC3B,QAAQ,EAAE,CAAC,GAAG,EAAE;gBACd,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;oBAAE,OAAO,EAAE,CAAC;gBACjD,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,GAAG,IAAI,EAAE,GACvD,IAA+B,CAAC;gBAClC,KAAK,WAAW,CAAC;gBACjB,KAAK,WAAW,CAAC;gBACjB,KAAK,YAAY,CAAC;gBAClB,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,EAAE;YACJ,UAAU,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SACpC,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
package/package.json
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@vivantel/rag-store-chromadb",
|
|
3
|
+
"version": "1.2.1",
|
|
4
|
+
"description": "ChromaDB vector store for @vivantel/rag-core (local or hosted)",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"import": "./dist/index.js",
|
|
11
|
+
"types": "./dist/index.d.ts"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"files": [
|
|
15
|
+
"dist",
|
|
16
|
+
"README.md"
|
|
17
|
+
],
|
|
18
|
+
"sideEffects": false,
|
|
19
|
+
"publishConfig": {
|
|
20
|
+
"access": "public"
|
|
21
|
+
},
|
|
22
|
+
"rag-plugin": {
|
|
23
|
+
"type": "vectorStore",
|
|
24
|
+
"label": "ChromaDB (local or hosted)",
|
|
25
|
+
"key": "chromadb",
|
|
26
|
+
"envVars": [],
|
|
27
|
+
"defaultConfig": {
|
|
28
|
+
"path": "http://localhost:8000"
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"scripts": {
|
|
32
|
+
"build": "tsc",
|
|
33
|
+
"type-check": "tsc --noEmit",
|
|
34
|
+
"test": "vitest run",
|
|
35
|
+
"prepublishOnly": "npm run build",
|
|
36
|
+
"lint": "eslint src/",
|
|
37
|
+
"lint:fix": "eslint src/ --fix",
|
|
38
|
+
"format": "prettier --write \"src/**/*.ts\"",
|
|
39
|
+
"fix": "npm run lint:fix && npm run format"
|
|
40
|
+
},
|
|
41
|
+
"keywords": [
|
|
42
|
+
"rag",
|
|
43
|
+
"vector-store",
|
|
44
|
+
"chromadb",
|
|
45
|
+
"vector-database",
|
|
46
|
+
"semantic-search"
|
|
47
|
+
],
|
|
48
|
+
"author": "Vivantel",
|
|
49
|
+
"license": "MIT",
|
|
50
|
+
"repository": {
|
|
51
|
+
"type": "git",
|
|
52
|
+
"url": "https://github.com/vivantel/rag_core",
|
|
53
|
+
"directory": "packages/rag-store-chromadb"
|
|
54
|
+
},
|
|
55
|
+
"dependencies": {
|
|
56
|
+
"chromadb": "^1.10.0"
|
|
57
|
+
},
|
|
58
|
+
"peerDependencies": {
|
|
59
|
+
"@vivantel/rag-core": "^2.0.0"
|
|
60
|
+
},
|
|
61
|
+
"devDependencies": {
|
|
62
|
+
"@types/node": "^25.9.1",
|
|
63
|
+
"@vivantel/rag-core": "file:../rag-core",
|
|
64
|
+
"typescript": "^6.0.3",
|
|
65
|
+
"vitest": "^4.1.8"
|
|
66
|
+
},
|
|
67
|
+
"engines": {
|
|
68
|
+
"node": ">=18.0.0"
|
|
69
|
+
}
|
|
70
|
+
}
|