@ixo/data-store 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +9 -0
- package/.prettierignore +3 -0
- package/.prettierrc.js +4 -0
- package/.turbo/turbo-build.log +4 -0
- package/CHANGELOG.md +7 -0
- package/README.md +276 -0
- package/dist/airtable-store/index.d.ts +42 -0
- package/dist/airtable-store/index.d.ts.map +1 -0
- package/dist/airtable-store/index.js +79 -0
- package/dist/chroma/chroma-data-store.d.ts +74 -0
- package/dist/chroma/chroma-data-store.d.ts.map +1 -0
- package/dist/chroma/chroma-data-store.js +196 -0
- package/dist/chroma/chroma-data-store.test.d.ts +2 -0
- package/dist/chroma/chroma-data-store.test.d.ts.map +1 -0
- package/dist/chroma/chroma-data-store.test.js +176 -0
- package/dist/chroma/chroma.types.d.ts +24 -0
- package/dist/chroma/chroma.types.d.ts.map +1 -0
- package/dist/chroma/chroma.types.js +2 -0
- package/dist/chroma/embedding-function.d.ts +14 -0
- package/dist/chroma/embedding-function.d.ts.map +1 -0
- package/dist/chroma/embedding-function.js +26 -0
- package/dist/chroma/index.d.ts +3 -0
- package/dist/chroma/index.d.ts.map +1 -0
- package/dist/chroma/index.js +18 -0
- package/dist/chroma/utils.d.ts +6 -0
- package/dist/chroma/utils.d.ts.map +1 -0
- package/dist/chroma/utils.js +31 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +19 -0
- package/dist/types/index.d.ts +3 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +18 -0
- package/dist/types/structured-data-store.d.ts +24 -0
- package/dist/types/structured-data-store.d.ts.map +1 -0
- package/dist/types/structured-data-store.js +2 -0
- package/dist/types/vector-db-data-store.d.ts +58 -0
- package/dist/types/vector-db-data-store.d.ts.map +1 -0
- package/dist/types/vector-db-data-store.js +10 -0
- package/dist/utils/index.d.ts +2 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +17 -0
- package/dist/utils/with-report-error.d.ts +5 -0
- package/dist/utils/with-report-error.d.ts.map +1 -0
- package/dist/utils/with-report-error.js +21 -0
- package/jest.config.js +6 -0
- package/package.json +49 -0
- package/src/airtable-store/index.ts +141 -0
- package/src/chroma/chroma-data-store.test.ts +210 -0
- package/src/chroma/chroma-data-store.ts +254 -0
- package/src/chroma/chroma.types.ts +28 -0
- package/src/chroma/embedding-function.ts +26 -0
- package/src/chroma/index.ts +2 -0
- package/src/chroma/utils.ts +40 -0
- package/src/index.ts +4 -0
- package/src/types/index.ts +2 -0
- package/src/types/structured-data-store.ts +34 -0
- package/src/types/vector-db-data-store.ts +78 -0
- package/src/utils/index.ts +1 -0
- package/src/utils/with-report-error.ts +18 -0
- package/tsconfig.json +7 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ChromaDataStore = void 0;
|
|
4
|
+
const chromadb_1 = require("chromadb");
|
|
5
|
+
const vector_db_data_store_1 = require("../types/vector-db-data-store");
|
|
6
|
+
const embedding_function_1 = require("./embedding-function");
|
|
7
|
+
const utils_1 = require("./utils");
|
|
8
|
+
/**
|
|
9
|
+
* ChromaDataStore class for managing vector storage and retrieval using ChromaDB
|
|
10
|
+
*/
|
|
11
|
+
class ChromaDataStore extends vector_db_data_store_1.VectorDBDataStore {
|
|
12
|
+
client;
|
|
13
|
+
collection;
|
|
14
|
+
/**
|
|
15
|
+
* Creates a new ChromaDataStore instance
|
|
16
|
+
* @param options - Configuration options for the vector store
|
|
17
|
+
* @throws Error if OPENAI_API_KEY is not set and no embedding function is provided
|
|
18
|
+
*/
|
|
19
|
+
constructor(options) {
|
|
20
|
+
if (typeof process.env.OPENAI_API_KEY !== 'string' &&
|
|
21
|
+
!options.embeddingFunction) {
|
|
22
|
+
throw new Error('OPENAI_API_KEY is not set and no embedding function is provided');
|
|
23
|
+
}
|
|
24
|
+
options.embeddingFunction =
|
|
25
|
+
options.embeddingFunction ||
|
|
26
|
+
new embedding_function_1.OpenAIEmbeddingFunction({
|
|
27
|
+
openai_api_key: process.env.OPENAI_API_KEY ?? '',
|
|
28
|
+
openai_model: 'text-embedding-3-small',
|
|
29
|
+
});
|
|
30
|
+
options.url = options.url || 'http://localhost:8000';
|
|
31
|
+
options.collectionName = options.collectionName || 'default-vector-store';
|
|
32
|
+
super(options);
|
|
33
|
+
this.client = new chromadb_1.ChromaClient({
|
|
34
|
+
path: options.url,
|
|
35
|
+
});
|
|
36
|
+
// this is a workaround to avoid setting collection as optional
|
|
37
|
+
this.collection = undefined;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Checks if the ChromaDB collection is initialized
|
|
41
|
+
* @throws Error if collection is not initialized
|
|
42
|
+
*/
|
|
43
|
+
checkIsInitialized() {
|
|
44
|
+
if (!this.collection) {
|
|
45
|
+
throw new Error('ChromaDataStore is not initialized');
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Initializes the ChromaDB collection
|
|
50
|
+
*/
|
|
51
|
+
async init() {
|
|
52
|
+
if (this.collection) {
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
const collection = await this.client.getOrCreateCollection({
|
|
56
|
+
name: this.options.collectionName,
|
|
57
|
+
embeddingFunction: this.options.embeddingFunction,
|
|
58
|
+
});
|
|
59
|
+
this.collection = collection;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Queries the vector store using text
|
|
63
|
+
* @param query - Text query to search for
|
|
64
|
+
* @param options - Query options including filters and top-k results
|
|
65
|
+
* @returns Array of matching documents
|
|
66
|
+
*/
|
|
67
|
+
async query(query, options) {
|
|
68
|
+
this.checkIsInitialized();
|
|
69
|
+
const params = {
|
|
70
|
+
queryTexts: [query],
|
|
71
|
+
nResults: options?.topK || 10,
|
|
72
|
+
};
|
|
73
|
+
if (options?.filters) {
|
|
74
|
+
params.where = options.filters;
|
|
75
|
+
}
|
|
76
|
+
const result = await this.collection.query(params);
|
|
77
|
+
return (0, utils_1.createVectorStoreSearchResult)(result);
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Queries the vector store and filters results by similarity threshold
|
|
81
|
+
* @param query - Text query to search for
|
|
82
|
+
* @param options - Query options including similarity threshold
|
|
83
|
+
* @returns Array of documents meeting the similarity threshold
|
|
84
|
+
*/
|
|
85
|
+
async queryWithSimilarity(query, options) {
|
|
86
|
+
this.checkIsInitialized();
|
|
87
|
+
const result = await this.query(query, options);
|
|
88
|
+
return result.filter((doc) => (doc.score && doc.score >= (options?.similarityThreshold ?? 0.5)) ??
|
|
89
|
+
false);
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Upserts (inserts or updates) documents into the vector store
|
|
93
|
+
* @param documents - Array of documents to upsert
|
|
94
|
+
* @throws Error if any document is missing an ID
|
|
95
|
+
*/
|
|
96
|
+
async upsert(documents) {
|
|
97
|
+
this.checkIsInitialized();
|
|
98
|
+
const [ids, contents, metadatas] = documents.reduce((acc, doc) => {
|
|
99
|
+
const id = doc.id;
|
|
100
|
+
const metadata = doc.metadata ?? {};
|
|
101
|
+
const content = doc.content;
|
|
102
|
+
if (!id) {
|
|
103
|
+
throw new Error('Document ID is required');
|
|
104
|
+
}
|
|
105
|
+
acc[0].push(id);
|
|
106
|
+
acc[1].push(content);
|
|
107
|
+
acc[2].push(metadata);
|
|
108
|
+
return acc;
|
|
109
|
+
}, [[], [], []]);
|
|
110
|
+
await this.collection.upsert({
|
|
111
|
+
ids,
|
|
112
|
+
documents: contents,
|
|
113
|
+
metadatas,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
async addDocumentsWithEmbeddings(documents) {
|
|
117
|
+
this.checkIsInitialized();
|
|
118
|
+
const [ids, embeddings, metadatas, contents] = documents.reduce((acc, doc) => {
|
|
119
|
+
const id = doc.id;
|
|
120
|
+
const metadata = doc.metadata ?? {};
|
|
121
|
+
const embedding = doc.embedding;
|
|
122
|
+
if (!id) {
|
|
123
|
+
throw new Error('Document ID is required');
|
|
124
|
+
}
|
|
125
|
+
if (embedding.length < 1536) {
|
|
126
|
+
throw new Error('Embedding is required');
|
|
127
|
+
}
|
|
128
|
+
acc[0].push(id);
|
|
129
|
+
acc[1].push(embedding);
|
|
130
|
+
acc[2].push(metadata);
|
|
131
|
+
acc[3].push(doc.content);
|
|
132
|
+
return acc;
|
|
133
|
+
}, [[], [], [], []]);
|
|
134
|
+
await this.collection.add({
|
|
135
|
+
ids,
|
|
136
|
+
embeddings,
|
|
137
|
+
metadatas,
|
|
138
|
+
documents: contents,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Deletes documents from the vector store by their IDs
|
|
143
|
+
* @param ids - Array of document IDs to delete
|
|
144
|
+
*/
|
|
145
|
+
async delete(ids) {
|
|
146
|
+
this.checkIsInitialized();
|
|
147
|
+
await this.collection.delete({
|
|
148
|
+
ids,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Retrieves a document by its ID
|
|
153
|
+
* @param id - Document ID to retrieve
|
|
154
|
+
* @returns The document if found, null otherwise
|
|
155
|
+
*/
|
|
156
|
+
async getById(id) {
|
|
157
|
+
const result = await this.collection.get({
|
|
158
|
+
ids: [id],
|
|
159
|
+
});
|
|
160
|
+
return result.documents.at(0)
|
|
161
|
+
? {
|
|
162
|
+
id: result.ids.at(0)?.toString() ?? '',
|
|
163
|
+
content: result.documents.at(0) ?? '',
|
|
164
|
+
metadata: result.metadatas.at(0) ?? {},
|
|
165
|
+
score: undefined,
|
|
166
|
+
}
|
|
167
|
+
: null;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Queries the vector store using a pre-computed embedding vector
|
|
171
|
+
* @param vector - The embedding vector to search with
|
|
172
|
+
* @param options - Query options including top-k results
|
|
173
|
+
* @returns Array of matching documents
|
|
174
|
+
*/
|
|
175
|
+
async queryByVector(vector, options) {
|
|
176
|
+
this.checkIsInitialized();
|
|
177
|
+
const result = await this.collection.query({
|
|
178
|
+
queryEmbeddings: [vector],
|
|
179
|
+
nResults: options?.topK || 10,
|
|
180
|
+
});
|
|
181
|
+
return (0, utils_1.createVectorStoreSearchResult)(result);
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Updates the metadata for a document by its ID
|
|
185
|
+
* @param ids - Array of document IDs to update
|
|
186
|
+
* @param metadata - Metadata to update
|
|
187
|
+
*/
|
|
188
|
+
async updateMetadata(ids, metadatas) {
|
|
189
|
+
this.checkIsInitialized();
|
|
190
|
+
await this.collection.update({
|
|
191
|
+
ids,
|
|
192
|
+
metadatas,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
exports.ChromaDataStore = ChromaDataStore;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chroma-data-store.test.d.ts","sourceRoot":"","sources":["../../src/chroma/chroma-data-store.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
/* eslint-disable jest/unbound-method -- ignore this for now */
|
|
4
|
+
const chromadb_1 = require("chromadb");
|
|
5
|
+
const chroma_data_store_1 = require("./chroma-data-store");
|
|
6
|
+
jest.mock('chromadb');
|
|
7
|
+
const embeddingFunction = {
|
|
8
|
+
generate: jest.fn(),
|
|
9
|
+
};
|
|
10
|
+
jest
|
|
11
|
+
.mocked(chromadb_1.ChromaClient.prototype.getOrCreateCollection)
|
|
12
|
+
.mockResolvedValue(new chromadb_1.Collection('test', 'test', jest.fn(), embeddingFunction));
|
|
13
|
+
describe('ChromaDataStore', () => {
|
|
14
|
+
beforeAll(() => {
|
|
15
|
+
process.env.OPENAI_API_KEY = 'test';
|
|
16
|
+
});
|
|
17
|
+
afterAll(() => {
|
|
18
|
+
process.env.OPENAI_API_KEY = undefined;
|
|
19
|
+
});
|
|
20
|
+
it('should be defined', () => {
|
|
21
|
+
expect(chroma_data_store_1.ChromaDataStore).toBeDefined();
|
|
22
|
+
});
|
|
23
|
+
describe('Initialization', () => {
|
|
24
|
+
it('should construct and initialize without errors', async () => {
|
|
25
|
+
// Should not throw during construction
|
|
26
|
+
const chromaDataStore = new chroma_data_store_1.ChromaDataStore({
|
|
27
|
+
collectionName: 'test',
|
|
28
|
+
url: 'http://localhost:8000',
|
|
29
|
+
});
|
|
30
|
+
expect(chromaDataStore).toBeInstanceOf(chroma_data_store_1.ChromaDataStore);
|
|
31
|
+
// Should not throw during initialization
|
|
32
|
+
await expect(chromaDataStore.init()).resolves.not.toThrow();
|
|
33
|
+
});
|
|
34
|
+
it('should throw error when OPENAI_API_KEY is not set and no embedding function is provided', () => {
|
|
35
|
+
// Temporarily remove API key
|
|
36
|
+
const originalKey = process.env.OPENAI_API_KEY;
|
|
37
|
+
delete process.env.OPENAI_API_KEY;
|
|
38
|
+
// Should throw during construction
|
|
39
|
+
expect(() => new chroma_data_store_1.ChromaDataStore({
|
|
40
|
+
collectionName: 'test',
|
|
41
|
+
url: 'http://localhost:8000',
|
|
42
|
+
})).toThrow('OPENAI_API_KEY is not set');
|
|
43
|
+
// Restore API key
|
|
44
|
+
process.env.OPENAI_API_KEY = originalKey;
|
|
45
|
+
});
|
|
46
|
+
it('should create collection or use existing collection', async () => {
|
|
47
|
+
jest.spyOn(chromadb_1.ChromaClient.prototype, 'getOrCreateCollection');
|
|
48
|
+
const chromaDataStore = new chroma_data_store_1.ChromaDataStore({
|
|
49
|
+
collectionName: 'test',
|
|
50
|
+
url: 'http://localhost:8000',
|
|
51
|
+
embeddingFunction,
|
|
52
|
+
});
|
|
53
|
+
await chromaDataStore.init();
|
|
54
|
+
expect(chromadb_1.ChromaClient.prototype.getOrCreateCollection).toHaveBeenCalledWith({
|
|
55
|
+
name: 'test',
|
|
56
|
+
embeddingFunction,
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
describe('Query', () => {
|
|
61
|
+
let chromaDataStore;
|
|
62
|
+
beforeAll(async () => {
|
|
63
|
+
chromaDataStore = new chroma_data_store_1.ChromaDataStore({
|
|
64
|
+
collectionName: 'test',
|
|
65
|
+
url: 'http://localhost:8000',
|
|
66
|
+
embeddingFunction,
|
|
67
|
+
});
|
|
68
|
+
await chromaDataStore.init();
|
|
69
|
+
await chromaDataStore.init();
|
|
70
|
+
jest.mocked(chromadb_1.Collection.prototype.query).mockResolvedValue({
|
|
71
|
+
ids: [['1', '2']],
|
|
72
|
+
embeddings: null,
|
|
73
|
+
documents: [['Hello, world!', 'Hello, world! 2']],
|
|
74
|
+
metadatas: [[{ title: 'Hello, world!' }, { title: 'Hello, world! 2' }]],
|
|
75
|
+
distances: [[3.3435163194430493e-16, 0.30934086831133445]],
|
|
76
|
+
included: [],
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
it('should query documents', async () => {
|
|
80
|
+
const documents = await chromaDataStore.query('Hello, world!');
|
|
81
|
+
expect(documents).toHaveLength(2);
|
|
82
|
+
expect(documents[0]?.id).toBe('1');
|
|
83
|
+
expect(documents[0]?.content).toBe('Hello, world!');
|
|
84
|
+
});
|
|
85
|
+
it('should query documents by vector', async () => {
|
|
86
|
+
const vector = [1, 2, 3];
|
|
87
|
+
const documents = await chromaDataStore.queryByVector(vector);
|
|
88
|
+
expect(documents).toHaveLength(2);
|
|
89
|
+
expect(documents[0]?.id).toBe('1');
|
|
90
|
+
expect(documents[0]?.content).toBe('Hello, world!');
|
|
91
|
+
});
|
|
92
|
+
it('should query documents by similarity', async () => {
|
|
93
|
+
const documents = await chromaDataStore.queryWithSimilarity('Hello, world!', { similarityThreshold: 0.9 });
|
|
94
|
+
expect(documents).toHaveLength(1);
|
|
95
|
+
expect(documents[0]?.id).toBe('1');
|
|
96
|
+
expect(documents[0]?.content).toBe('Hello, world!');
|
|
97
|
+
});
|
|
98
|
+
it('should get document by id', async () => {
|
|
99
|
+
jest.spyOn(chromadb_1.Collection.prototype, 'get');
|
|
100
|
+
jest.mocked(chromadb_1.Collection.prototype.get).mockResolvedValue({
|
|
101
|
+
ids: ['1'],
|
|
102
|
+
embeddings: null,
|
|
103
|
+
documents: ['Hello, world!'],
|
|
104
|
+
metadatas: [{ title: 'Hello, world!' }],
|
|
105
|
+
included: [],
|
|
106
|
+
});
|
|
107
|
+
const document = await chromaDataStore.getById('1');
|
|
108
|
+
expect(chromadb_1.Collection.prototype.get).toHaveBeenCalledWith({
|
|
109
|
+
ids: ['1'],
|
|
110
|
+
});
|
|
111
|
+
expect(document?.id).toBe('1');
|
|
112
|
+
expect(document?.content).toBe('Hello, world!');
|
|
113
|
+
});
|
|
114
|
+
it('should handle empty results', async () => {
|
|
115
|
+
jest.mocked(chromadb_1.Collection.prototype.query).mockResolvedValue({
|
|
116
|
+
ids: [],
|
|
117
|
+
embeddings: null,
|
|
118
|
+
documents: [],
|
|
119
|
+
metadatas: [],
|
|
120
|
+
distances: [],
|
|
121
|
+
included: [],
|
|
122
|
+
});
|
|
123
|
+
const documents = await chromaDataStore.query('Hello, world!');
|
|
124
|
+
expect(documents).toHaveLength(0);
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
describe('Upsert', () => {
|
|
128
|
+
let chromaDataStore;
|
|
129
|
+
beforeAll(async () => {
|
|
130
|
+
chromaDataStore = new chroma_data_store_1.ChromaDataStore({
|
|
131
|
+
collectionName: 'test',
|
|
132
|
+
url: 'http://localhost:8000',
|
|
133
|
+
embeddingFunction,
|
|
134
|
+
});
|
|
135
|
+
await chromaDataStore.init();
|
|
136
|
+
});
|
|
137
|
+
it('should upsert documents', async () => {
|
|
138
|
+
jest.spyOn(chromadb_1.Collection.prototype, 'upsert');
|
|
139
|
+
await chromaDataStore.upsert([
|
|
140
|
+
{
|
|
141
|
+
id: '1',
|
|
142
|
+
content: 'Hello, world!',
|
|
143
|
+
metadata: { title: 'Hello, world!' },
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
id: '2',
|
|
147
|
+
content: 'Hello, world! 2',
|
|
148
|
+
metadata: { title: 'Hello, world! 2' },
|
|
149
|
+
},
|
|
150
|
+
]);
|
|
151
|
+
expect(chromadb_1.Collection.prototype.upsert).toHaveBeenCalledWith({
|
|
152
|
+
ids: ['1', '2'],
|
|
153
|
+
documents: ['Hello, world!', 'Hello, world! 2'],
|
|
154
|
+
metadatas: [{ title: 'Hello, world!' }, { title: 'Hello, world! 2' }],
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
});
|
|
158
|
+
describe('Delete', () => {
|
|
159
|
+
let chromaDataStore;
|
|
160
|
+
beforeAll(async () => {
|
|
161
|
+
chromaDataStore = new chroma_data_store_1.ChromaDataStore({
|
|
162
|
+
collectionName: 'test',
|
|
163
|
+
url: 'http://localhost:8000',
|
|
164
|
+
embeddingFunction,
|
|
165
|
+
});
|
|
166
|
+
await chromaDataStore.init();
|
|
167
|
+
});
|
|
168
|
+
it('should delete documents', async () => {
|
|
169
|
+
jest.spyOn(chromadb_1.Collection.prototype, 'delete');
|
|
170
|
+
await chromaDataStore.delete(['1', '2']);
|
|
171
|
+
expect(chromadb_1.Collection.prototype.delete).toHaveBeenCalledWith({
|
|
172
|
+
ids: ['1', '2'],
|
|
173
|
+
});
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chroma-specific document content filter operators
|
|
3
|
+
*/
|
|
4
|
+
export interface IChromaDocumentFilter {
|
|
5
|
+
$contains?: string;
|
|
6
|
+
$not_contains?: string;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Chroma-specific metadata filter operators
|
|
10
|
+
*/
|
|
11
|
+
export type IChromaMetadataFilter = Record<string, string | number | boolean | null | IChromaMetadataOperators>;
|
|
12
|
+
export interface IChromaMetadataOperators {
|
|
13
|
+
$eq?: unknown;
|
|
14
|
+
$ne?: unknown;
|
|
15
|
+
$gt?: number;
|
|
16
|
+
$gte?: number;
|
|
17
|
+
$lt?: number;
|
|
18
|
+
$lte?: number;
|
|
19
|
+
$in?: unknown[];
|
|
20
|
+
$nin?: unknown[];
|
|
21
|
+
$and?: IChromaMetadataFilter[];
|
|
22
|
+
$or?: IChromaMetadataFilter[];
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=chroma.types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chroma.types.d.ts","sourceRoot":"","sources":["../../src/chroma/chroma.types.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,MAAM,qBAAqB,GAAG,MAAM,CACxC,MAAM,EACN,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,wBAAwB,CAC5D,CAAC;AAEF,MAAM,WAAW,wBAAwB;IACvC,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC;IAChB,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC;IACjB,IAAI,CAAC,EAAE,qBAAqB,EAAE,CAAC;IAC/B,GAAG,CAAC,EAAE,qBAAqB,EAAE,CAAC;CAC/B"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type IEmbeddingFunction } from 'chromadb';
|
|
2
|
+
interface IOpenAIEmbeddingFunctionOptions {
|
|
3
|
+
openai_api_key: string;
|
|
4
|
+
openai_model: string;
|
|
5
|
+
openai_organization_id?: string;
|
|
6
|
+
}
|
|
7
|
+
export declare class OpenAIEmbeddingFunction implements IEmbeddingFunction {
|
|
8
|
+
private readonly options;
|
|
9
|
+
private readonly openai;
|
|
10
|
+
constructor(options: IOpenAIEmbeddingFunctionOptions);
|
|
11
|
+
generate(texts: string[]): Promise<number[][]>;
|
|
12
|
+
}
|
|
13
|
+
export {};
|
|
14
|
+
//# sourceMappingURL=embedding-function.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-function.d.ts","sourceRoot":"","sources":["../../src/chroma/embedding-function.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAGnD,UAAU,+BAA+B;IACvC,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,qBAAa,uBAAwB,YAAW,kBAAkB;IAEpD,OAAO,CAAC,QAAQ,CAAC,OAAO;IADpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBACH,OAAO,EAAE,+BAA+B;IAO/D,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAOrD"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.OpenAIEmbeddingFunction = void 0;
|
|
7
|
+
const openai_1 = __importDefault(require("openai"));
|
|
8
|
+
class OpenAIEmbeddingFunction {
|
|
9
|
+
options;
|
|
10
|
+
openai;
|
|
11
|
+
constructor(options) {
|
|
12
|
+
this.options = options;
|
|
13
|
+
this.openai = new openai_1.default({
|
|
14
|
+
apiKey: options.openai_api_key,
|
|
15
|
+
organization: options.openai_organization_id,
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
async generate(texts) {
|
|
19
|
+
const embeddings = await this.openai.embeddings.create({
|
|
20
|
+
model: this.options.openai_model,
|
|
21
|
+
input: texts,
|
|
22
|
+
});
|
|
23
|
+
return embeddings.data.map((embedding) => embedding.embedding);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
exports.OpenAIEmbeddingFunction = OpenAIEmbeddingFunction;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chroma/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,gBAAgB,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./chroma-data-store"), exports);
|
|
18
|
+
__exportStar(require("./chroma.types"), exports);
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { type Collection } from 'chromadb';
|
|
2
|
+
import { type IVectorStoreDocument } from 'types/vector-db-data-store';
|
|
3
|
+
type MultiQueryResponse = Awaited<ReturnType<typeof Collection.prototype.query>>;
|
|
4
|
+
export declare const createVectorStoreSearchResult: (result: MultiQueryResponse) => IVectorStoreDocument[];
|
|
5
|
+
export {};
|
|
6
|
+
//# sourceMappingURL=utils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/chroma/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AAC3C,OAAO,EAAE,KAAK,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AAEvE,KAAK,kBAAkB,GAAG,OAAO,CAC/B,UAAU,CAAC,OAAO,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAC9C,CAAC;AAEF,eAAO,MAAM,6BAA6B,GACxC,QAAQ,kBAAkB,KACzB,oBAAoB,EAWtB,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createVectorStoreSearchResult = void 0;
|
|
4
|
+
const createVectorStoreSearchResult = (result) => {
|
|
5
|
+
const scores = result.distances
|
|
6
|
+
? convertCosineDistancesToScores(result.distances)
|
|
7
|
+
: undefined;
|
|
8
|
+
const metadatas = result.metadatas.flat().filter(Boolean);
|
|
9
|
+
return result.documents.flat().map((document, idx) => ({
|
|
10
|
+
id: result.ids.flat().at(idx)?.toString() ?? '',
|
|
11
|
+
content: document ?? '',
|
|
12
|
+
metadata: metadatas.at(idx) ?? {},
|
|
13
|
+
score: scores?.at(idx),
|
|
14
|
+
}));
|
|
15
|
+
};
|
|
16
|
+
exports.createVectorStoreSearchResult = createVectorStoreSearchResult;
|
|
17
|
+
/**
|
|
18
|
+
* Converts a 2D array of cosine distances into a flattened array of similarity scores.
|
|
19
|
+
* @param distances - A 2D array of cosine distances (e.g., [[1.04, 1.24], [0.8, 1.5]]).
|
|
20
|
+
* @returns A flattened array of similarity scores (e.g., [0.48, 0.38, 0.6, 0.25]).
|
|
21
|
+
*/
|
|
22
|
+
function convertCosineDistancesToScores(distances) {
|
|
23
|
+
return distances.flat().map((distance) => {
|
|
24
|
+
// 1. Validate that the distance is within the valid range [0, 2].
|
|
25
|
+
if (distance < 0 || distance > 2) {
|
|
26
|
+
throw new Error(`Cosine distance must be between 0 and 2. Received: ${distance}`);
|
|
27
|
+
}
|
|
28
|
+
// 2. Convert cosine distance (d) to similarity score (s): s = 1 - (d / 2).
|
|
29
|
+
return 1 - distance / 2;
|
|
30
|
+
});
|
|
31
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AAEzB,cAAc,SAAS,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./airtable-store"), exports);
|
|
18
|
+
__exportStar(require("./chroma"), exports);
|
|
19
|
+
__exportStar(require("./types"), exports);
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./structured-data-store"), exports);
|
|
18
|
+
__exportStar(require("./vector-db-data-store"), exports);
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
interface IQueryParams<T> {
|
|
2
|
+
filterByFormula?: string;
|
|
3
|
+
maxRecords?: number;
|
|
4
|
+
pageSize?: number;
|
|
5
|
+
fields?: (keyof T)[];
|
|
6
|
+
view?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface IDataStore<T> {
|
|
9
|
+
getAllRecords: (tableName: string, selectOptions: IQueryParams<T>) => Promise<T[]>;
|
|
10
|
+
/**
|
|
11
|
+
* Get a record by its ID or throw an error if not found
|
|
12
|
+
*/
|
|
13
|
+
getRecord: (tableName: string, recordId: string) => Promise<T>;
|
|
14
|
+
createRecord: (tableName: string, recordData: T) => Promise<T>;
|
|
15
|
+
updateRecord: (tableName: string, recordId: string, recordData: T) => Promise<T>;
|
|
16
|
+
batchUpdateRecords: (tableName: string, records: {
|
|
17
|
+
id: string;
|
|
18
|
+
fields: T;
|
|
19
|
+
}[]) => Promise<T[]>;
|
|
20
|
+
deleteRecord: (tableName: string, recordId: string) => Promise<T>;
|
|
21
|
+
getRecordByField: (tableName: string, fieldName: string, fieldValue: string) => Promise<T[]>;
|
|
22
|
+
}
|
|
23
|
+
export {};
|
|
24
|
+
//# sourceMappingURL=structured-data-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"structured-data-store.d.ts","sourceRoot":"","sources":["../../src/types/structured-data-store.ts"],"names":[],"mappings":"AAAA,UAAU,YAAY,CAAC,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,aAAa,EAAE,CACb,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,YAAY,CAAC,CAAC,CAAC,KAC3B,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IAClB;;OAEG;IACH,SAAS,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/D,YAAY,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/D,YAAY,EAAE,CACZ,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,CAAC,KACV,OAAO,CAAC,CAAC,CAAC,CAAC;IAChB,kBAAkB,EAAE,CAClB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,CAAC,CAAA;KAAE,EAAE,KACjC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IAClB,YAAY,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC;IAClE,gBAAgB,EAAE,CAChB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,KACf,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;CACnB"}
|