@sprucelabs/chroma-data-store 0.0.3 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/.spruce/errors/chromaDataStore/featureNotSupported.schema.js +0 -1
- package/build/.spruce/errors/errors.types.js +0 -1
- package/build/.spruce/errors/options.types.js +0 -1
- package/build/.spruce/schemas/fields/fieldClassMap.js +0 -1
- package/build/.spruce/schemas/fields/fields.types.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/choice.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/link.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/location.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/message.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/messageSource.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/messageTarget.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/organization.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/person.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/personLocation.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/personOrganization.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/role.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/sendMessage.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/skill.schema.js +0 -1
- package/build/.spruce/schemas/spruce/v2020_07_22/skillCreator.schema.js +0 -1
- package/build/ChromaDatabase.js +0 -1
- package/build/chroma.types.js +0 -1
- package/build/errors/SpruceError.js +0 -1
- package/build/errors/featureNotSupported.builder.js +0 -1
- package/build/esm/.spruce/errors/chromaDataStore/featureNotSupported.schema.d.ts +3 -0
- package/build/esm/.spruce/errors/chromaDataStore/featureNotSupported.schema.js +18 -0
- package/build/esm/.spruce/errors/errors.types.d.ts +21 -0
- package/build/esm/.spruce/errors/errors.types.js +1 -0
- package/build/esm/.spruce/errors/options.types.d.ts +7 -0
- package/build/esm/.spruce/errors/options.types.js +1 -0
- package/build/esm/.spruce/schemas/fields/fieldClassMap.d.ts +2 -0
- package/build/esm/.spruce/schemas/fields/fieldClassMap.js +4 -0
- package/build/esm/.spruce/schemas/fields/fields.types.d.ts +1 -0
- package/build/esm/.spruce/schemas/fields/fields.types.js +2 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/choice.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/choice.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/link.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/link.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/location.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/location.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/message.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/message.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/messageSource.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/messageSource.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/messageTarget.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/messageTarget.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/organization.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/organization.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/person.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/person.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/personLocation.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/personLocation.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/personOrganization.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/personOrganization.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/role.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/role.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/sendMessage.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/sendMessage.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/skill.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/skill.schema.js +5 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/skillCreator.schema.d.ts +1 -0
- package/build/esm/.spruce/schemas/spruce/v2020_07_22/skillCreator.schema.js +5 -0
- package/build/esm/.spruce/settings.json +13 -0
- package/build/esm/ChromaDatabase.d.ts +46 -0
- package/build/esm/ChromaDatabase.js +445 -0
- package/build/esm/chroma.types.d.ts +177 -0
- package/build/esm/chroma.types.js +1 -0
- package/build/esm/errors/SpruceError.d.ts +6 -0
- package/build/esm/errors/SpruceError.js +19 -0
- package/build/esm/errors/featureNotSupported.builder.d.ts +11 -0
- package/build/esm/errors/featureNotSupported.builder.js +11 -0
- package/build/esm/index.d.ts +2 -0
- package/build/esm/index.js +5 -0
- package/build/index.js +0 -1
- package/package.json +1 -1
- package/build/__tests__/behavioral/ChromaDatabase.test.d.ts +0 -39
- package/build/__tests__/behavioral/ChromaDatabase.test.js +0 -351
- package/build/__tests__/behavioral/ChromaDatabase.test.js.map +0 -1
- /package/build/{.spruce → esm/.spruce}/errors/chromaDataStore/featureNotSupported.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/errors/errors.types.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/errors/options.types.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/fields/fieldClassMap.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/fields/fields.types.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/choice.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/link.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/location.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/message.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/messageSource.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/messageTarget.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/organization.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/person.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/personLocation.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/personOrganization.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/role.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/sendMessage.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/skill.schema.js.map +0 -0
- /package/build/{.spruce → esm/.spruce}/schemas/spruce/v2020_07_22/skillCreator.schema.js.map +0 -0
- /package/build/{ChromaDatabase.js.map → esm/ChromaDatabase.js.map} +0 -0
- /package/build/{chroma.types.js.map → esm/chroma.types.js.map} +0 -0
- /package/build/{errors → esm/errors}/SpruceError.js.map +0 -0
- /package/build/{errors → esm/errors}/featureNotSupported.builder.js.map +0 -0
- /package/build/{index.js.map → esm/index.js.map} +0 -0
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
var __rest = (this && this.__rest) || function (s, e) {
|
|
11
|
+
var t = {};
|
|
12
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
|
|
13
|
+
t[p] = s[p];
|
|
14
|
+
if (s != null && typeof Object.getOwnPropertySymbols === "function")
|
|
15
|
+
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
|
|
16
|
+
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
|
|
17
|
+
t[p[i]] = s[p[i]];
|
|
18
|
+
}
|
|
19
|
+
return t;
|
|
20
|
+
};
|
|
21
|
+
import { DataStoresError, generateId, } from '@sprucelabs/data-stores';
|
|
22
|
+
import { assertOptions, flattenValues, expandValues } from '@sprucelabs/schema';
|
|
23
|
+
import { NULL_PLACEHOLDER } from '@sprucelabs/test-utils';
|
|
24
|
+
import { ChromaClient, OllamaEmbeddingFunction, } from 'chromadb';
|
|
25
|
+
import SpruceError from './errors/SpruceError.js';
|
|
26
|
+
export default class ChromaDatabase {
|
|
27
|
+
constructor(connectionString) {
|
|
28
|
+
this._isConnected = false;
|
|
29
|
+
this.collections = {};
|
|
30
|
+
assertOptions({ connectionString }, ['connectionString']);
|
|
31
|
+
this.embeddings = new OllamaEmbeddingFunction({
|
|
32
|
+
model: 'llama3.2',
|
|
33
|
+
url: 'http://localhost:11434/api/embeddings',
|
|
34
|
+
});
|
|
35
|
+
if (!connectionString.startsWith('chroma://')) {
|
|
36
|
+
throw new DataStoresError({
|
|
37
|
+
code: 'INVALID_DB_CONNECTION_STRING',
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
this.connectionString = connectionString.replace('chroma://', 'http://');
|
|
41
|
+
}
|
|
42
|
+
static setEmbeddingsFields(collectionName, fields) {
|
|
43
|
+
if (!this.embeddingFields) {
|
|
44
|
+
this.embeddingFields = {};
|
|
45
|
+
}
|
|
46
|
+
this.embeddingFields[collectionName] = fields;
|
|
47
|
+
}
|
|
48
|
+
static clearEmbeddingsFields() {
|
|
49
|
+
delete this.embeddingFields;
|
|
50
|
+
}
|
|
51
|
+
syncUniqueIndexes(_collectionName, _indexes) {
|
|
52
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
53
|
+
throw new SpruceError({
|
|
54
|
+
code: 'FEATURE_NOT_SUPPORTED',
|
|
55
|
+
operation: 'syncUniqueIndexes',
|
|
56
|
+
});
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
syncIndexes(_collectionName, _indexes) {
|
|
60
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
61
|
+
throw new SpruceError({
|
|
62
|
+
code: 'FEATURE_NOT_SUPPORTED',
|
|
63
|
+
operation: 'syncIndexes',
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
dropIndex(_collectionName, _index) {
|
|
68
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
69
|
+
throw new SpruceError({
|
|
70
|
+
code: 'FEATURE_NOT_SUPPORTED',
|
|
71
|
+
operation: 'dropIndex',
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
getUniqueIndexes(_collectionName) {
|
|
76
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
77
|
+
throw new SpruceError({
|
|
78
|
+
code: 'FEATURE_NOT_SUPPORTED',
|
|
79
|
+
operation: 'getUniqueIndexes',
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
getIndexes(_collectionName) {
|
|
84
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
85
|
+
throw new SpruceError({
|
|
86
|
+
code: 'FEATURE_NOT_SUPPORTED',
|
|
87
|
+
operation: 'getIndexes',
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
isConnected() {
|
|
92
|
+
return this._isConnected;
|
|
93
|
+
}
|
|
94
|
+
generateId() {
|
|
95
|
+
return generateId();
|
|
96
|
+
}
|
|
97
|
+
connect() {
|
|
98
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
99
|
+
this.client = new ChromaClient({ path: this.connectionString });
|
|
100
|
+
try {
|
|
101
|
+
yield this.client.heartbeat();
|
|
102
|
+
}
|
|
103
|
+
catch (err) {
|
|
104
|
+
throw new DataStoresError({
|
|
105
|
+
code: 'UNABLE_TO_CONNECT_TO_DB',
|
|
106
|
+
originalError: err,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
this._isConnected = true;
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
close() {
|
|
113
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
114
|
+
this._isConnected = false;
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
getShouldAutoGenerateId() {
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
createOne(collection, values, _options) {
|
|
121
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
122
|
+
const results = yield this.create(collection, [values]);
|
|
123
|
+
return results[0];
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
getCollection(collection) {
|
|
127
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
128
|
+
if (!this.collections[collection]) {
|
|
129
|
+
this.collections[collection] =
|
|
130
|
+
yield this.client.getOrCreateCollection({
|
|
131
|
+
name: collection,
|
|
132
|
+
embeddingFunction: this.embeddings,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
return this.collections[collection];
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
buildPrompt(values, collectionName) {
|
|
139
|
+
var _a;
|
|
140
|
+
const fields = (_a = ChromaDatabase.embeddingFields) === null || _a === void 0 ? void 0 : _a[collectionName];
|
|
141
|
+
if ((fields === null || fields === void 0 ? void 0 : fields.length) === 1) {
|
|
142
|
+
const field = fields[0];
|
|
143
|
+
return values[field];
|
|
144
|
+
}
|
|
145
|
+
let prompt = '';
|
|
146
|
+
const keys = Object.keys(values);
|
|
147
|
+
for (const key of keys) {
|
|
148
|
+
const value = values[key];
|
|
149
|
+
if (value && typeof value === 'object') {
|
|
150
|
+
prompt += `${key}:\n\t${this.buildPrompt(value, collectionName)}\n`;
|
|
151
|
+
}
|
|
152
|
+
else if (value) {
|
|
153
|
+
prompt += `${key}: ${value}\n`;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return prompt.trim();
|
|
157
|
+
}
|
|
158
|
+
create(collection, values) {
|
|
159
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
160
|
+
const col = yield this.getCollection(collection);
|
|
161
|
+
const { documents, ids, metadatas } = this.splitValues(values, collection);
|
|
162
|
+
yield col.add({
|
|
163
|
+
documents,
|
|
164
|
+
ids,
|
|
165
|
+
metadatas,
|
|
166
|
+
});
|
|
167
|
+
return this.find(collection, { id: { $in: ids } });
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
splitValues(values, collectionName) {
|
|
171
|
+
const ids = [];
|
|
172
|
+
const documents = [];
|
|
173
|
+
const metadatas = [];
|
|
174
|
+
for (const v of values) {
|
|
175
|
+
const { id = this.generateId() } = v, values = __rest(v, ["id"]);
|
|
176
|
+
ids.push(id);
|
|
177
|
+
documents.push(this.buildPrompt(values, collectionName));
|
|
178
|
+
const flattened = this.flattenValues(values);
|
|
179
|
+
metadatas.push(flattened);
|
|
180
|
+
}
|
|
181
|
+
return { documents, ids, metadatas };
|
|
182
|
+
}
|
|
183
|
+
flattenValues(values) {
|
|
184
|
+
const flattened = flattenValues(values, [
|
|
185
|
+
'$or',
|
|
186
|
+
'*.$gt',
|
|
187
|
+
'*.$gte',
|
|
188
|
+
'*.$lt',
|
|
189
|
+
'*.$lte',
|
|
190
|
+
'*.$ne',
|
|
191
|
+
]);
|
|
192
|
+
this.dropInNullPlaceholders(flattened);
|
|
193
|
+
return flattened;
|
|
194
|
+
}
|
|
195
|
+
dropInNullPlaceholders(flattened) {
|
|
196
|
+
for (const key in flattened) {
|
|
197
|
+
if (flattened[key] === undefined) {
|
|
198
|
+
flattened[key] = null;
|
|
199
|
+
}
|
|
200
|
+
if (flattened[key] === null) {
|
|
201
|
+
flattened[key] = NULL_PLACEHOLDER;
|
|
202
|
+
}
|
|
203
|
+
if (typeof flattened[key] === 'object') {
|
|
204
|
+
this.dropInNullPlaceholders(flattened[key]);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
dropCollection(name) {
|
|
209
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
210
|
+
yield this.client.deleteCollection({ name });
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
dropDatabase() {
|
|
214
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
215
|
+
const collections = yield this.client.listCollections();
|
|
216
|
+
yield Promise.all(collections.map((col) => this.dropCollection(col.name)));
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
findOne(collection, query, options, dbOptions) {
|
|
220
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
221
|
+
var _a;
|
|
222
|
+
const matches = yield this.find(collection, query, Object.assign(Object.assign({}, options), { limit: 1 }), dbOptions);
|
|
223
|
+
return (_a = matches[0]) !== null && _a !== void 0 ? _a : null;
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
find(collection, query, options, _dbOptions) {
|
|
227
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
228
|
+
let { ids, where, skipIds } = this.buildQuery(query);
|
|
229
|
+
let matches;
|
|
230
|
+
const { limit, includeFields } = options !== null && options !== void 0 ? options : {};
|
|
231
|
+
const col = yield this.getCollection(collection);
|
|
232
|
+
if (where === null || where === void 0 ? void 0 : where.$prompt) {
|
|
233
|
+
const { $prompt } = where, rest = __rest(where, ["$prompt"]);
|
|
234
|
+
const queryResults = yield col.query({
|
|
235
|
+
queryTexts: [$prompt],
|
|
236
|
+
nResults: 1,
|
|
237
|
+
where: Object.keys(rest).length > 0 ? rest : undefined,
|
|
238
|
+
});
|
|
239
|
+
matches = {
|
|
240
|
+
ids: queryResults.ids[0],
|
|
241
|
+
metadatas: queryResults.metadatas[0],
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
else {
|
|
245
|
+
matches = yield col.get({
|
|
246
|
+
ids,
|
|
247
|
+
include: ['metadatas'],
|
|
248
|
+
where,
|
|
249
|
+
limit: limit == 0 ? 1 : limit,
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
if (!matches || !matches.ids[0]) {
|
|
253
|
+
return [];
|
|
254
|
+
}
|
|
255
|
+
let records = this.mapResultsToRecords(limit, matches, skipIds);
|
|
256
|
+
if (includeFields) {
|
|
257
|
+
records = this.filterFieldsByInclude(records, includeFields);
|
|
258
|
+
}
|
|
259
|
+
return records;
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
mapResultsToRecords(limit, matches, skipIds) {
|
|
263
|
+
var _a;
|
|
264
|
+
let records = [];
|
|
265
|
+
const total = limit !== null && limit !== void 0 ? limit : matches.ids.length;
|
|
266
|
+
for (let i = 0; i < total; i++) {
|
|
267
|
+
const values = (_a = matches.metadatas[i]) !== null && _a !== void 0 ? _a : {};
|
|
268
|
+
const id = matches.ids[i];
|
|
269
|
+
if (!skipIds.includes(id)) {
|
|
270
|
+
records.push(Object.assign({ id }, this.expandValues(values)));
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
return records;
|
|
274
|
+
}
|
|
275
|
+
filterFieldsByInclude(records, includeFields) {
|
|
276
|
+
const trimmedRecords = [];
|
|
277
|
+
// eslint-disable-next-line @typescript-eslint/prefer-for-of
|
|
278
|
+
for (let c = 0; c < records.length; c++) {
|
|
279
|
+
const record = {};
|
|
280
|
+
for (const key of includeFields) {
|
|
281
|
+
record[key] = records[c][key];
|
|
282
|
+
}
|
|
283
|
+
trimmedRecords.push(record);
|
|
284
|
+
}
|
|
285
|
+
return trimmedRecords;
|
|
286
|
+
}
|
|
287
|
+
buildQuery(query) {
|
|
288
|
+
const _a = query !== null && query !== void 0 ? query : {}, { id } = _a, rest = __rest(_a, ["id"]);
|
|
289
|
+
let ids;
|
|
290
|
+
let where;
|
|
291
|
+
if (id === null || id === void 0 ? void 0 : id.$in) {
|
|
292
|
+
ids = id.$in;
|
|
293
|
+
}
|
|
294
|
+
else if (id) {
|
|
295
|
+
ids = [id];
|
|
296
|
+
}
|
|
297
|
+
else if (query) {
|
|
298
|
+
where = rest;
|
|
299
|
+
}
|
|
300
|
+
if ((where === null || where === void 0 ? void 0 : where.$or) && where.$or.length === 1) {
|
|
301
|
+
//@ts-ignore
|
|
302
|
+
where = where.$or[0];
|
|
303
|
+
}
|
|
304
|
+
if (where) {
|
|
305
|
+
const { $or, $prompt } = where, rest = __rest(where, ["$or", "$prompt"]);
|
|
306
|
+
if ($or) {
|
|
307
|
+
where.$or = $or;
|
|
308
|
+
}
|
|
309
|
+
else {
|
|
310
|
+
where = this.flattenValues(rest);
|
|
311
|
+
if (Object.keys(where).length > 1) {
|
|
312
|
+
const $and = [];
|
|
313
|
+
for (const key in where) {
|
|
314
|
+
//@ts-ignore
|
|
315
|
+
$and.push({ [key]: where[key] });
|
|
316
|
+
}
|
|
317
|
+
where = {
|
|
318
|
+
$and,
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
if ($prompt) {
|
|
323
|
+
where.$prompt = $prompt;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
const firstId = ids === null || ids === void 0 ? void 0 : ids[0];
|
|
327
|
+
const skipIds = [];
|
|
328
|
+
//@ts-ignore
|
|
329
|
+
if (firstId === null || firstId === void 0 ? void 0 : firstId['$ne']) {
|
|
330
|
+
//@ts-ignore
|
|
331
|
+
skipIds.push(firstId.$ne);
|
|
332
|
+
ids = undefined;
|
|
333
|
+
}
|
|
334
|
+
if (Object.keys(where !== null && where !== void 0 ? where : {}).length === 0) {
|
|
335
|
+
where = undefined;
|
|
336
|
+
}
|
|
337
|
+
return { ids, where, skipIds };
|
|
338
|
+
}
|
|
339
|
+
expandValues(values) {
|
|
340
|
+
const nulledValues = {};
|
|
341
|
+
for (const key in values) {
|
|
342
|
+
if (values[key] === NULL_PLACEHOLDER) {
|
|
343
|
+
nulledValues[key] = null;
|
|
344
|
+
}
|
|
345
|
+
else {
|
|
346
|
+
nulledValues[key] = values[key];
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
return expandValues(nulledValues);
|
|
350
|
+
}
|
|
351
|
+
updateOne(collection, query, updates, _dbOptions) {
|
|
352
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
353
|
+
const col = yield this.getCollection(collection);
|
|
354
|
+
const match = yield this.findOne(collection, query);
|
|
355
|
+
if (!match) {
|
|
356
|
+
throw new DataStoresError({
|
|
357
|
+
code: 'RECORD_NOT_FOUND',
|
|
358
|
+
query,
|
|
359
|
+
storeName: collection,
|
|
360
|
+
});
|
|
361
|
+
}
|
|
362
|
+
const { documents, metadatas } = this.splitValues([updates], collection);
|
|
363
|
+
yield col.update({
|
|
364
|
+
ids: [match.id],
|
|
365
|
+
documents,
|
|
366
|
+
metadatas,
|
|
367
|
+
});
|
|
368
|
+
const { id } = match, values = __rest(match, ["id"]);
|
|
369
|
+
return Object.assign(Object.assign({ id }, values), updates);
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
update(collection, query, updates) {
|
|
373
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
374
|
+
const matches = yield this.find(collection, query);
|
|
375
|
+
for (const match of matches) {
|
|
376
|
+
yield this.updateOne(collection, { id: match.id }, updates);
|
|
377
|
+
}
|
|
378
|
+
return matches.length;
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
upsertOne(collection, query, updates) {
|
|
382
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
383
|
+
const col = yield this.getCollection(collection);
|
|
384
|
+
const match = yield this.findOne(collection, query);
|
|
385
|
+
let { documents, ids, metadatas } = this.splitValues([updates], collection);
|
|
386
|
+
if (match === null || match === void 0 ? void 0 : match.id) {
|
|
387
|
+
ids = [match.id];
|
|
388
|
+
}
|
|
389
|
+
yield col.upsert({
|
|
390
|
+
documents,
|
|
391
|
+
ids,
|
|
392
|
+
metadatas,
|
|
393
|
+
});
|
|
394
|
+
const updated = yield this.findOne(collection, { id: ids[0] });
|
|
395
|
+
return updated;
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
delete(collection, query) {
|
|
399
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
400
|
+
const matches = yield this.find(collection, query);
|
|
401
|
+
const col = yield this.getCollection(collection);
|
|
402
|
+
yield col.delete({
|
|
403
|
+
ids: matches.map((m) => m.id),
|
|
404
|
+
});
|
|
405
|
+
return matches.length;
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
deleteOne(collection, query) {
|
|
409
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
410
|
+
const match = yield this.findOne(collection, query);
|
|
411
|
+
yield this.delete(collection, { id: match === null || match === void 0 ? void 0 : match.id });
|
|
412
|
+
return match ? 1 : 0;
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
count(collection, query) {
|
|
416
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
417
|
+
const matches = yield this.find(collection, query);
|
|
418
|
+
return matches.length;
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
createUniqueIndex(_collection, _index) {
|
|
422
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
423
|
+
throw new SpruceError({
|
|
424
|
+
code: 'FEATURE_NOT_SUPPORTED',
|
|
425
|
+
operation: 'createUniqueIndex',
|
|
426
|
+
});
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
createIndex(_collection, _index) {
|
|
430
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
431
|
+
throw new SpruceError({
|
|
432
|
+
code: 'FEATURE_NOT_SUPPORTED',
|
|
433
|
+
operation: 'createIndex',
|
|
434
|
+
});
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
query(_query, _params) {
|
|
438
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
439
|
+
throw new SpruceError({
|
|
440
|
+
code: 'FEATURE_NOT_SUPPORTED',
|
|
441
|
+
operation: 'query',
|
|
442
|
+
});
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { AddRecordsParams, UpsertRecordsParams, GetResponse, UpdateRecordsParams, QueryRecordsParams, QueryResponse, CollectionMetadata, CollectionParams, PeekParams, DeleteParams, ID, IDs, Where, IncludeEnum, WhereDocument } from 'chromadb';
|
|
2
|
+
export interface Collection {
|
|
3
|
+
add(params: AddRecordsParams): Promise<void>;
|
|
4
|
+
/**
|
|
5
|
+
* Upsert items to the collection
|
|
6
|
+
* @param {Object} params - The parameters for the query.
|
|
7
|
+
* @param {ID | IDs} [params.ids] - IDs of the items to add.
|
|
8
|
+
* @param {Embedding | Embeddings} [params.embeddings] - Optional embeddings of the items to add.
|
|
9
|
+
* @param {Metadata | Metadatas} [params.metadatas] - Optional metadata of the items to add.
|
|
10
|
+
* @param {Document | Documents} [params.documents] - Optional documents of the items to add.
|
|
11
|
+
* @returns {Promise<void>}
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```typescript
|
|
15
|
+
* const response = await collection.upsert({
|
|
16
|
+
* ids: ["id1", "id2"],
|
|
17
|
+
* embeddings: [[1, 2, 3], [4, 5, 6]],
|
|
18
|
+
* metadatas: [{ "key": "value" }, { "key": "value" }],
|
|
19
|
+
* documents: ["document1", "document2"],
|
|
20
|
+
* });
|
|
21
|
+
* ```
|
|
22
|
+
*/
|
|
23
|
+
upsert(params: UpsertRecordsParams): Promise<void>;
|
|
24
|
+
/**
|
|
25
|
+
* Count the number of items in the collection
|
|
26
|
+
* @returns {Promise<number>} - The number of items in the collection.
|
|
27
|
+
*
|
|
28
|
+
* @example
|
|
29
|
+
* ```typescript
|
|
30
|
+
* const count = await collection.count();
|
|
31
|
+
* ```
|
|
32
|
+
*/
|
|
33
|
+
count(): Promise<number>;
|
|
34
|
+
/**
|
|
35
|
+
* Get items from the collection
|
|
36
|
+
* @param {Object} params - The parameters for the query.
|
|
37
|
+
* @param {ID | IDs} [params.ids] - Optional IDs of the items to get.
|
|
38
|
+
* @param {Where} [params.where] - Optional where clause to filter items by.
|
|
39
|
+
* @param {PositiveInteger} [params.limit] - Optional limit on the number of items to get.
|
|
40
|
+
* @param {PositiveInteger} [params.offset] - Optional offset on the items to get.
|
|
41
|
+
* @param {IncludeEnum[]} [params.include] - Optional list of items to include in the response.
|
|
42
|
+
* @param {WhereDocument} [params.whereDocument] - Optional where clause to filter items by.
|
|
43
|
+
* @returns {Promise<GetResponse>} - The response from the server.
|
|
44
|
+
*
|
|
45
|
+
* @example
|
|
46
|
+
* ```typescript
|
|
47
|
+
* const response = await collection.get({
|
|
48
|
+
* ids: ["id1", "id2"],
|
|
49
|
+
* where: { "key": "value" },
|
|
50
|
+
* limit: 10,
|
|
51
|
+
* offset: 0,
|
|
52
|
+
* include: ["embeddings", "metadatas", "documents"],
|
|
53
|
+
* whereDocument: { $contains: "value" },
|
|
54
|
+
* });
|
|
55
|
+
* ```
|
|
56
|
+
*/
|
|
57
|
+
get(options?: BaseGetParams): Promise<GetResponse>;
|
|
58
|
+
/**
|
|
59
|
+
* Update items in the collection
|
|
60
|
+
* @param {Object} params - The parameters for the query.
|
|
61
|
+
* @param {ID | IDs} [params.ids] - IDs of the items to add.
|
|
62
|
+
* @param {Embedding | Embeddings} [params.embeddings] - Optional embeddings of the items to add.
|
|
63
|
+
* @param {Metadata | Metadatas} [params.metadatas] - Optional metadata of the items to add.
|
|
64
|
+
* @param {Document | Documents} [params.documents] - Optional documents of the items to add.
|
|
65
|
+
* @returns {Promise<void>}
|
|
66
|
+
*
|
|
67
|
+
* @example
|
|
68
|
+
* ```typescript
|
|
69
|
+
* const response = await collection.update({
|
|
70
|
+
* ids: ["id1", "id2"],
|
|
71
|
+
* embeddings: [[1, 2, 3], [4, 5, 6]],
|
|
72
|
+
* metadatas: [{ "key": "value" }, { "key": "value" }],
|
|
73
|
+
* documents: ["document1", "document2"],
|
|
74
|
+
* });
|
|
75
|
+
* ```
|
|
76
|
+
*/
|
|
77
|
+
update(params: UpdateRecordsParams): Promise<void>;
|
|
78
|
+
/**
|
|
79
|
+
* Performs a query on the collection using the specified parameters.
|
|
80
|
+
*
|
|
81
|
+
* @param {Object} params - The parameters for the query.
|
|
82
|
+
* @param {Embedding | Embeddings} [params.queryEmbeddings] - Optional query embeddings to use for the search.
|
|
83
|
+
* @param {PositiveInteger} [params.nResults] - Optional number of results to return (default is 10).
|
|
84
|
+
* @param {Where} [params.where] - Optional query condition to filter results based on metadata values.
|
|
85
|
+
* @param {string | string[]} [params.queryTexts] - Optional query text(s) to search for in the collection.
|
|
86
|
+
* @param {WhereDocument} [params.whereDocument] - Optional query condition to filter results based on document content.
|
|
87
|
+
* @param {IncludeEnum[]} [params.include] - Optional array of fields to include in the result, such as "metadata" and "document".
|
|
88
|
+
*
|
|
89
|
+
* @returns {Promise<QueryResponse>} A promise that resolves to the query results.
|
|
90
|
+
* @throws {Error} If there is an issue executing the query.
|
|
91
|
+
* @example
|
|
92
|
+
* // Query the collection using embeddings
|
|
93
|
+
* const results = await collection.query({
|
|
94
|
+
* queryEmbeddings: [[0.1, 0.2, ...], ...],
|
|
95
|
+
* nResults: 10,
|
|
96
|
+
* where: {"name": {"$eq": "John Doe"}},
|
|
97
|
+
* include: ["metadata", "document"]
|
|
98
|
+
* });
|
|
99
|
+
* @example
|
|
100
|
+
* ```js
|
|
101
|
+
* // Query the collection using query text
|
|
102
|
+
* const results = await collection.query({
|
|
103
|
+
* queryTexts: "some text",
|
|
104
|
+
* nResults: 10,
|
|
105
|
+
* where: {"name": {"$eq": "John Doe"}},
|
|
106
|
+
* include: ["metadata", "document"]
|
|
107
|
+
* });
|
|
108
|
+
* ```
|
|
109
|
+
*
|
|
110
|
+
*/
|
|
111
|
+
query({ nResults, where, whereDocument, include, queryTexts, queryEmbeddings, }: QueryRecordsParams): Promise<QueryResponse>;
|
|
112
|
+
/**
|
|
113
|
+
* Modify the collection name or metadata
|
|
114
|
+
* @param {Object} params - The parameters for the query.
|
|
115
|
+
* @param {string} [params.name] - Optional new name for the collection.
|
|
116
|
+
* @param {CollectionMetadata} [params.metadata] - Optional new metadata for the collection.
|
|
117
|
+
* @returns {Promise<void>} - The response from the API.
|
|
118
|
+
*
|
|
119
|
+
* @example
|
|
120
|
+
* ```typescript
|
|
121
|
+
* const response = await client.updateCollection({
|
|
122
|
+
* name: "new name",
|
|
123
|
+
* metadata: { "key": "value" },
|
|
124
|
+
* });
|
|
125
|
+
* ```
|
|
126
|
+
*/
|
|
127
|
+
modify({ name, metadata, }: {
|
|
128
|
+
name?: string;
|
|
129
|
+
metadata?: CollectionMetadata;
|
|
130
|
+
}): Promise<CollectionParams>;
|
|
131
|
+
/**
|
|
132
|
+
* Peek inside the collection
|
|
133
|
+
* @param {Object} params - The parameters for the query.
|
|
134
|
+
* @param {PositiveInteger} [params.limit] - Optional number of results to return (default is 10).
|
|
135
|
+
* @returns {Promise<GetResponse>} A promise that resolves to the query results.
|
|
136
|
+
* @throws {Error} If there is an issue executing the query.
|
|
137
|
+
*
|
|
138
|
+
* @example
|
|
139
|
+
* ```typescript
|
|
140
|
+
* const results = await collection.peek({
|
|
141
|
+
* limit: 10
|
|
142
|
+
* });
|
|
143
|
+
* ```
|
|
144
|
+
*/
|
|
145
|
+
peek(options?: PeekParams): Promise<GetResponse>;
|
|
146
|
+
/**
|
|
147
|
+
* Deletes items from the collection.
|
|
148
|
+
* @param {Object} params - The parameters for deleting items from the collection.
|
|
149
|
+
* @param {ID | IDs} [params.ids] - Optional ID or array of IDs of items to delete.
|
|
150
|
+
* @param {Where} [params.where] - Optional query condition to filter items to delete based on metadata values.
|
|
151
|
+
* @param {WhereDocument} [params.whereDocument] - Optional query condition to filter items to delete based on document content.
|
|
152
|
+
* @returns {Promise<string[]>} A promise that resolves to the IDs of the deleted items.
|
|
153
|
+
* @throws {Error} If there is an issue deleting items from the collection.
|
|
154
|
+
*
|
|
155
|
+
* @example
|
|
156
|
+
* ```typescript
|
|
157
|
+
* const results = await collection.delete({
|
|
158
|
+
* ids: "some_id",
|
|
159
|
+
* where: {"name": {"$eq": "John Doe"}},
|
|
160
|
+
* whereDocument: {"$contains":"search_string"}
|
|
161
|
+
* });
|
|
162
|
+
* ```
|
|
163
|
+
*/
|
|
164
|
+
delete(options?: DeleteParams): Promise<void>;
|
|
165
|
+
}
|
|
166
|
+
interface BaseGetParams {
|
|
167
|
+
ids?: ID | IDs;
|
|
168
|
+
where?: Where;
|
|
169
|
+
limit?: number;
|
|
170
|
+
offset?: number;
|
|
171
|
+
include?: IncludeEnum[];
|
|
172
|
+
whereDocument?: WhereDocument;
|
|
173
|
+
}
|
|
174
|
+
export type WhereWithPrompt = Where & {
|
|
175
|
+
$prompt?: string;
|
|
176
|
+
};
|
|
177
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import BaseSpruceError from '@sprucelabs/error';
|
|
2
|
+
import ErrorOptions from './../.spruce/errors/options.types';
|
|
3
|
+
export default class SpruceError extends BaseSpruceError<ErrorOptions> {
|
|
4
|
+
/** an easy to understand version of the errors */
|
|
5
|
+
friendlyMessage(): string;
|
|
6
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import BaseSpruceError from '@sprucelabs/error';
|
|
2
|
+
export default class SpruceError extends BaseSpruceError {
|
|
3
|
+
/** an easy to understand version of the errors */
|
|
4
|
+
friendlyMessage() {
|
|
5
|
+
const { options } = this;
|
|
6
|
+
let message;
|
|
7
|
+
switch (options === null || options === void 0 ? void 0 : options.code) {
|
|
8
|
+
case 'FEATURE_NOT_SUPPORTED':
|
|
9
|
+
message = `I'm sorry, but you can't run ${options.operation} operation on the Chromadb`;
|
|
10
|
+
break;
|
|
11
|
+
default:
|
|
12
|
+
message = super.friendlyMessage();
|
|
13
|
+
}
|
|
14
|
+
const fullMessage = options.friendlyMessage
|
|
15
|
+
? options.friendlyMessage
|
|
16
|
+
: message;
|
|
17
|
+
return fullMessage;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { default as ChromaDatabase } from './ChromaDatabase.js';
|
|
2
|
+
import { DatabaseFactory } from '@sprucelabs/data-stores';
|
|
3
|
+
import ChromaDatabase from './ChromaDatabase.js';
|
|
4
|
+
export * from './ChromaDatabase.js';
|
|
5
|
+
DatabaseFactory.addAdapter('chroma://', ChromaDatabase);
|
package/build/index.js
CHANGED
|
@@ -24,4 +24,3 @@ const data_stores_1 = require("@sprucelabs/data-stores");
|
|
|
24
24
|
const ChromaDatabase_2 = __importDefault(require("./ChromaDatabase"));
|
|
25
25
|
__exportStar(require("./ChromaDatabase"), exports);
|
|
26
26
|
data_stores_1.DatabaseFactory.addAdapter('chroma://', ChromaDatabase_2.default);
|
|
27
|
-
//# sourceMappingURL=index.js.map
|