langchain 0.0.197-rc.1 → 0.0.198
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chains/openai_moderation.cjs +2 -2
- package/dist/chains/openai_moderation.d.ts +1 -1
- package/dist/chains/openai_moderation.js +1 -1
- package/dist/chat_models/anthropic.cjs +351 -15
- package/dist/chat_models/anthropic.d.ts +157 -1
- package/dist/chat_models/anthropic.js +348 -1
- package/dist/chat_models/cloudflare_workersai.cjs +5 -0
- package/dist/chat_models/cloudflare_workersai.d.ts +3 -0
- package/dist/chat_models/cloudflare_workersai.js +5 -0
- package/dist/chat_models/fireworks.d.ts +1 -1
- package/dist/chat_models/iflytek_xinghuo/common.d.ts +1 -1
- package/dist/chat_models/llama_cpp.cjs +24 -0
- package/dist/chat_models/llama_cpp.d.ts +3 -1
- package/dist/chat_models/llama_cpp.js +24 -0
- package/dist/chat_models/minimax.d.ts +1 -1
- package/dist/chat_models/openai.cjs +698 -4
- package/dist/chat_models/openai.d.ts +137 -4
- package/dist/chat_models/openai.js +695 -2
- package/dist/document_loaders/fs/openai_whisper_audio.cjs +2 -2
- package/dist/document_loaders/fs/openai_whisper_audio.d.ts +1 -1
- package/dist/document_loaders/fs/openai_whisper_audio.js +1 -1
- package/dist/document_loaders/fs/pptx.cjs +39 -0
- package/dist/document_loaders/fs/pptx.d.ts +23 -0
- package/dist/document_loaders/fs/pptx.js +35 -0
- package/dist/embeddings/openai.cjs +240 -2
- package/dist/embeddings/openai.d.ts +82 -1
- package/dist/embeddings/openai.js +239 -1
- package/dist/experimental/openai_assistant/index.cjs +35 -3
- package/dist/experimental/openai_assistant/index.d.ts +27 -1
- package/dist/experimental/openai_assistant/index.js +33 -1
- package/dist/experimental/openai_assistant/schema.d.ts +1 -1
- package/dist/experimental/openai_files/index.cjs +2 -2
- package/dist/experimental/openai_files/index.d.ts +1 -1
- package/dist/experimental/openai_files/index.js +1 -1
- package/dist/experimental/tools/pyinterpreter.cjs +248 -0
- package/dist/experimental/tools/pyinterpreter.d.ts +18 -0
- package/dist/experimental/tools/pyinterpreter.js +244 -0
- package/dist/graphs/neo4j_graph.cjs +49 -14
- package/dist/graphs/neo4j_graph.d.ts +30 -0
- package/dist/graphs/neo4j_graph.js +49 -14
- package/dist/llms/fireworks.d.ts +1 -1
- package/dist/llms/hf.cjs +13 -2
- package/dist/llms/hf.d.ts +5 -0
- package/dist/llms/hf.js +13 -2
- package/dist/llms/llama_cpp.cjs +17 -3
- package/dist/llms/llama_cpp.d.ts +4 -1
- package/dist/llms/llama_cpp.js +17 -3
- package/dist/llms/openai-chat.cjs +445 -3
- package/dist/llms/openai-chat.d.ts +123 -4
- package/dist/llms/openai-chat.js +443 -2
- package/dist/llms/openai.cjs +530 -6
- package/dist/llms/openai.d.ts +123 -4
- package/dist/llms/openai.js +525 -2
- package/dist/load/import_constants.cjs +3 -0
- package/dist/load/import_constants.js +3 -0
- package/dist/output_parsers/json.cjs +4 -0
- package/dist/output_parsers/json.js +4 -0
- package/dist/schema/index.d.ts +1 -1
- package/dist/tools/convert_to_openai.cjs +38 -4
- package/dist/tools/convert_to_openai.d.ts +11 -1
- package/dist/tools/convert_to_openai.js +35 -1
- package/dist/types/openai-types.d.ts +133 -1
- package/dist/util/env.cjs +9 -70
- package/dist/util/env.d.ts +1 -21
- package/dist/util/env.js +1 -62
- package/dist/util/openai-format-fndef.cjs +81 -0
- package/dist/util/openai-format-fndef.d.ts +44 -0
- package/dist/util/openai-format-fndef.js +77 -0
- package/dist/util/openai.cjs +18 -2
- package/dist/util/openai.d.ts +1 -1
- package/dist/util/openai.js +17 -1
- package/dist/util/openapi.d.ts +2 -2
- package/dist/util/prompt-layer.d.ts +1 -1
- package/dist/vectorstores/clickhouse.cjs +286 -0
- package/dist/vectorstores/clickhouse.d.ts +126 -0
- package/dist/vectorstores/clickhouse.js +259 -0
- package/dist/vectorstores/pgvector.cjs +142 -18
- package/dist/vectorstores/pgvector.d.ts +21 -0
- package/dist/vectorstores/pgvector.js +142 -18
- package/dist/vectorstores/weaviate.cjs +45 -2
- package/dist/vectorstores/weaviate.d.ts +27 -1
- package/dist/vectorstores/weaviate.js +45 -2
- package/document_loaders/fs/pptx.cjs +1 -0
- package/document_loaders/fs/pptx.d.ts +1 -0
- package/document_loaders/fs/pptx.js +1 -0
- package/experimental/tools/pyinterpreter.cjs +1 -0
- package/experimental/tools/pyinterpreter.d.ts +1 -0
- package/experimental/tools/pyinterpreter.js +1 -0
- package/package.json +41 -9
- package/vectorstores/clickhouse.cjs +1 -0
- package/vectorstores/clickhouse.d.ts +1 -0
- package/vectorstores/clickhouse.js +1 -0
|
@@ -26,6 +26,24 @@ class PGVectorStore extends base_js_1.VectorStore {
|
|
|
26
26
|
writable: true,
|
|
27
27
|
value: void 0
|
|
28
28
|
});
|
|
29
|
+
Object.defineProperty(this, "collectionTableName", {
|
|
30
|
+
enumerable: true,
|
|
31
|
+
configurable: true,
|
|
32
|
+
writable: true,
|
|
33
|
+
value: void 0
|
|
34
|
+
});
|
|
35
|
+
Object.defineProperty(this, "collectionName", {
|
|
36
|
+
enumerable: true,
|
|
37
|
+
configurable: true,
|
|
38
|
+
writable: true,
|
|
39
|
+
value: "langchain"
|
|
40
|
+
});
|
|
41
|
+
Object.defineProperty(this, "collectionMetadata", {
|
|
42
|
+
enumerable: true,
|
|
43
|
+
configurable: true,
|
|
44
|
+
writable: true,
|
|
45
|
+
value: void 0
|
|
46
|
+
});
|
|
29
47
|
Object.defineProperty(this, "idColumnName", {
|
|
30
48
|
enumerable: true,
|
|
31
49
|
configurable: true,
|
|
@@ -81,6 +99,9 @@ class PGVectorStore extends base_js_1.VectorStore {
|
|
|
81
99
|
value: 500
|
|
82
100
|
});
|
|
83
101
|
this.tableName = config.tableName;
|
|
102
|
+
this.collectionTableName = config.collectionTableName;
|
|
103
|
+
this.collectionName = config.collectionName ?? "langchain";
|
|
104
|
+
this.collectionMetadata = config.collectionMetadata ?? null;
|
|
84
105
|
this.filter = config.filter;
|
|
85
106
|
this.vectorColumnName = config.columns?.vectorColumnName ?? "embedding";
|
|
86
107
|
this.contentColumnName = config.columns?.contentColumnName ?? "text";
|
|
@@ -106,6 +127,9 @@ class PGVectorStore extends base_js_1.VectorStore {
|
|
|
106
127
|
const postgresqlVectorStore = new PGVectorStore(embeddings, config);
|
|
107
128
|
await postgresqlVectorStore._initializeClient();
|
|
108
129
|
await postgresqlVectorStore.ensureTableInDatabase();
|
|
130
|
+
if (postgresqlVectorStore.collectionTableName) {
|
|
131
|
+
await postgresqlVectorStore.ensureCollectionTableInDatabase();
|
|
132
|
+
}
|
|
109
133
|
return postgresqlVectorStore;
|
|
110
134
|
}
|
|
111
135
|
async _initializeClient() {
|
|
@@ -122,15 +146,56 @@ class PGVectorStore extends base_js_1.VectorStore {
|
|
|
122
146
|
const texts = documents.map(({ pageContent }) => pageContent);
|
|
123
147
|
return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
|
|
124
148
|
}
|
|
149
|
+
/**
|
|
150
|
+
* Inserts a row for the collectionName provided at initialization if it does not
|
|
151
|
+
* exist and returns the collectionId.
|
|
152
|
+
*
|
|
153
|
+
* @returns The collectionId for the given collectionName.
|
|
154
|
+
*/
|
|
155
|
+
async getOrCreateCollection() {
|
|
156
|
+
const queryString = `
|
|
157
|
+
SELECT uuid from ${this.collectionTableName}
|
|
158
|
+
WHERE name = $1;
|
|
159
|
+
`;
|
|
160
|
+
const queryResult = await this.pool.query(queryString, [
|
|
161
|
+
this.collectionName,
|
|
162
|
+
]);
|
|
163
|
+
let collectionId = queryResult.rows[0]?.uuid;
|
|
164
|
+
if (!collectionId) {
|
|
165
|
+
const insertString = `
|
|
166
|
+
INSERT INTO ${this.collectionTableName}(
|
|
167
|
+
uuid,
|
|
168
|
+
name,
|
|
169
|
+
cmetadata
|
|
170
|
+
)
|
|
171
|
+
VALUES (
|
|
172
|
+
uuid_generate_v4(),
|
|
173
|
+
$1,
|
|
174
|
+
$2
|
|
175
|
+
)
|
|
176
|
+
RETURNING uuid;
|
|
177
|
+
`;
|
|
178
|
+
const insertResult = await this.pool.query(insertString, [
|
|
179
|
+
this.collectionName,
|
|
180
|
+
this.collectionMetadata,
|
|
181
|
+
]);
|
|
182
|
+
collectionId = insertResult.rows[0]?.uuid;
|
|
183
|
+
}
|
|
184
|
+
return collectionId;
|
|
185
|
+
}
|
|
125
186
|
/**
|
|
126
187
|
* Generates the SQL placeholders for a specific row at the provided index.
|
|
127
188
|
*
|
|
128
189
|
* @param index - The index of the row for which placeholders need to be generated.
|
|
190
|
+
* @param numOfColumns - The number of columns we are inserting data into.
|
|
129
191
|
* @returns The SQL placeholders for the row values.
|
|
130
192
|
*/
|
|
131
|
-
generatePlaceholderForRowAt(index) {
|
|
132
|
-
const
|
|
133
|
-
|
|
193
|
+
generatePlaceholderForRowAt(index, numOfColumns) {
|
|
194
|
+
const placeholders = [];
|
|
195
|
+
for (let i = 0; i < numOfColumns; i += 1) {
|
|
196
|
+
placeholders.push(`$${index * numOfColumns + i + 1}`);
|
|
197
|
+
}
|
|
198
|
+
return `(${placeholders.join(", ")})`;
|
|
134
199
|
}
|
|
135
200
|
/**
|
|
136
201
|
* Constructs the SQL query for inserting rows into the specified table.
|
|
@@ -139,15 +204,25 @@ class PGVectorStore extends base_js_1.VectorStore {
|
|
|
139
204
|
* @param chunkIndex - The starting index for generating query placeholders based on chunk positioning.
|
|
140
205
|
* @returns The complete SQL INSERT INTO query string.
|
|
141
206
|
*/
|
|
142
|
-
buildInsertQuery(rows) {
|
|
207
|
+
async buildInsertQuery(rows) {
|
|
208
|
+
let collectionId;
|
|
209
|
+
if (this.collectionTableName) {
|
|
210
|
+
collectionId = await this.getOrCreateCollection();
|
|
211
|
+
}
|
|
212
|
+
const columns = [
|
|
213
|
+
this.contentColumnName,
|
|
214
|
+
this.vectorColumnName,
|
|
215
|
+
this.metadataColumnName,
|
|
216
|
+
];
|
|
217
|
+
if (collectionId) {
|
|
218
|
+
columns.push("collection_id");
|
|
219
|
+
}
|
|
143
220
|
const valuesPlaceholders = rows
|
|
144
|
-
.map((_, j) => this.generatePlaceholderForRowAt(j))
|
|
221
|
+
.map((_, j) => this.generatePlaceholderForRowAt(j, columns.length))
|
|
145
222
|
.join(", ");
|
|
146
223
|
const text = `
|
|
147
224
|
INSERT INTO ${this.tableName}(
|
|
148
|
-
${
|
|
149
|
-
${this.vectorColumnName},
|
|
150
|
-
${this.metadataColumnName}
|
|
225
|
+
${columns}
|
|
151
226
|
)
|
|
152
227
|
VALUES ${valuesPlaceholders}
|
|
153
228
|
`;
|
|
@@ -162,17 +237,24 @@ class PGVectorStore extends base_js_1.VectorStore {
|
|
|
162
237
|
* @returns Promise that resolves when the vectors have been added.
|
|
163
238
|
*/
|
|
164
239
|
async addVectors(vectors, documents) {
|
|
165
|
-
const rows =
|
|
240
|
+
const rows = [];
|
|
241
|
+
let collectionId;
|
|
242
|
+
if (this.collectionTableName) {
|
|
243
|
+
collectionId = await this.getOrCreateCollection();
|
|
244
|
+
}
|
|
245
|
+
for (let i = 0; i < vectors.length; i += 1) {
|
|
246
|
+
const values = [];
|
|
247
|
+
const embedding = vectors[i];
|
|
166
248
|
const embeddingString = `[${embedding.join(",")}]`;
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
}
|
|
249
|
+
values.push(documents[i].pageContent, embeddingString, documents[i].metadata);
|
|
250
|
+
if (collectionId) {
|
|
251
|
+
values.push(collectionId);
|
|
252
|
+
}
|
|
253
|
+
rows.push(values);
|
|
254
|
+
}
|
|
173
255
|
for (let i = 0; i < rows.length; i += this.chunkSize) {
|
|
174
256
|
const chunk = rows.slice(i, i + this.chunkSize);
|
|
175
|
-
const insertQuery = this.buildInsertQuery(chunk);
|
|
257
|
+
const insertQuery = await this.buildInsertQuery(chunk);
|
|
176
258
|
const flatValues = chunk.flat();
|
|
177
259
|
try {
|
|
178
260
|
await this.pool.query(insertQuery, flatValues);
|
|
@@ -196,13 +278,23 @@ class PGVectorStore extends base_js_1.VectorStore {
|
|
|
196
278
|
async similaritySearchVectorWithScore(query, k, filter) {
|
|
197
279
|
const embeddingString = `[${query.join(",")}]`;
|
|
198
280
|
const _filter = filter ?? "{}";
|
|
281
|
+
let collectionId;
|
|
282
|
+
if (this.collectionTableName) {
|
|
283
|
+
collectionId = await this.getOrCreateCollection();
|
|
284
|
+
}
|
|
285
|
+
const parameters = [embeddingString, _filter, k];
|
|
286
|
+
if (collectionId) {
|
|
287
|
+
parameters.push(collectionId);
|
|
288
|
+
}
|
|
199
289
|
const queryString = `
|
|
200
290
|
SELECT *, ${this.vectorColumnName} <=> $1 as "_distance"
|
|
201
291
|
FROM ${this.tableName}
|
|
202
292
|
WHERE ${this.metadataColumnName}::jsonb @> $2
|
|
293
|
+
${collectionId ? "AND collection_id = $4" : ""}
|
|
203
294
|
ORDER BY "_distance" ASC
|
|
204
|
-
LIMIT $3
|
|
205
|
-
|
|
295
|
+
LIMIT $3;
|
|
296
|
+
`;
|
|
297
|
+
const documents = (await this.pool.query(queryString, parameters)).rows;
|
|
206
298
|
const results = [];
|
|
207
299
|
for (const doc of documents) {
|
|
208
300
|
if (doc._distance != null && doc[this.contentColumnName] != null) {
|
|
@@ -233,6 +325,38 @@ class PGVectorStore extends base_js_1.VectorStore {
|
|
|
233
325
|
);
|
|
234
326
|
`);
|
|
235
327
|
}
|
|
328
|
+
/**
|
|
329
|
+
* Method to ensure the existence of the collection table in the database.
|
|
330
|
+
* It creates the table if it does not already exist.
|
|
331
|
+
*
|
|
332
|
+
* @returns Promise that resolves when the collection table has been ensured.
|
|
333
|
+
*/
|
|
334
|
+
async ensureCollectionTableInDatabase() {
|
|
335
|
+
try {
|
|
336
|
+
await this.pool.query(`
|
|
337
|
+
CREATE TABLE IF NOT EXISTS ${this.collectionTableName} (
|
|
338
|
+
uuid uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY,
|
|
339
|
+
name character varying,
|
|
340
|
+
cmetadata jsonb
|
|
341
|
+
);
|
|
342
|
+
|
|
343
|
+
ALTER TABLE ${this.tableName}
|
|
344
|
+
ADD COLUMN collection_id uuid;
|
|
345
|
+
|
|
346
|
+
ALTER TABLE ${this.tableName}
|
|
347
|
+
ADD CONSTRAINT ${this.tableName}_collection_id_fkey
|
|
348
|
+
FOREIGN KEY (collection_id)
|
|
349
|
+
REFERENCES ${this.collectionTableName}(uuid)
|
|
350
|
+
ON DELETE CASCADE;
|
|
351
|
+
`);
|
|
352
|
+
}
|
|
353
|
+
catch (e) {
|
|
354
|
+
if (!e.message.includes("already exists")) {
|
|
355
|
+
console.error(e);
|
|
356
|
+
throw new Error(`Error adding column: ${e.message}`);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
}
|
|
236
360
|
/**
|
|
237
361
|
* Static method to create a new `PGVectorStore` instance from an
|
|
238
362
|
* array of texts and their metadata. It converts the texts into
|
|
@@ -11,6 +11,9 @@ type Metadata = Record<string, unknown>;
|
|
|
11
11
|
export interface PGVectorStoreArgs {
|
|
12
12
|
postgresConnectionOptions: PoolConfig;
|
|
13
13
|
tableName: string;
|
|
14
|
+
collectionTableName?: string;
|
|
15
|
+
collectionName?: string;
|
|
16
|
+
collectionMetadata?: Metadata | null;
|
|
14
17
|
columns?: {
|
|
15
18
|
idColumnName?: string;
|
|
16
19
|
vectorColumnName?: string;
|
|
@@ -35,6 +38,9 @@ export interface PGVectorStoreArgs {
|
|
|
35
38
|
export declare class PGVectorStore extends VectorStore {
|
|
36
39
|
FilterType: Metadata;
|
|
37
40
|
tableName: string;
|
|
41
|
+
collectionTableName?: string;
|
|
42
|
+
collectionName: string;
|
|
43
|
+
collectionMetadata: Metadata | null;
|
|
38
44
|
idColumnName: string;
|
|
39
45
|
vectorColumnName: string;
|
|
40
46
|
contentColumnName: string;
|
|
@@ -65,10 +71,18 @@ export declare class PGVectorStore extends VectorStore {
|
|
|
65
71
|
* @returns Promise that resolves when the documents have been added.
|
|
66
72
|
*/
|
|
67
73
|
addDocuments(documents: Document[]): Promise<void>;
|
|
74
|
+
/**
|
|
75
|
+
* Inserts a row for the collectionName provided at initialization if it does not
|
|
76
|
+
* exist and returns the collectionId.
|
|
77
|
+
*
|
|
78
|
+
* @returns The collectionId for the given collectionName.
|
|
79
|
+
*/
|
|
80
|
+
getOrCreateCollection(): Promise<string>;
|
|
68
81
|
/**
|
|
69
82
|
* Generates the SQL placeholders for a specific row at the provided index.
|
|
70
83
|
*
|
|
71
84
|
* @param index - The index of the row for which placeholders need to be generated.
|
|
85
|
+
* @param numOfColumns - The number of columns we are inserting data into.
|
|
72
86
|
* @returns The SQL placeholders for the row values.
|
|
73
87
|
*/
|
|
74
88
|
private generatePlaceholderForRowAt;
|
|
@@ -107,6 +121,13 @@ export declare class PGVectorStore extends VectorStore {
|
|
|
107
121
|
* @returns Promise that resolves when the table has been ensured.
|
|
108
122
|
*/
|
|
109
123
|
ensureTableInDatabase(): Promise<void>;
|
|
124
|
+
/**
|
|
125
|
+
* Method to ensure the existence of the collection table in the database.
|
|
126
|
+
* It creates the table if it does not already exist.
|
|
127
|
+
*
|
|
128
|
+
* @returns Promise that resolves when the collection table has been ensured.
|
|
129
|
+
*/
|
|
130
|
+
ensureCollectionTableInDatabase(): Promise<void>;
|
|
110
131
|
/**
|
|
111
132
|
* Static method to create a new `PGVectorStore` instance from an
|
|
112
133
|
* array of texts and their metadata. It converts the texts into
|
|
@@ -20,6 +20,24 @@ export class PGVectorStore extends VectorStore {
|
|
|
20
20
|
writable: true,
|
|
21
21
|
value: void 0
|
|
22
22
|
});
|
|
23
|
+
Object.defineProperty(this, "collectionTableName", {
|
|
24
|
+
enumerable: true,
|
|
25
|
+
configurable: true,
|
|
26
|
+
writable: true,
|
|
27
|
+
value: void 0
|
|
28
|
+
});
|
|
29
|
+
Object.defineProperty(this, "collectionName", {
|
|
30
|
+
enumerable: true,
|
|
31
|
+
configurable: true,
|
|
32
|
+
writable: true,
|
|
33
|
+
value: "langchain"
|
|
34
|
+
});
|
|
35
|
+
Object.defineProperty(this, "collectionMetadata", {
|
|
36
|
+
enumerable: true,
|
|
37
|
+
configurable: true,
|
|
38
|
+
writable: true,
|
|
39
|
+
value: void 0
|
|
40
|
+
});
|
|
23
41
|
Object.defineProperty(this, "idColumnName", {
|
|
24
42
|
enumerable: true,
|
|
25
43
|
configurable: true,
|
|
@@ -75,6 +93,9 @@ export class PGVectorStore extends VectorStore {
|
|
|
75
93
|
value: 500
|
|
76
94
|
});
|
|
77
95
|
this.tableName = config.tableName;
|
|
96
|
+
this.collectionTableName = config.collectionTableName;
|
|
97
|
+
this.collectionName = config.collectionName ?? "langchain";
|
|
98
|
+
this.collectionMetadata = config.collectionMetadata ?? null;
|
|
78
99
|
this.filter = config.filter;
|
|
79
100
|
this.vectorColumnName = config.columns?.vectorColumnName ?? "embedding";
|
|
80
101
|
this.contentColumnName = config.columns?.contentColumnName ?? "text";
|
|
@@ -100,6 +121,9 @@ export class PGVectorStore extends VectorStore {
|
|
|
100
121
|
const postgresqlVectorStore = new PGVectorStore(embeddings, config);
|
|
101
122
|
await postgresqlVectorStore._initializeClient();
|
|
102
123
|
await postgresqlVectorStore.ensureTableInDatabase();
|
|
124
|
+
if (postgresqlVectorStore.collectionTableName) {
|
|
125
|
+
await postgresqlVectorStore.ensureCollectionTableInDatabase();
|
|
126
|
+
}
|
|
103
127
|
return postgresqlVectorStore;
|
|
104
128
|
}
|
|
105
129
|
async _initializeClient() {
|
|
@@ -116,15 +140,56 @@ export class PGVectorStore extends VectorStore {
|
|
|
116
140
|
const texts = documents.map(({ pageContent }) => pageContent);
|
|
117
141
|
return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
|
|
118
142
|
}
|
|
143
|
+
/**
|
|
144
|
+
* Inserts a row for the collectionName provided at initialization if it does not
|
|
145
|
+
* exist and returns the collectionId.
|
|
146
|
+
*
|
|
147
|
+
* @returns The collectionId for the given collectionName.
|
|
148
|
+
*/
|
|
149
|
+
async getOrCreateCollection() {
|
|
150
|
+
const queryString = `
|
|
151
|
+
SELECT uuid from ${this.collectionTableName}
|
|
152
|
+
WHERE name = $1;
|
|
153
|
+
`;
|
|
154
|
+
const queryResult = await this.pool.query(queryString, [
|
|
155
|
+
this.collectionName,
|
|
156
|
+
]);
|
|
157
|
+
let collectionId = queryResult.rows[0]?.uuid;
|
|
158
|
+
if (!collectionId) {
|
|
159
|
+
const insertString = `
|
|
160
|
+
INSERT INTO ${this.collectionTableName}(
|
|
161
|
+
uuid,
|
|
162
|
+
name,
|
|
163
|
+
cmetadata
|
|
164
|
+
)
|
|
165
|
+
VALUES (
|
|
166
|
+
uuid_generate_v4(),
|
|
167
|
+
$1,
|
|
168
|
+
$2
|
|
169
|
+
)
|
|
170
|
+
RETURNING uuid;
|
|
171
|
+
`;
|
|
172
|
+
const insertResult = await this.pool.query(insertString, [
|
|
173
|
+
this.collectionName,
|
|
174
|
+
this.collectionMetadata,
|
|
175
|
+
]);
|
|
176
|
+
collectionId = insertResult.rows[0]?.uuid;
|
|
177
|
+
}
|
|
178
|
+
return collectionId;
|
|
179
|
+
}
|
|
119
180
|
/**
|
|
120
181
|
* Generates the SQL placeholders for a specific row at the provided index.
|
|
121
182
|
*
|
|
122
183
|
* @param index - The index of the row for which placeholders need to be generated.
|
|
184
|
+
* @param numOfColumns - The number of columns we are inserting data into.
|
|
123
185
|
* @returns The SQL placeholders for the row values.
|
|
124
186
|
*/
|
|
125
|
-
generatePlaceholderForRowAt(index) {
|
|
126
|
-
const
|
|
127
|
-
|
|
187
|
+
generatePlaceholderForRowAt(index, numOfColumns) {
|
|
188
|
+
const placeholders = [];
|
|
189
|
+
for (let i = 0; i < numOfColumns; i += 1) {
|
|
190
|
+
placeholders.push(`$${index * numOfColumns + i + 1}`);
|
|
191
|
+
}
|
|
192
|
+
return `(${placeholders.join(", ")})`;
|
|
128
193
|
}
|
|
129
194
|
/**
|
|
130
195
|
* Constructs the SQL query for inserting rows into the specified table.
|
|
@@ -133,15 +198,25 @@ export class PGVectorStore extends VectorStore {
|
|
|
133
198
|
* @param chunkIndex - The starting index for generating query placeholders based on chunk positioning.
|
|
134
199
|
* @returns The complete SQL INSERT INTO query string.
|
|
135
200
|
*/
|
|
136
|
-
buildInsertQuery(rows) {
|
|
201
|
+
async buildInsertQuery(rows) {
|
|
202
|
+
let collectionId;
|
|
203
|
+
if (this.collectionTableName) {
|
|
204
|
+
collectionId = await this.getOrCreateCollection();
|
|
205
|
+
}
|
|
206
|
+
const columns = [
|
|
207
|
+
this.contentColumnName,
|
|
208
|
+
this.vectorColumnName,
|
|
209
|
+
this.metadataColumnName,
|
|
210
|
+
];
|
|
211
|
+
if (collectionId) {
|
|
212
|
+
columns.push("collection_id");
|
|
213
|
+
}
|
|
137
214
|
const valuesPlaceholders = rows
|
|
138
|
-
.map((_, j) => this.generatePlaceholderForRowAt(j))
|
|
215
|
+
.map((_, j) => this.generatePlaceholderForRowAt(j, columns.length))
|
|
139
216
|
.join(", ");
|
|
140
217
|
const text = `
|
|
141
218
|
INSERT INTO ${this.tableName}(
|
|
142
|
-
${
|
|
143
|
-
${this.vectorColumnName},
|
|
144
|
-
${this.metadataColumnName}
|
|
219
|
+
${columns}
|
|
145
220
|
)
|
|
146
221
|
VALUES ${valuesPlaceholders}
|
|
147
222
|
`;
|
|
@@ -156,17 +231,24 @@ export class PGVectorStore extends VectorStore {
|
|
|
156
231
|
* @returns Promise that resolves when the vectors have been added.
|
|
157
232
|
*/
|
|
158
233
|
async addVectors(vectors, documents) {
|
|
159
|
-
const rows =
|
|
234
|
+
const rows = [];
|
|
235
|
+
let collectionId;
|
|
236
|
+
if (this.collectionTableName) {
|
|
237
|
+
collectionId = await this.getOrCreateCollection();
|
|
238
|
+
}
|
|
239
|
+
for (let i = 0; i < vectors.length; i += 1) {
|
|
240
|
+
const values = [];
|
|
241
|
+
const embedding = vectors[i];
|
|
160
242
|
const embeddingString = `[${embedding.join(",")}]`;
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
}
|
|
243
|
+
values.push(documents[i].pageContent, embeddingString, documents[i].metadata);
|
|
244
|
+
if (collectionId) {
|
|
245
|
+
values.push(collectionId);
|
|
246
|
+
}
|
|
247
|
+
rows.push(values);
|
|
248
|
+
}
|
|
167
249
|
for (let i = 0; i < rows.length; i += this.chunkSize) {
|
|
168
250
|
const chunk = rows.slice(i, i + this.chunkSize);
|
|
169
|
-
const insertQuery = this.buildInsertQuery(chunk);
|
|
251
|
+
const insertQuery = await this.buildInsertQuery(chunk);
|
|
170
252
|
const flatValues = chunk.flat();
|
|
171
253
|
try {
|
|
172
254
|
await this.pool.query(insertQuery, flatValues);
|
|
@@ -190,13 +272,23 @@ export class PGVectorStore extends VectorStore {
|
|
|
190
272
|
async similaritySearchVectorWithScore(query, k, filter) {
|
|
191
273
|
const embeddingString = `[${query.join(",")}]`;
|
|
192
274
|
const _filter = filter ?? "{}";
|
|
275
|
+
let collectionId;
|
|
276
|
+
if (this.collectionTableName) {
|
|
277
|
+
collectionId = await this.getOrCreateCollection();
|
|
278
|
+
}
|
|
279
|
+
const parameters = [embeddingString, _filter, k];
|
|
280
|
+
if (collectionId) {
|
|
281
|
+
parameters.push(collectionId);
|
|
282
|
+
}
|
|
193
283
|
const queryString = `
|
|
194
284
|
SELECT *, ${this.vectorColumnName} <=> $1 as "_distance"
|
|
195
285
|
FROM ${this.tableName}
|
|
196
286
|
WHERE ${this.metadataColumnName}::jsonb @> $2
|
|
287
|
+
${collectionId ? "AND collection_id = $4" : ""}
|
|
197
288
|
ORDER BY "_distance" ASC
|
|
198
|
-
LIMIT $3
|
|
199
|
-
|
|
289
|
+
LIMIT $3;
|
|
290
|
+
`;
|
|
291
|
+
const documents = (await this.pool.query(queryString, parameters)).rows;
|
|
200
292
|
const results = [];
|
|
201
293
|
for (const doc of documents) {
|
|
202
294
|
if (doc._distance != null && doc[this.contentColumnName] != null) {
|
|
@@ -227,6 +319,38 @@ export class PGVectorStore extends VectorStore {
|
|
|
227
319
|
);
|
|
228
320
|
`);
|
|
229
321
|
}
|
|
322
|
+
/**
|
|
323
|
+
* Method to ensure the existence of the collection table in the database.
|
|
324
|
+
* It creates the table if it does not already exist.
|
|
325
|
+
*
|
|
326
|
+
* @returns Promise that resolves when the collection table has been ensured.
|
|
327
|
+
*/
|
|
328
|
+
async ensureCollectionTableInDatabase() {
|
|
329
|
+
try {
|
|
330
|
+
await this.pool.query(`
|
|
331
|
+
CREATE TABLE IF NOT EXISTS ${this.collectionTableName} (
|
|
332
|
+
uuid uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY,
|
|
333
|
+
name character varying,
|
|
334
|
+
cmetadata jsonb
|
|
335
|
+
);
|
|
336
|
+
|
|
337
|
+
ALTER TABLE ${this.tableName}
|
|
338
|
+
ADD COLUMN collection_id uuid;
|
|
339
|
+
|
|
340
|
+
ALTER TABLE ${this.tableName}
|
|
341
|
+
ADD CONSTRAINT ${this.tableName}_collection_id_fkey
|
|
342
|
+
FOREIGN KEY (collection_id)
|
|
343
|
+
REFERENCES ${this.collectionTableName}(uuid)
|
|
344
|
+
ON DELETE CASCADE;
|
|
345
|
+
`);
|
|
346
|
+
}
|
|
347
|
+
catch (e) {
|
|
348
|
+
if (!e.message.includes("already exists")) {
|
|
349
|
+
console.error(e);
|
|
350
|
+
throw new Error(`Error adding column: ${e.message}`);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
}
|
|
230
354
|
/**
|
|
231
355
|
* Static method to create a new `PGVectorStore` instance from an
|
|
232
356
|
* array of texts and their metadata. It converts the texts into
|
|
@@ -27,6 +27,7 @@ exports.WeaviateStore = exports.flattenObjectForWeaviate = void 0;
|
|
|
27
27
|
const uuid = __importStar(require("uuid"));
|
|
28
28
|
const base_js_1 = require("./base.cjs");
|
|
29
29
|
const document_js_1 = require("../document.cjs");
|
|
30
|
+
const math_js_1 = require("../util/math.cjs");
|
|
30
31
|
// Note this function is not generic, it is designed specifically for Weaviate
|
|
31
32
|
// https://weaviate.io/developers/weaviate/config-refs/datatypes#introduction
|
|
32
33
|
const flattenObjectForWeaviate = (
|
|
@@ -235,11 +236,27 @@ class WeaviateStore extends base_js_1.VectorStore {
|
|
|
235
236
|
* @returns An array of tuples, where each tuple contains a document and its similarity score.
|
|
236
237
|
*/
|
|
237
238
|
async similaritySearchVectorWithScore(query, k, filter) {
|
|
239
|
+
const resultsWithEmbedding = await this.similaritySearchVectorWithScoreAndEmbedding(query, k, filter);
|
|
240
|
+
return resultsWithEmbedding.map(([document, score, _embedding]) => [
|
|
241
|
+
document,
|
|
242
|
+
score,
|
|
243
|
+
]);
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Method to perform a similarity search on the stored vectors in the
|
|
247
|
+
* Weaviate index. It returns the top k most similar documents, their
|
|
248
|
+
* similarity scores and embedding vectors.
|
|
249
|
+
* @param query The query vector.
|
|
250
|
+
* @param k The number of most similar documents to return.
|
|
251
|
+
* @param filter Optional filter to apply to the search.
|
|
252
|
+
* @returns An array of tuples, where each tuple contains a document, its similarity score and its embedding vector.
|
|
253
|
+
*/
|
|
254
|
+
async similaritySearchVectorWithScoreAndEmbedding(query, k, filter) {
|
|
238
255
|
try {
|
|
239
|
-
let builder =
|
|
256
|
+
let builder = this.client.graphql
|
|
240
257
|
.get()
|
|
241
258
|
.withClassName(this.indexName)
|
|
242
|
-
.withFields(`${this.queryAttrs.join(" ")} _additional { distance }`)
|
|
259
|
+
.withFields(`${this.queryAttrs.join(" ")} _additional { distance vector }`)
|
|
243
260
|
.withNearVector({
|
|
244
261
|
vector: query,
|
|
245
262
|
distance: filter?.distance,
|
|
@@ -261,6 +278,7 @@ class WeaviateStore extends base_js_1.VectorStore {
|
|
|
261
278
|
metadata: rest,
|
|
262
279
|
}),
|
|
263
280
|
_additional.distance,
|
|
281
|
+
_additional.vector,
|
|
264
282
|
]);
|
|
265
283
|
}
|
|
266
284
|
return documents;
|
|
@@ -269,6 +287,31 @@ class WeaviateStore extends base_js_1.VectorStore {
|
|
|
269
287
|
throw Error(`'Error in similaritySearch' ${e}`);
|
|
270
288
|
}
|
|
271
289
|
}
|
|
290
|
+
/**
|
|
291
|
+
* Return documents selected using the maximal marginal relevance.
|
|
292
|
+
* Maximal marginal relevance optimizes for similarity to the query AND diversity
|
|
293
|
+
* among selected documents.
|
|
294
|
+
*
|
|
295
|
+
* @param {string} query - Text to look up documents similar to.
|
|
296
|
+
* @param {number} options.k - Number of documents to return.
|
|
297
|
+
* @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
|
|
298
|
+
* @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
|
|
299
|
+
* where 0 corresponds to maximum diversity and 1 to minimum diversity.
|
|
300
|
+
* @param {this["FilterType"]} options.filter - Optional filter
|
|
301
|
+
* @param _callbacks
|
|
302
|
+
*
|
|
303
|
+
* @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
|
|
304
|
+
*/
|
|
305
|
+
async maxMarginalRelevanceSearch(query, options, _callbacks) {
|
|
306
|
+
const { k, fetchK = 20, lambda = 0.5, filter } = options;
|
|
307
|
+
const queryEmbedding = await this.embeddings.embedQuery(query);
|
|
308
|
+
const allResults = await this.similaritySearchVectorWithScoreAndEmbedding(queryEmbedding, fetchK, filter);
|
|
309
|
+
const embeddingList = allResults.map(([_doc, _score, embedding]) => embedding);
|
|
310
|
+
const mmrIndexes = (0, math_js_1.maximalMarginalRelevance)(queryEmbedding, embeddingList, lambda, k);
|
|
311
|
+
return mmrIndexes
|
|
312
|
+
.filter((idx) => idx !== -1)
|
|
313
|
+
.map((idx) => allResults[idx][0]);
|
|
314
|
+
}
|
|
272
315
|
/**
|
|
273
316
|
* Static method to create a new `WeaviateStore` instance from a list of
|
|
274
317
|
* texts. It first creates documents from the texts and metadata, then
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { WeaviateClient, WhereFilter } from "weaviate-ts-client";
|
|
2
|
-
import { VectorStore } from "./base.js";
|
|
2
|
+
import { MaxMarginalRelevanceSearchOptions, VectorStore } from "./base.js";
|
|
3
3
|
import { Embeddings } from "../embeddings/base.js";
|
|
4
4
|
import { Document } from "../document.js";
|
|
5
5
|
export declare const flattenObjectForWeaviate: (obj: Record<string, any>) => Record<string, any>;
|
|
@@ -83,6 +83,32 @@ export declare class WeaviateStore extends VectorStore {
|
|
|
83
83
|
* @returns An array of tuples, where each tuple contains a document and its similarity score.
|
|
84
84
|
*/
|
|
85
85
|
similaritySearchVectorWithScore(query: number[], k: number, filter?: WeaviateFilter): Promise<[Document, number][]>;
|
|
86
|
+
/**
|
|
87
|
+
* Method to perform a similarity search on the stored vectors in the
|
|
88
|
+
* Weaviate index. It returns the top k most similar documents, their
|
|
89
|
+
* similarity scores and embedding vectors.
|
|
90
|
+
* @param query The query vector.
|
|
91
|
+
* @param k The number of most similar documents to return.
|
|
92
|
+
* @param filter Optional filter to apply to the search.
|
|
93
|
+
* @returns An array of tuples, where each tuple contains a document, its similarity score and its embedding vector.
|
|
94
|
+
*/
|
|
95
|
+
similaritySearchVectorWithScoreAndEmbedding(query: number[], k: number, filter?: WeaviateFilter): Promise<[Document, number, number[]][]>;
|
|
96
|
+
/**
|
|
97
|
+
* Return documents selected using the maximal marginal relevance.
|
|
98
|
+
* Maximal marginal relevance optimizes for similarity to the query AND diversity
|
|
99
|
+
* among selected documents.
|
|
100
|
+
*
|
|
101
|
+
* @param {string} query - Text to look up documents similar to.
|
|
102
|
+
* @param {number} options.k - Number of documents to return.
|
|
103
|
+
* @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
|
|
104
|
+
* @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
|
|
105
|
+
* where 0 corresponds to maximum diversity and 1 to minimum diversity.
|
|
106
|
+
* @param {this["FilterType"]} options.filter - Optional filter
|
|
107
|
+
* @param _callbacks
|
|
108
|
+
*
|
|
109
|
+
* @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
|
|
110
|
+
*/
|
|
111
|
+
maxMarginalRelevanceSearch(query: string, options: MaxMarginalRelevanceSearchOptions<this["FilterType"]>, _callbacks: undefined): Promise<Document[]>;
|
|
86
112
|
/**
|
|
87
113
|
* Static method to create a new `WeaviateStore` instance from a list of
|
|
88
114
|
* texts. It first creates documents from the texts and metadata, then
|