langchain 0.0.197 → 0.0.198
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chat_models/llama_cpp.cjs +24 -0
- package/dist/chat_models/llama_cpp.d.ts +3 -1
- package/dist/chat_models/llama_cpp.js +24 -0
- package/dist/document_loaders/fs/pptx.cjs +39 -0
- package/dist/document_loaders/fs/pptx.d.ts +23 -0
- package/dist/document_loaders/fs/pptx.js +35 -0
- package/dist/experimental/openai_assistant/index.cjs +32 -0
- package/dist/experimental/openai_assistant/index.d.ts +26 -0
- package/dist/experimental/openai_assistant/index.js +32 -0
- package/dist/experimental/tools/pyinterpreter.cjs +248 -0
- package/dist/experimental/tools/pyinterpreter.d.ts +18 -0
- package/dist/experimental/tools/pyinterpreter.js +244 -0
- package/dist/graphs/neo4j_graph.cjs +49 -14
- package/dist/graphs/neo4j_graph.d.ts +30 -0
- package/dist/graphs/neo4j_graph.js +49 -14
- package/dist/llms/hf.cjs +13 -2
- package/dist/llms/hf.d.ts +5 -0
- package/dist/llms/hf.js +13 -2
- package/dist/llms/llama_cpp.cjs +17 -3
- package/dist/llms/llama_cpp.d.ts +4 -1
- package/dist/llms/llama_cpp.js +17 -3
- package/dist/load/import_constants.cjs +3 -0
- package/dist/load/import_constants.js +3 -0
- package/dist/output_parsers/json.cjs +4 -0
- package/dist/output_parsers/json.js +4 -0
- package/dist/vectorstores/clickhouse.cjs +286 -0
- package/dist/vectorstores/clickhouse.d.ts +126 -0
- package/dist/vectorstores/clickhouse.js +259 -0
- package/dist/vectorstores/pgvector.cjs +142 -18
- package/dist/vectorstores/pgvector.d.ts +21 -0
- package/dist/vectorstores/pgvector.js +142 -18
- package/dist/vectorstores/weaviate.cjs +45 -2
- package/dist/vectorstores/weaviate.d.ts +27 -1
- package/dist/vectorstores/weaviate.js +45 -2
- package/document_loaders/fs/pptx.cjs +1 -0
- package/document_loaders/fs/pptx.d.ts +1 -0
- package/document_loaders/fs/pptx.js +1 -0
- package/experimental/tools/pyinterpreter.cjs +1 -0
- package/experimental/tools/pyinterpreter.d.ts +1 -0
- package/experimental/tools/pyinterpreter.js +1 -0
- package/package.json +39 -4
- package/vectorstores/clickhouse.cjs +1 -0
- package/vectorstores/clickhouse.d.ts +1 -0
- package/vectorstores/clickhouse.js +1 -0
|
@@ -20,6 +20,24 @@ export class PGVectorStore extends VectorStore {
|
|
|
20
20
|
writable: true,
|
|
21
21
|
value: void 0
|
|
22
22
|
});
|
|
23
|
+
Object.defineProperty(this, "collectionTableName", {
|
|
24
|
+
enumerable: true,
|
|
25
|
+
configurable: true,
|
|
26
|
+
writable: true,
|
|
27
|
+
value: void 0
|
|
28
|
+
});
|
|
29
|
+
Object.defineProperty(this, "collectionName", {
|
|
30
|
+
enumerable: true,
|
|
31
|
+
configurable: true,
|
|
32
|
+
writable: true,
|
|
33
|
+
value: "langchain"
|
|
34
|
+
});
|
|
35
|
+
Object.defineProperty(this, "collectionMetadata", {
|
|
36
|
+
enumerable: true,
|
|
37
|
+
configurable: true,
|
|
38
|
+
writable: true,
|
|
39
|
+
value: void 0
|
|
40
|
+
});
|
|
23
41
|
Object.defineProperty(this, "idColumnName", {
|
|
24
42
|
enumerable: true,
|
|
25
43
|
configurable: true,
|
|
@@ -75,6 +93,9 @@ export class PGVectorStore extends VectorStore {
|
|
|
75
93
|
value: 500
|
|
76
94
|
});
|
|
77
95
|
this.tableName = config.tableName;
|
|
96
|
+
this.collectionTableName = config.collectionTableName;
|
|
97
|
+
this.collectionName = config.collectionName ?? "langchain";
|
|
98
|
+
this.collectionMetadata = config.collectionMetadata ?? null;
|
|
78
99
|
this.filter = config.filter;
|
|
79
100
|
this.vectorColumnName = config.columns?.vectorColumnName ?? "embedding";
|
|
80
101
|
this.contentColumnName = config.columns?.contentColumnName ?? "text";
|
|
@@ -100,6 +121,9 @@ export class PGVectorStore extends VectorStore {
|
|
|
100
121
|
const postgresqlVectorStore = new PGVectorStore(embeddings, config);
|
|
101
122
|
await postgresqlVectorStore._initializeClient();
|
|
102
123
|
await postgresqlVectorStore.ensureTableInDatabase();
|
|
124
|
+
if (postgresqlVectorStore.collectionTableName) {
|
|
125
|
+
await postgresqlVectorStore.ensureCollectionTableInDatabase();
|
|
126
|
+
}
|
|
103
127
|
return postgresqlVectorStore;
|
|
104
128
|
}
|
|
105
129
|
async _initializeClient() {
|
|
@@ -116,15 +140,56 @@ export class PGVectorStore extends VectorStore {
|
|
|
116
140
|
const texts = documents.map(({ pageContent }) => pageContent);
|
|
117
141
|
return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
|
|
118
142
|
}
|
|
143
|
+
/**
|
|
144
|
+
* Inserts a row for the collectionName provided at initialization if it does not
|
|
145
|
+
* exist and returns the collectionId.
|
|
146
|
+
*
|
|
147
|
+
* @returns The collectionId for the given collectionName.
|
|
148
|
+
*/
|
|
149
|
+
async getOrCreateCollection() {
|
|
150
|
+
const queryString = `
|
|
151
|
+
SELECT uuid from ${this.collectionTableName}
|
|
152
|
+
WHERE name = $1;
|
|
153
|
+
`;
|
|
154
|
+
const queryResult = await this.pool.query(queryString, [
|
|
155
|
+
this.collectionName,
|
|
156
|
+
]);
|
|
157
|
+
let collectionId = queryResult.rows[0]?.uuid;
|
|
158
|
+
if (!collectionId) {
|
|
159
|
+
const insertString = `
|
|
160
|
+
INSERT INTO ${this.collectionTableName}(
|
|
161
|
+
uuid,
|
|
162
|
+
name,
|
|
163
|
+
cmetadata
|
|
164
|
+
)
|
|
165
|
+
VALUES (
|
|
166
|
+
uuid_generate_v4(),
|
|
167
|
+
$1,
|
|
168
|
+
$2
|
|
169
|
+
)
|
|
170
|
+
RETURNING uuid;
|
|
171
|
+
`;
|
|
172
|
+
const insertResult = await this.pool.query(insertString, [
|
|
173
|
+
this.collectionName,
|
|
174
|
+
this.collectionMetadata,
|
|
175
|
+
]);
|
|
176
|
+
collectionId = insertResult.rows[0]?.uuid;
|
|
177
|
+
}
|
|
178
|
+
return collectionId;
|
|
179
|
+
}
|
|
119
180
|
/**
|
|
120
181
|
* Generates the SQL placeholders for a specific row at the provided index.
|
|
121
182
|
*
|
|
122
183
|
* @param index - The index of the row for which placeholders need to be generated.
|
|
184
|
+
* @param numOfColumns - The number of columns we are inserting data into.
|
|
123
185
|
* @returns The SQL placeholders for the row values.
|
|
124
186
|
*/
|
|
125
|
-
generatePlaceholderForRowAt(index) {
|
|
126
|
-
const
|
|
127
|
-
|
|
187
|
+
generatePlaceholderForRowAt(index, numOfColumns) {
|
|
188
|
+
const placeholders = [];
|
|
189
|
+
for (let i = 0; i < numOfColumns; i += 1) {
|
|
190
|
+
placeholders.push(`$${index * numOfColumns + i + 1}`);
|
|
191
|
+
}
|
|
192
|
+
return `(${placeholders.join(", ")})`;
|
|
128
193
|
}
|
|
129
194
|
/**
|
|
130
195
|
* Constructs the SQL query for inserting rows into the specified table.
|
|
@@ -133,15 +198,25 @@ export class PGVectorStore extends VectorStore {
|
|
|
133
198
|
* @param chunkIndex - The starting index for generating query placeholders based on chunk positioning.
|
|
134
199
|
* @returns The complete SQL INSERT INTO query string.
|
|
135
200
|
*/
|
|
136
|
-
buildInsertQuery(rows) {
|
|
201
|
+
async buildInsertQuery(rows) {
|
|
202
|
+
let collectionId;
|
|
203
|
+
if (this.collectionTableName) {
|
|
204
|
+
collectionId = await this.getOrCreateCollection();
|
|
205
|
+
}
|
|
206
|
+
const columns = [
|
|
207
|
+
this.contentColumnName,
|
|
208
|
+
this.vectorColumnName,
|
|
209
|
+
this.metadataColumnName,
|
|
210
|
+
];
|
|
211
|
+
if (collectionId) {
|
|
212
|
+
columns.push("collection_id");
|
|
213
|
+
}
|
|
137
214
|
const valuesPlaceholders = rows
|
|
138
|
-
.map((_, j) => this.generatePlaceholderForRowAt(j))
|
|
215
|
+
.map((_, j) => this.generatePlaceholderForRowAt(j, columns.length))
|
|
139
216
|
.join(", ");
|
|
140
217
|
const text = `
|
|
141
218
|
INSERT INTO ${this.tableName}(
|
|
142
|
-
${
|
|
143
|
-
${this.vectorColumnName},
|
|
144
|
-
${this.metadataColumnName}
|
|
219
|
+
${columns}
|
|
145
220
|
)
|
|
146
221
|
VALUES ${valuesPlaceholders}
|
|
147
222
|
`;
|
|
@@ -156,17 +231,24 @@ export class PGVectorStore extends VectorStore {
|
|
|
156
231
|
* @returns Promise that resolves when the vectors have been added.
|
|
157
232
|
*/
|
|
158
233
|
async addVectors(vectors, documents) {
|
|
159
|
-
const rows =
|
|
234
|
+
const rows = [];
|
|
235
|
+
let collectionId;
|
|
236
|
+
if (this.collectionTableName) {
|
|
237
|
+
collectionId = await this.getOrCreateCollection();
|
|
238
|
+
}
|
|
239
|
+
for (let i = 0; i < vectors.length; i += 1) {
|
|
240
|
+
const values = [];
|
|
241
|
+
const embedding = vectors[i];
|
|
160
242
|
const embeddingString = `[${embedding.join(",")}]`;
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
}
|
|
243
|
+
values.push(documents[i].pageContent, embeddingString, documents[i].metadata);
|
|
244
|
+
if (collectionId) {
|
|
245
|
+
values.push(collectionId);
|
|
246
|
+
}
|
|
247
|
+
rows.push(values);
|
|
248
|
+
}
|
|
167
249
|
for (let i = 0; i < rows.length; i += this.chunkSize) {
|
|
168
250
|
const chunk = rows.slice(i, i + this.chunkSize);
|
|
169
|
-
const insertQuery = this.buildInsertQuery(chunk);
|
|
251
|
+
const insertQuery = await this.buildInsertQuery(chunk);
|
|
170
252
|
const flatValues = chunk.flat();
|
|
171
253
|
try {
|
|
172
254
|
await this.pool.query(insertQuery, flatValues);
|
|
@@ -190,13 +272,23 @@ export class PGVectorStore extends VectorStore {
|
|
|
190
272
|
async similaritySearchVectorWithScore(query, k, filter) {
|
|
191
273
|
const embeddingString = `[${query.join(",")}]`;
|
|
192
274
|
const _filter = filter ?? "{}";
|
|
275
|
+
let collectionId;
|
|
276
|
+
if (this.collectionTableName) {
|
|
277
|
+
collectionId = await this.getOrCreateCollection();
|
|
278
|
+
}
|
|
279
|
+
const parameters = [embeddingString, _filter, k];
|
|
280
|
+
if (collectionId) {
|
|
281
|
+
parameters.push(collectionId);
|
|
282
|
+
}
|
|
193
283
|
const queryString = `
|
|
194
284
|
SELECT *, ${this.vectorColumnName} <=> $1 as "_distance"
|
|
195
285
|
FROM ${this.tableName}
|
|
196
286
|
WHERE ${this.metadataColumnName}::jsonb @> $2
|
|
287
|
+
${collectionId ? "AND collection_id = $4" : ""}
|
|
197
288
|
ORDER BY "_distance" ASC
|
|
198
|
-
LIMIT $3
|
|
199
|
-
|
|
289
|
+
LIMIT $3;
|
|
290
|
+
`;
|
|
291
|
+
const documents = (await this.pool.query(queryString, parameters)).rows;
|
|
200
292
|
const results = [];
|
|
201
293
|
for (const doc of documents) {
|
|
202
294
|
if (doc._distance != null && doc[this.contentColumnName] != null) {
|
|
@@ -227,6 +319,38 @@ export class PGVectorStore extends VectorStore {
|
|
|
227
319
|
);
|
|
228
320
|
`);
|
|
229
321
|
}
|
|
322
|
+
/**
|
|
323
|
+
* Method to ensure the existence of the collection table in the database.
|
|
324
|
+
* It creates the table if it does not already exist.
|
|
325
|
+
*
|
|
326
|
+
* @returns Promise that resolves when the collection table has been ensured.
|
|
327
|
+
*/
|
|
328
|
+
async ensureCollectionTableInDatabase() {
|
|
329
|
+
try {
|
|
330
|
+
await this.pool.query(`
|
|
331
|
+
CREATE TABLE IF NOT EXISTS ${this.collectionTableName} (
|
|
332
|
+
uuid uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY,
|
|
333
|
+
name character varying,
|
|
334
|
+
cmetadata jsonb
|
|
335
|
+
);
|
|
336
|
+
|
|
337
|
+
ALTER TABLE ${this.tableName}
|
|
338
|
+
ADD COLUMN collection_id uuid;
|
|
339
|
+
|
|
340
|
+
ALTER TABLE ${this.tableName}
|
|
341
|
+
ADD CONSTRAINT ${this.tableName}_collection_id_fkey
|
|
342
|
+
FOREIGN KEY (collection_id)
|
|
343
|
+
REFERENCES ${this.collectionTableName}(uuid)
|
|
344
|
+
ON DELETE CASCADE;
|
|
345
|
+
`);
|
|
346
|
+
}
|
|
347
|
+
catch (e) {
|
|
348
|
+
if (!e.message.includes("already exists")) {
|
|
349
|
+
console.error(e);
|
|
350
|
+
throw new Error(`Error adding column: ${e.message}`);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
}
|
|
230
354
|
/**
|
|
231
355
|
* Static method to create a new `PGVectorStore` instance from an
|
|
232
356
|
* array of texts and their metadata. It converts the texts into
|
|
@@ -27,6 +27,7 @@ exports.WeaviateStore = exports.flattenObjectForWeaviate = void 0;
|
|
|
27
27
|
const uuid = __importStar(require("uuid"));
|
|
28
28
|
const base_js_1 = require("./base.cjs");
|
|
29
29
|
const document_js_1 = require("../document.cjs");
|
|
30
|
+
const math_js_1 = require("../util/math.cjs");
|
|
30
31
|
// Note this function is not generic, it is designed specifically for Weaviate
|
|
31
32
|
// https://weaviate.io/developers/weaviate/config-refs/datatypes#introduction
|
|
32
33
|
const flattenObjectForWeaviate = (
|
|
@@ -235,11 +236,27 @@ class WeaviateStore extends base_js_1.VectorStore {
|
|
|
235
236
|
* @returns An array of tuples, where each tuple contains a document and its similarity score.
|
|
236
237
|
*/
|
|
237
238
|
async similaritySearchVectorWithScore(query, k, filter) {
|
|
239
|
+
const resultsWithEmbedding = await this.similaritySearchVectorWithScoreAndEmbedding(query, k, filter);
|
|
240
|
+
return resultsWithEmbedding.map(([document, score, _embedding]) => [
|
|
241
|
+
document,
|
|
242
|
+
score,
|
|
243
|
+
]);
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Method to perform a similarity search on the stored vectors in the
|
|
247
|
+
* Weaviate index. It returns the top k most similar documents, their
|
|
248
|
+
* similarity scores and embedding vectors.
|
|
249
|
+
* @param query The query vector.
|
|
250
|
+
* @param k The number of most similar documents to return.
|
|
251
|
+
* @param filter Optional filter to apply to the search.
|
|
252
|
+
* @returns An array of tuples, where each tuple contains a document, its similarity score and its embedding vector.
|
|
253
|
+
*/
|
|
254
|
+
async similaritySearchVectorWithScoreAndEmbedding(query, k, filter) {
|
|
238
255
|
try {
|
|
239
|
-
let builder =
|
|
256
|
+
let builder = this.client.graphql
|
|
240
257
|
.get()
|
|
241
258
|
.withClassName(this.indexName)
|
|
242
|
-
.withFields(`${this.queryAttrs.join(" ")} _additional { distance }`)
|
|
259
|
+
.withFields(`${this.queryAttrs.join(" ")} _additional { distance vector }`)
|
|
243
260
|
.withNearVector({
|
|
244
261
|
vector: query,
|
|
245
262
|
distance: filter?.distance,
|
|
@@ -261,6 +278,7 @@ class WeaviateStore extends base_js_1.VectorStore {
|
|
|
261
278
|
metadata: rest,
|
|
262
279
|
}),
|
|
263
280
|
_additional.distance,
|
|
281
|
+
_additional.vector,
|
|
264
282
|
]);
|
|
265
283
|
}
|
|
266
284
|
return documents;
|
|
@@ -269,6 +287,31 @@ class WeaviateStore extends base_js_1.VectorStore {
|
|
|
269
287
|
throw Error(`'Error in similaritySearch' ${e}`);
|
|
270
288
|
}
|
|
271
289
|
}
|
|
290
|
+
/**
|
|
291
|
+
* Return documents selected using the maximal marginal relevance.
|
|
292
|
+
* Maximal marginal relevance optimizes for similarity to the query AND diversity
|
|
293
|
+
* among selected documents.
|
|
294
|
+
*
|
|
295
|
+
* @param {string} query - Text to look up documents similar to.
|
|
296
|
+
* @param {number} options.k - Number of documents to return.
|
|
297
|
+
* @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
|
|
298
|
+
* @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
|
|
299
|
+
* where 0 corresponds to maximum diversity and 1 to minimum diversity.
|
|
300
|
+
* @param {this["FilterType"]} options.filter - Optional filter
|
|
301
|
+
* @param _callbacks
|
|
302
|
+
*
|
|
303
|
+
* @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
|
|
304
|
+
*/
|
|
305
|
+
async maxMarginalRelevanceSearch(query, options, _callbacks) {
|
|
306
|
+
const { k, fetchK = 20, lambda = 0.5, filter } = options;
|
|
307
|
+
const queryEmbedding = await this.embeddings.embedQuery(query);
|
|
308
|
+
const allResults = await this.similaritySearchVectorWithScoreAndEmbedding(queryEmbedding, fetchK, filter);
|
|
309
|
+
const embeddingList = allResults.map(([_doc, _score, embedding]) => embedding);
|
|
310
|
+
const mmrIndexes = (0, math_js_1.maximalMarginalRelevance)(queryEmbedding, embeddingList, lambda, k);
|
|
311
|
+
return mmrIndexes
|
|
312
|
+
.filter((idx) => idx !== -1)
|
|
313
|
+
.map((idx) => allResults[idx][0]);
|
|
314
|
+
}
|
|
272
315
|
/**
|
|
273
316
|
* Static method to create a new `WeaviateStore` instance from a list of
|
|
274
317
|
* texts. It first creates documents from the texts and metadata, then
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { WeaviateClient, WhereFilter } from "weaviate-ts-client";
|
|
2
|
-
import { VectorStore } from "./base.js";
|
|
2
|
+
import { MaxMarginalRelevanceSearchOptions, VectorStore } from "./base.js";
|
|
3
3
|
import { Embeddings } from "../embeddings/base.js";
|
|
4
4
|
import { Document } from "../document.js";
|
|
5
5
|
export declare const flattenObjectForWeaviate: (obj: Record<string, any>) => Record<string, any>;
|
|
@@ -83,6 +83,32 @@ export declare class WeaviateStore extends VectorStore {
|
|
|
83
83
|
* @returns An array of tuples, where each tuple contains a document and its similarity score.
|
|
84
84
|
*/
|
|
85
85
|
similaritySearchVectorWithScore(query: number[], k: number, filter?: WeaviateFilter): Promise<[Document, number][]>;
|
|
86
|
+
/**
|
|
87
|
+
* Method to perform a similarity search on the stored vectors in the
|
|
88
|
+
* Weaviate index. It returns the top k most similar documents, their
|
|
89
|
+
* similarity scores and embedding vectors.
|
|
90
|
+
* @param query The query vector.
|
|
91
|
+
* @param k The number of most similar documents to return.
|
|
92
|
+
* @param filter Optional filter to apply to the search.
|
|
93
|
+
* @returns An array of tuples, where each tuple contains a document, its similarity score and its embedding vector.
|
|
94
|
+
*/
|
|
95
|
+
similaritySearchVectorWithScoreAndEmbedding(query: number[], k: number, filter?: WeaviateFilter): Promise<[Document, number, number[]][]>;
|
|
96
|
+
/**
|
|
97
|
+
* Return documents selected using the maximal marginal relevance.
|
|
98
|
+
* Maximal marginal relevance optimizes for similarity to the query AND diversity
|
|
99
|
+
* among selected documents.
|
|
100
|
+
*
|
|
101
|
+
* @param {string} query - Text to look up documents similar to.
|
|
102
|
+
* @param {number} options.k - Number of documents to return.
|
|
103
|
+
* @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
|
|
104
|
+
* @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
|
|
105
|
+
* where 0 corresponds to maximum diversity and 1 to minimum diversity.
|
|
106
|
+
* @param {this["FilterType"]} options.filter - Optional filter
|
|
107
|
+
* @param _callbacks
|
|
108
|
+
*
|
|
109
|
+
* @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
|
|
110
|
+
*/
|
|
111
|
+
maxMarginalRelevanceSearch(query: string, options: MaxMarginalRelevanceSearchOptions<this["FilterType"]>, _callbacks: undefined): Promise<Document[]>;
|
|
86
112
|
/**
|
|
87
113
|
* Static method to create a new `WeaviateStore` instance from a list of
|
|
88
114
|
* texts. It first creates documents from the texts and metadata, then
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as uuid from "uuid";
|
|
2
2
|
import { VectorStore } from "./base.js";
|
|
3
3
|
import { Document } from "../document.js";
|
|
4
|
+
import { maximalMarginalRelevance } from "../util/math.js";
|
|
4
5
|
// Note this function is not generic, it is designed specifically for Weaviate
|
|
5
6
|
// https://weaviate.io/developers/weaviate/config-refs/datatypes#introduction
|
|
6
7
|
export const flattenObjectForWeaviate = (
|
|
@@ -208,11 +209,27 @@ export class WeaviateStore extends VectorStore {
|
|
|
208
209
|
* @returns An array of tuples, where each tuple contains a document and its similarity score.
|
|
209
210
|
*/
|
|
210
211
|
async similaritySearchVectorWithScore(query, k, filter) {
|
|
212
|
+
const resultsWithEmbedding = await this.similaritySearchVectorWithScoreAndEmbedding(query, k, filter);
|
|
213
|
+
return resultsWithEmbedding.map(([document, score, _embedding]) => [
|
|
214
|
+
document,
|
|
215
|
+
score,
|
|
216
|
+
]);
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Method to perform a similarity search on the stored vectors in the
|
|
220
|
+
* Weaviate index. It returns the top k most similar documents, their
|
|
221
|
+
* similarity scores and embedding vectors.
|
|
222
|
+
* @param query The query vector.
|
|
223
|
+
* @param k The number of most similar documents to return.
|
|
224
|
+
* @param filter Optional filter to apply to the search.
|
|
225
|
+
* @returns An array of tuples, where each tuple contains a document, its similarity score and its embedding vector.
|
|
226
|
+
*/
|
|
227
|
+
async similaritySearchVectorWithScoreAndEmbedding(query, k, filter) {
|
|
211
228
|
try {
|
|
212
|
-
let builder =
|
|
229
|
+
let builder = this.client.graphql
|
|
213
230
|
.get()
|
|
214
231
|
.withClassName(this.indexName)
|
|
215
|
-
.withFields(`${this.queryAttrs.join(" ")} _additional { distance }`)
|
|
232
|
+
.withFields(`${this.queryAttrs.join(" ")} _additional { distance vector }`)
|
|
216
233
|
.withNearVector({
|
|
217
234
|
vector: query,
|
|
218
235
|
distance: filter?.distance,
|
|
@@ -234,6 +251,7 @@ export class WeaviateStore extends VectorStore {
|
|
|
234
251
|
metadata: rest,
|
|
235
252
|
}),
|
|
236
253
|
_additional.distance,
|
|
254
|
+
_additional.vector,
|
|
237
255
|
]);
|
|
238
256
|
}
|
|
239
257
|
return documents;
|
|
@@ -242,6 +260,31 @@ export class WeaviateStore extends VectorStore {
|
|
|
242
260
|
throw Error(`'Error in similaritySearch' ${e}`);
|
|
243
261
|
}
|
|
244
262
|
}
|
|
263
|
+
/**
|
|
264
|
+
* Return documents selected using the maximal marginal relevance.
|
|
265
|
+
* Maximal marginal relevance optimizes for similarity to the query AND diversity
|
|
266
|
+
* among selected documents.
|
|
267
|
+
*
|
|
268
|
+
* @param {string} query - Text to look up documents similar to.
|
|
269
|
+
* @param {number} options.k - Number of documents to return.
|
|
270
|
+
* @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
|
|
271
|
+
* @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
|
|
272
|
+
* where 0 corresponds to maximum diversity and 1 to minimum diversity.
|
|
273
|
+
* @param {this["FilterType"]} options.filter - Optional filter
|
|
274
|
+
* @param _callbacks
|
|
275
|
+
*
|
|
276
|
+
* @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
|
|
277
|
+
*/
|
|
278
|
+
async maxMarginalRelevanceSearch(query, options, _callbacks) {
|
|
279
|
+
const { k, fetchK = 20, lambda = 0.5, filter } = options;
|
|
280
|
+
const queryEmbedding = await this.embeddings.embedQuery(query);
|
|
281
|
+
const allResults = await this.similaritySearchVectorWithScoreAndEmbedding(queryEmbedding, fetchK, filter);
|
|
282
|
+
const embeddingList = allResults.map(([_doc, _score, embedding]) => embedding);
|
|
283
|
+
const mmrIndexes = maximalMarginalRelevance(queryEmbedding, embeddingList, lambda, k);
|
|
284
|
+
return mmrIndexes
|
|
285
|
+
.filter((idx) => idx !== -1)
|
|
286
|
+
.map((idx) => allResults[idx][0]);
|
|
287
|
+
}
|
|
245
288
|
/**
|
|
246
289
|
* Static method to create a new `WeaviateStore` instance from a list of
|
|
247
290
|
* texts. It first creates documents from the texts and metadata, then
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = require('../../dist/document_loaders/fs/pptx.cjs');
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../../dist/document_loaders/fs/pptx.js'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../../dist/document_loaders/fs/pptx.js'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = require('../../dist/experimental/tools/pyinterpreter.cjs');
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../../dist/experimental/tools/pyinterpreter.js'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../../dist/experimental/tools/pyinterpreter.js'
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "langchain",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.198",
|
|
4
4
|
"description": "Typescript bindings for langchain",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"engines": {
|
|
@@ -229,6 +229,9 @@
|
|
|
229
229
|
"prompts/load.cjs",
|
|
230
230
|
"prompts/load.js",
|
|
231
231
|
"prompts/load.d.ts",
|
|
232
|
+
"vectorstores/clickhouse.cjs",
|
|
233
|
+
"vectorstores/clickhouse.js",
|
|
234
|
+
"vectorstores/clickhouse.d.ts",
|
|
232
235
|
"vectorstores/analyticdb.cjs",
|
|
233
236
|
"vectorstores/analyticdb.js",
|
|
234
237
|
"vectorstores/analyticdb.d.ts",
|
|
@@ -469,6 +472,9 @@
|
|
|
469
472
|
"document_loaders/fs/openai_whisper_audio.cjs",
|
|
470
473
|
"document_loaders/fs/openai_whisper_audio.js",
|
|
471
474
|
"document_loaders/fs/openai_whisper_audio.d.ts",
|
|
475
|
+
"document_loaders/fs/pptx.cjs",
|
|
476
|
+
"document_loaders/fs/pptx.js",
|
|
477
|
+
"document_loaders/fs/pptx.d.ts",
|
|
472
478
|
"document_transformers/html_to_text.cjs",
|
|
473
479
|
"document_transformers/html_to_text.js",
|
|
474
480
|
"document_transformers/html_to_text.d.ts",
|
|
@@ -802,6 +808,9 @@
|
|
|
802
808
|
"experimental/chains/violation_of_expectations.cjs",
|
|
803
809
|
"experimental/chains/violation_of_expectations.js",
|
|
804
810
|
"experimental/chains/violation_of_expectations.d.ts",
|
|
811
|
+
"experimental/tools/pyinterpreter.cjs",
|
|
812
|
+
"experimental/tools/pyinterpreter.js",
|
|
813
|
+
"experimental/tools/pyinterpreter.d.ts",
|
|
805
814
|
"evaluation.cjs",
|
|
806
815
|
"evaluation.js",
|
|
807
816
|
"evaluation.d.ts",
|
|
@@ -852,7 +861,7 @@
|
|
|
852
861
|
"@aws-sdk/credential-provider-node": "^3.388.0",
|
|
853
862
|
"@aws-sdk/types": "^3.357.0",
|
|
854
863
|
"@azure/storage-blob": "^12.15.0",
|
|
855
|
-
"@clickhouse/client": "^0.
|
|
864
|
+
"@clickhouse/client": "^0.2.5",
|
|
856
865
|
"@cloudflare/ai": "^1.0.12",
|
|
857
866
|
"@cloudflare/workers-types": "^4.20230922.0",
|
|
858
867
|
"@elastic/elasticsearch": "^8.4.0",
|
|
@@ -927,6 +936,7 @@
|
|
|
927
936
|
"eslint-config-airbnb-base": "^15.0.0",
|
|
928
937
|
"eslint-config-prettier": "^8.6.0",
|
|
929
938
|
"eslint-plugin-import": "^2.27.5",
|
|
939
|
+
"eslint-plugin-jest": "^27.6.0",
|
|
930
940
|
"eslint-plugin-no-instanceof": "^1.0.1",
|
|
931
941
|
"eslint-plugin-prettier": "^4.2.1",
|
|
932
942
|
"faiss-node": "^0.5.1",
|
|
@@ -951,6 +961,7 @@
|
|
|
951
961
|
"neo4j-driver": "^5.12.0",
|
|
952
962
|
"node-llama-cpp": "2.7.3",
|
|
953
963
|
"notion-to-md": "^3.1.0",
|
|
964
|
+
"officeparser": "^4.0.4",
|
|
954
965
|
"pdf-parse": "1.1.1",
|
|
955
966
|
"peggy": "^3.0.2",
|
|
956
967
|
"pg": "^8.11.0",
|
|
@@ -960,6 +971,7 @@
|
|
|
960
971
|
"portkey-ai": "^0.1.11",
|
|
961
972
|
"prettier": "^2.8.3",
|
|
962
973
|
"puppeteer": "^19.7.2",
|
|
974
|
+
"pyodide": "^0.24.1",
|
|
963
975
|
"redis": "^4.6.6",
|
|
964
976
|
"release-it": "^15.10.1",
|
|
965
977
|
"replicate": "^0.18.0",
|
|
@@ -991,7 +1003,7 @@
|
|
|
991
1003
|
"@aws-sdk/client-sfn": "^3.310.0",
|
|
992
1004
|
"@aws-sdk/credential-provider-node": "^3.388.0",
|
|
993
1005
|
"@azure/storage-blob": "^12.15.0",
|
|
994
|
-
"@clickhouse/client": "^0.
|
|
1006
|
+
"@clickhouse/client": "^0.2.5",
|
|
995
1007
|
"@cloudflare/ai": "^1.0.12",
|
|
996
1008
|
"@elastic/elasticsearch": "^8.4.0",
|
|
997
1009
|
"@getmetal/metal-sdk": "*",
|
|
@@ -1057,6 +1069,7 @@
|
|
|
1057
1069
|
"neo4j-driver": "*",
|
|
1058
1070
|
"node-llama-cpp": "*",
|
|
1059
1071
|
"notion-to-md": "^3.1.0",
|
|
1072
|
+
"officeparser": "^4.0.4",
|
|
1060
1073
|
"pdf-parse": "1.1.1",
|
|
1061
1074
|
"peggy": "^3.0.2",
|
|
1062
1075
|
"pg": "^8.11.0",
|
|
@@ -1065,6 +1078,7 @@
|
|
|
1065
1078
|
"playwright": "^1.32.1",
|
|
1066
1079
|
"portkey-ai": "^0.1.11",
|
|
1067
1080
|
"puppeteer": "^19.7.2",
|
|
1081
|
+
"pyodide": "^0.24.1",
|
|
1068
1082
|
"redis": "^4.6.4",
|
|
1069
1083
|
"replicate": "^0.18.0",
|
|
1070
1084
|
"sonix-speech-recognition": "^2.1.1",
|
|
@@ -1309,6 +1323,9 @@
|
|
|
1309
1323
|
"notion-to-md": {
|
|
1310
1324
|
"optional": true
|
|
1311
1325
|
},
|
|
1326
|
+
"officeparser": {
|
|
1327
|
+
"optional": true
|
|
1328
|
+
},
|
|
1312
1329
|
"pdf-parse": {
|
|
1313
1330
|
"optional": true
|
|
1314
1331
|
},
|
|
@@ -1333,6 +1350,9 @@
|
|
|
1333
1350
|
"puppeteer": {
|
|
1334
1351
|
"optional": true
|
|
1335
1352
|
},
|
|
1353
|
+
"pyodide": {
|
|
1354
|
+
"optional": true
|
|
1355
|
+
},
|
|
1336
1356
|
"redis": {
|
|
1337
1357
|
"optional": true
|
|
1338
1358
|
},
|
|
@@ -1378,7 +1398,7 @@
|
|
|
1378
1398
|
},
|
|
1379
1399
|
"dependencies": {
|
|
1380
1400
|
"@anthropic-ai/sdk": "^0.9.1",
|
|
1381
|
-
"@langchain/core": "
|
|
1401
|
+
"@langchain/core": "~0.0.2",
|
|
1382
1402
|
"binary-extensions": "^2.2.0",
|
|
1383
1403
|
"expr-eval": "^2.0.2",
|
|
1384
1404
|
"flat": "^5.0.2",
|
|
@@ -1779,6 +1799,11 @@
|
|
|
1779
1799
|
"import": "./prompts/load.js",
|
|
1780
1800
|
"require": "./prompts/load.cjs"
|
|
1781
1801
|
},
|
|
1802
|
+
"./vectorstores/clickhouse": {
|
|
1803
|
+
"types": "./vectorstores/clickhouse.d.ts",
|
|
1804
|
+
"import": "./vectorstores/clickhouse.js",
|
|
1805
|
+
"require": "./vectorstores/clickhouse.cjs"
|
|
1806
|
+
},
|
|
1782
1807
|
"./vectorstores/analyticdb": {
|
|
1783
1808
|
"types": "./vectorstores/analyticdb.d.ts",
|
|
1784
1809
|
"import": "./vectorstores/analyticdb.js",
|
|
@@ -2179,6 +2204,11 @@
|
|
|
2179
2204
|
"import": "./document_loaders/fs/openai_whisper_audio.js",
|
|
2180
2205
|
"require": "./document_loaders/fs/openai_whisper_audio.cjs"
|
|
2181
2206
|
},
|
|
2207
|
+
"./document_loaders/fs/pptx": {
|
|
2208
|
+
"types": "./document_loaders/fs/pptx.d.ts",
|
|
2209
|
+
"import": "./document_loaders/fs/pptx.js",
|
|
2210
|
+
"require": "./document_loaders/fs/pptx.cjs"
|
|
2211
|
+
},
|
|
2182
2212
|
"./document_transformers/html_to_text": {
|
|
2183
2213
|
"types": "./document_transformers/html_to_text.d.ts",
|
|
2184
2214
|
"import": "./document_transformers/html_to_text.js",
|
|
@@ -2734,6 +2764,11 @@
|
|
|
2734
2764
|
"import": "./experimental/chains/violation_of_expectations.js",
|
|
2735
2765
|
"require": "./experimental/chains/violation_of_expectations.cjs"
|
|
2736
2766
|
},
|
|
2767
|
+
"./experimental/tools/pyinterpreter": {
|
|
2768
|
+
"types": "./experimental/tools/pyinterpreter.d.ts",
|
|
2769
|
+
"import": "./experimental/tools/pyinterpreter.js",
|
|
2770
|
+
"require": "./experimental/tools/pyinterpreter.cjs"
|
|
2771
|
+
},
|
|
2737
2772
|
"./evaluation": {
|
|
2738
2773
|
"types": "./evaluation.d.ts",
|
|
2739
2774
|
"import": "./evaluation.js",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = require('../dist/vectorstores/clickhouse.cjs');
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/vectorstores/clickhouse.js'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/vectorstores/clickhouse.js'
|