langchain 0.0.128 → 0.0.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embeddings/cache_backed.cjs +109 -0
- package/dist/embeddings/cache_backed.d.ts +60 -0
- package/dist/embeddings/cache_backed.js +102 -0
- package/dist/load/import_constants.cjs +1 -0
- package/dist/load/import_constants.js +1 -0
- package/dist/load/import_map.cjs +5 -2
- package/dist/load/import_map.d.ts +3 -0
- package/dist/load/import_map.js +3 -0
- package/dist/schema/runnable.cjs +177 -22
- package/dist/schema/runnable.d.ts +89 -0
- package/dist/schema/runnable.js +175 -21
- package/dist/schema/storage.cjs +10 -0
- package/dist/schema/storage.d.ts +10 -0
- package/dist/schema/storage.js +6 -0
- package/dist/storage/encoder_backed.cjs +66 -0
- package/dist/storage/encoder_backed.d.ts +18 -0
- package/dist/storage/encoder_backed.js +62 -0
- package/dist/storage/in_memory.cjs +44 -0
- package/dist/storage/in_memory.d.ts +9 -0
- package/dist/storage/in_memory.js +40 -0
- package/dist/storage/ioredis.cjs +110 -0
- package/dist/storage/ioredis.d.ts +22 -0
- package/dist/storage/ioredis.js +106 -0
- package/dist/vectorstores/zep.cjs +101 -23
- package/dist/vectorstores/zep.d.ts +32 -2
- package/dist/vectorstores/zep.js +101 -23
- package/embeddings/cache_backed.cjs +1 -0
- package/embeddings/cache_backed.d.ts +1 -0
- package/embeddings/cache_backed.js +1 -0
- package/package.json +33 -1
- package/schema/storage.cjs +1 -0
- package/schema/storage.d.ts +1 -0
- package/schema/storage.js +1 -0
- package/storage/in_memory.cjs +1 -0
- package/storage/in_memory.d.ts +1 -0
- package/storage/in_memory.js +1 -0
- package/storage/ioredis.cjs +1 -0
- package/storage/ioredis.d.ts +1 -0
- package/storage/ioredis.js +1 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/// <reference types="node" resolution-mode="require"/>
|
|
2
|
+
import type { Redis } from "ioredis";
|
|
3
|
+
import { BaseStore } from "../schema/storage.js";
|
|
4
|
+
export declare class RedisByteStore extends BaseStore<string, Uint8Array> {
|
|
5
|
+
lc_namespace: string[];
|
|
6
|
+
protected client: Redis;
|
|
7
|
+
protected ttl?: number;
|
|
8
|
+
protected namespace?: string;
|
|
9
|
+
protected yieldKeysScanBatchSize: number;
|
|
10
|
+
constructor(fields: {
|
|
11
|
+
client: Redis;
|
|
12
|
+
ttl?: number;
|
|
13
|
+
namespace?: string;
|
|
14
|
+
yieldKeysScanBatchSize?: number;
|
|
15
|
+
});
|
|
16
|
+
_getPrefixedKey(key: string): string;
|
|
17
|
+
_getDeprefixedKey(key: string): string;
|
|
18
|
+
mget(keys: string[]): Promise<(Buffer | undefined)[]>;
|
|
19
|
+
mset(keyValuePairs: [string, Uint8Array][]): Promise<void>;
|
|
20
|
+
mdelete(keys: string[]): Promise<void>;
|
|
21
|
+
yieldKeys(prefix?: string): AsyncGenerator<string>;
|
|
22
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { BaseStore } from "../schema/storage.js";
|
|
2
|
+
export class RedisByteStore extends BaseStore {
|
|
3
|
+
constructor(fields) {
|
|
4
|
+
super(fields);
|
|
5
|
+
Object.defineProperty(this, "lc_namespace", {
|
|
6
|
+
enumerable: true,
|
|
7
|
+
configurable: true,
|
|
8
|
+
writable: true,
|
|
9
|
+
value: ["langchain", "storage", "ioredis"]
|
|
10
|
+
});
|
|
11
|
+
Object.defineProperty(this, "client", {
|
|
12
|
+
enumerable: true,
|
|
13
|
+
configurable: true,
|
|
14
|
+
writable: true,
|
|
15
|
+
value: void 0
|
|
16
|
+
});
|
|
17
|
+
Object.defineProperty(this, "ttl", {
|
|
18
|
+
enumerable: true,
|
|
19
|
+
configurable: true,
|
|
20
|
+
writable: true,
|
|
21
|
+
value: void 0
|
|
22
|
+
});
|
|
23
|
+
Object.defineProperty(this, "namespace", {
|
|
24
|
+
enumerable: true,
|
|
25
|
+
configurable: true,
|
|
26
|
+
writable: true,
|
|
27
|
+
value: void 0
|
|
28
|
+
});
|
|
29
|
+
Object.defineProperty(this, "yieldKeysScanBatchSize", {
|
|
30
|
+
enumerable: true,
|
|
31
|
+
configurable: true,
|
|
32
|
+
writable: true,
|
|
33
|
+
value: 1000
|
|
34
|
+
});
|
|
35
|
+
this.client = fields.client;
|
|
36
|
+
this.ttl = fields.ttl;
|
|
37
|
+
this.namespace = fields.namespace;
|
|
38
|
+
this.yieldKeysScanBatchSize =
|
|
39
|
+
fields.yieldKeysScanBatchSize ?? this.yieldKeysScanBatchSize;
|
|
40
|
+
}
|
|
41
|
+
_getPrefixedKey(key) {
|
|
42
|
+
if (this.namespace) {
|
|
43
|
+
const delimiter = "/";
|
|
44
|
+
return `${this.namespace}${delimiter}${key}`;
|
|
45
|
+
}
|
|
46
|
+
return key;
|
|
47
|
+
}
|
|
48
|
+
_getDeprefixedKey(key) {
|
|
49
|
+
if (this.namespace) {
|
|
50
|
+
const delimiter = "/";
|
|
51
|
+
return key.slice(this.namespace.length + delimiter.length);
|
|
52
|
+
}
|
|
53
|
+
return key;
|
|
54
|
+
}
|
|
55
|
+
async mget(keys) {
|
|
56
|
+
const prefixedKeys = keys.map(this._getPrefixedKey.bind(this));
|
|
57
|
+
const retrievedValues = await this.client.mgetBuffer(prefixedKeys);
|
|
58
|
+
return retrievedValues.map((key) => {
|
|
59
|
+
if (!key) {
|
|
60
|
+
return undefined;
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
return key;
|
|
64
|
+
}
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
async mset(keyValuePairs) {
|
|
68
|
+
const decoder = new TextDecoder();
|
|
69
|
+
const encodedKeyValuePairs = keyValuePairs.map(([key, value]) => [
|
|
70
|
+
this._getPrefixedKey(key),
|
|
71
|
+
decoder.decode(value),
|
|
72
|
+
]);
|
|
73
|
+
const pipeline = this.client.pipeline();
|
|
74
|
+
for (const [key, value] of encodedKeyValuePairs) {
|
|
75
|
+
if (this.ttl) {
|
|
76
|
+
pipeline.set(key, value, "EX", this.ttl);
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
pipeline.set(key, value);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
await pipeline.exec();
|
|
83
|
+
}
|
|
84
|
+
async mdelete(keys) {
|
|
85
|
+
await this.client.del(...keys.map(this._getPrefixedKey.bind(this)));
|
|
86
|
+
}
|
|
87
|
+
async *yieldKeys(prefix) {
|
|
88
|
+
let pattern;
|
|
89
|
+
if (prefix) {
|
|
90
|
+
pattern = this._getPrefixedKey(prefix);
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
pattern = this._getPrefixedKey("*");
|
|
94
|
+
}
|
|
95
|
+
let [cursor, batch] = await this.client.scan(0, "MATCH", pattern, "COUNT", this.yieldKeysScanBatchSize);
|
|
96
|
+
for (const key of batch) {
|
|
97
|
+
yield this._getDeprefixedKey(key);
|
|
98
|
+
}
|
|
99
|
+
while (cursor !== "0") {
|
|
100
|
+
[cursor, batch] = await this.client.scan(cursor, "MATCH", pattern, "COUNT", this.yieldKeysScanBatchSize);
|
|
101
|
+
for (const key of batch) {
|
|
102
|
+
yield this._getDeprefixedKey(key);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -5,6 +5,7 @@ const zep_js_1 = require("@getzep/zep-js");
|
|
|
5
5
|
const base_js_1 = require("./base.cjs");
|
|
6
6
|
const document_js_1 = require("../document.cjs");
|
|
7
7
|
const fake_js_1 = require("../embeddings/fake.cjs");
|
|
8
|
+
const math_js_1 = require("../util/math.cjs");
|
|
8
9
|
/**
|
|
9
10
|
* ZepVectorStore is a VectorStore implementation that uses the Zep long-term memory store as a backend.
|
|
10
11
|
*
|
|
@@ -52,7 +53,8 @@ class ZepVectorStore extends base_js_1.VectorStore {
|
|
|
52
53
|
this.autoEmbed = true;
|
|
53
54
|
}
|
|
54
55
|
this.initPromise = this.initCollection(args).catch((err) => {
|
|
55
|
-
console.error("Error
|
|
56
|
+
console.error("Error initializing collection:", err);
|
|
57
|
+
throw err;
|
|
56
58
|
});
|
|
57
59
|
}
|
|
58
60
|
/**
|
|
@@ -160,13 +162,6 @@ class ZepVectorStore extends base_js_1.VectorStore {
|
|
|
160
162
|
await this.collection.deleteDocument(uuid);
|
|
161
163
|
}
|
|
162
164
|
}
|
|
163
|
-
isRecord(value) {
|
|
164
|
-
return (typeof value === "object" &&
|
|
165
|
-
value !== null &&
|
|
166
|
-
!Array.isArray(value) &&
|
|
167
|
-
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
168
|
-
!(value instanceof Function));
|
|
169
|
-
}
|
|
170
165
|
/**
|
|
171
166
|
* Performs a similarity search in the collection and returns the results with their scores.
|
|
172
167
|
*
|
|
@@ -176,25 +171,90 @@ class ZepVectorStore extends base_js_1.VectorStore {
|
|
|
176
171
|
* @returns {Promise<[Document, number][]>} - A promise that resolves with the search results and their scores.
|
|
177
172
|
*/
|
|
178
173
|
async similaritySearchVectorWithScore(query, k, filter) {
|
|
179
|
-
if (filter && !this.isRecord(filter)) {
|
|
180
|
-
throw new Error(`Filter must be a record, got ${filter}`);
|
|
181
|
-
}
|
|
182
174
|
await this.initPromise;
|
|
183
175
|
const results = await this.collection.search({
|
|
184
176
|
embedding: new Float32Array(query),
|
|
185
|
-
metadata: filter,
|
|
177
|
+
metadata: assignMetadata(filter),
|
|
186
178
|
}, k);
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
179
|
+
return zepDocsToDocumentsAndScore(results);
|
|
180
|
+
}
|
|
181
|
+
async _similaritySearchWithScore(query, k, filter) {
|
|
182
|
+
await this.initPromise;
|
|
183
|
+
const results = await this.collection.search({
|
|
184
|
+
text: query,
|
|
185
|
+
metadata: assignMetadata(filter),
|
|
186
|
+
}, k);
|
|
187
|
+
return zepDocsToDocumentsAndScore(results);
|
|
188
|
+
}
|
|
189
|
+
async similaritySearchWithScore(query, k = 4, filter = undefined, _callbacks = undefined // implement passing to embedQuery later
|
|
190
|
+
) {
|
|
191
|
+
if (this.autoEmbed) {
|
|
192
|
+
return this._similaritySearchWithScore(query, k, filter);
|
|
193
|
+
}
|
|
194
|
+
else {
|
|
195
|
+
return this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k, filter);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Performs a similarity search on the Zep collection.
|
|
200
|
+
*
|
|
201
|
+
* @param {string} query - The query string to search for.
|
|
202
|
+
* @param {number} [k=4] - The number of results to return. Defaults to 4.
|
|
203
|
+
* @param {this["FilterType"] | undefined} [filter=undefined] - An optional set of JSONPath filters to apply to the search.
|
|
204
|
+
* @param {Callbacks | undefined} [_callbacks=undefined] - Optional callbacks. Currently not implemented.
|
|
205
|
+
* @returns {Promise<Document[]>} - A promise that resolves to an array of Documents that are similar to the query.
|
|
206
|
+
*
|
|
207
|
+
* @async
|
|
208
|
+
*/
|
|
209
|
+
async similaritySearch(query, k = 4, filter = undefined, _callbacks = undefined // implement passing to embedQuery later
|
|
210
|
+
) {
|
|
211
|
+
await this.initPromise;
|
|
212
|
+
let results;
|
|
213
|
+
if (this.autoEmbed) {
|
|
214
|
+
const zepResults = await this.collection.search({ text: query, metadata: assignMetadata(filter) }, k);
|
|
215
|
+
results = zepDocsToDocumentsAndScore(zepResults);
|
|
216
|
+
}
|
|
217
|
+
else {
|
|
218
|
+
results = await this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k, assignMetadata(filter));
|
|
219
|
+
}
|
|
220
|
+
return results.map((result) => result[0]);
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Return documents selected using the maximal marginal relevance.
|
|
224
|
+
* Maximal marginal relevance optimizes for similarity to the query AND diversity
|
|
225
|
+
* among selected documents.
|
|
226
|
+
*
|
|
227
|
+
* @param {string} query - Text to look up documents similar to.
|
|
228
|
+
* @param options
|
|
229
|
+
* @param {number} options.k - Number of documents to return.
|
|
230
|
+
* @param {number} options.fetchK=20- Number of documents to fetch before passing to the MMR algorithm.
|
|
231
|
+
* @param {number} options.lambda=0.5 - Number between 0 and 1 that determines the degree of diversity among the results,
|
|
232
|
+
* where 0 corresponds to maximum diversity and 1 to minimum diversity.
|
|
233
|
+
* @param {Record<string, any>} options.filter - Optional Zep JSONPath query to pre-filter on document metadata field
|
|
234
|
+
*
|
|
235
|
+
* @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
|
|
236
|
+
*/
|
|
237
|
+
async maxMarginalRelevanceSearch(query, options) {
|
|
238
|
+
const { k, fetchK = 20, lambda = 0.5, filter } = options;
|
|
239
|
+
let queryEmbedding;
|
|
240
|
+
let zepResults;
|
|
241
|
+
if (!this.autoEmbed) {
|
|
242
|
+
queryEmbedding = await this.embeddings.embedQuery(query);
|
|
243
|
+
zepResults = await this.collection.search({
|
|
244
|
+
embedding: new Float32Array(queryEmbedding),
|
|
245
|
+
metadata: assignMetadata(filter),
|
|
246
|
+
}, fetchK);
|
|
247
|
+
}
|
|
248
|
+
else {
|
|
249
|
+
let queryEmbeddingArray;
|
|
250
|
+
[zepResults, queryEmbeddingArray] =
|
|
251
|
+
await this.collection.searchReturnQueryVector({ text: query, metadata: assignMetadata(filter) }, fetchK);
|
|
252
|
+
queryEmbedding = Array.from(queryEmbeddingArray);
|
|
253
|
+
}
|
|
254
|
+
const results = zepDocsToDocumentsAndScore(zepResults);
|
|
255
|
+
const embeddingList = zepResults.map((doc) => Array.from(doc.embedding ? doc.embedding : []));
|
|
256
|
+
const mmrIndexes = (0, math_js_1.maximalMarginalRelevance)(queryEmbedding, embeddingList, lambda, k);
|
|
257
|
+
return mmrIndexes.filter((idx) => idx !== -1).map((idx) => results[idx][0]);
|
|
198
258
|
}
|
|
199
259
|
/**
|
|
200
260
|
* Creates a new ZepVectorStore instance from an array of texts. Each text is converted into a Document and added to the collection.
|
|
@@ -234,3 +294,21 @@ class ZepVectorStore extends base_js_1.VectorStore {
|
|
|
234
294
|
}
|
|
235
295
|
}
|
|
236
296
|
exports.ZepVectorStore = ZepVectorStore;
|
|
297
|
+
function zepDocsToDocumentsAndScore(results) {
|
|
298
|
+
return results.map((d) => [
|
|
299
|
+
new document_js_1.Document({
|
|
300
|
+
pageContent: d.content,
|
|
301
|
+
metadata: d.metadata,
|
|
302
|
+
}),
|
|
303
|
+
d.score ? d.score : 0,
|
|
304
|
+
]);
|
|
305
|
+
}
|
|
306
|
+
function assignMetadata(value) {
|
|
307
|
+
if (typeof value === "object" && value !== null) {
|
|
308
|
+
return value;
|
|
309
|
+
}
|
|
310
|
+
if (value !== undefined) {
|
|
311
|
+
console.warn("Metadata filters must be an object, Record, or undefined.");
|
|
312
|
+
}
|
|
313
|
+
return undefined;
|
|
314
|
+
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { DocumentCollection, ZepClient } from "@getzep/zep-js";
|
|
2
|
-
import { VectorStore } from "./base.js";
|
|
2
|
+
import { MaxMarginalRelevanceSearchOptions, VectorStore } from "./base.js";
|
|
3
3
|
import { Embeddings } from "../embeddings/base.js";
|
|
4
4
|
import { Document } from "../document.js";
|
|
5
|
+
import { Callbacks } from "../callbacks/index.js";
|
|
5
6
|
export interface IZepArgs {
|
|
6
7
|
collection: DocumentCollection;
|
|
7
8
|
}
|
|
@@ -73,7 +74,6 @@ export declare class ZepVectorStore extends VectorStore {
|
|
|
73
74
|
* @returns {Promise<void>}
|
|
74
75
|
*/
|
|
75
76
|
delete(params: IZepDeleteParams): Promise<void>;
|
|
76
|
-
private isRecord;
|
|
77
77
|
/**
|
|
78
78
|
* Performs a similarity search in the collection and returns the results with their scores.
|
|
79
79
|
*
|
|
@@ -83,6 +83,36 @@ export declare class ZepVectorStore extends VectorStore {
|
|
|
83
83
|
* @returns {Promise<[Document, number][]>} - A promise that resolves with the search results and their scores.
|
|
84
84
|
*/
|
|
85
85
|
similaritySearchVectorWithScore(query: number[], k: number, filter?: Record<string, unknown> | undefined): Promise<[Document, number][]>;
|
|
86
|
+
_similaritySearchWithScore(query: string, k: number, filter?: Record<string, unknown> | undefined): Promise<[Document, number][]>;
|
|
87
|
+
similaritySearchWithScore(query: string, k?: number, filter?: Record<string, unknown> | undefined, _callbacks?: undefined): Promise<[Document, number][]>;
|
|
88
|
+
/**
|
|
89
|
+
* Performs a similarity search on the Zep collection.
|
|
90
|
+
*
|
|
91
|
+
* @param {string} query - The query string to search for.
|
|
92
|
+
* @param {number} [k=4] - The number of results to return. Defaults to 4.
|
|
93
|
+
* @param {this["FilterType"] | undefined} [filter=undefined] - An optional set of JSONPath filters to apply to the search.
|
|
94
|
+
* @param {Callbacks | undefined} [_callbacks=undefined] - Optional callbacks. Currently not implemented.
|
|
95
|
+
* @returns {Promise<Document[]>} - A promise that resolves to an array of Documents that are similar to the query.
|
|
96
|
+
*
|
|
97
|
+
* @async
|
|
98
|
+
*/
|
|
99
|
+
similaritySearch(query: string, k?: number, filter?: this["FilterType"] | undefined, _callbacks?: Callbacks | undefined): Promise<Document[]>;
|
|
100
|
+
/**
|
|
101
|
+
* Return documents selected using the maximal marginal relevance.
|
|
102
|
+
* Maximal marginal relevance optimizes for similarity to the query AND diversity
|
|
103
|
+
* among selected documents.
|
|
104
|
+
*
|
|
105
|
+
* @param {string} query - Text to look up documents similar to.
|
|
106
|
+
* @param options
|
|
107
|
+
* @param {number} options.k - Number of documents to return.
|
|
108
|
+
* @param {number} options.fetchK=20- Number of documents to fetch before passing to the MMR algorithm.
|
|
109
|
+
* @param {number} options.lambda=0.5 - Number between 0 and 1 that determines the degree of diversity among the results,
|
|
110
|
+
* where 0 corresponds to maximum diversity and 1 to minimum diversity.
|
|
111
|
+
* @param {Record<string, any>} options.filter - Optional Zep JSONPath query to pre-filter on document metadata field
|
|
112
|
+
*
|
|
113
|
+
* @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
|
|
114
|
+
*/
|
|
115
|
+
maxMarginalRelevanceSearch(query: string, options: MaxMarginalRelevanceSearchOptions<this["FilterType"]>): Promise<Document[]>;
|
|
86
116
|
/**
|
|
87
117
|
* Creates a new ZepVectorStore instance from an array of texts. Each text is converted into a Document and added to the collection.
|
|
88
118
|
*
|
package/dist/vectorstores/zep.js
CHANGED
|
@@ -2,6 +2,7 @@ import { ZepClient } from "@getzep/zep-js";
|
|
|
2
2
|
import { VectorStore } from "./base.js";
|
|
3
3
|
import { Document } from "../document.js";
|
|
4
4
|
import { FakeEmbeddings } from "../embeddings/fake.js";
|
|
5
|
+
import { maximalMarginalRelevance } from "../util/math.js";
|
|
5
6
|
/**
|
|
6
7
|
* ZepVectorStore is a VectorStore implementation that uses the Zep long-term memory store as a backend.
|
|
7
8
|
*
|
|
@@ -49,7 +50,8 @@ export class ZepVectorStore extends VectorStore {
|
|
|
49
50
|
this.autoEmbed = true;
|
|
50
51
|
}
|
|
51
52
|
this.initPromise = this.initCollection(args).catch((err) => {
|
|
52
|
-
console.error("Error
|
|
53
|
+
console.error("Error initializing collection:", err);
|
|
54
|
+
throw err;
|
|
53
55
|
});
|
|
54
56
|
}
|
|
55
57
|
/**
|
|
@@ -157,13 +159,6 @@ export class ZepVectorStore extends VectorStore {
|
|
|
157
159
|
await this.collection.deleteDocument(uuid);
|
|
158
160
|
}
|
|
159
161
|
}
|
|
160
|
-
isRecord(value) {
|
|
161
|
-
return (typeof value === "object" &&
|
|
162
|
-
value !== null &&
|
|
163
|
-
!Array.isArray(value) &&
|
|
164
|
-
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
165
|
-
!(value instanceof Function));
|
|
166
|
-
}
|
|
167
162
|
/**
|
|
168
163
|
* Performs a similarity search in the collection and returns the results with their scores.
|
|
169
164
|
*
|
|
@@ -173,25 +168,90 @@ export class ZepVectorStore extends VectorStore {
|
|
|
173
168
|
* @returns {Promise<[Document, number][]>} - A promise that resolves with the search results and their scores.
|
|
174
169
|
*/
|
|
175
170
|
async similaritySearchVectorWithScore(query, k, filter) {
|
|
176
|
-
if (filter && !this.isRecord(filter)) {
|
|
177
|
-
throw new Error(`Filter must be a record, got ${filter}`);
|
|
178
|
-
}
|
|
179
171
|
await this.initPromise;
|
|
180
172
|
const results = await this.collection.search({
|
|
181
173
|
embedding: new Float32Array(query),
|
|
182
|
-
metadata: filter,
|
|
174
|
+
metadata: assignMetadata(filter),
|
|
183
175
|
}, k);
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
176
|
+
return zepDocsToDocumentsAndScore(results);
|
|
177
|
+
}
|
|
178
|
+
async _similaritySearchWithScore(query, k, filter) {
|
|
179
|
+
await this.initPromise;
|
|
180
|
+
const results = await this.collection.search({
|
|
181
|
+
text: query,
|
|
182
|
+
metadata: assignMetadata(filter),
|
|
183
|
+
}, k);
|
|
184
|
+
return zepDocsToDocumentsAndScore(results);
|
|
185
|
+
}
|
|
186
|
+
async similaritySearchWithScore(query, k = 4, filter = undefined, _callbacks = undefined // implement passing to embedQuery later
|
|
187
|
+
) {
|
|
188
|
+
if (this.autoEmbed) {
|
|
189
|
+
return this._similaritySearchWithScore(query, k, filter);
|
|
190
|
+
}
|
|
191
|
+
else {
|
|
192
|
+
return this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k, filter);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Performs a similarity search on the Zep collection.
|
|
197
|
+
*
|
|
198
|
+
* @param {string} query - The query string to search for.
|
|
199
|
+
* @param {number} [k=4] - The number of results to return. Defaults to 4.
|
|
200
|
+
* @param {this["FilterType"] | undefined} [filter=undefined] - An optional set of JSONPath filters to apply to the search.
|
|
201
|
+
* @param {Callbacks | undefined} [_callbacks=undefined] - Optional callbacks. Currently not implemented.
|
|
202
|
+
* @returns {Promise<Document[]>} - A promise that resolves to an array of Documents that are similar to the query.
|
|
203
|
+
*
|
|
204
|
+
* @async
|
|
205
|
+
*/
|
|
206
|
+
async similaritySearch(query, k = 4, filter = undefined, _callbacks = undefined // implement passing to embedQuery later
|
|
207
|
+
) {
|
|
208
|
+
await this.initPromise;
|
|
209
|
+
let results;
|
|
210
|
+
if (this.autoEmbed) {
|
|
211
|
+
const zepResults = await this.collection.search({ text: query, metadata: assignMetadata(filter) }, k);
|
|
212
|
+
results = zepDocsToDocumentsAndScore(zepResults);
|
|
213
|
+
}
|
|
214
|
+
else {
|
|
215
|
+
results = await this.similaritySearchVectorWithScore(await this.embeddings.embedQuery(query), k, assignMetadata(filter));
|
|
216
|
+
}
|
|
217
|
+
return results.map((result) => result[0]);
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Return documents selected using the maximal marginal relevance.
|
|
221
|
+
* Maximal marginal relevance optimizes for similarity to the query AND diversity
|
|
222
|
+
* among selected documents.
|
|
223
|
+
*
|
|
224
|
+
* @param {string} query - Text to look up documents similar to.
|
|
225
|
+
* @param options
|
|
226
|
+
* @param {number} options.k - Number of documents to return.
|
|
227
|
+
* @param {number} options.fetchK=20- Number of documents to fetch before passing to the MMR algorithm.
|
|
228
|
+
* @param {number} options.lambda=0.5 - Number between 0 and 1 that determines the degree of diversity among the results,
|
|
229
|
+
* where 0 corresponds to maximum diversity and 1 to minimum diversity.
|
|
230
|
+
* @param {Record<string, any>} options.filter - Optional Zep JSONPath query to pre-filter on document metadata field
|
|
231
|
+
*
|
|
232
|
+
* @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
|
|
233
|
+
*/
|
|
234
|
+
async maxMarginalRelevanceSearch(query, options) {
|
|
235
|
+
const { k, fetchK = 20, lambda = 0.5, filter } = options;
|
|
236
|
+
let queryEmbedding;
|
|
237
|
+
let zepResults;
|
|
238
|
+
if (!this.autoEmbed) {
|
|
239
|
+
queryEmbedding = await this.embeddings.embedQuery(query);
|
|
240
|
+
zepResults = await this.collection.search({
|
|
241
|
+
embedding: new Float32Array(queryEmbedding),
|
|
242
|
+
metadata: assignMetadata(filter),
|
|
243
|
+
}, fetchK);
|
|
244
|
+
}
|
|
245
|
+
else {
|
|
246
|
+
let queryEmbeddingArray;
|
|
247
|
+
[zepResults, queryEmbeddingArray] =
|
|
248
|
+
await this.collection.searchReturnQueryVector({ text: query, metadata: assignMetadata(filter) }, fetchK);
|
|
249
|
+
queryEmbedding = Array.from(queryEmbeddingArray);
|
|
250
|
+
}
|
|
251
|
+
const results = zepDocsToDocumentsAndScore(zepResults);
|
|
252
|
+
const embeddingList = zepResults.map((doc) => Array.from(doc.embedding ? doc.embedding : []));
|
|
253
|
+
const mmrIndexes = maximalMarginalRelevance(queryEmbedding, embeddingList, lambda, k);
|
|
254
|
+
return mmrIndexes.filter((idx) => idx !== -1).map((idx) => results[idx][0]);
|
|
195
255
|
}
|
|
196
256
|
/**
|
|
197
257
|
* Creates a new ZepVectorStore instance from an array of texts. Each text is converted into a Document and added to the collection.
|
|
@@ -230,3 +290,21 @@ export class ZepVectorStore extends VectorStore {
|
|
|
230
290
|
return instance;
|
|
231
291
|
}
|
|
232
292
|
}
|
|
293
|
+
function zepDocsToDocumentsAndScore(results) {
|
|
294
|
+
return results.map((d) => [
|
|
295
|
+
new Document({
|
|
296
|
+
pageContent: d.content,
|
|
297
|
+
metadata: d.metadata,
|
|
298
|
+
}),
|
|
299
|
+
d.score ? d.score : 0,
|
|
300
|
+
]);
|
|
301
|
+
}
|
|
302
|
+
function assignMetadata(value) {
|
|
303
|
+
if (typeof value === "object" && value !== null) {
|
|
304
|
+
return value;
|
|
305
|
+
}
|
|
306
|
+
if (value !== undefined) {
|
|
307
|
+
console.warn("Metadata filters must be an object, Record, or undefined.");
|
|
308
|
+
}
|
|
309
|
+
return undefined;
|
|
310
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = require('../dist/embeddings/cache_backed.cjs');
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/embeddings/cache_backed.js'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/embeddings/cache_backed.js'
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "langchain",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.129",
|
|
4
4
|
"description": "Typescript bindings for langchain",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"engines": {
|
|
@@ -76,6 +76,9 @@
|
|
|
76
76
|
"embeddings/base.cjs",
|
|
77
77
|
"embeddings/base.js",
|
|
78
78
|
"embeddings/base.d.ts",
|
|
79
|
+
"embeddings/cache_backed.cjs",
|
|
80
|
+
"embeddings/cache_backed.js",
|
|
81
|
+
"embeddings/cache_backed.d.ts",
|
|
79
82
|
"embeddings/fake.cjs",
|
|
80
83
|
"embeddings/fake.js",
|
|
81
84
|
"embeddings/fake.d.ts",
|
|
@@ -391,6 +394,9 @@
|
|
|
391
394
|
"schema/runnable.cjs",
|
|
392
395
|
"schema/runnable.js",
|
|
393
396
|
"schema/runnable.d.ts",
|
|
397
|
+
"schema/storage.cjs",
|
|
398
|
+
"schema/storage.js",
|
|
399
|
+
"schema/storage.d.ts",
|
|
394
400
|
"sql_db.cjs",
|
|
395
401
|
"sql_db.js",
|
|
396
402
|
"sql_db.d.ts",
|
|
@@ -517,6 +523,12 @@
|
|
|
517
523
|
"stores/message/xata.cjs",
|
|
518
524
|
"stores/message/xata.js",
|
|
519
525
|
"stores/message/xata.d.ts",
|
|
526
|
+
"storage/in_memory.cjs",
|
|
527
|
+
"storage/in_memory.js",
|
|
528
|
+
"storage/in_memory.d.ts",
|
|
529
|
+
"storage/ioredis.cjs",
|
|
530
|
+
"storage/ioredis.js",
|
|
531
|
+
"storage/ioredis.d.ts",
|
|
520
532
|
"util/math.cjs",
|
|
521
533
|
"util/math.js",
|
|
522
534
|
"util/math.d.ts",
|
|
@@ -1111,6 +1123,11 @@
|
|
|
1111
1123
|
"import": "./embeddings/base.js",
|
|
1112
1124
|
"require": "./embeddings/base.cjs"
|
|
1113
1125
|
},
|
|
1126
|
+
"./embeddings/cache_backed": {
|
|
1127
|
+
"types": "./embeddings/cache_backed.d.ts",
|
|
1128
|
+
"import": "./embeddings/cache_backed.js",
|
|
1129
|
+
"require": "./embeddings/cache_backed.cjs"
|
|
1130
|
+
},
|
|
1114
1131
|
"./embeddings/fake": {
|
|
1115
1132
|
"types": "./embeddings/fake.d.ts",
|
|
1116
1133
|
"import": "./embeddings/fake.js",
|
|
@@ -1644,6 +1661,11 @@
|
|
|
1644
1661
|
"import": "./schema/runnable.js",
|
|
1645
1662
|
"require": "./schema/runnable.cjs"
|
|
1646
1663
|
},
|
|
1664
|
+
"./schema/storage": {
|
|
1665
|
+
"types": "./schema/storage.d.ts",
|
|
1666
|
+
"import": "./schema/storage.js",
|
|
1667
|
+
"require": "./schema/storage.cjs"
|
|
1668
|
+
},
|
|
1647
1669
|
"./sql_db": {
|
|
1648
1670
|
"types": "./sql_db.d.ts",
|
|
1649
1671
|
"import": "./sql_db.js",
|
|
@@ -1856,6 +1878,16 @@
|
|
|
1856
1878
|
"import": "./stores/message/xata.js",
|
|
1857
1879
|
"require": "./stores/message/xata.cjs"
|
|
1858
1880
|
},
|
|
1881
|
+
"./storage/in_memory": {
|
|
1882
|
+
"types": "./storage/in_memory.d.ts",
|
|
1883
|
+
"import": "./storage/in_memory.js",
|
|
1884
|
+
"require": "./storage/in_memory.cjs"
|
|
1885
|
+
},
|
|
1886
|
+
"./storage/ioredis": {
|
|
1887
|
+
"types": "./storage/ioredis.d.ts",
|
|
1888
|
+
"import": "./storage/ioredis.js",
|
|
1889
|
+
"require": "./storage/ioredis.cjs"
|
|
1890
|
+
},
|
|
1859
1891
|
"./util/math": {
|
|
1860
1892
|
"types": "./util/math.d.ts",
|
|
1861
1893
|
"import": "./util/math.js",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = require('../dist/schema/storage.cjs');
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/schema/storage.js'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/schema/storage.js'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = require('../dist/storage/in_memory.cjs');
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/storage/in_memory.js'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/storage/in_memory.js'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = require('../dist/storage/ioredis.cjs');
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/storage/ioredis.js'
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../dist/storage/ioredis.js'
|