langchain 0.0.164 → 0.0.166

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/chat_models/portkey.cjs +1 -0
  2. package/chat_models/portkey.d.ts +1 -0
  3. package/chat_models/portkey.js +1 -0
  4. package/dist/cache/momento.cjs +7 -7
  5. package/dist/cache/momento.d.ts +1 -1
  6. package/dist/cache/momento.js +1 -1
  7. package/dist/chat_models/portkey.cjs +159 -0
  8. package/dist/chat_models/portkey.d.ts +17 -0
  9. package/dist/chat_models/portkey.js +155 -0
  10. package/dist/document_loaders/fs/unstructured.cjs +40 -0
  11. package/dist/document_loaders/fs/unstructured.d.ts +27 -0
  12. package/dist/document_loaders/fs/unstructured.js +40 -0
  13. package/dist/embeddings/minimax.cjs +1 -1
  14. package/dist/embeddings/minimax.js +1 -1
  15. package/dist/experimental/chat_models/anthropic_functions.cjs +34 -2
  16. package/dist/experimental/chat_models/anthropic_functions.d.ts +15 -5
  17. package/dist/experimental/chat_models/anthropic_functions.js +34 -2
  18. package/dist/llms/portkey.cjs +147 -0
  19. package/dist/llms/portkey.d.ts +33 -0
  20. package/dist/llms/portkey.js +138 -0
  21. package/dist/llms/sagemaker_endpoint.cjs +76 -14
  22. package/dist/llms/sagemaker_endpoint.d.ts +39 -20
  23. package/dist/llms/sagemaker_endpoint.js +77 -15
  24. package/dist/load/import_constants.cjs +4 -0
  25. package/dist/load/import_constants.js +4 -0
  26. package/dist/load/import_map.cjs +3 -1
  27. package/dist/load/import_map.d.ts +2 -0
  28. package/dist/load/import_map.js +2 -0
  29. package/dist/output_parsers/list.cjs +1 -1
  30. package/dist/output_parsers/list.js +1 -1
  31. package/dist/prompts/chat.cjs +19 -2
  32. package/dist/prompts/chat.d.ts +1 -0
  33. package/dist/prompts/chat.js +19 -2
  34. package/dist/retrievers/self_query/base.cjs +4 -1
  35. package/dist/retrievers/self_query/base.d.ts +3 -2
  36. package/dist/retrievers/self_query/base.js +4 -1
  37. package/dist/retrievers/self_query/index.cjs +2 -2
  38. package/dist/retrievers/self_query/index.d.ts +2 -0
  39. package/dist/retrievers/self_query/index.js +2 -2
  40. package/dist/runnables/remote.cjs +225 -0
  41. package/dist/runnables/remote.d.ts +28 -0
  42. package/dist/runnables/remote.js +221 -0
  43. package/dist/schema/index.cjs +1 -1
  44. package/dist/schema/index.d.ts +1 -1
  45. package/dist/schema/index.js +1 -1
  46. package/dist/schema/runnable/base.cjs +4 -4
  47. package/dist/schema/runnable/base.d.ts +9 -7
  48. package/dist/schema/runnable/base.js +4 -4
  49. package/dist/schema/runnable/remote.cjs +225 -0
  50. package/dist/schema/runnable/remote.d.ts +28 -0
  51. package/dist/schema/runnable/remote.js +221 -0
  52. package/dist/stores/message/momento.cjs +11 -11
  53. package/dist/stores/message/momento.d.ts +1 -1
  54. package/dist/stores/message/momento.js +1 -1
  55. package/dist/util/time.cjs +14 -0
  56. package/dist/util/time.d.ts +6 -0
  57. package/dist/util/time.js +10 -0
  58. package/dist/vectorstores/cassandra.cjs +212 -0
  59. package/dist/vectorstores/cassandra.d.ts +98 -0
  60. package/dist/vectorstores/cassandra.js +208 -0
  61. package/dist/vectorstores/momento_vector_index.cjs +292 -0
  62. package/dist/vectorstores/momento_vector_index.d.ts +135 -0
  63. package/dist/vectorstores/momento_vector_index.js +265 -0
  64. package/dist/vectorstores/mongodb_atlas.cjs +29 -39
  65. package/dist/vectorstores/mongodb_atlas.js +29 -39
  66. package/dist/vectorstores/supabase.cjs +37 -8
  67. package/dist/vectorstores/supabase.d.ts +28 -1
  68. package/dist/vectorstores/supabase.js +37 -8
  69. package/llms/portkey.cjs +1 -0
  70. package/llms/portkey.d.ts +1 -0
  71. package/llms/portkey.js +1 -0
  72. package/package.json +77 -4
  73. package/runnables/remote.cjs +1 -0
  74. package/runnables/remote.d.ts +1 -0
  75. package/runnables/remote.js +1 -0
  76. package/util/time.cjs +1 -0
  77. package/util/time.d.ts +1 -0
  78. package/util/time.js +1 -0
  79. package/vectorstores/cassandra.cjs +1 -0
  80. package/vectorstores/cassandra.d.ts +1 -0
  81. package/vectorstores/cassandra.js +1 -0
  82. package/vectorstores/momento_vector_index.cjs +1 -0
  83. package/vectorstores/momento_vector_index.d.ts +1 -0
  84. package/vectorstores/momento_vector_index.js +1 -0
@@ -0,0 +1,292 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.MomentoVectorIndex = void 0;
27
+ /* eslint-disable no-instanceof/no-instanceof */
28
+ /* eslint-disable @typescript-eslint/no-explicit-any */
29
+ const sdk_core_1 = require("@gomomento/sdk-core");
30
+ const uuid = __importStar(require("uuid"));
31
+ const document_js_1 = require("../document.cjs");
32
+ const base_js_1 = require("./base.cjs");
33
+ /**
34
+ * A vector store that uses the Momento Vector Index.
35
+ *
36
+ * @remarks
37
+ * To sign up for a free Momento account, visit https://console.gomomento.com.
38
+ */
39
+ class MomentoVectorIndex extends base_js_1.VectorStore {
40
+ _vectorstoreType() {
41
+ return "momento";
42
+ }
43
+ /**
44
+ * Creates a new `MomentoVectorIndex` instance.
45
+ * @param embeddings The embeddings instance to use to generate embeddings from documents.
46
+ * @param args The arguments to use to configure the vector store.
47
+ */
48
+ constructor(embeddings, args) {
49
+ super(embeddings, args);
50
+ Object.defineProperty(this, "client", {
51
+ enumerable: true,
52
+ configurable: true,
53
+ writable: true,
54
+ value: void 0
55
+ });
56
+ Object.defineProperty(this, "indexName", {
57
+ enumerable: true,
58
+ configurable: true,
59
+ writable: true,
60
+ value: void 0
61
+ });
62
+ Object.defineProperty(this, "textField", {
63
+ enumerable: true,
64
+ configurable: true,
65
+ writable: true,
66
+ value: void 0
67
+ });
68
+ Object.defineProperty(this, "_ensureIndexExists", {
69
+ enumerable: true,
70
+ configurable: true,
71
+ writable: true,
72
+ value: void 0
73
+ });
74
+ this.embeddings = embeddings;
75
+ this.client = args.client;
76
+ this.indexName = args.indexName ?? "default";
77
+ this.textField = args.textField ?? "text";
78
+ this._ensureIndexExists = args.ensureIndexExists ?? true;
79
+ }
80
+ /**
81
+ * Returns the Momento Vector Index client.
82
+ * @returns The Momento Vector Index client.
83
+ */
84
+ getClient() {
85
+ return this.client;
86
+ }
87
+ /**
88
+ * Creates the index if it does not already exist.
89
+ * @param numDimensions The number of dimensions of the vectors to be stored in the index.
90
+ * @returns Promise that resolves to true if the index was created, false if it already existed.
91
+ */
92
+ async ensureIndexExists(numDimensions) {
93
+ const response = await this.client.createIndex(this.indexName, numDimensions);
94
+ if (response instanceof sdk_core_1.CreateVectorIndex.Success) {
95
+ return true;
96
+ }
97
+ else if (response instanceof sdk_core_1.CreateVectorIndex.AlreadyExists) {
98
+ return false;
99
+ }
100
+ else if (response instanceof sdk_core_1.CreateVectorIndex.Error) {
101
+ throw new Error(response.toString());
102
+ }
103
+ else {
104
+ throw new Error(`Unknown response type: ${response.toString()}`);
105
+ }
106
+ }
107
+ /**
108
+ * Converts the metadata to a format that can be stored in the index.
109
+ *
110
+ * @remarks stringifies all values in the metadata object
111
+ * @param metadata The metadata to convert.
112
+ * @returns The converted metadata.
113
+ */
114
+ static prepareMetadata(metadata) {
115
+ return Object.fromEntries(Object.entries(metadata).map(([key, val]) => [key, JSON.stringify(val)]));
116
+ }
117
+ /**
118
+ * Converts the documents to a format that can be stored in the index.
119
+ *
120
+ * This is necessary because the Momento Vector Index requires that the metadata
121
+ * be a map of strings to strings.
122
+ * @param vectors The vectors to convert.
123
+ * @param documents The documents to convert.
124
+ * @param ids The ids to convert.
125
+ * @returns The converted documents.
126
+ */
127
+ prepareItemBatch(vectors, documents, ids) {
128
+ return vectors.map((vector, idx) => ({
129
+ id: ids[idx],
130
+ vector,
131
+ metadata: {
132
+ ...MomentoVectorIndex.prepareMetadata(documents[idx].metadata),
133
+ [this.textField]: documents[idx].pageContent,
134
+ },
135
+ }));
136
+ }
137
+ /**
138
+ * Adds vectors to the index.
139
+ *
140
+ * @remarks If the index does not already exist, it will be created if `ensureIndexExists` is true.
141
+ * @param vectors The vectors to add to the index.
142
+ * @param documents The documents to add to the index.
143
+ * @param documentProps The properties of the documents to add to the index, specifically the ids.
144
+ * @returns Promise that resolves when the vectors have been added to the index. Also returns the ids of the
145
+ * documents that were added.
146
+ */
147
+ async addVectors(vectors, documents, documentProps) {
148
+ if (vectors.length === 0) {
149
+ return;
150
+ }
151
+ if (documents.length !== vectors.length) {
152
+ throw new Error(`Number of vectors (${vectors.length}) does not equal number of documents (${documents.length})`);
153
+ }
154
+ if (vectors.some((v) => v.length !== vectors[0].length)) {
155
+ throw new Error("All vectors must have the same length");
156
+ }
157
+ if (documentProps?.ids !== undefined &&
158
+ documentProps.ids.length !== vectors.length) {
159
+ throw new Error(`Number of ids (${documentProps?.ids?.length || "null"}) does not equal number of vectors (${vectors.length})`);
160
+ }
161
+ if (this._ensureIndexExists) {
162
+ await this.ensureIndexExists(vectors[0].length);
163
+ }
164
+ const documentIds = documentProps?.ids ?? documents.map(() => uuid.v4());
165
+ const batchSize = 128;
166
+ const numBatches = Math.ceil(vectors.length / batchSize);
167
+ // Add each batch of vectors to the index
168
+ for (let i = 0; i < numBatches; i += 1) {
169
+ const [startIndex, endIndex] = [
170
+ i * batchSize,
171
+ Math.min((i + 1) * batchSize, vectors.length),
172
+ ];
173
+ const batchVectors = vectors.slice(startIndex, endIndex);
174
+ const batchDocuments = documents.slice(startIndex, endIndex);
175
+ const batchDocumentIds = documentIds.slice(startIndex, endIndex);
176
+ // Insert the items to the index
177
+ const response = await this.client.upsertItemBatch(this.indexName, this.prepareItemBatch(batchVectors, batchDocuments, batchDocumentIds));
178
+ if (response instanceof sdk_core_1.VectorUpsertItemBatch.Success) {
179
+ // eslint-disable-next-line no-continue
180
+ continue;
181
+ }
182
+ else if (response instanceof sdk_core_1.VectorUpsertItemBatch.Error) {
183
+ throw new Error(response.toString());
184
+ }
185
+ else {
186
+ throw new Error(`Unknown response type: ${response.toString()}`);
187
+ }
188
+ }
189
+ }
190
+ /**
191
+ * Adds vectors to the index. Generates embeddings from the documents
192
+ * using the `Embeddings` instance passed to the constructor.
193
+ * @param documents Array of `Document` instances to be added to the index.
194
+ * @returns Promise that resolves when the documents have been added to the index.
195
+ */
196
+ async addDocuments(documents, documentProps) {
197
+ const texts = documents.map(({ pageContent }) => pageContent);
198
+ await this.addVectors(await this.embeddings.embedDocuments(texts), documents, documentProps);
199
+ }
200
+ /**
201
+ * Deletes vectors from the index by id.
202
+ * @param params The parameters to use to delete the vectors, specifically the ids.
203
+ */
204
+ async delete(params) {
205
+ const response = await this.client.deleteItemBatch(this.indexName, params.ids);
206
+ if (response instanceof sdk_core_1.VectorDeleteItemBatch.Success) {
207
+ // pass
208
+ }
209
+ else if (response instanceof sdk_core_1.VectorDeleteItemBatch.Error) {
210
+ throw new Error(response.toString());
211
+ }
212
+ else {
213
+ throw new Error(`Unknown response type: ${response.toString()}`);
214
+ }
215
+ }
216
+ /**
217
+ * Searches the index for the most similar vectors to the query vector.
218
+ * @param query The query vector.
219
+ * @param k The number of results to return.
220
+ * @returns Promise that resolves to the documents of the most similar vectors
221
+ * to the query vector.
222
+ */
223
+ async similaritySearchVectorWithScore(query, k) {
224
+ const response = await this.client.search(this.indexName, query, {
225
+ topK: k,
226
+ metadataFields: sdk_core_1.ALL_VECTOR_METADATA,
227
+ });
228
+ if (response instanceof sdk_core_1.VectorSearch.Success) {
229
+ if (response.hits === undefined) {
230
+ return [];
231
+ }
232
+ return response.hits().map((hit) => [
233
+ new document_js_1.Document({
234
+ pageContent: hit.metadata[this.textField] ?? "",
235
+ metadata: Object.fromEntries(Object.entries(hit.metadata)
236
+ .filter(([key]) => key !== this.textField)
237
+ .map(([key, val]) => [key, JSON.parse(val)])),
238
+ }),
239
+ hit.distance,
240
+ ]);
241
+ }
242
+ else if (response instanceof sdk_core_1.VectorSearch.Error) {
243
+ throw new Error(response.toString());
244
+ }
245
+ else {
246
+ throw new Error(`Unknown response type: ${response.toString()}`);
247
+ }
248
+ }
249
+ /**
250
+ * Stores the documents in the index.
251
+ *
252
+ * Converts the documents to vectors using the `Embeddings` instance passed.
253
+ * @param texts The texts to store in the index.
254
+ * @param metadatas The metadata to store in the index.
255
+ * @param embeddings The embeddings instance to use to generate embeddings from the documents.
256
+ * @param dbConfig The configuration to use to instantiate the vector store.
257
+ * @param documentProps The properties of the documents to add to the index, specifically the ids.
258
+ * @returns Promise that resolves to the vector store.
259
+ */
260
+ static async fromTexts(texts, metadatas, embeddings, dbConfig, documentProps) {
261
+ if (Array.isArray(metadatas) && texts.length !== metadatas.length) {
262
+ throw new Error(`Number of texts (${texts.length}) does not equal number of metadatas (${metadatas.length})`);
263
+ }
264
+ const docs = [];
265
+ for (let i = 0; i < texts.length; i += 1) {
266
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
267
+ const metadata = Array.isArray(metadatas)
268
+ ? metadatas[i]
269
+ : metadatas;
270
+ const newDoc = new document_js_1.Document({
271
+ pageContent: texts[i],
272
+ metadata,
273
+ });
274
+ docs.push(newDoc);
275
+ }
276
+ return await this.fromDocuments(docs, embeddings, dbConfig, documentProps);
277
+ }
278
+ /**
279
+ * Stores the documents in the index.
280
+ * @param docs The documents to store in the index.
281
+ * @param embeddings The embeddings instance to use to generate embeddings from the documents.
282
+ * @param dbConfig The configuration to use to instantiate the vector store.
283
+ * @param documentProps The properties of the documents to add to the index, specifically the ids.
284
+ * @returns Promise that resolves to the vector store.
285
+ */
286
+ static async fromDocuments(docs, embeddings, dbConfig, documentProps) {
287
+ const vectorStore = new MomentoVectorIndex(embeddings, dbConfig);
288
+ await vectorStore.addDocuments(docs, documentProps);
289
+ return vectorStore;
290
+ }
291
+ }
292
+ exports.MomentoVectorIndex = MomentoVectorIndex;
@@ -0,0 +1,135 @@
1
+ import { IVectorIndexClient } from "@gomomento/sdk-core";
2
+ import { Document } from "../document.js";
3
+ import { Embeddings } from "../embeddings/base.js";
4
+ import { VectorStore } from "./base.js";
5
+ export interface DocumentProps {
6
+ ids: string[];
7
+ }
8
+ export interface MomentoVectorIndexLibArgs {
9
+ /**
10
+ * The Momento Vector Index client.
11
+ */
12
+ client: IVectorIndexClient;
13
+ /**
14
+ * The name of the index to use to store the data.
15
+ * Defaults to "default".
16
+ */
17
+ indexName?: string;
18
+ /**
19
+ * The name of the metadata field to use to store the text of the document.
20
+ * Defaults to "text".
21
+ */
22
+ textField?: string;
23
+ /**
24
+ * Whether to create the index if it does not already exist.
25
+ * Defaults to true.
26
+ */
27
+ ensureIndexExists?: boolean;
28
+ }
29
+ export interface DeleteProps {
30
+ /**
31
+ * The ids of the documents to delete.
32
+ */
33
+ ids: string[];
34
+ }
35
+ /**
36
+ * A vector store that uses the Momento Vector Index.
37
+ *
38
+ * @remarks
39
+ * To sign up for a free Momento account, visit https://console.gomomento.com.
40
+ */
41
+ export declare class MomentoVectorIndex extends VectorStore {
42
+ private client;
43
+ private indexName;
44
+ private textField;
45
+ private _ensureIndexExists;
46
+ _vectorstoreType(): string;
47
+ /**
48
+ * Creates a new `MomentoVectorIndex` instance.
49
+ * @param embeddings The embeddings instance to use to generate embeddings from documents.
50
+ * @param args The arguments to use to configure the vector store.
51
+ */
52
+ constructor(embeddings: Embeddings, args: MomentoVectorIndexLibArgs);
53
+ /**
54
+ * Returns the Momento Vector Index client.
55
+ * @returns The Momento Vector Index client.
56
+ */
57
+ getClient(): IVectorIndexClient;
58
+ /**
59
+ * Creates the index if it does not already exist.
60
+ * @param numDimensions The number of dimensions of the vectors to be stored in the index.
61
+ * @returns Promise that resolves to true if the index was created, false if it already existed.
62
+ */
63
+ private ensureIndexExists;
64
+ /**
65
+ * Converts the metadata to a format that can be stored in the index.
66
+ *
67
+ * @remarks stringifies all values in the metadata object
68
+ * @param metadata The metadata to convert.
69
+ * @returns The converted metadata.
70
+ */
71
+ private static prepareMetadata;
72
+ /**
73
+ * Converts the documents to a format that can be stored in the index.
74
+ *
75
+ * This is necessary because the Momento Vector Index requires that the metadata
76
+ * be a map of strings to strings.
77
+ * @param vectors The vectors to convert.
78
+ * @param documents The documents to convert.
79
+ * @param ids The ids to convert.
80
+ * @returns The converted documents.
81
+ */
82
+ private prepareItemBatch;
83
+ /**
84
+ * Adds vectors to the index.
85
+ *
86
+ * @remarks If the index does not already exist, it will be created if `ensureIndexExists` is true.
87
+ * @param vectors The vectors to add to the index.
88
+ * @param documents The documents to add to the index.
89
+ * @param documentProps The properties of the documents to add to the index, specifically the ids.
90
+ * @returns Promise that resolves when the vectors have been added to the index. Also returns the ids of the
91
+ * documents that were added.
92
+ */
93
+ addVectors(vectors: number[][], documents: Document<Record<string, any>>[], documentProps?: DocumentProps): Promise<void | string[]>;
94
+ /**
95
+ * Adds vectors to the index. Generates embeddings from the documents
96
+ * using the `Embeddings` instance passed to the constructor.
97
+ * @param documents Array of `Document` instances to be added to the index.
98
+ * @returns Promise that resolves when the documents have been added to the index.
99
+ */
100
+ addDocuments(documents: Document[], documentProps?: DocumentProps): Promise<void>;
101
+ /**
102
+ * Deletes vectors from the index by id.
103
+ * @param params The parameters to use to delete the vectors, specifically the ids.
104
+ */
105
+ delete(params: DeleteProps): Promise<void>;
106
+ /**
107
+ * Searches the index for the most similar vectors to the query vector.
108
+ * @param query The query vector.
109
+ * @param k The number of results to return.
110
+ * @returns Promise that resolves to the documents of the most similar vectors
111
+ * to the query vector.
112
+ */
113
+ similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document<Record<string, any>>, number][]>;
114
+ /**
115
+ * Stores the documents in the index.
116
+ *
117
+ * Converts the documents to vectors using the `Embeddings` instance passed.
118
+ * @param texts The texts to store in the index.
119
+ * @param metadatas The metadata to store in the index.
120
+ * @param embeddings The embeddings instance to use to generate embeddings from the documents.
121
+ * @param dbConfig The configuration to use to instantiate the vector store.
122
+ * @param documentProps The properties of the documents to add to the index, specifically the ids.
123
+ * @returns Promise that resolves to the vector store.
124
+ */
125
+ static fromTexts(texts: string[], metadatas: object[] | object, embeddings: Embeddings, dbConfig: MomentoVectorIndexLibArgs, documentProps?: DocumentProps): Promise<MomentoVectorIndex>;
126
+ /**
127
+ * Stores the documents in the index.
128
+ * @param docs The documents to store in the index.
129
+ * @param embeddings The embeddings instance to use to generate embeddings from the documents.
130
+ * @param dbConfig The configuration to use to instantiate the vector store.
131
+ * @param documentProps The properties of the documents to add to the index, specifically the ids.
132
+ * @returns Promise that resolves to the vector store.
133
+ */
134
+ static fromDocuments(docs: Document[], embeddings: Embeddings, dbConfig: MomentoVectorIndexLibArgs, documentProps?: DocumentProps): Promise<MomentoVectorIndex>;
135
+ }
@@ -0,0 +1,265 @@
1
+ /* eslint-disable no-instanceof/no-instanceof */
2
+ /* eslint-disable @typescript-eslint/no-explicit-any */
3
+ import { ALL_VECTOR_METADATA, CreateVectorIndex, VectorUpsertItemBatch, VectorDeleteItemBatch, VectorSearch, } from "@gomomento/sdk-core";
4
+ import * as uuid from "uuid";
5
+ import { Document } from "../document.js";
6
+ import { VectorStore } from "./base.js";
7
+ /**
8
+ * A vector store that uses the Momento Vector Index.
9
+ *
10
+ * @remarks
11
+ * To sign up for a free Momento account, visit https://console.gomomento.com.
12
+ */
13
+ export class MomentoVectorIndex extends VectorStore {
14
+ _vectorstoreType() {
15
+ return "momento";
16
+ }
17
+ /**
18
+ * Creates a new `MomentoVectorIndex` instance.
19
+ * @param embeddings The embeddings instance to use to generate embeddings from documents.
20
+ * @param args The arguments to use to configure the vector store.
21
+ */
22
+ constructor(embeddings, args) {
23
+ super(embeddings, args);
24
+ Object.defineProperty(this, "client", {
25
+ enumerable: true,
26
+ configurable: true,
27
+ writable: true,
28
+ value: void 0
29
+ });
30
+ Object.defineProperty(this, "indexName", {
31
+ enumerable: true,
32
+ configurable: true,
33
+ writable: true,
34
+ value: void 0
35
+ });
36
+ Object.defineProperty(this, "textField", {
37
+ enumerable: true,
38
+ configurable: true,
39
+ writable: true,
40
+ value: void 0
41
+ });
42
+ Object.defineProperty(this, "_ensureIndexExists", {
43
+ enumerable: true,
44
+ configurable: true,
45
+ writable: true,
46
+ value: void 0
47
+ });
48
+ this.embeddings = embeddings;
49
+ this.client = args.client;
50
+ this.indexName = args.indexName ?? "default";
51
+ this.textField = args.textField ?? "text";
52
+ this._ensureIndexExists = args.ensureIndexExists ?? true;
53
+ }
54
+ /**
55
+ * Returns the Momento Vector Index client.
56
+ * @returns The Momento Vector Index client.
57
+ */
58
+ getClient() {
59
+ return this.client;
60
+ }
61
+ /**
62
+ * Creates the index if it does not already exist.
63
+ * @param numDimensions The number of dimensions of the vectors to be stored in the index.
64
+ * @returns Promise that resolves to true if the index was created, false if it already existed.
65
+ */
66
+ async ensureIndexExists(numDimensions) {
67
+ const response = await this.client.createIndex(this.indexName, numDimensions);
68
+ if (response instanceof CreateVectorIndex.Success) {
69
+ return true;
70
+ }
71
+ else if (response instanceof CreateVectorIndex.AlreadyExists) {
72
+ return false;
73
+ }
74
+ else if (response instanceof CreateVectorIndex.Error) {
75
+ throw new Error(response.toString());
76
+ }
77
+ else {
78
+ throw new Error(`Unknown response type: ${response.toString()}`);
79
+ }
80
+ }
81
+ /**
82
+ * Converts the metadata to a format that can be stored in the index.
83
+ *
84
+ * @remarks stringifies all values in the metadata object
85
+ * @param metadata The metadata to convert.
86
+ * @returns The converted metadata.
87
+ */
88
+ static prepareMetadata(metadata) {
89
+ return Object.fromEntries(Object.entries(metadata).map(([key, val]) => [key, JSON.stringify(val)]));
90
+ }
91
+ /**
92
+ * Converts the documents to a format that can be stored in the index.
93
+ *
94
+ * This is necessary because the Momento Vector Index requires that the metadata
95
+ * be a map of strings to strings.
96
+ * @param vectors The vectors to convert.
97
+ * @param documents The documents to convert.
98
+ * @param ids The ids to convert.
99
+ * @returns The converted documents.
100
+ */
101
+ prepareItemBatch(vectors, documents, ids) {
102
+ return vectors.map((vector, idx) => ({
103
+ id: ids[idx],
104
+ vector,
105
+ metadata: {
106
+ ...MomentoVectorIndex.prepareMetadata(documents[idx].metadata),
107
+ [this.textField]: documents[idx].pageContent,
108
+ },
109
+ }));
110
+ }
111
+ /**
112
+ * Adds vectors to the index.
113
+ *
114
+ * @remarks If the index does not already exist, it will be created if `ensureIndexExists` is true.
115
+ * @param vectors The vectors to add to the index.
116
+ * @param documents The documents to add to the index.
117
+ * @param documentProps The properties of the documents to add to the index, specifically the ids.
118
+ * @returns Promise that resolves when the vectors have been added to the index. Also returns the ids of the
119
+ * documents that were added.
120
+ */
121
+ async addVectors(vectors, documents, documentProps) {
122
+ if (vectors.length === 0) {
123
+ return;
124
+ }
125
+ if (documents.length !== vectors.length) {
126
+ throw new Error(`Number of vectors (${vectors.length}) does not equal number of documents (${documents.length})`);
127
+ }
128
+ if (vectors.some((v) => v.length !== vectors[0].length)) {
129
+ throw new Error("All vectors must have the same length");
130
+ }
131
+ if (documentProps?.ids !== undefined &&
132
+ documentProps.ids.length !== vectors.length) {
133
+ throw new Error(`Number of ids (${documentProps?.ids?.length || "null"}) does not equal number of vectors (${vectors.length})`);
134
+ }
135
+ if (this._ensureIndexExists) {
136
+ await this.ensureIndexExists(vectors[0].length);
137
+ }
138
+ const documentIds = documentProps?.ids ?? documents.map(() => uuid.v4());
139
+ const batchSize = 128;
140
+ const numBatches = Math.ceil(vectors.length / batchSize);
141
+ // Add each batch of vectors to the index
142
+ for (let i = 0; i < numBatches; i += 1) {
143
+ const [startIndex, endIndex] = [
144
+ i * batchSize,
145
+ Math.min((i + 1) * batchSize, vectors.length),
146
+ ];
147
+ const batchVectors = vectors.slice(startIndex, endIndex);
148
+ const batchDocuments = documents.slice(startIndex, endIndex);
149
+ const batchDocumentIds = documentIds.slice(startIndex, endIndex);
150
+ // Insert the items to the index
151
+ const response = await this.client.upsertItemBatch(this.indexName, this.prepareItemBatch(batchVectors, batchDocuments, batchDocumentIds));
152
+ if (response instanceof VectorUpsertItemBatch.Success) {
153
+ // eslint-disable-next-line no-continue
154
+ continue;
155
+ }
156
+ else if (response instanceof VectorUpsertItemBatch.Error) {
157
+ throw new Error(response.toString());
158
+ }
159
+ else {
160
+ throw new Error(`Unknown response type: ${response.toString()}`);
161
+ }
162
+ }
163
+ }
164
+ /**
165
+ * Adds vectors to the index. Generates embeddings from the documents
166
+ * using the `Embeddings` instance passed to the constructor.
167
+ * @param documents Array of `Document` instances to be added to the index.
168
+ * @returns Promise that resolves when the documents have been added to the index.
169
+ */
170
+ async addDocuments(documents, documentProps) {
171
+ const texts = documents.map(({ pageContent }) => pageContent);
172
+ await this.addVectors(await this.embeddings.embedDocuments(texts), documents, documentProps);
173
+ }
174
+ /**
175
+ * Deletes vectors from the index by id.
176
+ * @param params The parameters to use to delete the vectors, specifically the ids.
177
+ */
178
+ async delete(params) {
179
+ const response = await this.client.deleteItemBatch(this.indexName, params.ids);
180
+ if (response instanceof VectorDeleteItemBatch.Success) {
181
+ // pass
182
+ }
183
+ else if (response instanceof VectorDeleteItemBatch.Error) {
184
+ throw new Error(response.toString());
185
+ }
186
+ else {
187
+ throw new Error(`Unknown response type: ${response.toString()}`);
188
+ }
189
+ }
190
+ /**
191
+ * Searches the index for the most similar vectors to the query vector.
192
+ * @param query The query vector.
193
+ * @param k The number of results to return.
194
+ * @returns Promise that resolves to the documents of the most similar vectors
195
+ * to the query vector.
196
+ */
197
+ async similaritySearchVectorWithScore(query, k) {
198
+ const response = await this.client.search(this.indexName, query, {
199
+ topK: k,
200
+ metadataFields: ALL_VECTOR_METADATA,
201
+ });
202
+ if (response instanceof VectorSearch.Success) {
203
+ if (response.hits === undefined) {
204
+ return [];
205
+ }
206
+ return response.hits().map((hit) => [
207
+ new Document({
208
+ pageContent: hit.metadata[this.textField] ?? "",
209
+ metadata: Object.fromEntries(Object.entries(hit.metadata)
210
+ .filter(([key]) => key !== this.textField)
211
+ .map(([key, val]) => [key, JSON.parse(val)])),
212
+ }),
213
+ hit.distance,
214
+ ]);
215
+ }
216
+ else if (response instanceof VectorSearch.Error) {
217
+ throw new Error(response.toString());
218
+ }
219
+ else {
220
+ throw new Error(`Unknown response type: ${response.toString()}`);
221
+ }
222
+ }
223
+ /**
224
+ * Stores the documents in the index.
225
+ *
226
+ * Converts the documents to vectors using the `Embeddings` instance passed.
227
+ * @param texts The texts to store in the index.
228
+ * @param metadatas The metadata to store in the index.
229
+ * @param embeddings The embeddings instance to use to generate embeddings from the documents.
230
+ * @param dbConfig The configuration to use to instantiate the vector store.
231
+ * @param documentProps The properties of the documents to add to the index, specifically the ids.
232
+ * @returns Promise that resolves to the vector store.
233
+ */
234
+ static async fromTexts(texts, metadatas, embeddings, dbConfig, documentProps) {
235
+ if (Array.isArray(metadatas) && texts.length !== metadatas.length) {
236
+ throw new Error(`Number of texts (${texts.length}) does not equal number of metadatas (${metadatas.length})`);
237
+ }
238
+ const docs = [];
239
+ for (let i = 0; i < texts.length; i += 1) {
240
+ // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
241
+ const metadata = Array.isArray(metadatas)
242
+ ? metadatas[i]
243
+ : metadatas;
244
+ const newDoc = new Document({
245
+ pageContent: texts[i],
246
+ metadata,
247
+ });
248
+ docs.push(newDoc);
249
+ }
250
+ return await this.fromDocuments(docs, embeddings, dbConfig, documentProps);
251
+ }
252
+ /**
253
+ * Stores the documents in the index.
254
+ * @param docs The documents to store in the index.
255
+ * @param embeddings The embeddings instance to use to generate embeddings from the documents.
256
+ * @param dbConfig The configuration to use to instantiate the vector store.
257
+ * @param documentProps The properties of the documents to add to the index, specifically the ids.
258
+ * @returns Promise that resolves to the vector store.
259
+ */
260
+ static async fromDocuments(docs, embeddings, dbConfig, documentProps) {
261
+ const vectorStore = new MomentoVectorIndex(embeddings, dbConfig);
262
+ await vectorStore.addDocuments(docs, documentProps);
263
+ return vectorStore;
264
+ }
265
+ }