langchain 0.0.197-rc.1 → 0.0.198

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/dist/chains/openai_moderation.cjs +2 -2
  2. package/dist/chains/openai_moderation.d.ts +1 -1
  3. package/dist/chains/openai_moderation.js +1 -1
  4. package/dist/chat_models/anthropic.cjs +351 -15
  5. package/dist/chat_models/anthropic.d.ts +157 -1
  6. package/dist/chat_models/anthropic.js +348 -1
  7. package/dist/chat_models/cloudflare_workersai.cjs +5 -0
  8. package/dist/chat_models/cloudflare_workersai.d.ts +3 -0
  9. package/dist/chat_models/cloudflare_workersai.js +5 -0
  10. package/dist/chat_models/fireworks.d.ts +1 -1
  11. package/dist/chat_models/iflytek_xinghuo/common.d.ts +1 -1
  12. package/dist/chat_models/llama_cpp.cjs +24 -0
  13. package/dist/chat_models/llama_cpp.d.ts +3 -1
  14. package/dist/chat_models/llama_cpp.js +24 -0
  15. package/dist/chat_models/minimax.d.ts +1 -1
  16. package/dist/chat_models/openai.cjs +698 -4
  17. package/dist/chat_models/openai.d.ts +137 -4
  18. package/dist/chat_models/openai.js +695 -2
  19. package/dist/document_loaders/fs/openai_whisper_audio.cjs +2 -2
  20. package/dist/document_loaders/fs/openai_whisper_audio.d.ts +1 -1
  21. package/dist/document_loaders/fs/openai_whisper_audio.js +1 -1
  22. package/dist/document_loaders/fs/pptx.cjs +39 -0
  23. package/dist/document_loaders/fs/pptx.d.ts +23 -0
  24. package/dist/document_loaders/fs/pptx.js +35 -0
  25. package/dist/embeddings/openai.cjs +240 -2
  26. package/dist/embeddings/openai.d.ts +82 -1
  27. package/dist/embeddings/openai.js +239 -1
  28. package/dist/experimental/openai_assistant/index.cjs +35 -3
  29. package/dist/experimental/openai_assistant/index.d.ts +27 -1
  30. package/dist/experimental/openai_assistant/index.js +33 -1
  31. package/dist/experimental/openai_assistant/schema.d.ts +1 -1
  32. package/dist/experimental/openai_files/index.cjs +2 -2
  33. package/dist/experimental/openai_files/index.d.ts +1 -1
  34. package/dist/experimental/openai_files/index.js +1 -1
  35. package/dist/experimental/tools/pyinterpreter.cjs +248 -0
  36. package/dist/experimental/tools/pyinterpreter.d.ts +18 -0
  37. package/dist/experimental/tools/pyinterpreter.js +244 -0
  38. package/dist/graphs/neo4j_graph.cjs +49 -14
  39. package/dist/graphs/neo4j_graph.d.ts +30 -0
  40. package/dist/graphs/neo4j_graph.js +49 -14
  41. package/dist/llms/fireworks.d.ts +1 -1
  42. package/dist/llms/hf.cjs +13 -2
  43. package/dist/llms/hf.d.ts +5 -0
  44. package/dist/llms/hf.js +13 -2
  45. package/dist/llms/llama_cpp.cjs +17 -3
  46. package/dist/llms/llama_cpp.d.ts +4 -1
  47. package/dist/llms/llama_cpp.js +17 -3
  48. package/dist/llms/openai-chat.cjs +445 -3
  49. package/dist/llms/openai-chat.d.ts +123 -4
  50. package/dist/llms/openai-chat.js +443 -2
  51. package/dist/llms/openai.cjs +530 -6
  52. package/dist/llms/openai.d.ts +123 -4
  53. package/dist/llms/openai.js +525 -2
  54. package/dist/load/import_constants.cjs +3 -0
  55. package/dist/load/import_constants.js +3 -0
  56. package/dist/output_parsers/json.cjs +4 -0
  57. package/dist/output_parsers/json.js +4 -0
  58. package/dist/schema/index.d.ts +1 -1
  59. package/dist/tools/convert_to_openai.cjs +38 -4
  60. package/dist/tools/convert_to_openai.d.ts +11 -1
  61. package/dist/tools/convert_to_openai.js +35 -1
  62. package/dist/types/openai-types.d.ts +133 -1
  63. package/dist/util/env.cjs +9 -70
  64. package/dist/util/env.d.ts +1 -21
  65. package/dist/util/env.js +1 -62
  66. package/dist/util/openai-format-fndef.cjs +81 -0
  67. package/dist/util/openai-format-fndef.d.ts +44 -0
  68. package/dist/util/openai-format-fndef.js +77 -0
  69. package/dist/util/openai.cjs +18 -2
  70. package/dist/util/openai.d.ts +1 -1
  71. package/dist/util/openai.js +17 -1
  72. package/dist/util/openapi.d.ts +2 -2
  73. package/dist/util/prompt-layer.d.ts +1 -1
  74. package/dist/vectorstores/clickhouse.cjs +286 -0
  75. package/dist/vectorstores/clickhouse.d.ts +126 -0
  76. package/dist/vectorstores/clickhouse.js +259 -0
  77. package/dist/vectorstores/pgvector.cjs +142 -18
  78. package/dist/vectorstores/pgvector.d.ts +21 -0
  79. package/dist/vectorstores/pgvector.js +142 -18
  80. package/dist/vectorstores/weaviate.cjs +45 -2
  81. package/dist/vectorstores/weaviate.d.ts +27 -1
  82. package/dist/vectorstores/weaviate.js +45 -2
  83. package/document_loaders/fs/pptx.cjs +1 -0
  84. package/document_loaders/fs/pptx.d.ts +1 -0
  85. package/document_loaders/fs/pptx.js +1 -0
  86. package/experimental/tools/pyinterpreter.cjs +1 -0
  87. package/experimental/tools/pyinterpreter.d.ts +1 -0
  88. package/experimental/tools/pyinterpreter.js +1 -0
  89. package/package.json +41 -9
  90. package/vectorstores/clickhouse.cjs +1 -0
  91. package/vectorstores/clickhouse.d.ts +1 -0
  92. package/vectorstores/clickhouse.js +1 -0
@@ -26,6 +26,24 @@ class PGVectorStore extends base_js_1.VectorStore {
26
26
  writable: true,
27
27
  value: void 0
28
28
  });
29
+ Object.defineProperty(this, "collectionTableName", {
30
+ enumerable: true,
31
+ configurable: true,
32
+ writable: true,
33
+ value: void 0
34
+ });
35
+ Object.defineProperty(this, "collectionName", {
36
+ enumerable: true,
37
+ configurable: true,
38
+ writable: true,
39
+ value: "langchain"
40
+ });
41
+ Object.defineProperty(this, "collectionMetadata", {
42
+ enumerable: true,
43
+ configurable: true,
44
+ writable: true,
45
+ value: void 0
46
+ });
29
47
  Object.defineProperty(this, "idColumnName", {
30
48
  enumerable: true,
31
49
  configurable: true,
@@ -81,6 +99,9 @@ class PGVectorStore extends base_js_1.VectorStore {
81
99
  value: 500
82
100
  });
83
101
  this.tableName = config.tableName;
102
+ this.collectionTableName = config.collectionTableName;
103
+ this.collectionName = config.collectionName ?? "langchain";
104
+ this.collectionMetadata = config.collectionMetadata ?? null;
84
105
  this.filter = config.filter;
85
106
  this.vectorColumnName = config.columns?.vectorColumnName ?? "embedding";
86
107
  this.contentColumnName = config.columns?.contentColumnName ?? "text";
@@ -106,6 +127,9 @@ class PGVectorStore extends base_js_1.VectorStore {
106
127
  const postgresqlVectorStore = new PGVectorStore(embeddings, config);
107
128
  await postgresqlVectorStore._initializeClient();
108
129
  await postgresqlVectorStore.ensureTableInDatabase();
130
+ if (postgresqlVectorStore.collectionTableName) {
131
+ await postgresqlVectorStore.ensureCollectionTableInDatabase();
132
+ }
109
133
  return postgresqlVectorStore;
110
134
  }
111
135
  async _initializeClient() {
@@ -122,15 +146,56 @@ class PGVectorStore extends base_js_1.VectorStore {
122
146
  const texts = documents.map(({ pageContent }) => pageContent);
123
147
  return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
124
148
  }
149
+ /**
150
+ * Inserts a row for the collectionName provided at initialization if it does not
151
+ * exist and returns the collectionId.
152
+ *
153
+ * @returns The collectionId for the given collectionName.
154
+ */
155
+ async getOrCreateCollection() {
156
+ const queryString = `
157
+ SELECT uuid from ${this.collectionTableName}
158
+ WHERE name = $1;
159
+ `;
160
+ const queryResult = await this.pool.query(queryString, [
161
+ this.collectionName,
162
+ ]);
163
+ let collectionId = queryResult.rows[0]?.uuid;
164
+ if (!collectionId) {
165
+ const insertString = `
166
+ INSERT INTO ${this.collectionTableName}(
167
+ uuid,
168
+ name,
169
+ cmetadata
170
+ )
171
+ VALUES (
172
+ uuid_generate_v4(),
173
+ $1,
174
+ $2
175
+ )
176
+ RETURNING uuid;
177
+ `;
178
+ const insertResult = await this.pool.query(insertString, [
179
+ this.collectionName,
180
+ this.collectionMetadata,
181
+ ]);
182
+ collectionId = insertResult.rows[0]?.uuid;
183
+ }
184
+ return collectionId;
185
+ }
125
186
  /**
126
187
  * Generates the SQL placeholders for a specific row at the provided index.
127
188
  *
128
189
  * @param index - The index of the row for which placeholders need to be generated.
190
+ * @param numOfColumns - The number of columns we are inserting data into.
129
191
  * @returns The SQL placeholders for the row values.
130
192
  */
131
- generatePlaceholderForRowAt(index) {
132
- const base = index * 3;
133
- return `($${base + 1}, $${base + 2}, $${base + 3})`;
193
+ generatePlaceholderForRowAt(index, numOfColumns) {
194
+ const placeholders = [];
195
+ for (let i = 0; i < numOfColumns; i += 1) {
196
+ placeholders.push(`$${index * numOfColumns + i + 1}`);
197
+ }
198
+ return `(${placeholders.join(", ")})`;
134
199
  }
135
200
  /**
136
201
  * Constructs the SQL query for inserting rows into the specified table.
@@ -139,15 +204,25 @@ class PGVectorStore extends base_js_1.VectorStore {
139
204
  * @param chunkIndex - The starting index for generating query placeholders based on chunk positioning.
140
205
  * @returns The complete SQL INSERT INTO query string.
141
206
  */
142
- buildInsertQuery(rows) {
207
+ async buildInsertQuery(rows) {
208
+ let collectionId;
209
+ if (this.collectionTableName) {
210
+ collectionId = await this.getOrCreateCollection();
211
+ }
212
+ const columns = [
213
+ this.contentColumnName,
214
+ this.vectorColumnName,
215
+ this.metadataColumnName,
216
+ ];
217
+ if (collectionId) {
218
+ columns.push("collection_id");
219
+ }
143
220
  const valuesPlaceholders = rows
144
- .map((_, j) => this.generatePlaceholderForRowAt(j))
221
+ .map((_, j) => this.generatePlaceholderForRowAt(j, columns.length))
145
222
  .join(", ");
146
223
  const text = `
147
224
  INSERT INTO ${this.tableName}(
148
- ${this.contentColumnName},
149
- ${this.vectorColumnName},
150
- ${this.metadataColumnName}
225
+ ${columns}
151
226
  )
152
227
  VALUES ${valuesPlaceholders}
153
228
  `;
@@ -162,17 +237,24 @@ class PGVectorStore extends base_js_1.VectorStore {
162
237
  * @returns Promise that resolves when the vectors have been added.
163
238
  */
164
239
  async addVectors(vectors, documents) {
165
- const rows = vectors.map((embedding, idx) => {
240
+ const rows = [];
241
+ let collectionId;
242
+ if (this.collectionTableName) {
243
+ collectionId = await this.getOrCreateCollection();
244
+ }
245
+ for (let i = 0; i < vectors.length; i += 1) {
246
+ const values = [];
247
+ const embedding = vectors[i];
166
248
  const embeddingString = `[${embedding.join(",")}]`;
167
- return [
168
- documents[idx].pageContent,
169
- embeddingString,
170
- documents[idx].metadata,
171
- ];
172
- });
249
+ values.push(documents[i].pageContent, embeddingString, documents[i].metadata);
250
+ if (collectionId) {
251
+ values.push(collectionId);
252
+ }
253
+ rows.push(values);
254
+ }
173
255
  for (let i = 0; i < rows.length; i += this.chunkSize) {
174
256
  const chunk = rows.slice(i, i + this.chunkSize);
175
- const insertQuery = this.buildInsertQuery(chunk);
257
+ const insertQuery = await this.buildInsertQuery(chunk);
176
258
  const flatValues = chunk.flat();
177
259
  try {
178
260
  await this.pool.query(insertQuery, flatValues);
@@ -196,13 +278,23 @@ class PGVectorStore extends base_js_1.VectorStore {
196
278
  async similaritySearchVectorWithScore(query, k, filter) {
197
279
  const embeddingString = `[${query.join(",")}]`;
198
280
  const _filter = filter ?? "{}";
281
+ let collectionId;
282
+ if (this.collectionTableName) {
283
+ collectionId = await this.getOrCreateCollection();
284
+ }
285
+ const parameters = [embeddingString, _filter, k];
286
+ if (collectionId) {
287
+ parameters.push(collectionId);
288
+ }
199
289
  const queryString = `
200
290
  SELECT *, ${this.vectorColumnName} <=> $1 as "_distance"
201
291
  FROM ${this.tableName}
202
292
  WHERE ${this.metadataColumnName}::jsonb @> $2
293
+ ${collectionId ? "AND collection_id = $4" : ""}
203
294
  ORDER BY "_distance" ASC
204
- LIMIT $3;`;
205
- const documents = (await this.pool.query(queryString, [embeddingString, _filter, k])).rows;
295
+ LIMIT $3;
296
+ `;
297
+ const documents = (await this.pool.query(queryString, parameters)).rows;
206
298
  const results = [];
207
299
  for (const doc of documents) {
208
300
  if (doc._distance != null && doc[this.contentColumnName] != null) {
@@ -233,6 +325,38 @@ class PGVectorStore extends base_js_1.VectorStore {
233
325
  );
234
326
  `);
235
327
  }
328
+ /**
329
+ * Method to ensure the existence of the collection table in the database.
330
+ * It creates the table if it does not already exist.
331
+ *
332
+ * @returns Promise that resolves when the collection table has been ensured.
333
+ */
334
+ async ensureCollectionTableInDatabase() {
335
+ try {
336
+ await this.pool.query(`
337
+ CREATE TABLE IF NOT EXISTS ${this.collectionTableName} (
338
+ uuid uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY,
339
+ name character varying,
340
+ cmetadata jsonb
341
+ );
342
+
343
+ ALTER TABLE ${this.tableName}
344
+ ADD COLUMN collection_id uuid;
345
+
346
+ ALTER TABLE ${this.tableName}
347
+ ADD CONSTRAINT ${this.tableName}_collection_id_fkey
348
+ FOREIGN KEY (collection_id)
349
+ REFERENCES ${this.collectionTableName}(uuid)
350
+ ON DELETE CASCADE;
351
+ `);
352
+ }
353
+ catch (e) {
354
+ if (!e.message.includes("already exists")) {
355
+ console.error(e);
356
+ throw new Error(`Error adding column: ${e.message}`);
357
+ }
358
+ }
359
+ }
236
360
  /**
237
361
  * Static method to create a new `PGVectorStore` instance from an
238
362
  * array of texts and their metadata. It converts the texts into
@@ -11,6 +11,9 @@ type Metadata = Record<string, unknown>;
11
11
  export interface PGVectorStoreArgs {
12
12
  postgresConnectionOptions: PoolConfig;
13
13
  tableName: string;
14
+ collectionTableName?: string;
15
+ collectionName?: string;
16
+ collectionMetadata?: Metadata | null;
14
17
  columns?: {
15
18
  idColumnName?: string;
16
19
  vectorColumnName?: string;
@@ -35,6 +38,9 @@ export interface PGVectorStoreArgs {
35
38
  export declare class PGVectorStore extends VectorStore {
36
39
  FilterType: Metadata;
37
40
  tableName: string;
41
+ collectionTableName?: string;
42
+ collectionName: string;
43
+ collectionMetadata: Metadata | null;
38
44
  idColumnName: string;
39
45
  vectorColumnName: string;
40
46
  contentColumnName: string;
@@ -65,10 +71,18 @@ export declare class PGVectorStore extends VectorStore {
65
71
  * @returns Promise that resolves when the documents have been added.
66
72
  */
67
73
  addDocuments(documents: Document[]): Promise<void>;
74
+ /**
75
+ * Inserts a row for the collectionName provided at initialization if it does not
76
+ * exist and returns the collectionId.
77
+ *
78
+ * @returns The collectionId for the given collectionName.
79
+ */
80
+ getOrCreateCollection(): Promise<string>;
68
81
  /**
69
82
  * Generates the SQL placeholders for a specific row at the provided index.
70
83
  *
71
84
  * @param index - The index of the row for which placeholders need to be generated.
85
+ * @param numOfColumns - The number of columns we are inserting data into.
72
86
  * @returns The SQL placeholders for the row values.
73
87
  */
74
88
  private generatePlaceholderForRowAt;
@@ -107,6 +121,13 @@ export declare class PGVectorStore extends VectorStore {
107
121
  * @returns Promise that resolves when the table has been ensured.
108
122
  */
109
123
  ensureTableInDatabase(): Promise<void>;
124
+ /**
125
+ * Method to ensure the existence of the collection table in the database.
126
+ * It creates the table if it does not already exist.
127
+ *
128
+ * @returns Promise that resolves when the collection table has been ensured.
129
+ */
130
+ ensureCollectionTableInDatabase(): Promise<void>;
110
131
  /**
111
132
  * Static method to create a new `PGVectorStore` instance from an
112
133
  * array of texts and their metadata. It converts the texts into
@@ -20,6 +20,24 @@ export class PGVectorStore extends VectorStore {
20
20
  writable: true,
21
21
  value: void 0
22
22
  });
23
+ Object.defineProperty(this, "collectionTableName", {
24
+ enumerable: true,
25
+ configurable: true,
26
+ writable: true,
27
+ value: void 0
28
+ });
29
+ Object.defineProperty(this, "collectionName", {
30
+ enumerable: true,
31
+ configurable: true,
32
+ writable: true,
33
+ value: "langchain"
34
+ });
35
+ Object.defineProperty(this, "collectionMetadata", {
36
+ enumerable: true,
37
+ configurable: true,
38
+ writable: true,
39
+ value: void 0
40
+ });
23
41
  Object.defineProperty(this, "idColumnName", {
24
42
  enumerable: true,
25
43
  configurable: true,
@@ -75,6 +93,9 @@ export class PGVectorStore extends VectorStore {
75
93
  value: 500
76
94
  });
77
95
  this.tableName = config.tableName;
96
+ this.collectionTableName = config.collectionTableName;
97
+ this.collectionName = config.collectionName ?? "langchain";
98
+ this.collectionMetadata = config.collectionMetadata ?? null;
78
99
  this.filter = config.filter;
79
100
  this.vectorColumnName = config.columns?.vectorColumnName ?? "embedding";
80
101
  this.contentColumnName = config.columns?.contentColumnName ?? "text";
@@ -100,6 +121,9 @@ export class PGVectorStore extends VectorStore {
100
121
  const postgresqlVectorStore = new PGVectorStore(embeddings, config);
101
122
  await postgresqlVectorStore._initializeClient();
102
123
  await postgresqlVectorStore.ensureTableInDatabase();
124
+ if (postgresqlVectorStore.collectionTableName) {
125
+ await postgresqlVectorStore.ensureCollectionTableInDatabase();
126
+ }
103
127
  return postgresqlVectorStore;
104
128
  }
105
129
  async _initializeClient() {
@@ -116,15 +140,56 @@ export class PGVectorStore extends VectorStore {
116
140
  const texts = documents.map(({ pageContent }) => pageContent);
117
141
  return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
118
142
  }
143
+ /**
144
+ * Inserts a row for the collectionName provided at initialization if it does not
145
+ * exist and returns the collectionId.
146
+ *
147
+ * @returns The collectionId for the given collectionName.
148
+ */
149
+ async getOrCreateCollection() {
150
+ const queryString = `
151
+ SELECT uuid from ${this.collectionTableName}
152
+ WHERE name = $1;
153
+ `;
154
+ const queryResult = await this.pool.query(queryString, [
155
+ this.collectionName,
156
+ ]);
157
+ let collectionId = queryResult.rows[0]?.uuid;
158
+ if (!collectionId) {
159
+ const insertString = `
160
+ INSERT INTO ${this.collectionTableName}(
161
+ uuid,
162
+ name,
163
+ cmetadata
164
+ )
165
+ VALUES (
166
+ uuid_generate_v4(),
167
+ $1,
168
+ $2
169
+ )
170
+ RETURNING uuid;
171
+ `;
172
+ const insertResult = await this.pool.query(insertString, [
173
+ this.collectionName,
174
+ this.collectionMetadata,
175
+ ]);
176
+ collectionId = insertResult.rows[0]?.uuid;
177
+ }
178
+ return collectionId;
179
+ }
119
180
  /**
120
181
  * Generates the SQL placeholders for a specific row at the provided index.
121
182
  *
122
183
  * @param index - The index of the row for which placeholders need to be generated.
184
+ * @param numOfColumns - The number of columns we are inserting data into.
123
185
  * @returns The SQL placeholders for the row values.
124
186
  */
125
- generatePlaceholderForRowAt(index) {
126
- const base = index * 3;
127
- return `($${base + 1}, $${base + 2}, $${base + 3})`;
187
+ generatePlaceholderForRowAt(index, numOfColumns) {
188
+ const placeholders = [];
189
+ for (let i = 0; i < numOfColumns; i += 1) {
190
+ placeholders.push(`$${index * numOfColumns + i + 1}`);
191
+ }
192
+ return `(${placeholders.join(", ")})`;
128
193
  }
129
194
  /**
130
195
  * Constructs the SQL query for inserting rows into the specified table.
@@ -133,15 +198,25 @@ export class PGVectorStore extends VectorStore {
133
198
  * @param chunkIndex - The starting index for generating query placeholders based on chunk positioning.
134
199
  * @returns The complete SQL INSERT INTO query string.
135
200
  */
136
- buildInsertQuery(rows) {
201
+ async buildInsertQuery(rows) {
202
+ let collectionId;
203
+ if (this.collectionTableName) {
204
+ collectionId = await this.getOrCreateCollection();
205
+ }
206
+ const columns = [
207
+ this.contentColumnName,
208
+ this.vectorColumnName,
209
+ this.metadataColumnName,
210
+ ];
211
+ if (collectionId) {
212
+ columns.push("collection_id");
213
+ }
137
214
  const valuesPlaceholders = rows
138
- .map((_, j) => this.generatePlaceholderForRowAt(j))
215
+ .map((_, j) => this.generatePlaceholderForRowAt(j, columns.length))
139
216
  .join(", ");
140
217
  const text = `
141
218
  INSERT INTO ${this.tableName}(
142
- ${this.contentColumnName},
143
- ${this.vectorColumnName},
144
- ${this.metadataColumnName}
219
+ ${columns}
145
220
  )
146
221
  VALUES ${valuesPlaceholders}
147
222
  `;
@@ -156,17 +231,24 @@ export class PGVectorStore extends VectorStore {
156
231
  * @returns Promise that resolves when the vectors have been added.
157
232
  */
158
233
  async addVectors(vectors, documents) {
159
- const rows = vectors.map((embedding, idx) => {
234
+ const rows = [];
235
+ let collectionId;
236
+ if (this.collectionTableName) {
237
+ collectionId = await this.getOrCreateCollection();
238
+ }
239
+ for (let i = 0; i < vectors.length; i += 1) {
240
+ const values = [];
241
+ const embedding = vectors[i];
160
242
  const embeddingString = `[${embedding.join(",")}]`;
161
- return [
162
- documents[idx].pageContent,
163
- embeddingString,
164
- documents[idx].metadata,
165
- ];
166
- });
243
+ values.push(documents[i].pageContent, embeddingString, documents[i].metadata);
244
+ if (collectionId) {
245
+ values.push(collectionId);
246
+ }
247
+ rows.push(values);
248
+ }
167
249
  for (let i = 0; i < rows.length; i += this.chunkSize) {
168
250
  const chunk = rows.slice(i, i + this.chunkSize);
169
- const insertQuery = this.buildInsertQuery(chunk);
251
+ const insertQuery = await this.buildInsertQuery(chunk);
170
252
  const flatValues = chunk.flat();
171
253
  try {
172
254
  await this.pool.query(insertQuery, flatValues);
@@ -190,13 +272,23 @@ export class PGVectorStore extends VectorStore {
190
272
  async similaritySearchVectorWithScore(query, k, filter) {
191
273
  const embeddingString = `[${query.join(",")}]`;
192
274
  const _filter = filter ?? "{}";
275
+ let collectionId;
276
+ if (this.collectionTableName) {
277
+ collectionId = await this.getOrCreateCollection();
278
+ }
279
+ const parameters = [embeddingString, _filter, k];
280
+ if (collectionId) {
281
+ parameters.push(collectionId);
282
+ }
193
283
  const queryString = `
194
284
  SELECT *, ${this.vectorColumnName} <=> $1 as "_distance"
195
285
  FROM ${this.tableName}
196
286
  WHERE ${this.metadataColumnName}::jsonb @> $2
287
+ ${collectionId ? "AND collection_id = $4" : ""}
197
288
  ORDER BY "_distance" ASC
198
- LIMIT $3;`;
199
- const documents = (await this.pool.query(queryString, [embeddingString, _filter, k])).rows;
289
+ LIMIT $3;
290
+ `;
291
+ const documents = (await this.pool.query(queryString, parameters)).rows;
200
292
  const results = [];
201
293
  for (const doc of documents) {
202
294
  if (doc._distance != null && doc[this.contentColumnName] != null) {
@@ -227,6 +319,38 @@ export class PGVectorStore extends VectorStore {
227
319
  );
228
320
  `);
229
321
  }
322
+ /**
323
+ * Method to ensure the existence of the collection table in the database.
324
+ * It creates the table if it does not already exist.
325
+ *
326
+ * @returns Promise that resolves when the collection table has been ensured.
327
+ */
328
+ async ensureCollectionTableInDatabase() {
329
+ try {
330
+ await this.pool.query(`
331
+ CREATE TABLE IF NOT EXISTS ${this.collectionTableName} (
332
+ uuid uuid NOT NULL DEFAULT uuid_generate_v4() PRIMARY KEY,
333
+ name character varying,
334
+ cmetadata jsonb
335
+ );
336
+
337
+ ALTER TABLE ${this.tableName}
338
+ ADD COLUMN collection_id uuid;
339
+
340
+ ALTER TABLE ${this.tableName}
341
+ ADD CONSTRAINT ${this.tableName}_collection_id_fkey
342
+ FOREIGN KEY (collection_id)
343
+ REFERENCES ${this.collectionTableName}(uuid)
344
+ ON DELETE CASCADE;
345
+ `);
346
+ }
347
+ catch (e) {
348
+ if (!e.message.includes("already exists")) {
349
+ console.error(e);
350
+ throw new Error(`Error adding column: ${e.message}`);
351
+ }
352
+ }
353
+ }
230
354
  /**
231
355
  * Static method to create a new `PGVectorStore` instance from an
232
356
  * array of texts and their metadata. It converts the texts into
@@ -27,6 +27,7 @@ exports.WeaviateStore = exports.flattenObjectForWeaviate = void 0;
27
27
  const uuid = __importStar(require("uuid"));
28
28
  const base_js_1 = require("./base.cjs");
29
29
  const document_js_1 = require("../document.cjs");
30
+ const math_js_1 = require("../util/math.cjs");
30
31
  // Note this function is not generic, it is designed specifically for Weaviate
31
32
  // https://weaviate.io/developers/weaviate/config-refs/datatypes#introduction
32
33
  const flattenObjectForWeaviate = (
@@ -235,11 +236,27 @@ class WeaviateStore extends base_js_1.VectorStore {
235
236
  * @returns An array of tuples, where each tuple contains a document and its similarity score.
236
237
  */
237
238
  async similaritySearchVectorWithScore(query, k, filter) {
239
+ const resultsWithEmbedding = await this.similaritySearchVectorWithScoreAndEmbedding(query, k, filter);
240
+ return resultsWithEmbedding.map(([document, score, _embedding]) => [
241
+ document,
242
+ score,
243
+ ]);
244
+ }
245
+ /**
246
+ * Method to perform a similarity search on the stored vectors in the
247
+ * Weaviate index. It returns the top k most similar documents, their
248
+ * similarity scores and embedding vectors.
249
+ * @param query The query vector.
250
+ * @param k The number of most similar documents to return.
251
+ * @param filter Optional filter to apply to the search.
252
+ * @returns An array of tuples, where each tuple contains a document, its similarity score and its embedding vector.
253
+ */
254
+ async similaritySearchVectorWithScoreAndEmbedding(query, k, filter) {
238
255
  try {
239
- let builder = await this.client.graphql
256
+ let builder = this.client.graphql
240
257
  .get()
241
258
  .withClassName(this.indexName)
242
- .withFields(`${this.queryAttrs.join(" ")} _additional { distance }`)
259
+ .withFields(`${this.queryAttrs.join(" ")} _additional { distance vector }`)
243
260
  .withNearVector({
244
261
  vector: query,
245
262
  distance: filter?.distance,
@@ -261,6 +278,7 @@ class WeaviateStore extends base_js_1.VectorStore {
261
278
  metadata: rest,
262
279
  }),
263
280
  _additional.distance,
281
+ _additional.vector,
264
282
  ]);
265
283
  }
266
284
  return documents;
@@ -269,6 +287,31 @@ class WeaviateStore extends base_js_1.VectorStore {
269
287
  throw Error(`'Error in similaritySearch' ${e}`);
270
288
  }
271
289
  }
290
+ /**
291
+ * Return documents selected using the maximal marginal relevance.
292
+ * Maximal marginal relevance optimizes for similarity to the query AND diversity
293
+ * among selected documents.
294
+ *
295
+ * @param {string} query - Text to look up documents similar to.
296
+ * @param {number} options.k - Number of documents to return.
297
+ * @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
298
+ * @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
299
+ * where 0 corresponds to maximum diversity and 1 to minimum diversity.
300
+ * @param {this["FilterType"]} options.filter - Optional filter
301
+ * @param _callbacks
302
+ *
303
+ * @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
304
+ */
305
+ async maxMarginalRelevanceSearch(query, options, _callbacks) {
306
+ const { k, fetchK = 20, lambda = 0.5, filter } = options;
307
+ const queryEmbedding = await this.embeddings.embedQuery(query);
308
+ const allResults = await this.similaritySearchVectorWithScoreAndEmbedding(queryEmbedding, fetchK, filter);
309
+ const embeddingList = allResults.map(([_doc, _score, embedding]) => embedding);
310
+ const mmrIndexes = (0, math_js_1.maximalMarginalRelevance)(queryEmbedding, embeddingList, lambda, k);
311
+ return mmrIndexes
312
+ .filter((idx) => idx !== -1)
313
+ .map((idx) => allResults[idx][0]);
314
+ }
272
315
  /**
273
316
  * Static method to create a new `WeaviateStore` instance from a list of
274
317
  * texts. It first creates documents from the texts and metadata, then
@@ -1,5 +1,5 @@
1
1
  import type { WeaviateClient, WhereFilter } from "weaviate-ts-client";
2
- import { VectorStore } from "./base.js";
2
+ import { MaxMarginalRelevanceSearchOptions, VectorStore } from "./base.js";
3
3
  import { Embeddings } from "../embeddings/base.js";
4
4
  import { Document } from "../document.js";
5
5
  export declare const flattenObjectForWeaviate: (obj: Record<string, any>) => Record<string, any>;
@@ -83,6 +83,32 @@ export declare class WeaviateStore extends VectorStore {
83
83
  * @returns An array of tuples, where each tuple contains a document and its similarity score.
84
84
  */
85
85
  similaritySearchVectorWithScore(query: number[], k: number, filter?: WeaviateFilter): Promise<[Document, number][]>;
86
+ /**
87
+ * Method to perform a similarity search on the stored vectors in the
88
+ * Weaviate index. It returns the top k most similar documents, their
89
+ * similarity scores and embedding vectors.
90
+ * @param query The query vector.
91
+ * @param k The number of most similar documents to return.
92
+ * @param filter Optional filter to apply to the search.
93
+ * @returns An array of tuples, where each tuple contains a document, its similarity score and its embedding vector.
94
+ */
95
+ similaritySearchVectorWithScoreAndEmbedding(query: number[], k: number, filter?: WeaviateFilter): Promise<[Document, number, number[]][]>;
96
+ /**
97
+ * Return documents selected using the maximal marginal relevance.
98
+ * Maximal marginal relevance optimizes for similarity to the query AND diversity
99
+ * among selected documents.
100
+ *
101
+ * @param {string} query - Text to look up documents similar to.
102
+ * @param {number} options.k - Number of documents to return.
103
+ * @param {number} options.fetchK - Number of documents to fetch before passing to the MMR algorithm.
104
+ * @param {number} options.lambda - Number between 0 and 1 that determines the degree of diversity among the results,
105
+ * where 0 corresponds to maximum diversity and 1 to minimum diversity.
106
+ * @param {this["FilterType"]} options.filter - Optional filter
107
+ * @param _callbacks
108
+ *
109
+ * @returns {Promise<Document[]>} - List of documents selected by maximal marginal relevance.
110
+ */
111
+ maxMarginalRelevanceSearch(query: string, options: MaxMarginalRelevanceSearchOptions<this["FilterType"]>, _callbacks: undefined): Promise<Document[]>;
86
112
  /**
87
113
  * Static method to create a new `WeaviateStore` instance from a list of
88
114
  * texts. It first creates documents from the texts and metadata, then