langchain 0.0.197-rc.1 → 0.0.198

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/dist/chains/openai_moderation.cjs +2 -2
  2. package/dist/chains/openai_moderation.d.ts +1 -1
  3. package/dist/chains/openai_moderation.js +1 -1
  4. package/dist/chat_models/anthropic.cjs +351 -15
  5. package/dist/chat_models/anthropic.d.ts +157 -1
  6. package/dist/chat_models/anthropic.js +348 -1
  7. package/dist/chat_models/cloudflare_workersai.cjs +5 -0
  8. package/dist/chat_models/cloudflare_workersai.d.ts +3 -0
  9. package/dist/chat_models/cloudflare_workersai.js +5 -0
  10. package/dist/chat_models/fireworks.d.ts +1 -1
  11. package/dist/chat_models/iflytek_xinghuo/common.d.ts +1 -1
  12. package/dist/chat_models/llama_cpp.cjs +24 -0
  13. package/dist/chat_models/llama_cpp.d.ts +3 -1
  14. package/dist/chat_models/llama_cpp.js +24 -0
  15. package/dist/chat_models/minimax.d.ts +1 -1
  16. package/dist/chat_models/openai.cjs +698 -4
  17. package/dist/chat_models/openai.d.ts +137 -4
  18. package/dist/chat_models/openai.js +695 -2
  19. package/dist/document_loaders/fs/openai_whisper_audio.cjs +2 -2
  20. package/dist/document_loaders/fs/openai_whisper_audio.d.ts +1 -1
  21. package/dist/document_loaders/fs/openai_whisper_audio.js +1 -1
  22. package/dist/document_loaders/fs/pptx.cjs +39 -0
  23. package/dist/document_loaders/fs/pptx.d.ts +23 -0
  24. package/dist/document_loaders/fs/pptx.js +35 -0
  25. package/dist/embeddings/openai.cjs +240 -2
  26. package/dist/embeddings/openai.d.ts +82 -1
  27. package/dist/embeddings/openai.js +239 -1
  28. package/dist/experimental/openai_assistant/index.cjs +35 -3
  29. package/dist/experimental/openai_assistant/index.d.ts +27 -1
  30. package/dist/experimental/openai_assistant/index.js +33 -1
  31. package/dist/experimental/openai_assistant/schema.d.ts +1 -1
  32. package/dist/experimental/openai_files/index.cjs +2 -2
  33. package/dist/experimental/openai_files/index.d.ts +1 -1
  34. package/dist/experimental/openai_files/index.js +1 -1
  35. package/dist/experimental/tools/pyinterpreter.cjs +248 -0
  36. package/dist/experimental/tools/pyinterpreter.d.ts +18 -0
  37. package/dist/experimental/tools/pyinterpreter.js +244 -0
  38. package/dist/graphs/neo4j_graph.cjs +49 -14
  39. package/dist/graphs/neo4j_graph.d.ts +30 -0
  40. package/dist/graphs/neo4j_graph.js +49 -14
  41. package/dist/llms/fireworks.d.ts +1 -1
  42. package/dist/llms/hf.cjs +13 -2
  43. package/dist/llms/hf.d.ts +5 -0
  44. package/dist/llms/hf.js +13 -2
  45. package/dist/llms/llama_cpp.cjs +17 -3
  46. package/dist/llms/llama_cpp.d.ts +4 -1
  47. package/dist/llms/llama_cpp.js +17 -3
  48. package/dist/llms/openai-chat.cjs +445 -3
  49. package/dist/llms/openai-chat.d.ts +123 -4
  50. package/dist/llms/openai-chat.js +443 -2
  51. package/dist/llms/openai.cjs +530 -6
  52. package/dist/llms/openai.d.ts +123 -4
  53. package/dist/llms/openai.js +525 -2
  54. package/dist/load/import_constants.cjs +3 -0
  55. package/dist/load/import_constants.js +3 -0
  56. package/dist/output_parsers/json.cjs +4 -0
  57. package/dist/output_parsers/json.js +4 -0
  58. package/dist/schema/index.d.ts +1 -1
  59. package/dist/tools/convert_to_openai.cjs +38 -4
  60. package/dist/tools/convert_to_openai.d.ts +11 -1
  61. package/dist/tools/convert_to_openai.js +35 -1
  62. package/dist/types/openai-types.d.ts +133 -1
  63. package/dist/util/env.cjs +9 -70
  64. package/dist/util/env.d.ts +1 -21
  65. package/dist/util/env.js +1 -62
  66. package/dist/util/openai-format-fndef.cjs +81 -0
  67. package/dist/util/openai-format-fndef.d.ts +44 -0
  68. package/dist/util/openai-format-fndef.js +77 -0
  69. package/dist/util/openai.cjs +18 -2
  70. package/dist/util/openai.d.ts +1 -1
  71. package/dist/util/openai.js +17 -1
  72. package/dist/util/openapi.d.ts +2 -2
  73. package/dist/util/prompt-layer.d.ts +1 -1
  74. package/dist/vectorstores/clickhouse.cjs +286 -0
  75. package/dist/vectorstores/clickhouse.d.ts +126 -0
  76. package/dist/vectorstores/clickhouse.js +259 -0
  77. package/dist/vectorstores/pgvector.cjs +142 -18
  78. package/dist/vectorstores/pgvector.d.ts +21 -0
  79. package/dist/vectorstores/pgvector.js +142 -18
  80. package/dist/vectorstores/weaviate.cjs +45 -2
  81. package/dist/vectorstores/weaviate.d.ts +27 -1
  82. package/dist/vectorstores/weaviate.js +45 -2
  83. package/document_loaders/fs/pptx.cjs +1 -0
  84. package/document_loaders/fs/pptx.d.ts +1 -0
  85. package/document_loaders/fs/pptx.js +1 -0
  86. package/experimental/tools/pyinterpreter.cjs +1 -0
  87. package/experimental/tools/pyinterpreter.d.ts +1 -0
  88. package/experimental/tools/pyinterpreter.js +1 -0
  89. package/package.json +41 -9
  90. package/vectorstores/clickhouse.cjs +1 -0
  91. package/vectorstores/clickhouse.d.ts +1 -0
  92. package/vectorstores/clickhouse.js +1 -0
@@ -0,0 +1,286 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ Object.defineProperty(exports, "__esModule", { value: true });
26
+ exports.ClickHouseStore = void 0;
27
+ const uuid = __importStar(require("uuid"));
28
+ const client_1 = require("@clickhouse/client");
29
+ const mysql2_1 = require("mysql2");
30
+ const base_js_1 = require("./base.cjs");
31
+ const document_js_1 = require("../document.cjs");
32
+ /**
33
+ * Class for interacting with the ClickHouse database. It extends the
34
+ * VectorStore class and provides methods for adding vectors and
35
+ * documents, searching for similar vectors, and creating instances from
36
+ * texts or documents.
37
+ */
38
+ class ClickHouseStore extends base_js_1.VectorStore {
39
+ _vectorstoreType() {
40
+ return "clickhouse";
41
+ }
42
+ constructor(embeddings, args) {
43
+ super(embeddings, args);
44
+ Object.defineProperty(this, "client", {
45
+ enumerable: true,
46
+ configurable: true,
47
+ writable: true,
48
+ value: void 0
49
+ });
50
+ Object.defineProperty(this, "indexType", {
51
+ enumerable: true,
52
+ configurable: true,
53
+ writable: true,
54
+ value: void 0
55
+ });
56
+ Object.defineProperty(this, "indexParam", {
57
+ enumerable: true,
58
+ configurable: true,
59
+ writable: true,
60
+ value: void 0
61
+ });
62
+ Object.defineProperty(this, "indexQueryParams", {
63
+ enumerable: true,
64
+ configurable: true,
65
+ writable: true,
66
+ value: void 0
67
+ });
68
+ Object.defineProperty(this, "columnMap", {
69
+ enumerable: true,
70
+ configurable: true,
71
+ writable: true,
72
+ value: void 0
73
+ });
74
+ Object.defineProperty(this, "database", {
75
+ enumerable: true,
76
+ configurable: true,
77
+ writable: true,
78
+ value: void 0
79
+ });
80
+ Object.defineProperty(this, "table", {
81
+ enumerable: true,
82
+ configurable: true,
83
+ writable: true,
84
+ value: void 0
85
+ });
86
+ Object.defineProperty(this, "isInitialized", {
87
+ enumerable: true,
88
+ configurable: true,
89
+ writable: true,
90
+ value: false
91
+ });
92
+ this.indexType = args.indexType || "annoy";
93
+ this.indexParam = args.indexParam || { L2Distance: 100 };
94
+ this.indexQueryParams = args.indexQueryParams || {};
95
+ this.columnMap = args.columnMap || {
96
+ id: "id",
97
+ document: "document",
98
+ embedding: "embedding",
99
+ metadata: "metadata",
100
+ uuid: "uuid",
101
+ };
102
+ this.database = args.database || "default";
103
+ this.table = args.table || "vector_table";
104
+ this.client = (0, client_1.createClient)({
105
+ host: `${args.protocol ?? "https://"}${args.host}:${args.port}`,
106
+ username: args.username,
107
+ password: args.password,
108
+ session_id: uuid.v4(),
109
+ });
110
+ }
111
+ /**
112
+ * Method to add vectors to the ClickHouse database.
113
+ * @param vectors The vectors to add.
114
+ * @param documents The documents associated with the vectors.
115
+ * @returns Promise that resolves when the vectors have been added.
116
+ */
117
+ async addVectors(vectors, documents) {
118
+ if (vectors.length === 0) {
119
+ return;
120
+ }
121
+ if (!this.isInitialized) {
122
+ await this.initialize(vectors[0].length);
123
+ }
124
+ const queryStr = this.buildInsertQuery(vectors, documents);
125
+ await this.client.exec({ query: queryStr });
126
+ }
127
+ /**
128
+ * Method to add documents to the ClickHouse database.
129
+ * @param documents The documents to add.
130
+ * @returns Promise that resolves when the documents have been added.
131
+ */
132
+ async addDocuments(documents) {
133
+ return this.addVectors(await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)), documents);
134
+ }
135
+ /**
136
+ * Method to search for vectors that are similar to a given query vector.
137
+ * @param query The query vector.
138
+ * @param k The number of similar vectors to return.
139
+ * @param filter Optional filter for the search results.
140
+ * @returns Promise that resolves with an array of tuples, each containing a Document and a score.
141
+ */
142
+ async similaritySearchVectorWithScore(query, k, filter) {
143
+ if (!this.isInitialized) {
144
+ await this.initialize(query.length);
145
+ }
146
+ const queryStr = this.buildSearchQuery(query, k, filter);
147
+ const queryResultSet = await this.client.query({ query: queryStr });
148
+ const queryResult = await queryResultSet.json();
149
+ const result = queryResult.data.map((item) => [
150
+ new document_js_1.Document({ pageContent: item.document, metadata: item.metadata }),
151
+ item.dist,
152
+ ]);
153
+ return result;
154
+ }
155
+ /**
156
+ * Static method to create an instance of ClickHouseStore from texts.
157
+ * @param texts The texts to use.
158
+ * @param metadatas The metadata associated with the texts.
159
+ * @param embeddings The embeddings to use.
160
+ * @param args The arguments for the ClickHouseStore.
161
+ * @returns Promise that resolves with a new instance of ClickHouseStore.
162
+ */
163
+ static async fromTexts(texts, metadatas, embeddings, args) {
164
+ const docs = [];
165
+ for (let i = 0; i < texts.length; i += 1) {
166
+ const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
167
+ const newDoc = new document_js_1.Document({
168
+ pageContent: texts[i],
169
+ metadata,
170
+ });
171
+ docs.push(newDoc);
172
+ }
173
+ return ClickHouseStore.fromDocuments(docs, embeddings, args);
174
+ }
175
+ /**
176
+ * Static method to create an instance of ClickHouseStore from documents.
177
+ * @param docs The documents to use.
178
+ * @param embeddings The embeddings to use.
179
+ * @param args The arguments for the ClickHouseStore.
180
+ * @returns Promise that resolves with a new instance of ClickHouseStore.
181
+ */
182
+ static async fromDocuments(docs, embeddings, args) {
183
+ const instance = new this(embeddings, args);
184
+ await instance.addDocuments(docs);
185
+ return instance;
186
+ }
187
+ /**
188
+ * Static method to create an instance of ClickHouseStore from an existing
189
+ * index.
190
+ * @param embeddings The embeddings to use.
191
+ * @param args The arguments for the ClickHouseStore.
192
+ * @returns Promise that resolves with a new instance of ClickHouseStore.
193
+ */
194
+ static async fromExistingIndex(embeddings, args) {
195
+ const instance = new this(embeddings, args);
196
+ await instance.initialize();
197
+ return instance;
198
+ }
199
+ /**
200
+ * Method to initialize the ClickHouse database.
201
+ * @param dimension Optional dimension of the vectors.
202
+ * @returns Promise that resolves when the database has been initialized.
203
+ */
204
+ async initialize(dimension) {
205
+ const dim = dimension ?? (await this.embeddings.embedQuery("test")).length;
206
+ const indexParamStr = this.indexParam
207
+ ? Object.entries(this.indexParam)
208
+ .map(([key, value]) => `'${key}', ${value}`)
209
+ .join(", ")
210
+ : "";
211
+ const query = `
212
+ CREATE TABLE IF NOT EXISTS ${this.database}.${this.table}(
213
+ ${this.columnMap.id} Nullable(String),
214
+ ${this.columnMap.document} Nullable(String),
215
+ ${this.columnMap.embedding} Array(Float32),
216
+ ${this.columnMap.metadata} JSON,
217
+ ${this.columnMap.uuid} UUID DEFAULT generateUUIDv4(),
218
+ CONSTRAINT cons_vec_len CHECK length(${this.columnMap.embedding}) = ${dim},
219
+ INDEX vec_idx ${this.columnMap.embedding} TYPE ${this.indexType}(${indexParamStr}) GRANULARITY 1000
220
+ ) ENGINE = MergeTree ORDER BY ${this.columnMap.uuid} SETTINGS index_granularity = 8192;`;
221
+ await this.client.exec({
222
+ query,
223
+ clickhouse_settings: {
224
+ allow_experimental_object_type: 1,
225
+ allow_experimental_annoy_index: 1,
226
+ },
227
+ });
228
+ this.isInitialized = true;
229
+ }
230
+ /**
231
+ * Method to build an SQL query for inserting vectors and documents into
232
+ * the ClickHouse database.
233
+ * @param vectors The vectors to insert.
234
+ * @param documents The documents to insert.
235
+ * @returns The SQL query string.
236
+ */
237
+ buildInsertQuery(vectors, documents) {
238
+ const columnsStr = Object.values(Object.fromEntries(Object.entries(this.columnMap).filter(([key]) => key !== this.columnMap.uuid))).join(", ");
239
+ const placeholders = vectors.map(() => "(?, ?, ?, ?)").join(", ");
240
+ const values = [];
241
+ for (let i = 0; i < vectors.length; i += 1) {
242
+ const vector = vectors[i];
243
+ const document = documents[i];
244
+ values.push(uuid.v4(), this.escapeString(document.pageContent), JSON.stringify(vector), JSON.stringify(document.metadata));
245
+ }
246
+ const insertQueryStr = `
247
+ INSERT INTO TABLE ${this.database}.${this.table}(${columnsStr})
248
+ VALUES ${placeholders}
249
+ `;
250
+ const insertQuery = (0, mysql2_1.format)(insertQueryStr, values);
251
+ return insertQuery;
252
+ }
253
+ escapeString(str) {
254
+ return str.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
255
+ }
256
+ /**
257
+ * Method to build an SQL query for searching for similar vectors in the
258
+ * ClickHouse database.
259
+ * @param query The query vector.
260
+ * @param k The number of similar vectors to return.
261
+ * @param filter Optional filter for the search results.
262
+ * @returns The SQL query string.
263
+ */
264
+ buildSearchQuery(query, k, filter) {
265
+ const order = "ASC";
266
+ const whereStr = filter ? `PREWHERE ${filter.whereStr}` : "";
267
+ const placeholders = query.map(() => "?").join(", ");
268
+ const settingStrings = [];
269
+ if (this.indexQueryParams) {
270
+ for (const [key, value] of Object.entries(this.indexQueryParams)) {
271
+ settingStrings.push(`SETTING ${key}=${value}`);
272
+ }
273
+ }
274
+ const searchQueryStr = `
275
+ SELECT ${this.columnMap.document} AS document, ${this.columnMap.metadata} AS metadata, dist
276
+ FROM ${this.database}.${this.table}
277
+ ${whereStr}
278
+ ORDER BY L2Distance(${this.columnMap.embedding}, [${placeholders}]) AS dist ${order}
279
+ LIMIT ${k} ${settingStrings.join(" ")}
280
+ `;
281
+ // Format the query with actual values
282
+ const searchQuery = (0, mysql2_1.format)(searchQueryStr, query);
283
+ return searchQuery;
284
+ }
285
+ }
286
+ exports.ClickHouseStore = ClickHouseStore;
@@ -0,0 +1,126 @@
1
+ import { Embeddings } from "../embeddings/base.js";
2
+ import { VectorStore } from "./base.js";
3
+ import { Document } from "../document.js";
4
+ /**
5
+ * Arguments for the ClickHouseStore class, which include the host, port,
6
+ * protocol, username, password, index type, index parameters,
7
+ * index query params, column map, database, table.
8
+ */
9
+ export interface ClickHouseLibArgs {
10
+ host: string;
11
+ port: string | number;
12
+ protocol?: string;
13
+ username: string;
14
+ password: string;
15
+ indexType?: string;
16
+ indexParam?: Record<string, number>;
17
+ indexQueryParams?: Record<string, string>;
18
+ columnMap?: ColumnMap;
19
+ database?: string;
20
+ table?: string;
21
+ }
22
+ /**
23
+ * Mapping of columns in the ClickHouse database.
24
+ */
25
+ export interface ColumnMap {
26
+ id: string;
27
+ uuid: string;
28
+ document: string;
29
+ embedding: string;
30
+ metadata: string;
31
+ }
32
+ /**
33
+ * Type for filtering search results in the ClickHouse database.
34
+ */
35
+ export interface ClickHouseFilter {
36
+ whereStr: string;
37
+ }
38
+ /**
39
+ * Class for interacting with the ClickHouse database. It extends the
40
+ * VectorStore class and provides methods for adding vectors and
41
+ * documents, searching for similar vectors, and creating instances from
42
+ * texts or documents.
43
+ */
44
+ export declare class ClickHouseStore extends VectorStore {
45
+ FilterType: ClickHouseFilter;
46
+ private client;
47
+ private indexType;
48
+ private indexParam;
49
+ private indexQueryParams;
50
+ private columnMap;
51
+ private database;
52
+ private table;
53
+ private isInitialized;
54
+ _vectorstoreType(): string;
55
+ constructor(embeddings: Embeddings, args: ClickHouseLibArgs);
56
+ /**
57
+ * Method to add vectors to the ClickHouse database.
58
+ * @param vectors The vectors to add.
59
+ * @param documents The documents associated with the vectors.
60
+ * @returns Promise that resolves when the vectors have been added.
61
+ */
62
+ addVectors(vectors: number[][], documents: Document[]): Promise<void>;
63
+ /**
64
+ * Method to add documents to the ClickHouse database.
65
+ * @param documents The documents to add.
66
+ * @returns Promise that resolves when the documents have been added.
67
+ */
68
+ addDocuments(documents: Document[]): Promise<void>;
69
+ /**
70
+ * Method to search for vectors that are similar to a given query vector.
71
+ * @param query The query vector.
72
+ * @param k The number of similar vectors to return.
73
+ * @param filter Optional filter for the search results.
74
+ * @returns Promise that resolves with an array of tuples, each containing a Document and a score.
75
+ */
76
+ similaritySearchVectorWithScore(query: number[], k: number, filter?: this["FilterType"]): Promise<[Document, number][]>;
77
+ /**
78
+ * Static method to create an instance of ClickHouseStore from texts.
79
+ * @param texts The texts to use.
80
+ * @param metadatas The metadata associated with the texts.
81
+ * @param embeddings The embeddings to use.
82
+ * @param args The arguments for the ClickHouseStore.
83
+ * @returns Promise that resolves with a new instance of ClickHouseStore.
84
+ */
85
+ static fromTexts(texts: string[], metadatas: object | object[], embeddings: Embeddings, args: ClickHouseLibArgs): Promise<ClickHouseStore>;
86
+ /**
87
+ * Static method to create an instance of ClickHouseStore from documents.
88
+ * @param docs The documents to use.
89
+ * @param embeddings The embeddings to use.
90
+ * @param args The arguments for the ClickHouseStore.
91
+ * @returns Promise that resolves with a new instance of ClickHouseStore.
92
+ */
93
+ static fromDocuments(docs: Document[], embeddings: Embeddings, args: ClickHouseLibArgs): Promise<ClickHouseStore>;
94
+ /**
95
+ * Static method to create an instance of ClickHouseStore from an existing
96
+ * index.
97
+ * @param embeddings The embeddings to use.
98
+ * @param args The arguments for the ClickHouseStore.
99
+ * @returns Promise that resolves with a new instance of ClickHouseStore.
100
+ */
101
+ static fromExistingIndex(embeddings: Embeddings, args: ClickHouseLibArgs): Promise<ClickHouseStore>;
102
+ /**
103
+ * Method to initialize the ClickHouse database.
104
+ * @param dimension Optional dimension of the vectors.
105
+ * @returns Promise that resolves when the database has been initialized.
106
+ */
107
+ private initialize;
108
+ /**
109
+ * Method to build an SQL query for inserting vectors and documents into
110
+ * the ClickHouse database.
111
+ * @param vectors The vectors to insert.
112
+ * @param documents The documents to insert.
113
+ * @returns The SQL query string.
114
+ */
115
+ private buildInsertQuery;
116
+ private escapeString;
117
+ /**
118
+ * Method to build an SQL query for searching for similar vectors in the
119
+ * ClickHouse database.
120
+ * @param query The query vector.
121
+ * @param k The number of similar vectors to return.
122
+ * @param filter Optional filter for the search results.
123
+ * @returns The SQL query string.
124
+ */
125
+ private buildSearchQuery;
126
+ }
@@ -0,0 +1,259 @@
1
+ import * as uuid from "uuid";
2
+ import { createClient } from "@clickhouse/client";
3
+ import { format } from "mysql2";
4
+ import { VectorStore } from "./base.js";
5
+ import { Document } from "../document.js";
6
+ /**
7
+ * Class for interacting with the ClickHouse database. It extends the
8
+ * VectorStore class and provides methods for adding vectors and
9
+ * documents, searching for similar vectors, and creating instances from
10
+ * texts or documents.
11
+ */
12
+ export class ClickHouseStore extends VectorStore {
13
+ _vectorstoreType() {
14
+ return "clickhouse";
15
+ }
16
+ constructor(embeddings, args) {
17
+ super(embeddings, args);
18
+ Object.defineProperty(this, "client", {
19
+ enumerable: true,
20
+ configurable: true,
21
+ writable: true,
22
+ value: void 0
23
+ });
24
+ Object.defineProperty(this, "indexType", {
25
+ enumerable: true,
26
+ configurable: true,
27
+ writable: true,
28
+ value: void 0
29
+ });
30
+ Object.defineProperty(this, "indexParam", {
31
+ enumerable: true,
32
+ configurable: true,
33
+ writable: true,
34
+ value: void 0
35
+ });
36
+ Object.defineProperty(this, "indexQueryParams", {
37
+ enumerable: true,
38
+ configurable: true,
39
+ writable: true,
40
+ value: void 0
41
+ });
42
+ Object.defineProperty(this, "columnMap", {
43
+ enumerable: true,
44
+ configurable: true,
45
+ writable: true,
46
+ value: void 0
47
+ });
48
+ Object.defineProperty(this, "database", {
49
+ enumerable: true,
50
+ configurable: true,
51
+ writable: true,
52
+ value: void 0
53
+ });
54
+ Object.defineProperty(this, "table", {
55
+ enumerable: true,
56
+ configurable: true,
57
+ writable: true,
58
+ value: void 0
59
+ });
60
+ Object.defineProperty(this, "isInitialized", {
61
+ enumerable: true,
62
+ configurable: true,
63
+ writable: true,
64
+ value: false
65
+ });
66
+ this.indexType = args.indexType || "annoy";
67
+ this.indexParam = args.indexParam || { L2Distance: 100 };
68
+ this.indexQueryParams = args.indexQueryParams || {};
69
+ this.columnMap = args.columnMap || {
70
+ id: "id",
71
+ document: "document",
72
+ embedding: "embedding",
73
+ metadata: "metadata",
74
+ uuid: "uuid",
75
+ };
76
+ this.database = args.database || "default";
77
+ this.table = args.table || "vector_table";
78
+ this.client = createClient({
79
+ host: `${args.protocol ?? "https://"}${args.host}:${args.port}`,
80
+ username: args.username,
81
+ password: args.password,
82
+ session_id: uuid.v4(),
83
+ });
84
+ }
85
+ /**
86
+ * Method to add vectors to the ClickHouse database.
87
+ * @param vectors The vectors to add.
88
+ * @param documents The documents associated with the vectors.
89
+ * @returns Promise that resolves when the vectors have been added.
90
+ */
91
+ async addVectors(vectors, documents) {
92
+ if (vectors.length === 0) {
93
+ return;
94
+ }
95
+ if (!this.isInitialized) {
96
+ await this.initialize(vectors[0].length);
97
+ }
98
+ const queryStr = this.buildInsertQuery(vectors, documents);
99
+ await this.client.exec({ query: queryStr });
100
+ }
101
+ /**
102
+ * Method to add documents to the ClickHouse database.
103
+ * @param documents The documents to add.
104
+ * @returns Promise that resolves when the documents have been added.
105
+ */
106
+ async addDocuments(documents) {
107
+ return this.addVectors(await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)), documents);
108
+ }
109
+ /**
110
+ * Method to search for vectors that are similar to a given query vector.
111
+ * @param query The query vector.
112
+ * @param k The number of similar vectors to return.
113
+ * @param filter Optional filter for the search results.
114
+ * @returns Promise that resolves with an array of tuples, each containing a Document and a score.
115
+ */
116
+ async similaritySearchVectorWithScore(query, k, filter) {
117
+ if (!this.isInitialized) {
118
+ await this.initialize(query.length);
119
+ }
120
+ const queryStr = this.buildSearchQuery(query, k, filter);
121
+ const queryResultSet = await this.client.query({ query: queryStr });
122
+ const queryResult = await queryResultSet.json();
123
+ const result = queryResult.data.map((item) => [
124
+ new Document({ pageContent: item.document, metadata: item.metadata }),
125
+ item.dist,
126
+ ]);
127
+ return result;
128
+ }
129
+ /**
130
+ * Static method to create an instance of ClickHouseStore from texts.
131
+ * @param texts The texts to use.
132
+ * @param metadatas The metadata associated with the texts.
133
+ * @param embeddings The embeddings to use.
134
+ * @param args The arguments for the ClickHouseStore.
135
+ * @returns Promise that resolves with a new instance of ClickHouseStore.
136
+ */
137
+ static async fromTexts(texts, metadatas, embeddings, args) {
138
+ const docs = [];
139
+ for (let i = 0; i < texts.length; i += 1) {
140
+ const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;
141
+ const newDoc = new Document({
142
+ pageContent: texts[i],
143
+ metadata,
144
+ });
145
+ docs.push(newDoc);
146
+ }
147
+ return ClickHouseStore.fromDocuments(docs, embeddings, args);
148
+ }
149
+ /**
150
+ * Static method to create an instance of ClickHouseStore from documents.
151
+ * @param docs The documents to use.
152
+ * @param embeddings The embeddings to use.
153
+ * @param args The arguments for the ClickHouseStore.
154
+ * @returns Promise that resolves with a new instance of ClickHouseStore.
155
+ */
156
+ static async fromDocuments(docs, embeddings, args) {
157
+ const instance = new this(embeddings, args);
158
+ await instance.addDocuments(docs);
159
+ return instance;
160
+ }
161
+ /**
162
+ * Static method to create an instance of ClickHouseStore from an existing
163
+ * index.
164
+ * @param embeddings The embeddings to use.
165
+ * @param args The arguments for the ClickHouseStore.
166
+ * @returns Promise that resolves with a new instance of ClickHouseStore.
167
+ */
168
+ static async fromExistingIndex(embeddings, args) {
169
+ const instance = new this(embeddings, args);
170
+ await instance.initialize();
171
+ return instance;
172
+ }
173
+ /**
174
+ * Method to initialize the ClickHouse database.
175
+ * @param dimension Optional dimension of the vectors.
176
+ * @returns Promise that resolves when the database has been initialized.
177
+ */
178
+ async initialize(dimension) {
179
+ const dim = dimension ?? (await this.embeddings.embedQuery("test")).length;
180
+ const indexParamStr = this.indexParam
181
+ ? Object.entries(this.indexParam)
182
+ .map(([key, value]) => `'${key}', ${value}`)
183
+ .join(", ")
184
+ : "";
185
+ const query = `
186
+ CREATE TABLE IF NOT EXISTS ${this.database}.${this.table}(
187
+ ${this.columnMap.id} Nullable(String),
188
+ ${this.columnMap.document} Nullable(String),
189
+ ${this.columnMap.embedding} Array(Float32),
190
+ ${this.columnMap.metadata} JSON,
191
+ ${this.columnMap.uuid} UUID DEFAULT generateUUIDv4(),
192
+ CONSTRAINT cons_vec_len CHECK length(${this.columnMap.embedding}) = ${dim},
193
+ INDEX vec_idx ${this.columnMap.embedding} TYPE ${this.indexType}(${indexParamStr}) GRANULARITY 1000
194
+ ) ENGINE = MergeTree ORDER BY ${this.columnMap.uuid} SETTINGS index_granularity = 8192;`;
195
+ await this.client.exec({
196
+ query,
197
+ clickhouse_settings: {
198
+ allow_experimental_object_type: 1,
199
+ allow_experimental_annoy_index: 1,
200
+ },
201
+ });
202
+ this.isInitialized = true;
203
+ }
204
+ /**
205
+ * Method to build an SQL query for inserting vectors and documents into
206
+ * the ClickHouse database.
207
+ * @param vectors The vectors to insert.
208
+ * @param documents The documents to insert.
209
+ * @returns The SQL query string.
210
+ */
211
+ buildInsertQuery(vectors, documents) {
212
+ const columnsStr = Object.values(Object.fromEntries(Object.entries(this.columnMap).filter(([key]) => key !== this.columnMap.uuid))).join(", ");
213
+ const placeholders = vectors.map(() => "(?, ?, ?, ?)").join(", ");
214
+ const values = [];
215
+ for (let i = 0; i < vectors.length; i += 1) {
216
+ const vector = vectors[i];
217
+ const document = documents[i];
218
+ values.push(uuid.v4(), this.escapeString(document.pageContent), JSON.stringify(vector), JSON.stringify(document.metadata));
219
+ }
220
+ const insertQueryStr = `
221
+ INSERT INTO TABLE ${this.database}.${this.table}(${columnsStr})
222
+ VALUES ${placeholders}
223
+ `;
224
+ const insertQuery = format(insertQueryStr, values);
225
+ return insertQuery;
226
+ }
227
+ escapeString(str) {
228
+ return str.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
229
+ }
230
+ /**
231
+ * Method to build an SQL query for searching for similar vectors in the
232
+ * ClickHouse database.
233
+ * @param query The query vector.
234
+ * @param k The number of similar vectors to return.
235
+ * @param filter Optional filter for the search results.
236
+ * @returns The SQL query string.
237
+ */
238
+ buildSearchQuery(query, k, filter) {
239
+ const order = "ASC";
240
+ const whereStr = filter ? `PREWHERE ${filter.whereStr}` : "";
241
+ const placeholders = query.map(() => "?").join(", ");
242
+ const settingStrings = [];
243
+ if (this.indexQueryParams) {
244
+ for (const [key, value] of Object.entries(this.indexQueryParams)) {
245
+ settingStrings.push(`SETTING ${key}=${value}`);
246
+ }
247
+ }
248
+ const searchQueryStr = `
249
+ SELECT ${this.columnMap.document} AS document, ${this.columnMap.metadata} AS metadata, dist
250
+ FROM ${this.database}.${this.table}
251
+ ${whereStr}
252
+ ORDER BY L2Distance(${this.columnMap.embedding}, [${placeholders}]) AS dist ${order}
253
+ LIMIT ${k} ${settingStrings.join(" ")}
254
+ `;
255
+ // Format the query with actual values
256
+ const searchQuery = format(searchQueryStr, query);
257
+ return searchQuery;
258
+ }
259
+ }