langchain 0.0.154 → 0.0.155

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/dist/callbacks/base.d.ts +42 -28
  2. package/dist/callbacks/handlers/log_stream.cjs +283 -0
  3. package/dist/callbacks/handlers/log_stream.d.ts +99 -0
  4. package/dist/callbacks/handlers/log_stream.js +277 -0
  5. package/dist/callbacks/handlers/tracer.cjs +34 -18
  6. package/dist/callbacks/handlers/tracer.d.ts +18 -16
  7. package/dist/callbacks/handlers/tracer.js +34 -18
  8. package/dist/document_loaders/web/notionapi.cjs +8 -4
  9. package/dist/document_loaders/web/notionapi.js +8 -4
  10. package/dist/document_loaders/web/searchapi.cjs +134 -0
  11. package/dist/document_loaders/web/searchapi.d.ts +65 -0
  12. package/dist/document_loaders/web/searchapi.js +130 -0
  13. package/dist/load/import_constants.cjs +1 -0
  14. package/dist/load/import_constants.js +1 -0
  15. package/dist/load/import_map.cjs +3 -2
  16. package/dist/load/import_map.d.ts +1 -0
  17. package/dist/load/import_map.js +1 -0
  18. package/dist/schema/runnable/base.cjs +64 -5
  19. package/dist/schema/runnable/base.d.ts +13 -0
  20. package/dist/schema/runnable/base.js +64 -5
  21. package/dist/tools/index.cjs +3 -1
  22. package/dist/tools/index.d.ts +1 -0
  23. package/dist/tools/index.js +1 -0
  24. package/dist/tools/searchapi.cjs +139 -0
  25. package/dist/tools/searchapi.d.ts +64 -0
  26. package/dist/tools/searchapi.js +135 -0
  27. package/dist/util/fast-json-patch/index.cjs +48 -0
  28. package/dist/util/fast-json-patch/index.d.ts +21 -0
  29. package/dist/util/fast-json-patch/index.js +15 -0
  30. package/dist/util/fast-json-patch/src/core.cjs +469 -0
  31. package/dist/util/fast-json-patch/src/core.d.ts +111 -0
  32. package/dist/util/fast-json-patch/src/core.js +459 -0
  33. package/dist/util/fast-json-patch/src/helpers.cjs +194 -0
  34. package/dist/util/fast-json-patch/src/helpers.d.ts +36 -0
  35. package/dist/util/fast-json-patch/src/helpers.js +181 -0
  36. package/dist/util/googlevertexai-webauth.cjs +6 -2
  37. package/dist/util/googlevertexai-webauth.d.ts +1 -0
  38. package/dist/util/googlevertexai-webauth.js +6 -2
  39. package/dist/util/stream.cjs +2 -40
  40. package/dist/util/stream.d.ts +1 -2
  41. package/dist/util/stream.js +1 -38
  42. package/dist/vectorstores/pgvector.cjs +1 -1
  43. package/dist/vectorstores/pgvector.js +1 -1
  44. package/dist/vectorstores/vercel_postgres.cjs +300 -0
  45. package/dist/vectorstores/vercel_postgres.d.ts +145 -0
  46. package/dist/vectorstores/vercel_postgres.js +296 -0
  47. package/document_loaders/web/searchapi.cjs +1 -0
  48. package/document_loaders/web/searchapi.d.ts +1 -0
  49. package/document_loaders/web/searchapi.js +1 -0
  50. package/package.json +22 -1
  51. package/vectorstores/vercel_postgres.cjs +1 -0
  52. package/vectorstores/vercel_postgres.d.ts +1 -0
  53. package/vectorstores/vercel_postgres.js +1 -0
@@ -21,7 +21,7 @@ export class BaseTracer extends BaseCallbackHandler {
21
21
  _addChildRun(parentRun, childRun) {
22
22
  parentRun.child_runs.push(childRun);
23
23
  }
24
- _startTrace(run) {
24
+ async _startTrace(run) {
25
25
  if (run.parent_run_id !== undefined) {
26
26
  const parentRun = this.runMap.get(run.parent_run_id);
27
27
  if (parentRun) {
@@ -30,6 +30,7 @@ export class BaseTracer extends BaseCallbackHandler {
30
30
  }
31
31
  }
32
32
  this.runMap.set(run.id, run);
33
+ await this.onRunCreate?.(run);
33
34
  }
34
35
  async _endTrace(run) {
35
36
  const parentRun = run.parent_run_id !== undefined && this.runMap.get(run.parent_run_id);
@@ -40,6 +41,7 @@ export class BaseTracer extends BaseCallbackHandler {
40
41
  await this.persistRun(run);
41
42
  }
42
43
  this.runMap.delete(run.id);
44
+ await this.onRunUpdate?.(run);
43
45
  }
44
46
  _getExecutionOrder(parentRunId) {
45
47
  const parentRun = parentRunId !== undefined && this.runMap.get(parentRunId);
@@ -49,7 +51,7 @@ export class BaseTracer extends BaseCallbackHandler {
49
51
  }
50
52
  return parentRun.child_execution_order + 1;
51
53
  }
52
- async handleLLMStart(llm, prompts, runId, parentRunId, extraParams, tags, metadata) {
54
+ async handleLLMStart(llm, prompts, runId, parentRunId, extraParams, tags, metadata, name) {
53
55
  const execution_order = this._getExecutionOrder(parentRunId);
54
56
  const start_time = Date.now();
55
57
  const finalExtraParams = metadata
@@ -57,7 +59,7 @@ export class BaseTracer extends BaseCallbackHandler {
57
59
  : extraParams;
58
60
  const run = {
59
61
  id: runId,
60
- name: llm.id[llm.id.length - 1],
62
+ name: name ?? llm.id[llm.id.length - 1],
61
63
  parent_run_id: parentRunId,
62
64
  start_time,
63
65
  serialized: llm,
@@ -75,10 +77,11 @@ export class BaseTracer extends BaseCallbackHandler {
75
77
  extra: finalExtraParams ?? {},
76
78
  tags: tags || [],
77
79
  };
78
- this._startTrace(run);
80
+ await this._startTrace(run);
79
81
  await this.onLLMStart?.(run);
82
+ return run;
80
83
  }
81
- async handleChatModelStart(llm, messages, runId, parentRunId, extraParams, tags, metadata) {
84
+ async handleChatModelStart(llm, messages, runId, parentRunId, extraParams, tags, metadata, name) {
82
85
  const execution_order = this._getExecutionOrder(parentRunId);
83
86
  const start_time = Date.now();
84
87
  const finalExtraParams = metadata
@@ -86,7 +89,7 @@ export class BaseTracer extends BaseCallbackHandler {
86
89
  : extraParams;
87
90
  const run = {
88
91
  id: runId,
89
- name: llm.id[llm.id.length - 1],
92
+ name: name ?? llm.id[llm.id.length - 1],
90
93
  parent_run_id: parentRunId,
91
94
  start_time,
92
95
  serialized: llm,
@@ -104,8 +107,9 @@ export class BaseTracer extends BaseCallbackHandler {
104
107
  extra: finalExtraParams ?? {},
105
108
  tags: tags || [],
106
109
  };
107
- this._startTrace(run);
110
+ await this._startTrace(run);
108
111
  await this.onLLMStart?.(run);
112
+ return run;
109
113
  }
110
114
  async handleLLMEnd(output, runId) {
111
115
  const run = this.runMap.get(runId);
@@ -120,6 +124,7 @@ export class BaseTracer extends BaseCallbackHandler {
120
124
  });
121
125
  await this.onLLMEnd?.(run);
122
126
  await this._endTrace(run);
127
+ return run;
123
128
  }
124
129
  async handleLLMError(error, runId) {
125
130
  const run = this.runMap.get(runId);
@@ -134,13 +139,14 @@ export class BaseTracer extends BaseCallbackHandler {
134
139
  });
135
140
  await this.onLLMError?.(run);
136
141
  await this._endTrace(run);
142
+ return run;
137
143
  }
138
- async handleChainStart(chain, inputs, runId, parentRunId, tags, metadata, runType) {
144
+ async handleChainStart(chain, inputs, runId, parentRunId, tags, metadata, runType, name) {
139
145
  const execution_order = this._getExecutionOrder(parentRunId);
140
146
  const start_time = Date.now();
141
147
  const run = {
142
148
  id: runId,
143
- name: chain.id[chain.id.length - 1],
149
+ name: name ?? chain.id[chain.id.length - 1],
144
150
  parent_run_id: parentRunId,
145
151
  start_time,
146
152
  serialized: chain,
@@ -158,8 +164,9 @@ export class BaseTracer extends BaseCallbackHandler {
158
164
  extra: metadata ? { metadata } : {},
159
165
  tags: tags || [],
160
166
  };
161
- this._startTrace(run);
167
+ await this._startTrace(run);
162
168
  await this.onChainStart?.(run);
169
+ return run;
163
170
  }
164
171
  async handleChainEnd(outputs, runId, _parentRunId, _tags, kwargs) {
165
172
  const run = this.runMap.get(runId);
@@ -177,6 +184,7 @@ export class BaseTracer extends BaseCallbackHandler {
177
184
  }
178
185
  await this.onChainEnd?.(run);
179
186
  await this._endTrace(run);
187
+ return run;
180
188
  }
181
189
  async handleChainError(error, runId, _parentRunId, _tags, kwargs) {
182
190
  const run = this.runMap.get(runId);
@@ -194,13 +202,14 @@ export class BaseTracer extends BaseCallbackHandler {
194
202
  }
195
203
  await this.onChainError?.(run);
196
204
  await this._endTrace(run);
205
+ return run;
197
206
  }
198
- async handleToolStart(tool, input, runId, parentRunId, tags, metadata) {
207
+ async handleToolStart(tool, input, runId, parentRunId, tags, metadata, name) {
199
208
  const execution_order = this._getExecutionOrder(parentRunId);
200
209
  const start_time = Date.now();
201
210
  const run = {
202
211
  id: runId,
203
- name: tool.id[tool.id.length - 1],
212
+ name: name ?? tool.id[tool.id.length - 1],
204
213
  parent_run_id: parentRunId,
205
214
  start_time,
206
215
  serialized: tool,
@@ -218,8 +227,9 @@ export class BaseTracer extends BaseCallbackHandler {
218
227
  extra: metadata ? { metadata } : {},
219
228
  tags: tags || [],
220
229
  };
221
- this._startTrace(run);
230
+ await this._startTrace(run);
222
231
  await this.onToolStart?.(run);
232
+ return run;
223
233
  }
224
234
  async handleToolEnd(output, runId) {
225
235
  const run = this.runMap.get(runId);
@@ -234,6 +244,7 @@ export class BaseTracer extends BaseCallbackHandler {
234
244
  });
235
245
  await this.onToolEnd?.(run);
236
246
  await this._endTrace(run);
247
+ return run;
237
248
  }
238
249
  async handleToolError(error, runId) {
239
250
  const run = this.runMap.get(runId);
@@ -248,6 +259,7 @@ export class BaseTracer extends BaseCallbackHandler {
248
259
  });
249
260
  await this.onToolError?.(run);
250
261
  await this._endTrace(run);
262
+ return run;
251
263
  }
252
264
  async handleAgentAction(action, runId) {
253
265
  const run = this.runMap.get(runId);
@@ -276,12 +288,12 @@ export class BaseTracer extends BaseCallbackHandler {
276
288
  });
277
289
  await this.onAgentEnd?.(run);
278
290
  }
279
- async handleRetrieverStart(retriever, query, runId, parentRunId, tags, metadata) {
291
+ async handleRetrieverStart(retriever, query, runId, parentRunId, tags, metadata, name) {
280
292
  const execution_order = this._getExecutionOrder(parentRunId);
281
293
  const start_time = Date.now();
282
294
  const run = {
283
295
  id: runId,
284
- name: retriever.id[retriever.id.length - 1],
296
+ name: name ?? retriever.id[retriever.id.length - 1],
285
297
  parent_run_id: parentRunId,
286
298
  start_time,
287
299
  serialized: retriever,
@@ -299,8 +311,9 @@ export class BaseTracer extends BaseCallbackHandler {
299
311
  extra: metadata ? { metadata } : {},
300
312
  tags: tags || [],
301
313
  };
302
- this._startTrace(run);
314
+ await this._startTrace(run);
303
315
  await this.onRetrieverStart?.(run);
316
+ return run;
304
317
  }
305
318
  async handleRetrieverEnd(documents, runId) {
306
319
  const run = this.runMap.get(runId);
@@ -315,6 +328,7 @@ export class BaseTracer extends BaseCallbackHandler {
315
328
  });
316
329
  await this.onRetrieverEnd?.(run);
317
330
  await this._endTrace(run);
331
+ return run;
318
332
  }
319
333
  async handleRetrieverError(error, runId) {
320
334
  const run = this.runMap.get(runId);
@@ -329,6 +343,7 @@ export class BaseTracer extends BaseCallbackHandler {
329
343
  });
330
344
  await this.onRetrieverError?.(run);
331
345
  await this._endTrace(run);
346
+ return run;
332
347
  }
333
348
  async handleText(text, runId) {
334
349
  const run = this.runMap.get(runId);
@@ -345,13 +360,14 @@ export class BaseTracer extends BaseCallbackHandler {
345
360
  async handleLLMNewToken(token, idx, runId, _parentRunId, _tags, fields) {
346
361
  const run = this.runMap.get(runId);
347
362
  if (!run || run?.run_type !== "llm") {
348
- return;
363
+ throw new Error(`Invalid "runId" provided to "handleLLMNewToken" callback.`);
349
364
  }
350
365
  run.events.push({
351
366
  name: "new_token",
352
367
  time: new Date().toISOString(),
353
368
  kwargs: { token, idx, chunk: fields?.chunk },
354
369
  });
355
- await this.onLLMNewToken?.(run);
370
+ await this.onLLMNewToken?.(run, token);
371
+ return run;
356
372
  }
357
373
  }
@@ -119,11 +119,15 @@ class NotionAPILoader extends base_js_1.BaseDocumentLoader {
119
119
  * @returns The string of the title.
120
120
  */
121
121
  getTitle(obj) {
122
- if ((0, exports.isPage)(obj) && obj.properties.title.type === "title") {
123
- return obj.properties.title.title[0]?.plain_text;
122
+ if ((0, exports.isPage)(obj)) {
123
+ const titleProp = Object.values(obj.properties).find((prop) => prop.type === "title");
124
+ if (titleProp)
125
+ return this.getPropValue(titleProp);
124
126
  }
125
127
  if ((0, exports.isDatabase)(obj))
126
- return obj.title[0]?.plain_text;
128
+ return obj.title
129
+ .map((v) => this.n2mClient.annotatePlainText(v.plain_text, v.annotations))
130
+ .join("");
127
131
  return null;
128
132
  }
129
133
  /**
@@ -284,7 +288,7 @@ class NotionAPILoader extends base_js_1.BaseDocumentLoader {
284
288
  });
285
289
  this.documents.push(pageDocument);
286
290
  this.pageCompleted.push(pageId);
287
- this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, pageDocument.metadata.properties.title, this.rootTitle);
291
+ this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, this.getTitle(pageDetails) || undefined, this.rootTitle);
288
292
  }
289
293
  /**
290
294
  * Loads a Notion database and adds it's pages to the queue.
@@ -111,11 +111,15 @@ export class NotionAPILoader extends BaseDocumentLoader {
111
111
  * @returns The string of the title.
112
112
  */
113
113
  getTitle(obj) {
114
- if (isPage(obj) && obj.properties.title.type === "title") {
115
- return obj.properties.title.title[0]?.plain_text;
114
+ if (isPage(obj)) {
115
+ const titleProp = Object.values(obj.properties).find((prop) => prop.type === "title");
116
+ if (titleProp)
117
+ return this.getPropValue(titleProp);
116
118
  }
117
119
  if (isDatabase(obj))
118
- return obj.title[0]?.plain_text;
120
+ return obj.title
121
+ .map((v) => this.n2mClient.annotatePlainText(v.plain_text, v.annotations))
122
+ .join("");
119
123
  return null;
120
124
  }
121
125
  /**
@@ -276,7 +280,7 @@ export class NotionAPILoader extends BaseDocumentLoader {
276
280
  });
277
281
  this.documents.push(pageDocument);
278
282
  this.pageCompleted.push(pageId);
279
- this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, pageDocument.metadata.properties.title, this.rootTitle);
283
+ this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, this.getTitle(pageDetails) || undefined, this.rootTitle);
280
284
  }
281
285
  /**
282
286
  * Loads a Notion database and adds it's pages to the queue.
@@ -0,0 +1,134 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.SearchApiLoader = void 0;
4
+ const env_js_1 = require("../../util/env.cjs");
5
+ const document_js_1 = require("../../document.cjs");
6
+ const base_js_1 = require("../base.cjs");
7
+ /**
8
+ * Class representing a document loader for loading search results from
9
+ * the SearchApi. It extends the BaseDocumentLoader class.
10
+ */
11
+ class SearchApiLoader extends base_js_1.BaseDocumentLoader {
12
+ constructor(params) {
13
+ super();
14
+ Object.defineProperty(this, "apiKey", {
15
+ enumerable: true,
16
+ configurable: true,
17
+ writable: true,
18
+ value: void 0
19
+ });
20
+ Object.defineProperty(this, "parameters", {
21
+ enumerable: true,
22
+ configurable: true,
23
+ writable: true,
24
+ value: void 0
25
+ });
26
+ const { apiKey = (0, env_js_1.getEnvironmentVariable)("SEARCHAPI_API_KEY") } = params;
27
+ if (typeof apiKey !== "string") {
28
+ throw new Error("Invalid type for apiKey. Expected string.");
29
+ }
30
+ if (!apiKey) {
31
+ throw new Error("SearchApi API key not set. You can set it as SEARCHAPI_API_KEY in your .env file, or pass it to SearchApi.");
32
+ }
33
+ this.apiKey = apiKey;
34
+ this.parameters = { ...params };
35
+ }
36
+ /**
37
+ * Builds the URL for the SearchApi search request.
38
+ * @returns The URL for the search request.
39
+ */
40
+ buildUrl() {
41
+ this.parameters = {
42
+ engine: "google",
43
+ api_key: this.apiKey,
44
+ ...this.parameters,
45
+ };
46
+ const preparedParams = Object.entries(this.parameters)
47
+ .filter(([key, value]) => value !== undefined && value !== null && key !== "apiKey")
48
+ .map(([key, value]) => [key, `${value}`]);
49
+ const searchParams = new URLSearchParams(preparedParams);
50
+ return `https://www.searchapi.io/api/v1/search?${searchParams}`;
51
+ }
52
+ /**
53
+ * Extracts documents from the provided output.
54
+ * @param output - The output to extract documents from.
55
+ * @param responseType - The type of the response to extract documents from.
56
+ * @returns An array of Documents.
57
+ */
58
+ extractDocuments(output, responseType) {
59
+ const documents = [];
60
+ const results = Array.isArray(output) ? output : [output];
61
+ if (responseType === "transcripts") {
62
+ const pageContent = results.map((result) => result.text).join("\n");
63
+ const metadata = {
64
+ source: "SearchApi",
65
+ responseType,
66
+ };
67
+ documents.push(new document_js_1.Document({ pageContent, metadata }));
68
+ }
69
+ else {
70
+ for (const result of results) {
71
+ const pageContent = JSON.stringify(result);
72
+ const metadata = {
73
+ source: "SearchApi",
74
+ responseType,
75
+ };
76
+ documents.push(new document_js_1.Document({ pageContent, metadata }));
77
+ }
78
+ }
79
+ return documents;
80
+ }
81
+ /**
82
+ * Processes the response data from the SearchApi search request and converts it into an array of Documents.
83
+ * @param data - The response data from the SearchApi search request.
84
+ * @returns An array of Documents.
85
+ */
86
+ processResponseData(data) {
87
+ const documents = [];
88
+ const responseTypes = [
89
+ "answer_box",
90
+ "shopping_results",
91
+ "knowledge_graph",
92
+ "organic_results",
93
+ "transcripts",
94
+ ];
95
+ for (const responseType of responseTypes) {
96
+ if (responseType in data) {
97
+ documents.push(...this.extractDocuments(data[responseType], responseType));
98
+ }
99
+ }
100
+ return documents;
101
+ }
102
+ /**
103
+ * Fetches the data from the provided URL and returns it as a JSON object.
104
+ * If an error occurs during the fetch operation, an exception is thrown with the error message.
105
+ * @param url - The URL to fetch data from.
106
+ * @returns A promise that resolves to the fetched data as a JSON object.
107
+ * @throws An error if the fetch operation fails.
108
+ */
109
+ async fetchData(url) {
110
+ const response = await fetch(url);
111
+ const data = await response.json();
112
+ if (data.error) {
113
+ throw new Error(`Failed to load search results from SearchApi due to: ${data.error}`);
114
+ }
115
+ return data;
116
+ }
117
+ /**
118
+ * Loads the search results from the SearchApi.
119
+ * @returns An array of Documents representing the search results.
120
+ * @throws An error if the search results could not be loaded.
121
+ */
122
+ async load() {
123
+ const url = this.buildUrl();
124
+ const data = await this.fetchData(url);
125
+ try {
126
+ return this.processResponseData(data);
127
+ }
128
+ catch (error) {
129
+ console.error(error);
130
+ throw new Error(`Failed to process search results from SearchApi: ${error}`);
131
+ }
132
+ }
133
+ }
134
+ exports.SearchApiLoader = SearchApiLoader;
@@ -0,0 +1,65 @@
1
+ import { Document } from "../../document.js";
2
+ import { BaseDocumentLoader } from "../base.js";
3
+ type JSONPrimitive = string | number | boolean | null;
4
+ type JSONValue = JSONPrimitive | JSONObject | JSONArray;
5
+ interface JSONObject {
6
+ [key: string]: JSONValue;
7
+ }
8
+ interface JSONArray extends Array<JSONValue> {
9
+ }
10
+ /**
11
+ * SearchApiParameters Type Definition.
12
+ *
13
+ * For more parameters and supported search engines, refer specific engine documentation:
14
+ * Google - https://www.searchapi.io/docs/google
15
+ * Google News - https://www.searchapi.io/docs/google-news
16
+ * Google Scholar - https://www.searchapi.io/docs/google-scholar
17
+ * YouTube Transcripts - https://www.searchapi.io/docs/youtube-transcripts
18
+ * and others.
19
+ *
20
+ */
21
+ type SearchApiParameters = {
22
+ [key: string]: JSONValue;
23
+ };
24
+ /**
25
+ * Class representing a document loader for loading search results from
26
+ * the SearchApi. It extends the BaseDocumentLoader class.
27
+ */
28
+ export declare class SearchApiLoader extends BaseDocumentLoader {
29
+ private apiKey;
30
+ private parameters;
31
+ constructor(params: SearchApiParameters);
32
+ /**
33
+ * Builds the URL for the SearchApi search request.
34
+ * @returns The URL for the search request.
35
+ */
36
+ buildUrl(): string;
37
+ /**
38
+ * Extracts documents from the provided output.
39
+ * @param output - The output to extract documents from.
40
+ * @param responseType - The type of the response to extract documents from.
41
+ * @returns An array of Documents.
42
+ */
43
+ private extractDocuments;
44
+ /**
45
+ * Processes the response data from the SearchApi search request and converts it into an array of Documents.
46
+ * @param data - The response data from the SearchApi search request.
47
+ * @returns An array of Documents.
48
+ */
49
+ processResponseData(data: Record<string, unknown>): Document[];
50
+ /**
51
+ * Fetches the data from the provided URL and returns it as a JSON object.
52
+ * If an error occurs during the fetch operation, an exception is thrown with the error message.
53
+ * @param url - The URL to fetch data from.
54
+ * @returns A promise that resolves to the fetched data as a JSON object.
55
+ * @throws An error if the fetch operation fails.
56
+ */
57
+ private fetchData;
58
+ /**
59
+ * Loads the search results from the SearchApi.
60
+ * @returns An array of Documents representing the search results.
61
+ * @throws An error if the search results could not be loaded.
62
+ */
63
+ load(): Promise<Document[]>;
64
+ }
65
+ export {};
@@ -0,0 +1,130 @@
1
+ import { getEnvironmentVariable } from "../../util/env.js";
2
+ import { Document } from "../../document.js";
3
+ import { BaseDocumentLoader } from "../base.js";
4
+ /**
5
+ * Class representing a document loader for loading search results from
6
+ * the SearchApi. It extends the BaseDocumentLoader class.
7
+ */
8
+ export class SearchApiLoader extends BaseDocumentLoader {
9
+ constructor(params) {
10
+ super();
11
+ Object.defineProperty(this, "apiKey", {
12
+ enumerable: true,
13
+ configurable: true,
14
+ writable: true,
15
+ value: void 0
16
+ });
17
+ Object.defineProperty(this, "parameters", {
18
+ enumerable: true,
19
+ configurable: true,
20
+ writable: true,
21
+ value: void 0
22
+ });
23
+ const { apiKey = getEnvironmentVariable("SEARCHAPI_API_KEY") } = params;
24
+ if (typeof apiKey !== "string") {
25
+ throw new Error("Invalid type for apiKey. Expected string.");
26
+ }
27
+ if (!apiKey) {
28
+ throw new Error("SearchApi API key not set. You can set it as SEARCHAPI_API_KEY in your .env file, or pass it to SearchApi.");
29
+ }
30
+ this.apiKey = apiKey;
31
+ this.parameters = { ...params };
32
+ }
33
+ /**
34
+ * Builds the URL for the SearchApi search request.
35
+ * @returns The URL for the search request.
36
+ */
37
+ buildUrl() {
38
+ this.parameters = {
39
+ engine: "google",
40
+ api_key: this.apiKey,
41
+ ...this.parameters,
42
+ };
43
+ const preparedParams = Object.entries(this.parameters)
44
+ .filter(([key, value]) => value !== undefined && value !== null && key !== "apiKey")
45
+ .map(([key, value]) => [key, `${value}`]);
46
+ const searchParams = new URLSearchParams(preparedParams);
47
+ return `https://www.searchapi.io/api/v1/search?${searchParams}`;
48
+ }
49
+ /**
50
+ * Extracts documents from the provided output.
51
+ * @param output - The output to extract documents from.
52
+ * @param responseType - The type of the response to extract documents from.
53
+ * @returns An array of Documents.
54
+ */
55
+ extractDocuments(output, responseType) {
56
+ const documents = [];
57
+ const results = Array.isArray(output) ? output : [output];
58
+ if (responseType === "transcripts") {
59
+ const pageContent = results.map((result) => result.text).join("\n");
60
+ const metadata = {
61
+ source: "SearchApi",
62
+ responseType,
63
+ };
64
+ documents.push(new Document({ pageContent, metadata }));
65
+ }
66
+ else {
67
+ for (const result of results) {
68
+ const pageContent = JSON.stringify(result);
69
+ const metadata = {
70
+ source: "SearchApi",
71
+ responseType,
72
+ };
73
+ documents.push(new Document({ pageContent, metadata }));
74
+ }
75
+ }
76
+ return documents;
77
+ }
78
+ /**
79
+ * Processes the response data from the SearchApi search request and converts it into an array of Documents.
80
+ * @param data - The response data from the SearchApi search request.
81
+ * @returns An array of Documents.
82
+ */
83
+ processResponseData(data) {
84
+ const documents = [];
85
+ const responseTypes = [
86
+ "answer_box",
87
+ "shopping_results",
88
+ "knowledge_graph",
89
+ "organic_results",
90
+ "transcripts",
91
+ ];
92
+ for (const responseType of responseTypes) {
93
+ if (responseType in data) {
94
+ documents.push(...this.extractDocuments(data[responseType], responseType));
95
+ }
96
+ }
97
+ return documents;
98
+ }
99
+ /**
100
+ * Fetches the data from the provided URL and returns it as a JSON object.
101
+ * If an error occurs during the fetch operation, an exception is thrown with the error message.
102
+ * @param url - The URL to fetch data from.
103
+ * @returns A promise that resolves to the fetched data as a JSON object.
104
+ * @throws An error if the fetch operation fails.
105
+ */
106
+ async fetchData(url) {
107
+ const response = await fetch(url);
108
+ const data = await response.json();
109
+ if (data.error) {
110
+ throw new Error(`Failed to load search results from SearchApi due to: ${data.error}`);
111
+ }
112
+ return data;
113
+ }
114
+ /**
115
+ * Loads the search results from the SearchApi.
116
+ * @returns An array of Documents representing the search results.
117
+ * @throws An error if the search results could not be loaded.
118
+ */
119
+ async load() {
120
+ const url = this.buildUrl();
121
+ const data = await this.fetchData(url);
122
+ try {
123
+ return this.processResponseData(data);
124
+ }
125
+ catch (error) {
126
+ console.error(error);
127
+ throw new Error(`Failed to process search results from SearchApi: ${error}`);
128
+ }
129
+ }
130
+ }
@@ -57,6 +57,7 @@ exports.optionalImportEntrypoints = [
57
57
  "langchain/vectorstores/singlestore",
58
58
  "langchain/vectorstores/tigris",
59
59
  "langchain/vectorstores/usearch",
60
+ "langchain/vectorstores/vercel_postgres",
60
61
  "langchain/vectorstores/voy",
61
62
  "langchain/vectorstores/zep",
62
63
  "langchain/memory/zep",
@@ -54,6 +54,7 @@ export const optionalImportEntrypoints = [
54
54
  "langchain/vectorstores/singlestore",
55
55
  "langchain/vectorstores/tigris",
56
56
  "langchain/vectorstores/usearch",
57
+ "langchain/vectorstores/vercel_postgres",
57
58
  "langchain/vectorstores/voy",
58
59
  "langchain/vectorstores/zep",
59
60
  "langchain/memory/zep",
@@ -24,8 +24,8 @@ var __importStar = (this && this.__importStar) || function (mod) {
24
24
  return result;
25
25
  };
26
26
  Object.defineProperty(exports, "__esModule", { value: true });
27
- exports.retrievers__databerry = exports.retrievers__remote = exports.output_parsers = exports.callbacks = exports.schema__storage = exports.schema__runnable = exports.schema__retriever = exports.schema__query_constructor = exports.schema__output_parser = exports.schema__document = exports.schema = exports.chat_models__minimax = exports.chat_models__ollama = exports.chat_models__baiduwenxin = exports.chat_models__fireworks = exports.chat_models__anthropic = exports.chat_models__openai = exports.chat_models__base = exports.document_transformers__openai_functions = exports.document_loaders__web__sort_xyz_blockchain = exports.document_loaders__web__serpapi = exports.document_loaders__base = exports.document = exports.memory = exports.text_splitter = exports.vectorstores__xata = exports.vectorstores__vectara = exports.vectorstores__prisma = exports.vectorstores__memory = exports.vectorstores__base = exports.prompts = exports.llms__fireworks = exports.llms__ollama = exports.llms__aleph_alpha = exports.llms__ai21 = exports.llms__openai = exports.llms__base = exports.embeddings__minimax = exports.embeddings__openai = exports.embeddings__ollama = exports.embeddings__fake = exports.embeddings__cache_backed = exports.embeddings__base = exports.chains__openai_functions = exports.chains = exports.tools = exports.base_language = exports.agents__toolkits = exports.agents = exports.load__serializable = void 0;
28
- exports.evaluation = exports.experimental__chat_models__bittensor = exports.experimental__plan_and_execute = exports.experimental__generative_agents = exports.experimental__babyagi = exports.experimental__autogpt = exports.util__math = exports.storage__in_memory = exports.stores__message__in_memory = exports.stores__file__in_memory = exports.stores__doc__in_memory = exports.cache = exports.retrievers__vespa = exports.retrievers__score_threshold = exports.retrievers__hyde = exports.retrievers__document_compressors__chain_extract = exports.retrievers__time_weighted = exports.retrievers__parent_document = exports.retrievers__multi_vector = exports.retrievers__multi_query = exports.retrievers__document_compressors = exports.retrievers__contextual_compression = void 0;
27
+ exports.retrievers__remote = exports.output_parsers = exports.callbacks = exports.schema__storage = exports.schema__runnable = exports.schema__retriever = exports.schema__query_constructor = exports.schema__output_parser = exports.schema__document = exports.schema = exports.chat_models__minimax = exports.chat_models__ollama = exports.chat_models__baiduwenxin = exports.chat_models__fireworks = exports.chat_models__anthropic = exports.chat_models__openai = exports.chat_models__base = exports.document_transformers__openai_functions = exports.document_loaders__web__sort_xyz_blockchain = exports.document_loaders__web__serpapi = exports.document_loaders__web__searchapi = exports.document_loaders__base = exports.document = exports.memory = exports.text_splitter = exports.vectorstores__xata = exports.vectorstores__vectara = exports.vectorstores__prisma = exports.vectorstores__memory = exports.vectorstores__base = exports.prompts = exports.llms__fireworks = exports.llms__ollama = exports.llms__aleph_alpha = exports.llms__ai21 = exports.llms__openai = exports.llms__base = exports.embeddings__minimax = exports.embeddings__openai = exports.embeddings__ollama = exports.embeddings__fake = exports.embeddings__cache_backed = exports.embeddings__base = exports.chains__openai_functions = exports.chains = exports.tools = exports.base_language = exports.agents__toolkits = exports.agents = exports.load__serializable = void 0;
28
+ exports.evaluation = exports.experimental__chat_models__bittensor = exports.experimental__plan_and_execute = exports.experimental__generative_agents = exports.experimental__babyagi = exports.experimental__autogpt = exports.util__math = exports.storage__in_memory = exports.stores__message__in_memory = exports.stores__file__in_memory = exports.stores__doc__in_memory = exports.cache = exports.retrievers__vespa = exports.retrievers__score_threshold = exports.retrievers__hyde = exports.retrievers__document_compressors__chain_extract = exports.retrievers__time_weighted = exports.retrievers__parent_document = exports.retrievers__multi_vector = exports.retrievers__multi_query = exports.retrievers__document_compressors = exports.retrievers__contextual_compression = exports.retrievers__databerry = void 0;
29
29
  exports.load__serializable = __importStar(require("../load/serializable.cjs"));
30
30
  exports.agents = __importStar(require("../agents/index.cjs"));
31
31
  exports.agents__toolkits = __importStar(require("../agents/toolkits/index.cjs"));
@@ -55,6 +55,7 @@ exports.text_splitter = __importStar(require("../text_splitter.cjs"));
55
55
  exports.memory = __importStar(require("../memory/index.cjs"));
56
56
  exports.document = __importStar(require("../document.cjs"));
57
57
  exports.document_loaders__base = __importStar(require("../document_loaders/base.cjs"));
58
+ exports.document_loaders__web__searchapi = __importStar(require("../document_loaders/web/searchapi.cjs"));
58
59
  exports.document_loaders__web__serpapi = __importStar(require("../document_loaders/web/serpapi.cjs"));
59
60
  exports.document_loaders__web__sort_xyz_blockchain = __importStar(require("../document_loaders/web/sort_xyz_blockchain.cjs"));
60
61
  exports.document_transformers__openai_functions = __importStar(require("../document_transformers/openai_functions.cjs"));
@@ -27,6 +27,7 @@ export * as text_splitter from "../text_splitter.js";
27
27
  export * as memory from "../memory/index.js";
28
28
  export * as document from "../document.js";
29
29
  export * as document_loaders__base from "../document_loaders/base.js";
30
+ export * as document_loaders__web__searchapi from "../document_loaders/web/searchapi.js";
30
31
  export * as document_loaders__web__serpapi from "../document_loaders/web/serpapi.js";
31
32
  export * as document_loaders__web__sort_xyz_blockchain from "../document_loaders/web/sort_xyz_blockchain.js";
32
33
  export * as document_transformers__openai_functions from "../document_transformers/openai_functions.js";
@@ -28,6 +28,7 @@ export * as text_splitter from "../text_splitter.js";
28
28
  export * as memory from "../memory/index.js";
29
29
  export * as document from "../document.js";
30
30
  export * as document_loaders__base from "../document_loaders/base.js";
31
+ export * as document_loaders__web__searchapi from "../document_loaders/web/searchapi.js";
31
32
  export * as document_loaders__web__serpapi from "../document_loaders/web/serpapi.js";
32
33
  export * as document_loaders__web__sort_xyz_blockchain from "../document_loaders/web/sort_xyz_blockchain.js";
33
34
  export * as document_transformers__openai_functions from "../document_transformers/openai_functions.js";