langchain 0.0.154 → 0.0.155
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/callbacks/base.d.ts +42 -28
- package/dist/callbacks/handlers/log_stream.cjs +283 -0
- package/dist/callbacks/handlers/log_stream.d.ts +99 -0
- package/dist/callbacks/handlers/log_stream.js +277 -0
- package/dist/callbacks/handlers/tracer.cjs +34 -18
- package/dist/callbacks/handlers/tracer.d.ts +18 -16
- package/dist/callbacks/handlers/tracer.js +34 -18
- package/dist/document_loaders/web/notionapi.cjs +8 -4
- package/dist/document_loaders/web/notionapi.js +8 -4
- package/dist/document_loaders/web/searchapi.cjs +134 -0
- package/dist/document_loaders/web/searchapi.d.ts +65 -0
- package/dist/document_loaders/web/searchapi.js +130 -0
- package/dist/load/import_constants.cjs +1 -0
- package/dist/load/import_constants.js +1 -0
- package/dist/load/import_map.cjs +3 -2
- package/dist/load/import_map.d.ts +1 -0
- package/dist/load/import_map.js +1 -0
- package/dist/schema/runnable/base.cjs +64 -5
- package/dist/schema/runnable/base.d.ts +13 -0
- package/dist/schema/runnable/base.js +64 -5
- package/dist/tools/index.cjs +3 -1
- package/dist/tools/index.d.ts +1 -0
- package/dist/tools/index.js +1 -0
- package/dist/tools/searchapi.cjs +139 -0
- package/dist/tools/searchapi.d.ts +64 -0
- package/dist/tools/searchapi.js +135 -0
- package/dist/util/fast-json-patch/index.cjs +48 -0
- package/dist/util/fast-json-patch/index.d.ts +21 -0
- package/dist/util/fast-json-patch/index.js +15 -0
- package/dist/util/fast-json-patch/src/core.cjs +469 -0
- package/dist/util/fast-json-patch/src/core.d.ts +111 -0
- package/dist/util/fast-json-patch/src/core.js +459 -0
- package/dist/util/fast-json-patch/src/helpers.cjs +194 -0
- package/dist/util/fast-json-patch/src/helpers.d.ts +36 -0
- package/dist/util/fast-json-patch/src/helpers.js +181 -0
- package/dist/util/googlevertexai-webauth.cjs +6 -2
- package/dist/util/googlevertexai-webauth.d.ts +1 -0
- package/dist/util/googlevertexai-webauth.js +6 -2
- package/dist/util/stream.cjs +2 -40
- package/dist/util/stream.d.ts +1 -2
- package/dist/util/stream.js +1 -38
- package/dist/vectorstores/pgvector.cjs +1 -1
- package/dist/vectorstores/pgvector.js +1 -1
- package/dist/vectorstores/vercel_postgres.cjs +300 -0
- package/dist/vectorstores/vercel_postgres.d.ts +145 -0
- package/dist/vectorstores/vercel_postgres.js +296 -0
- package/document_loaders/web/searchapi.cjs +1 -0
- package/document_loaders/web/searchapi.d.ts +1 -0
- package/document_loaders/web/searchapi.js +1 -0
- package/package.json +22 -1
- package/vectorstores/vercel_postgres.cjs +1 -0
- package/vectorstores/vercel_postgres.d.ts +1 -0
- package/vectorstores/vercel_postgres.js +1 -0
|
@@ -21,7 +21,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
21
21
|
_addChildRun(parentRun, childRun) {
|
|
22
22
|
parentRun.child_runs.push(childRun);
|
|
23
23
|
}
|
|
24
|
-
_startTrace(run) {
|
|
24
|
+
async _startTrace(run) {
|
|
25
25
|
if (run.parent_run_id !== undefined) {
|
|
26
26
|
const parentRun = this.runMap.get(run.parent_run_id);
|
|
27
27
|
if (parentRun) {
|
|
@@ -30,6 +30,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
this.runMap.set(run.id, run);
|
|
33
|
+
await this.onRunCreate?.(run);
|
|
33
34
|
}
|
|
34
35
|
async _endTrace(run) {
|
|
35
36
|
const parentRun = run.parent_run_id !== undefined && this.runMap.get(run.parent_run_id);
|
|
@@ -40,6 +41,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
40
41
|
await this.persistRun(run);
|
|
41
42
|
}
|
|
42
43
|
this.runMap.delete(run.id);
|
|
44
|
+
await this.onRunUpdate?.(run);
|
|
43
45
|
}
|
|
44
46
|
_getExecutionOrder(parentRunId) {
|
|
45
47
|
const parentRun = parentRunId !== undefined && this.runMap.get(parentRunId);
|
|
@@ -49,7 +51,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
49
51
|
}
|
|
50
52
|
return parentRun.child_execution_order + 1;
|
|
51
53
|
}
|
|
52
|
-
async handleLLMStart(llm, prompts, runId, parentRunId, extraParams, tags, metadata) {
|
|
54
|
+
async handleLLMStart(llm, prompts, runId, parentRunId, extraParams, tags, metadata, name) {
|
|
53
55
|
const execution_order = this._getExecutionOrder(parentRunId);
|
|
54
56
|
const start_time = Date.now();
|
|
55
57
|
const finalExtraParams = metadata
|
|
@@ -57,7 +59,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
57
59
|
: extraParams;
|
|
58
60
|
const run = {
|
|
59
61
|
id: runId,
|
|
60
|
-
name: llm.id[llm.id.length - 1],
|
|
62
|
+
name: name ?? llm.id[llm.id.length - 1],
|
|
61
63
|
parent_run_id: parentRunId,
|
|
62
64
|
start_time,
|
|
63
65
|
serialized: llm,
|
|
@@ -75,10 +77,11 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
75
77
|
extra: finalExtraParams ?? {},
|
|
76
78
|
tags: tags || [],
|
|
77
79
|
};
|
|
78
|
-
this._startTrace(run);
|
|
80
|
+
await this._startTrace(run);
|
|
79
81
|
await this.onLLMStart?.(run);
|
|
82
|
+
return run;
|
|
80
83
|
}
|
|
81
|
-
async handleChatModelStart(llm, messages, runId, parentRunId, extraParams, tags, metadata) {
|
|
84
|
+
async handleChatModelStart(llm, messages, runId, parentRunId, extraParams, tags, metadata, name) {
|
|
82
85
|
const execution_order = this._getExecutionOrder(parentRunId);
|
|
83
86
|
const start_time = Date.now();
|
|
84
87
|
const finalExtraParams = metadata
|
|
@@ -86,7 +89,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
86
89
|
: extraParams;
|
|
87
90
|
const run = {
|
|
88
91
|
id: runId,
|
|
89
|
-
name: llm.id[llm.id.length - 1],
|
|
92
|
+
name: name ?? llm.id[llm.id.length - 1],
|
|
90
93
|
parent_run_id: parentRunId,
|
|
91
94
|
start_time,
|
|
92
95
|
serialized: llm,
|
|
@@ -104,8 +107,9 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
104
107
|
extra: finalExtraParams ?? {},
|
|
105
108
|
tags: tags || [],
|
|
106
109
|
};
|
|
107
|
-
this._startTrace(run);
|
|
110
|
+
await this._startTrace(run);
|
|
108
111
|
await this.onLLMStart?.(run);
|
|
112
|
+
return run;
|
|
109
113
|
}
|
|
110
114
|
async handleLLMEnd(output, runId) {
|
|
111
115
|
const run = this.runMap.get(runId);
|
|
@@ -120,6 +124,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
120
124
|
});
|
|
121
125
|
await this.onLLMEnd?.(run);
|
|
122
126
|
await this._endTrace(run);
|
|
127
|
+
return run;
|
|
123
128
|
}
|
|
124
129
|
async handleLLMError(error, runId) {
|
|
125
130
|
const run = this.runMap.get(runId);
|
|
@@ -134,13 +139,14 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
134
139
|
});
|
|
135
140
|
await this.onLLMError?.(run);
|
|
136
141
|
await this._endTrace(run);
|
|
142
|
+
return run;
|
|
137
143
|
}
|
|
138
|
-
async handleChainStart(chain, inputs, runId, parentRunId, tags, metadata, runType) {
|
|
144
|
+
async handleChainStart(chain, inputs, runId, parentRunId, tags, metadata, runType, name) {
|
|
139
145
|
const execution_order = this._getExecutionOrder(parentRunId);
|
|
140
146
|
const start_time = Date.now();
|
|
141
147
|
const run = {
|
|
142
148
|
id: runId,
|
|
143
|
-
name: chain.id[chain.id.length - 1],
|
|
149
|
+
name: name ?? chain.id[chain.id.length - 1],
|
|
144
150
|
parent_run_id: parentRunId,
|
|
145
151
|
start_time,
|
|
146
152
|
serialized: chain,
|
|
@@ -158,8 +164,9 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
158
164
|
extra: metadata ? { metadata } : {},
|
|
159
165
|
tags: tags || [],
|
|
160
166
|
};
|
|
161
|
-
this._startTrace(run);
|
|
167
|
+
await this._startTrace(run);
|
|
162
168
|
await this.onChainStart?.(run);
|
|
169
|
+
return run;
|
|
163
170
|
}
|
|
164
171
|
async handleChainEnd(outputs, runId, _parentRunId, _tags, kwargs) {
|
|
165
172
|
const run = this.runMap.get(runId);
|
|
@@ -177,6 +184,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
177
184
|
}
|
|
178
185
|
await this.onChainEnd?.(run);
|
|
179
186
|
await this._endTrace(run);
|
|
187
|
+
return run;
|
|
180
188
|
}
|
|
181
189
|
async handleChainError(error, runId, _parentRunId, _tags, kwargs) {
|
|
182
190
|
const run = this.runMap.get(runId);
|
|
@@ -194,13 +202,14 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
194
202
|
}
|
|
195
203
|
await this.onChainError?.(run);
|
|
196
204
|
await this._endTrace(run);
|
|
205
|
+
return run;
|
|
197
206
|
}
|
|
198
|
-
async handleToolStart(tool, input, runId, parentRunId, tags, metadata) {
|
|
207
|
+
async handleToolStart(tool, input, runId, parentRunId, tags, metadata, name) {
|
|
199
208
|
const execution_order = this._getExecutionOrder(parentRunId);
|
|
200
209
|
const start_time = Date.now();
|
|
201
210
|
const run = {
|
|
202
211
|
id: runId,
|
|
203
|
-
name: tool.id[tool.id.length - 1],
|
|
212
|
+
name: name ?? tool.id[tool.id.length - 1],
|
|
204
213
|
parent_run_id: parentRunId,
|
|
205
214
|
start_time,
|
|
206
215
|
serialized: tool,
|
|
@@ -218,8 +227,9 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
218
227
|
extra: metadata ? { metadata } : {},
|
|
219
228
|
tags: tags || [],
|
|
220
229
|
};
|
|
221
|
-
this._startTrace(run);
|
|
230
|
+
await this._startTrace(run);
|
|
222
231
|
await this.onToolStart?.(run);
|
|
232
|
+
return run;
|
|
223
233
|
}
|
|
224
234
|
async handleToolEnd(output, runId) {
|
|
225
235
|
const run = this.runMap.get(runId);
|
|
@@ -234,6 +244,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
234
244
|
});
|
|
235
245
|
await this.onToolEnd?.(run);
|
|
236
246
|
await this._endTrace(run);
|
|
247
|
+
return run;
|
|
237
248
|
}
|
|
238
249
|
async handleToolError(error, runId) {
|
|
239
250
|
const run = this.runMap.get(runId);
|
|
@@ -248,6 +259,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
248
259
|
});
|
|
249
260
|
await this.onToolError?.(run);
|
|
250
261
|
await this._endTrace(run);
|
|
262
|
+
return run;
|
|
251
263
|
}
|
|
252
264
|
async handleAgentAction(action, runId) {
|
|
253
265
|
const run = this.runMap.get(runId);
|
|
@@ -276,12 +288,12 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
276
288
|
});
|
|
277
289
|
await this.onAgentEnd?.(run);
|
|
278
290
|
}
|
|
279
|
-
async handleRetrieverStart(retriever, query, runId, parentRunId, tags, metadata) {
|
|
291
|
+
async handleRetrieverStart(retriever, query, runId, parentRunId, tags, metadata, name) {
|
|
280
292
|
const execution_order = this._getExecutionOrder(parentRunId);
|
|
281
293
|
const start_time = Date.now();
|
|
282
294
|
const run = {
|
|
283
295
|
id: runId,
|
|
284
|
-
name: retriever.id[retriever.id.length - 1],
|
|
296
|
+
name: name ?? retriever.id[retriever.id.length - 1],
|
|
285
297
|
parent_run_id: parentRunId,
|
|
286
298
|
start_time,
|
|
287
299
|
serialized: retriever,
|
|
@@ -299,8 +311,9 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
299
311
|
extra: metadata ? { metadata } : {},
|
|
300
312
|
tags: tags || [],
|
|
301
313
|
};
|
|
302
|
-
this._startTrace(run);
|
|
314
|
+
await this._startTrace(run);
|
|
303
315
|
await this.onRetrieverStart?.(run);
|
|
316
|
+
return run;
|
|
304
317
|
}
|
|
305
318
|
async handleRetrieverEnd(documents, runId) {
|
|
306
319
|
const run = this.runMap.get(runId);
|
|
@@ -315,6 +328,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
315
328
|
});
|
|
316
329
|
await this.onRetrieverEnd?.(run);
|
|
317
330
|
await this._endTrace(run);
|
|
331
|
+
return run;
|
|
318
332
|
}
|
|
319
333
|
async handleRetrieverError(error, runId) {
|
|
320
334
|
const run = this.runMap.get(runId);
|
|
@@ -329,6 +343,7 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
329
343
|
});
|
|
330
344
|
await this.onRetrieverError?.(run);
|
|
331
345
|
await this._endTrace(run);
|
|
346
|
+
return run;
|
|
332
347
|
}
|
|
333
348
|
async handleText(text, runId) {
|
|
334
349
|
const run = this.runMap.get(runId);
|
|
@@ -345,13 +360,14 @@ export class BaseTracer extends BaseCallbackHandler {
|
|
|
345
360
|
async handleLLMNewToken(token, idx, runId, _parentRunId, _tags, fields) {
|
|
346
361
|
const run = this.runMap.get(runId);
|
|
347
362
|
if (!run || run?.run_type !== "llm") {
|
|
348
|
-
|
|
363
|
+
throw new Error(`Invalid "runId" provided to "handleLLMNewToken" callback.`);
|
|
349
364
|
}
|
|
350
365
|
run.events.push({
|
|
351
366
|
name: "new_token",
|
|
352
367
|
time: new Date().toISOString(),
|
|
353
368
|
kwargs: { token, idx, chunk: fields?.chunk },
|
|
354
369
|
});
|
|
355
|
-
await this.onLLMNewToken?.(run);
|
|
370
|
+
await this.onLLMNewToken?.(run, token);
|
|
371
|
+
return run;
|
|
356
372
|
}
|
|
357
373
|
}
|
|
@@ -119,11 +119,15 @@ class NotionAPILoader extends base_js_1.BaseDocumentLoader {
|
|
|
119
119
|
* @returns The string of the title.
|
|
120
120
|
*/
|
|
121
121
|
getTitle(obj) {
|
|
122
|
-
if ((0, exports.isPage)(obj)
|
|
123
|
-
|
|
122
|
+
if ((0, exports.isPage)(obj)) {
|
|
123
|
+
const titleProp = Object.values(obj.properties).find((prop) => prop.type === "title");
|
|
124
|
+
if (titleProp)
|
|
125
|
+
return this.getPropValue(titleProp);
|
|
124
126
|
}
|
|
125
127
|
if ((0, exports.isDatabase)(obj))
|
|
126
|
-
return obj.title
|
|
128
|
+
return obj.title
|
|
129
|
+
.map((v) => this.n2mClient.annotatePlainText(v.plain_text, v.annotations))
|
|
130
|
+
.join("");
|
|
127
131
|
return null;
|
|
128
132
|
}
|
|
129
133
|
/**
|
|
@@ -284,7 +288,7 @@ class NotionAPILoader extends base_js_1.BaseDocumentLoader {
|
|
|
284
288
|
});
|
|
285
289
|
this.documents.push(pageDocument);
|
|
286
290
|
this.pageCompleted.push(pageId);
|
|
287
|
-
this.onDocumentLoaded(this.documents.length, this.pageQueueTotal,
|
|
291
|
+
this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, this.getTitle(pageDetails) || undefined, this.rootTitle);
|
|
288
292
|
}
|
|
289
293
|
/**
|
|
290
294
|
* Loads a Notion database and adds it's pages to the queue.
|
|
@@ -111,11 +111,15 @@ export class NotionAPILoader extends BaseDocumentLoader {
|
|
|
111
111
|
* @returns The string of the title.
|
|
112
112
|
*/
|
|
113
113
|
getTitle(obj) {
|
|
114
|
-
if (isPage(obj)
|
|
115
|
-
|
|
114
|
+
if (isPage(obj)) {
|
|
115
|
+
const titleProp = Object.values(obj.properties).find((prop) => prop.type === "title");
|
|
116
|
+
if (titleProp)
|
|
117
|
+
return this.getPropValue(titleProp);
|
|
116
118
|
}
|
|
117
119
|
if (isDatabase(obj))
|
|
118
|
-
return obj.title
|
|
120
|
+
return obj.title
|
|
121
|
+
.map((v) => this.n2mClient.annotatePlainText(v.plain_text, v.annotations))
|
|
122
|
+
.join("");
|
|
119
123
|
return null;
|
|
120
124
|
}
|
|
121
125
|
/**
|
|
@@ -276,7 +280,7 @@ export class NotionAPILoader extends BaseDocumentLoader {
|
|
|
276
280
|
});
|
|
277
281
|
this.documents.push(pageDocument);
|
|
278
282
|
this.pageCompleted.push(pageId);
|
|
279
|
-
this.onDocumentLoaded(this.documents.length, this.pageQueueTotal,
|
|
283
|
+
this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, this.getTitle(pageDetails) || undefined, this.rootTitle);
|
|
280
284
|
}
|
|
281
285
|
/**
|
|
282
286
|
* Loads a Notion database and adds it's pages to the queue.
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SearchApiLoader = void 0;
|
|
4
|
+
const env_js_1 = require("../../util/env.cjs");
|
|
5
|
+
const document_js_1 = require("../../document.cjs");
|
|
6
|
+
const base_js_1 = require("../base.cjs");
|
|
7
|
+
/**
|
|
8
|
+
* Class representing a document loader for loading search results from
|
|
9
|
+
* the SearchApi. It extends the BaseDocumentLoader class.
|
|
10
|
+
*/
|
|
11
|
+
class SearchApiLoader extends base_js_1.BaseDocumentLoader {
|
|
12
|
+
constructor(params) {
|
|
13
|
+
super();
|
|
14
|
+
Object.defineProperty(this, "apiKey", {
|
|
15
|
+
enumerable: true,
|
|
16
|
+
configurable: true,
|
|
17
|
+
writable: true,
|
|
18
|
+
value: void 0
|
|
19
|
+
});
|
|
20
|
+
Object.defineProperty(this, "parameters", {
|
|
21
|
+
enumerable: true,
|
|
22
|
+
configurable: true,
|
|
23
|
+
writable: true,
|
|
24
|
+
value: void 0
|
|
25
|
+
});
|
|
26
|
+
const { apiKey = (0, env_js_1.getEnvironmentVariable)("SEARCHAPI_API_KEY") } = params;
|
|
27
|
+
if (typeof apiKey !== "string") {
|
|
28
|
+
throw new Error("Invalid type for apiKey. Expected string.");
|
|
29
|
+
}
|
|
30
|
+
if (!apiKey) {
|
|
31
|
+
throw new Error("SearchApi API key not set. You can set it as SEARCHAPI_API_KEY in your .env file, or pass it to SearchApi.");
|
|
32
|
+
}
|
|
33
|
+
this.apiKey = apiKey;
|
|
34
|
+
this.parameters = { ...params };
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Builds the URL for the SearchApi search request.
|
|
38
|
+
* @returns The URL for the search request.
|
|
39
|
+
*/
|
|
40
|
+
buildUrl() {
|
|
41
|
+
this.parameters = {
|
|
42
|
+
engine: "google",
|
|
43
|
+
api_key: this.apiKey,
|
|
44
|
+
...this.parameters,
|
|
45
|
+
};
|
|
46
|
+
const preparedParams = Object.entries(this.parameters)
|
|
47
|
+
.filter(([key, value]) => value !== undefined && value !== null && key !== "apiKey")
|
|
48
|
+
.map(([key, value]) => [key, `${value}`]);
|
|
49
|
+
const searchParams = new URLSearchParams(preparedParams);
|
|
50
|
+
return `https://www.searchapi.io/api/v1/search?${searchParams}`;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Extracts documents from the provided output.
|
|
54
|
+
* @param output - The output to extract documents from.
|
|
55
|
+
* @param responseType - The type of the response to extract documents from.
|
|
56
|
+
* @returns An array of Documents.
|
|
57
|
+
*/
|
|
58
|
+
extractDocuments(output, responseType) {
|
|
59
|
+
const documents = [];
|
|
60
|
+
const results = Array.isArray(output) ? output : [output];
|
|
61
|
+
if (responseType === "transcripts") {
|
|
62
|
+
const pageContent = results.map((result) => result.text).join("\n");
|
|
63
|
+
const metadata = {
|
|
64
|
+
source: "SearchApi",
|
|
65
|
+
responseType,
|
|
66
|
+
};
|
|
67
|
+
documents.push(new document_js_1.Document({ pageContent, metadata }));
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
for (const result of results) {
|
|
71
|
+
const pageContent = JSON.stringify(result);
|
|
72
|
+
const metadata = {
|
|
73
|
+
source: "SearchApi",
|
|
74
|
+
responseType,
|
|
75
|
+
};
|
|
76
|
+
documents.push(new document_js_1.Document({ pageContent, metadata }));
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return documents;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Processes the response data from the SearchApi search request and converts it into an array of Documents.
|
|
83
|
+
* @param data - The response data from the SearchApi search request.
|
|
84
|
+
* @returns An array of Documents.
|
|
85
|
+
*/
|
|
86
|
+
processResponseData(data) {
|
|
87
|
+
const documents = [];
|
|
88
|
+
const responseTypes = [
|
|
89
|
+
"answer_box",
|
|
90
|
+
"shopping_results",
|
|
91
|
+
"knowledge_graph",
|
|
92
|
+
"organic_results",
|
|
93
|
+
"transcripts",
|
|
94
|
+
];
|
|
95
|
+
for (const responseType of responseTypes) {
|
|
96
|
+
if (responseType in data) {
|
|
97
|
+
documents.push(...this.extractDocuments(data[responseType], responseType));
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return documents;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Fetches the data from the provided URL and returns it as a JSON object.
|
|
104
|
+
* If an error occurs during the fetch operation, an exception is thrown with the error message.
|
|
105
|
+
* @param url - The URL to fetch data from.
|
|
106
|
+
* @returns A promise that resolves to the fetched data as a JSON object.
|
|
107
|
+
* @throws An error if the fetch operation fails.
|
|
108
|
+
*/
|
|
109
|
+
async fetchData(url) {
|
|
110
|
+
const response = await fetch(url);
|
|
111
|
+
const data = await response.json();
|
|
112
|
+
if (data.error) {
|
|
113
|
+
throw new Error(`Failed to load search results from SearchApi due to: ${data.error}`);
|
|
114
|
+
}
|
|
115
|
+
return data;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Loads the search results from the SearchApi.
|
|
119
|
+
* @returns An array of Documents representing the search results.
|
|
120
|
+
* @throws An error if the search results could not be loaded.
|
|
121
|
+
*/
|
|
122
|
+
async load() {
|
|
123
|
+
const url = this.buildUrl();
|
|
124
|
+
const data = await this.fetchData(url);
|
|
125
|
+
try {
|
|
126
|
+
return this.processResponseData(data);
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
console.error(error);
|
|
130
|
+
throw new Error(`Failed to process search results from SearchApi: ${error}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
exports.SearchApiLoader = SearchApiLoader;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { Document } from "../../document.js";
|
|
2
|
+
import { BaseDocumentLoader } from "../base.js";
|
|
3
|
+
type JSONPrimitive = string | number | boolean | null;
|
|
4
|
+
type JSONValue = JSONPrimitive | JSONObject | JSONArray;
|
|
5
|
+
interface JSONObject {
|
|
6
|
+
[key: string]: JSONValue;
|
|
7
|
+
}
|
|
8
|
+
interface JSONArray extends Array<JSONValue> {
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* SearchApiParameters Type Definition.
|
|
12
|
+
*
|
|
13
|
+
* For more parameters and supported search engines, refer specific engine documentation:
|
|
14
|
+
* Google - https://www.searchapi.io/docs/google
|
|
15
|
+
* Google News - https://www.searchapi.io/docs/google-news
|
|
16
|
+
* Google Scholar - https://www.searchapi.io/docs/google-scholar
|
|
17
|
+
* YouTube Transcripts - https://www.searchapi.io/docs/youtube-transcripts
|
|
18
|
+
* and others.
|
|
19
|
+
*
|
|
20
|
+
*/
|
|
21
|
+
type SearchApiParameters = {
|
|
22
|
+
[key: string]: JSONValue;
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Class representing a document loader for loading search results from
|
|
26
|
+
* the SearchApi. It extends the BaseDocumentLoader class.
|
|
27
|
+
*/
|
|
28
|
+
export declare class SearchApiLoader extends BaseDocumentLoader {
|
|
29
|
+
private apiKey;
|
|
30
|
+
private parameters;
|
|
31
|
+
constructor(params: SearchApiParameters);
|
|
32
|
+
/**
|
|
33
|
+
* Builds the URL for the SearchApi search request.
|
|
34
|
+
* @returns The URL for the search request.
|
|
35
|
+
*/
|
|
36
|
+
buildUrl(): string;
|
|
37
|
+
/**
|
|
38
|
+
* Extracts documents from the provided output.
|
|
39
|
+
* @param output - The output to extract documents from.
|
|
40
|
+
* @param responseType - The type of the response to extract documents from.
|
|
41
|
+
* @returns An array of Documents.
|
|
42
|
+
*/
|
|
43
|
+
private extractDocuments;
|
|
44
|
+
/**
|
|
45
|
+
* Processes the response data from the SearchApi search request and converts it into an array of Documents.
|
|
46
|
+
* @param data - The response data from the SearchApi search request.
|
|
47
|
+
* @returns An array of Documents.
|
|
48
|
+
*/
|
|
49
|
+
processResponseData(data: Record<string, unknown>): Document[];
|
|
50
|
+
/**
|
|
51
|
+
* Fetches the data from the provided URL and returns it as a JSON object.
|
|
52
|
+
* If an error occurs during the fetch operation, an exception is thrown with the error message.
|
|
53
|
+
* @param url - The URL to fetch data from.
|
|
54
|
+
* @returns A promise that resolves to the fetched data as a JSON object.
|
|
55
|
+
* @throws An error if the fetch operation fails.
|
|
56
|
+
*/
|
|
57
|
+
private fetchData;
|
|
58
|
+
/**
|
|
59
|
+
* Loads the search results from the SearchApi.
|
|
60
|
+
* @returns An array of Documents representing the search results.
|
|
61
|
+
* @throws An error if the search results could not be loaded.
|
|
62
|
+
*/
|
|
63
|
+
load(): Promise<Document[]>;
|
|
64
|
+
}
|
|
65
|
+
export {};
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { getEnvironmentVariable } from "../../util/env.js";
|
|
2
|
+
import { Document } from "../../document.js";
|
|
3
|
+
import { BaseDocumentLoader } from "../base.js";
|
|
4
|
+
/**
|
|
5
|
+
* Class representing a document loader for loading search results from
|
|
6
|
+
* the SearchApi. It extends the BaseDocumentLoader class.
|
|
7
|
+
*/
|
|
8
|
+
export class SearchApiLoader extends BaseDocumentLoader {
|
|
9
|
+
constructor(params) {
|
|
10
|
+
super();
|
|
11
|
+
Object.defineProperty(this, "apiKey", {
|
|
12
|
+
enumerable: true,
|
|
13
|
+
configurable: true,
|
|
14
|
+
writable: true,
|
|
15
|
+
value: void 0
|
|
16
|
+
});
|
|
17
|
+
Object.defineProperty(this, "parameters", {
|
|
18
|
+
enumerable: true,
|
|
19
|
+
configurable: true,
|
|
20
|
+
writable: true,
|
|
21
|
+
value: void 0
|
|
22
|
+
});
|
|
23
|
+
const { apiKey = getEnvironmentVariable("SEARCHAPI_API_KEY") } = params;
|
|
24
|
+
if (typeof apiKey !== "string") {
|
|
25
|
+
throw new Error("Invalid type for apiKey. Expected string.");
|
|
26
|
+
}
|
|
27
|
+
if (!apiKey) {
|
|
28
|
+
throw new Error("SearchApi API key not set. You can set it as SEARCHAPI_API_KEY in your .env file, or pass it to SearchApi.");
|
|
29
|
+
}
|
|
30
|
+
this.apiKey = apiKey;
|
|
31
|
+
this.parameters = { ...params };
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Builds the URL for the SearchApi search request.
|
|
35
|
+
* @returns The URL for the search request.
|
|
36
|
+
*/
|
|
37
|
+
buildUrl() {
|
|
38
|
+
this.parameters = {
|
|
39
|
+
engine: "google",
|
|
40
|
+
api_key: this.apiKey,
|
|
41
|
+
...this.parameters,
|
|
42
|
+
};
|
|
43
|
+
const preparedParams = Object.entries(this.parameters)
|
|
44
|
+
.filter(([key, value]) => value !== undefined && value !== null && key !== "apiKey")
|
|
45
|
+
.map(([key, value]) => [key, `${value}`]);
|
|
46
|
+
const searchParams = new URLSearchParams(preparedParams);
|
|
47
|
+
return `https://www.searchapi.io/api/v1/search?${searchParams}`;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Extracts documents from the provided output.
|
|
51
|
+
* @param output - The output to extract documents from.
|
|
52
|
+
* @param responseType - The type of the response to extract documents from.
|
|
53
|
+
* @returns An array of Documents.
|
|
54
|
+
*/
|
|
55
|
+
extractDocuments(output, responseType) {
|
|
56
|
+
const documents = [];
|
|
57
|
+
const results = Array.isArray(output) ? output : [output];
|
|
58
|
+
if (responseType === "transcripts") {
|
|
59
|
+
const pageContent = results.map((result) => result.text).join("\n");
|
|
60
|
+
const metadata = {
|
|
61
|
+
source: "SearchApi",
|
|
62
|
+
responseType,
|
|
63
|
+
};
|
|
64
|
+
documents.push(new Document({ pageContent, metadata }));
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
for (const result of results) {
|
|
68
|
+
const pageContent = JSON.stringify(result);
|
|
69
|
+
const metadata = {
|
|
70
|
+
source: "SearchApi",
|
|
71
|
+
responseType,
|
|
72
|
+
};
|
|
73
|
+
documents.push(new Document({ pageContent, metadata }));
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return documents;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Processes the response data from the SearchApi search request and converts it into an array of Documents.
|
|
80
|
+
* @param data - The response data from the SearchApi search request.
|
|
81
|
+
* @returns An array of Documents.
|
|
82
|
+
*/
|
|
83
|
+
processResponseData(data) {
|
|
84
|
+
const documents = [];
|
|
85
|
+
const responseTypes = [
|
|
86
|
+
"answer_box",
|
|
87
|
+
"shopping_results",
|
|
88
|
+
"knowledge_graph",
|
|
89
|
+
"organic_results",
|
|
90
|
+
"transcripts",
|
|
91
|
+
];
|
|
92
|
+
for (const responseType of responseTypes) {
|
|
93
|
+
if (responseType in data) {
|
|
94
|
+
documents.push(...this.extractDocuments(data[responseType], responseType));
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return documents;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Fetches the data from the provided URL and returns it as a JSON object.
|
|
101
|
+
* If an error occurs during the fetch operation, an exception is thrown with the error message.
|
|
102
|
+
* @param url - The URL to fetch data from.
|
|
103
|
+
* @returns A promise that resolves to the fetched data as a JSON object.
|
|
104
|
+
* @throws An error if the fetch operation fails.
|
|
105
|
+
*/
|
|
106
|
+
async fetchData(url) {
|
|
107
|
+
const response = await fetch(url);
|
|
108
|
+
const data = await response.json();
|
|
109
|
+
if (data.error) {
|
|
110
|
+
throw new Error(`Failed to load search results from SearchApi due to: ${data.error}`);
|
|
111
|
+
}
|
|
112
|
+
return data;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Loads the search results from the SearchApi.
|
|
116
|
+
* @returns An array of Documents representing the search results.
|
|
117
|
+
* @throws An error if the search results could not be loaded.
|
|
118
|
+
*/
|
|
119
|
+
async load() {
|
|
120
|
+
const url = this.buildUrl();
|
|
121
|
+
const data = await this.fetchData(url);
|
|
122
|
+
try {
|
|
123
|
+
return this.processResponseData(data);
|
|
124
|
+
}
|
|
125
|
+
catch (error) {
|
|
126
|
+
console.error(error);
|
|
127
|
+
throw new Error(`Failed to process search results from SearchApi: ${error}`);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
@@ -57,6 +57,7 @@ exports.optionalImportEntrypoints = [
|
|
|
57
57
|
"langchain/vectorstores/singlestore",
|
|
58
58
|
"langchain/vectorstores/tigris",
|
|
59
59
|
"langchain/vectorstores/usearch",
|
|
60
|
+
"langchain/vectorstores/vercel_postgres",
|
|
60
61
|
"langchain/vectorstores/voy",
|
|
61
62
|
"langchain/vectorstores/zep",
|
|
62
63
|
"langchain/memory/zep",
|
|
@@ -54,6 +54,7 @@ export const optionalImportEntrypoints = [
|
|
|
54
54
|
"langchain/vectorstores/singlestore",
|
|
55
55
|
"langchain/vectorstores/tigris",
|
|
56
56
|
"langchain/vectorstores/usearch",
|
|
57
|
+
"langchain/vectorstores/vercel_postgres",
|
|
57
58
|
"langchain/vectorstores/voy",
|
|
58
59
|
"langchain/vectorstores/zep",
|
|
59
60
|
"langchain/memory/zep",
|
package/dist/load/import_map.cjs
CHANGED
|
@@ -24,8 +24,8 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
24
24
|
return result;
|
|
25
25
|
};
|
|
26
26
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
27
|
-
exports.
|
|
28
|
-
exports.evaluation = exports.experimental__chat_models__bittensor = exports.experimental__plan_and_execute = exports.experimental__generative_agents = exports.experimental__babyagi = exports.experimental__autogpt = exports.util__math = exports.storage__in_memory = exports.stores__message__in_memory = exports.stores__file__in_memory = exports.stores__doc__in_memory = exports.cache = exports.retrievers__vespa = exports.retrievers__score_threshold = exports.retrievers__hyde = exports.retrievers__document_compressors__chain_extract = exports.retrievers__time_weighted = exports.retrievers__parent_document = exports.retrievers__multi_vector = exports.retrievers__multi_query = exports.retrievers__document_compressors = exports.retrievers__contextual_compression = void 0;
|
|
27
|
+
exports.retrievers__remote = exports.output_parsers = exports.callbacks = exports.schema__storage = exports.schema__runnable = exports.schema__retriever = exports.schema__query_constructor = exports.schema__output_parser = exports.schema__document = exports.schema = exports.chat_models__minimax = exports.chat_models__ollama = exports.chat_models__baiduwenxin = exports.chat_models__fireworks = exports.chat_models__anthropic = exports.chat_models__openai = exports.chat_models__base = exports.document_transformers__openai_functions = exports.document_loaders__web__sort_xyz_blockchain = exports.document_loaders__web__serpapi = exports.document_loaders__web__searchapi = exports.document_loaders__base = exports.document = exports.memory = exports.text_splitter = exports.vectorstores__xata = exports.vectorstores__vectara = exports.vectorstores__prisma = exports.vectorstores__memory = exports.vectorstores__base = exports.prompts = exports.llms__fireworks = exports.llms__ollama = exports.llms__aleph_alpha = exports.llms__ai21 = exports.llms__openai = exports.llms__base = exports.embeddings__minimax = exports.embeddings__openai = exports.embeddings__ollama = exports.embeddings__fake = exports.embeddings__cache_backed = exports.embeddings__base = exports.chains__openai_functions = exports.chains = exports.tools = exports.base_language = exports.agents__toolkits = exports.agents = exports.load__serializable = void 0;
|
|
28
|
+
exports.evaluation = exports.experimental__chat_models__bittensor = exports.experimental__plan_and_execute = exports.experimental__generative_agents = exports.experimental__babyagi = exports.experimental__autogpt = exports.util__math = exports.storage__in_memory = exports.stores__message__in_memory = exports.stores__file__in_memory = exports.stores__doc__in_memory = exports.cache = exports.retrievers__vespa = exports.retrievers__score_threshold = exports.retrievers__hyde = exports.retrievers__document_compressors__chain_extract = exports.retrievers__time_weighted = exports.retrievers__parent_document = exports.retrievers__multi_vector = exports.retrievers__multi_query = exports.retrievers__document_compressors = exports.retrievers__contextual_compression = exports.retrievers__databerry = void 0;
|
|
29
29
|
exports.load__serializable = __importStar(require("../load/serializable.cjs"));
|
|
30
30
|
exports.agents = __importStar(require("../agents/index.cjs"));
|
|
31
31
|
exports.agents__toolkits = __importStar(require("../agents/toolkits/index.cjs"));
|
|
@@ -55,6 +55,7 @@ exports.text_splitter = __importStar(require("../text_splitter.cjs"));
|
|
|
55
55
|
exports.memory = __importStar(require("../memory/index.cjs"));
|
|
56
56
|
exports.document = __importStar(require("../document.cjs"));
|
|
57
57
|
exports.document_loaders__base = __importStar(require("../document_loaders/base.cjs"));
|
|
58
|
+
exports.document_loaders__web__searchapi = __importStar(require("../document_loaders/web/searchapi.cjs"));
|
|
58
59
|
exports.document_loaders__web__serpapi = __importStar(require("../document_loaders/web/serpapi.cjs"));
|
|
59
60
|
exports.document_loaders__web__sort_xyz_blockchain = __importStar(require("../document_loaders/web/sort_xyz_blockchain.cjs"));
|
|
60
61
|
exports.document_transformers__openai_functions = __importStar(require("../document_transformers/openai_functions.cjs"));
|
|
@@ -27,6 +27,7 @@ export * as text_splitter from "../text_splitter.js";
|
|
|
27
27
|
export * as memory from "../memory/index.js";
|
|
28
28
|
export * as document from "../document.js";
|
|
29
29
|
export * as document_loaders__base from "../document_loaders/base.js";
|
|
30
|
+
export * as document_loaders__web__searchapi from "../document_loaders/web/searchapi.js";
|
|
30
31
|
export * as document_loaders__web__serpapi from "../document_loaders/web/serpapi.js";
|
|
31
32
|
export * as document_loaders__web__sort_xyz_blockchain from "../document_loaders/web/sort_xyz_blockchain.js";
|
|
32
33
|
export * as document_transformers__openai_functions from "../document_transformers/openai_functions.js";
|
package/dist/load/import_map.js
CHANGED
|
@@ -28,6 +28,7 @@ export * as text_splitter from "../text_splitter.js";
|
|
|
28
28
|
export * as memory from "../memory/index.js";
|
|
29
29
|
export * as document from "../document.js";
|
|
30
30
|
export * as document_loaders__base from "../document_loaders/base.js";
|
|
31
|
+
export * as document_loaders__web__searchapi from "../document_loaders/web/searchapi.js";
|
|
31
32
|
export * as document_loaders__web__serpapi from "../document_loaders/web/serpapi.js";
|
|
32
33
|
export * as document_loaders__web__sort_xyz_blockchain from "../document_loaders/web/sort_xyz_blockchain.js";
|
|
33
34
|
export * as document_transformers__openai_functions from "../document_transformers/openai_functions.js";
|