langchain 0.0.184 → 0.0.186
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/format_scratchpad/openai_tools.cjs +1 -0
- package/agents/format_scratchpad/openai_tools.d.ts +1 -0
- package/agents/format_scratchpad/openai_tools.js +1 -0
- package/dist/agents/format_scratchpad/openai_tools.cjs +19 -0
- package/dist/agents/format_scratchpad/openai_tools.d.ts +3 -0
- package/dist/agents/format_scratchpad/openai_tools.js +15 -0
- package/dist/agents/openai/index.cjs +2 -1
- package/dist/agents/openai/index.js +2 -1
- package/dist/agents/openai/output_parser.cjs +66 -1
- package/dist/agents/openai/output_parser.d.ts +26 -2
- package/dist/agents/openai/output_parser.js +65 -1
- package/dist/agents/structured_chat/index.cjs +1 -2
- package/dist/agents/structured_chat/index.d.ts +2 -0
- package/dist/agents/structured_chat/index.js +1 -2
- package/dist/agents/types.cjs +8 -1
- package/dist/agents/types.d.ts +6 -0
- package/dist/agents/types.js +6 -0
- package/dist/chains/combine_docs_chain.cjs +1 -1
- package/dist/chains/combine_docs_chain.js +1 -1
- package/dist/chains/llm_chain.cjs +52 -7
- package/dist/chains/llm_chain.d.ts +20 -12
- package/dist/chains/llm_chain.js +53 -8
- package/dist/chat_models/ollama.cjs +8 -0
- package/dist/chat_models/ollama.d.ts +3 -0
- package/dist/chat_models/ollama.js +8 -0
- package/dist/chat_models/openai.cjs +3 -0
- package/dist/chat_models/openai.js +3 -0
- package/dist/document_loaders/fs/pdf.cjs +17 -3
- package/dist/document_loaders/fs/pdf.js +17 -3
- package/dist/document_loaders/fs/unstructured.d.ts +1 -5
- package/dist/document_loaders/web/apify_dataset.cjs +12 -6
- package/dist/document_loaders/web/apify_dataset.d.ts +9 -6
- package/dist/document_loaders/web/apify_dataset.js +12 -6
- package/dist/document_loaders/web/pdf.cjs +17 -3
- package/dist/document_loaders/web/pdf.js +17 -3
- package/dist/document_loaders/web/puppeteer.cjs +37 -0
- package/dist/document_loaders/web/puppeteer.d.ts +17 -0
- package/dist/document_loaders/web/puppeteer.js +37 -0
- package/dist/embeddings/ollama.d.ts +1 -1
- package/dist/experimental/plan_and_execute/agent_executor.cjs +28 -2
- package/dist/experimental/plan_and_execute/agent_executor.d.ts +10 -3
- package/dist/experimental/plan_and_execute/agent_executor.js +26 -1
- package/dist/experimental/plan_and_execute/prompt.d.ts +2 -1
- package/dist/llms/ollama.cjs +8 -0
- package/dist/llms/ollama.d.ts +3 -0
- package/dist/llms/ollama.js +8 -0
- package/dist/llms/openai.cjs +1 -1
- package/dist/llms/openai.js +1 -1
- package/dist/load/import_map.cjs +3 -2
- package/dist/load/import_map.d.ts +1 -0
- package/dist/load/import_map.js +1 -0
- package/dist/output_parsers/index.cjs +3 -1
- package/dist/output_parsers/index.d.ts +1 -0
- package/dist/output_parsers/index.js +1 -0
- package/dist/output_parsers/openai_functions.cjs +3 -3
- package/dist/output_parsers/openai_functions.js +3 -3
- package/dist/output_parsers/openai_tools.cjs +53 -0
- package/dist/output_parsers/openai_tools.d.ts +22 -0
- package/dist/output_parsers/openai_tools.js +49 -0
- package/dist/prompts/base.d.ts +2 -1
- package/dist/prompts/chat.cjs +23 -2
- package/dist/prompts/chat.d.ts +1 -0
- package/dist/prompts/chat.js +23 -2
- package/dist/schema/index.d.ts +3 -4
- package/dist/schema/runnable/base.d.ts +2 -2
- package/dist/tools/convert_to_openai.cjs +2 -1
- package/dist/tools/convert_to_openai.js +2 -1
- package/dist/tools/index.cjs +2 -1
- package/dist/tools/index.d.ts +1 -1
- package/dist/tools/index.js +1 -1
- package/dist/util/ollama.d.ts +3 -0
- package/dist/util/types.cjs +5 -0
- package/dist/util/types.d.ts +4 -0
- package/dist/util/types.js +4 -0
- package/dist/vectorstores/momento_vector_index.cjs +1 -1
- package/dist/vectorstores/momento_vector_index.js +1 -1
- package/package.json +15 -7
|
@@ -213,6 +213,12 @@ class ChatOllama extends base_js_1.SimpleChatModel {
|
|
|
213
213
|
writable: true,
|
|
214
214
|
value: void 0
|
|
215
215
|
});
|
|
216
|
+
Object.defineProperty(this, "format", {
|
|
217
|
+
enumerable: true,
|
|
218
|
+
configurable: true,
|
|
219
|
+
writable: true,
|
|
220
|
+
value: void 0
|
|
221
|
+
});
|
|
216
222
|
this.model = fields.model ?? this.model;
|
|
217
223
|
this.baseUrl = fields.baseUrl?.endsWith("/")
|
|
218
224
|
? fields.baseUrl.slice(0, -1)
|
|
@@ -247,6 +253,7 @@ class ChatOllama extends base_js_1.SimpleChatModel {
|
|
|
247
253
|
this.useMLock = fields.useMLock;
|
|
248
254
|
this.useMMap = fields.useMMap;
|
|
249
255
|
this.vocabOnly = fields.vocabOnly;
|
|
256
|
+
this.format = fields.format;
|
|
250
257
|
}
|
|
251
258
|
_llmType() {
|
|
252
259
|
return "ollama";
|
|
@@ -260,6 +267,7 @@ class ChatOllama extends base_js_1.SimpleChatModel {
|
|
|
260
267
|
invocationParams(options) {
|
|
261
268
|
return {
|
|
262
269
|
model: this.model,
|
|
270
|
+
format: this.format,
|
|
263
271
|
options: {
|
|
264
272
|
embedding_only: this.embeddingOnly,
|
|
265
273
|
f16_kv: this.f16KV,
|
|
@@ -3,6 +3,7 @@ import { BaseLanguageModelCallOptions } from "../base_language/index.js";
|
|
|
3
3
|
import { OllamaInput } from "../util/ollama.js";
|
|
4
4
|
import { CallbackManagerForLLMRun } from "../callbacks/manager.js";
|
|
5
5
|
import { BaseMessage, ChatGenerationChunk } from "../schema/index.js";
|
|
6
|
+
import type { StringWithAutocomplete } from "../util/types.js";
|
|
6
7
|
/**
|
|
7
8
|
* An interface defining the options for an Ollama API call. It extends
|
|
8
9
|
* the BaseLanguageModelCallOptions interface.
|
|
@@ -49,6 +50,7 @@ export declare class ChatOllama extends SimpleChatModel<OllamaCallOptions> imple
|
|
|
49
50
|
useMLock?: boolean;
|
|
50
51
|
useMMap?: boolean;
|
|
51
52
|
vocabOnly?: boolean;
|
|
53
|
+
format?: StringWithAutocomplete<"json">;
|
|
52
54
|
constructor(fields: OllamaInput & BaseChatModelParams);
|
|
53
55
|
_llmType(): string;
|
|
54
56
|
/**
|
|
@@ -59,6 +61,7 @@ export declare class ChatOllama extends SimpleChatModel<OllamaCallOptions> imple
|
|
|
59
61
|
*/
|
|
60
62
|
invocationParams(options?: this["ParsedCallOptions"]): {
|
|
61
63
|
model: string;
|
|
64
|
+
format: StringWithAutocomplete<"json"> | undefined;
|
|
62
65
|
options: {
|
|
63
66
|
embedding_only: boolean | undefined;
|
|
64
67
|
f16_kv: boolean | undefined;
|
|
@@ -210,6 +210,12 @@ export class ChatOllama extends SimpleChatModel {
|
|
|
210
210
|
writable: true,
|
|
211
211
|
value: void 0
|
|
212
212
|
});
|
|
213
|
+
Object.defineProperty(this, "format", {
|
|
214
|
+
enumerable: true,
|
|
215
|
+
configurable: true,
|
|
216
|
+
writable: true,
|
|
217
|
+
value: void 0
|
|
218
|
+
});
|
|
213
219
|
this.model = fields.model ?? this.model;
|
|
214
220
|
this.baseUrl = fields.baseUrl?.endsWith("/")
|
|
215
221
|
? fields.baseUrl.slice(0, -1)
|
|
@@ -244,6 +250,7 @@ export class ChatOllama extends SimpleChatModel {
|
|
|
244
250
|
this.useMLock = fields.useMLock;
|
|
245
251
|
this.useMMap = fields.useMMap;
|
|
246
252
|
this.vocabOnly = fields.vocabOnly;
|
|
253
|
+
this.format = fields.format;
|
|
247
254
|
}
|
|
248
255
|
_llmType() {
|
|
249
256
|
return "ollama";
|
|
@@ -257,6 +264,7 @@ export class ChatOllama extends SimpleChatModel {
|
|
|
257
264
|
invocationParams(options) {
|
|
258
265
|
return {
|
|
259
266
|
model: this.model,
|
|
267
|
+
format: this.format,
|
|
260
268
|
options: {
|
|
261
269
|
embedding_only: this.embeddingOnly,
|
|
262
270
|
f16_kv: this.f16KV,
|
|
@@ -421,6 +421,9 @@ class ChatOpenAI extends base_js_1.BaseChatModel {
|
|
|
421
421
|
continue;
|
|
422
422
|
}
|
|
423
423
|
const { delta } = choice;
|
|
424
|
+
if (!delta) {
|
|
425
|
+
continue;
|
|
426
|
+
}
|
|
424
427
|
const chunk = _convertDeltaToMessageChunk(delta, defaultRole);
|
|
425
428
|
defaultRole = delta.role ?? defaultRole;
|
|
426
429
|
const newTokenIndices = {
|
|
@@ -418,6 +418,9 @@ export class ChatOpenAI extends BaseChatModel {
|
|
|
418
418
|
continue;
|
|
419
419
|
}
|
|
420
420
|
const { delta } = choice;
|
|
421
|
+
if (!delta) {
|
|
422
|
+
continue;
|
|
423
|
+
}
|
|
421
424
|
const chunk = _convertDeltaToMessageChunk(delta, defaultRole);
|
|
422
425
|
defaultRole = delta.role ?? defaultRole;
|
|
423
426
|
const newTokenIndices = {
|
|
@@ -59,9 +59,23 @@ class PDFLoader extends buffer_js_1.BufferLoader {
|
|
|
59
59
|
if (content.items.length === 0) {
|
|
60
60
|
continue;
|
|
61
61
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
// Eliminate excessive newlines
|
|
63
|
+
// Source: https://github.com/albertcui/pdf-parse/blob/7086fc1cc9058545cdf41dd0646d6ae5832c7107/lib/pdf-parse.js#L16
|
|
64
|
+
let lastY;
|
|
65
|
+
const textItems = [];
|
|
66
|
+
for (const item of content.items) {
|
|
67
|
+
if ("str" in item) {
|
|
68
|
+
if (lastY === item.transform[5] || !lastY) {
|
|
69
|
+
textItems.push(item.str);
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
textItems.push(`\n${item.str}`);
|
|
73
|
+
}
|
|
74
|
+
// eslint-disable-next-line prefer-destructuring
|
|
75
|
+
lastY = item.transform[5];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const text = textItems.join(" ");
|
|
65
79
|
documents.push(new document_js_1.Document({
|
|
66
80
|
pageContent: text,
|
|
67
81
|
metadata: {
|
|
@@ -56,9 +56,23 @@ export class PDFLoader extends BufferLoader {
|
|
|
56
56
|
if (content.items.length === 0) {
|
|
57
57
|
continue;
|
|
58
58
|
}
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
59
|
+
// Eliminate excessive newlines
|
|
60
|
+
// Source: https://github.com/albertcui/pdf-parse/blob/7086fc1cc9058545cdf41dd0646d6ae5832c7107/lib/pdf-parse.js#L16
|
|
61
|
+
let lastY;
|
|
62
|
+
const textItems = [];
|
|
63
|
+
for (const item of content.items) {
|
|
64
|
+
if ("str" in item) {
|
|
65
|
+
if (lastY === item.transform[5] || !lastY) {
|
|
66
|
+
textItems.push(item.str);
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
textItems.push(`\n${item.str}`);
|
|
70
|
+
}
|
|
71
|
+
// eslint-disable-next-line prefer-destructuring
|
|
72
|
+
lastY = item.transform[5];
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const text = textItems.join(" ");
|
|
62
76
|
documents.push(new Document({
|
|
63
77
|
pageContent: text,
|
|
64
78
|
metadata: {
|
|
@@ -5,6 +5,7 @@ import type { readFile as ReadFileT } from "node:fs/promises";
|
|
|
5
5
|
import { DirectoryLoader, UnknownHandling } from "./directory.js";
|
|
6
6
|
import { Document } from "../../document.js";
|
|
7
7
|
import { BaseDocumentLoader } from "../base.js";
|
|
8
|
+
import type { StringWithAutocomplete } from "../../util/types.js";
|
|
8
9
|
/**
|
|
9
10
|
* Represents an element returned by the Unstructured API. It has
|
|
10
11
|
* properties for the element type, text content, and metadata.
|
|
@@ -40,11 +41,6 @@ export type SkipInferTableTypes = "txt" | "text" | "pdf" | "docx" | "doc" | "jpg
|
|
|
40
41
|
* Set the chunking_strategy to chunk text into larger or smaller elements. Defaults to None with optional arg of by_title
|
|
41
42
|
*/
|
|
42
43
|
type ChunkingStrategy = "None" | "by_title";
|
|
43
|
-
/**
|
|
44
|
-
* Represents a string value with autocomplete suggestions. It is used for
|
|
45
|
-
* the `strategy` property in the UnstructuredLoaderOptions.
|
|
46
|
-
*/
|
|
47
|
-
type StringWithAutocomplete<T> = T | (string & Record<never, never>);
|
|
48
44
|
export type UnstructuredLoaderOptions = {
|
|
49
45
|
apiKey?: string;
|
|
50
46
|
apiUrl?: string;
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
4
|
exports.ApifyDatasetLoader = void 0;
|
|
5
5
|
const apify_client_1 = require("apify-client");
|
|
6
|
+
const async_caller_js_1 = require("../../util/async_caller.cjs");
|
|
6
7
|
const base_js_1 = require("../base.cjs");
|
|
7
8
|
const env_js_1 = require("../../util/env.cjs");
|
|
8
9
|
/**
|
|
@@ -31,13 +32,18 @@ class ApifyDatasetLoader extends base_js_1.BaseDocumentLoader {
|
|
|
31
32
|
writable: true,
|
|
32
33
|
value: void 0
|
|
33
34
|
});
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
Object.defineProperty(this, "caller", {
|
|
36
|
+
enumerable: true,
|
|
37
|
+
configurable: true,
|
|
38
|
+
writable: true,
|
|
39
|
+
value: void 0
|
|
38
40
|
});
|
|
41
|
+
const { clientOptions, datasetMappingFunction, ...asyncCallerParams } = config;
|
|
42
|
+
const token = ApifyDatasetLoader._getApifyApiToken(clientOptions);
|
|
43
|
+
this.apifyClient = new apify_client_1.ApifyClient({ ...clientOptions, token });
|
|
39
44
|
this.datasetId = datasetId;
|
|
40
|
-
this.datasetMappingFunction =
|
|
45
|
+
this.datasetMappingFunction = datasetMappingFunction;
|
|
46
|
+
this.caller = new async_caller_js_1.AsyncCaller(asyncCallerParams);
|
|
41
47
|
}
|
|
42
48
|
static _getApifyApiToken(config) {
|
|
43
49
|
return config?.token ?? (0, env_js_1.getEnvironmentVariable)("APIFY_API_TOKEN");
|
|
@@ -50,7 +56,7 @@ class ApifyDatasetLoader extends base_js_1.BaseDocumentLoader {
|
|
|
50
56
|
*/
|
|
51
57
|
async load() {
|
|
52
58
|
const datasetItems = (await this.apifyClient.dataset(this.datasetId).listItems({ clean: true })).items;
|
|
53
|
-
return datasetItems.map(this.datasetMappingFunction);
|
|
59
|
+
return await Promise.all(datasetItems.map((item) => this.caller.call(async () => this.datasetMappingFunction(item))));
|
|
54
60
|
}
|
|
55
61
|
/**
|
|
56
62
|
* Create an ApifyDatasetLoader by calling an Actor on the Apify platform and waiting for its results to be ready.
|
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import { ActorCallOptions, ApifyClient, ApifyClientOptions, TaskCallOptions } from "apify-client";
|
|
2
|
+
import { AsyncCaller, AsyncCallerParams } from "../../util/async_caller.js";
|
|
2
3
|
import { BaseDocumentLoader, DocumentLoader } from "../base.js";
|
|
3
4
|
import { Document } from "../../document.js";
|
|
4
5
|
/**
|
|
5
6
|
* A type that represents a function that takes a single object (an Apify
|
|
6
7
|
* dataset item) and converts it to an instance of the Document class.
|
|
7
8
|
*/
|
|
8
|
-
export type ApifyDatasetMappingFunction<Metadata extends Record<string, any>> = (item: Record<string | number, unknown>) => Document<Metadata
|
|
9
|
+
export type ApifyDatasetMappingFunction<Metadata extends Record<string, any>> = (item: Record<string | number, unknown>) => Document<Metadata> | Promise<Document<Metadata>>;
|
|
10
|
+
export interface ApifyDatasetLoaderConfig<Metadata extends Record<string, any>> extends AsyncCallerParams {
|
|
11
|
+
datasetMappingFunction: ApifyDatasetMappingFunction<Metadata>;
|
|
12
|
+
clientOptions?: ApifyClientOptions;
|
|
13
|
+
}
|
|
9
14
|
/**
|
|
10
15
|
* A class that extends the BaseDocumentLoader and implements the
|
|
11
16
|
* DocumentLoader interface. It represents a document loader that loads
|
|
@@ -14,11 +19,9 @@ export type ApifyDatasetMappingFunction<Metadata extends Record<string, any>> =
|
|
|
14
19
|
export declare class ApifyDatasetLoader<Metadata extends Record<string, any>> extends BaseDocumentLoader implements DocumentLoader {
|
|
15
20
|
protected apifyClient: ApifyClient;
|
|
16
21
|
protected datasetId: string;
|
|
17
|
-
protected datasetMappingFunction:
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
clientOptions?: ApifyClientOptions;
|
|
21
|
-
});
|
|
22
|
+
protected datasetMappingFunction: ApifyDatasetMappingFunction<Metadata>;
|
|
23
|
+
protected caller: AsyncCaller;
|
|
24
|
+
constructor(datasetId: string, config: ApifyDatasetLoaderConfig<Metadata>);
|
|
22
25
|
private static _getApifyApiToken;
|
|
23
26
|
/**
|
|
24
27
|
* Retrieves the dataset items from the Apify platform and applies the
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
2
2
|
import { ApifyClient, } from "apify-client";
|
|
3
|
+
import { AsyncCaller } from "../../util/async_caller.js";
|
|
3
4
|
import { BaseDocumentLoader } from "../base.js";
|
|
4
5
|
import { getEnvironmentVariable } from "../../util/env.js";
|
|
5
6
|
/**
|
|
@@ -28,13 +29,18 @@ export class ApifyDatasetLoader extends BaseDocumentLoader {
|
|
|
28
29
|
writable: true,
|
|
29
30
|
value: void 0
|
|
30
31
|
});
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
Object.defineProperty(this, "caller", {
|
|
33
|
+
enumerable: true,
|
|
34
|
+
configurable: true,
|
|
35
|
+
writable: true,
|
|
36
|
+
value: void 0
|
|
35
37
|
});
|
|
38
|
+
const { clientOptions, datasetMappingFunction, ...asyncCallerParams } = config;
|
|
39
|
+
const token = ApifyDatasetLoader._getApifyApiToken(clientOptions);
|
|
40
|
+
this.apifyClient = new ApifyClient({ ...clientOptions, token });
|
|
36
41
|
this.datasetId = datasetId;
|
|
37
|
-
this.datasetMappingFunction =
|
|
42
|
+
this.datasetMappingFunction = datasetMappingFunction;
|
|
43
|
+
this.caller = new AsyncCaller(asyncCallerParams);
|
|
38
44
|
}
|
|
39
45
|
static _getApifyApiToken(config) {
|
|
40
46
|
return config?.token ?? getEnvironmentVariable("APIFY_API_TOKEN");
|
|
@@ -47,7 +53,7 @@ export class ApifyDatasetLoader extends BaseDocumentLoader {
|
|
|
47
53
|
*/
|
|
48
54
|
async load() {
|
|
49
55
|
const datasetItems = (await this.apifyClient.dataset(this.datasetId).listItems({ clean: true })).items;
|
|
50
|
-
return datasetItems.map(this.datasetMappingFunction);
|
|
56
|
+
return await Promise.all(datasetItems.map((item) => this.caller.call(async () => this.datasetMappingFunction(item))));
|
|
51
57
|
}
|
|
52
58
|
/**
|
|
53
59
|
* Create an ApifyDatasetLoader by calling an Actor on the Apify platform and waiting for its results to be ready.
|
|
@@ -52,9 +52,23 @@ class WebPDFLoader extends base_js_1.BaseDocumentLoader {
|
|
|
52
52
|
if (content.items.length === 0) {
|
|
53
53
|
continue;
|
|
54
54
|
}
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
55
|
+
// Eliminate excessive newlines
|
|
56
|
+
// Source: https://github.com/albertcui/pdf-parse/blob/7086fc1cc9058545cdf41dd0646d6ae5832c7107/lib/pdf-parse.js#L16
|
|
57
|
+
let lastY;
|
|
58
|
+
const textItems = [];
|
|
59
|
+
for (const item of content.items) {
|
|
60
|
+
if ("str" in item) {
|
|
61
|
+
if (lastY === item.transform[5] || !lastY) {
|
|
62
|
+
textItems.push(item.str);
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
textItems.push(`\n${item.str}`);
|
|
66
|
+
}
|
|
67
|
+
// eslint-disable-next-line prefer-destructuring
|
|
68
|
+
lastY = item.transform[5];
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
const text = textItems.join(" ");
|
|
58
72
|
documents.push(new document_js_1.Document({
|
|
59
73
|
pageContent: text,
|
|
60
74
|
metadata: {
|
|
@@ -49,9 +49,23 @@ export class WebPDFLoader extends BaseDocumentLoader {
|
|
|
49
49
|
if (content.items.length === 0) {
|
|
50
50
|
continue;
|
|
51
51
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
52
|
+
// Eliminate excessive newlines
|
|
53
|
+
// Source: https://github.com/albertcui/pdf-parse/blob/7086fc1cc9058545cdf41dd0646d6ae5832c7107/lib/pdf-parse.js#L16
|
|
54
|
+
let lastY;
|
|
55
|
+
const textItems = [];
|
|
56
|
+
for (const item of content.items) {
|
|
57
|
+
if ("str" in item) {
|
|
58
|
+
if (lastY === item.transform[5] || !lastY) {
|
|
59
|
+
textItems.push(item.str);
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
textItems.push(`\n${item.str}`);
|
|
63
|
+
}
|
|
64
|
+
// eslint-disable-next-line prefer-destructuring
|
|
65
|
+
lastY = item.transform[5];
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
const text = textItems.join(" ");
|
|
55
69
|
documents.push(new Document({
|
|
56
70
|
pageContent: text,
|
|
57
71
|
metadata: {
|
|
@@ -63,6 +63,43 @@ class PuppeteerWebBaseLoader extends base_js_1.BaseDocumentLoader {
|
|
|
63
63
|
const metadata = { source: this.webPath };
|
|
64
64
|
return [new document_js_1.Document({ pageContent: text, metadata })];
|
|
65
65
|
}
|
|
66
|
+
/**
|
|
67
|
+
* Static class method used to screenshot a web page and return
|
|
68
|
+
* it as a {@link Document} object where the pageContent property
|
|
69
|
+
* is the screenshot encoded in base64.
|
|
70
|
+
*
|
|
71
|
+
* @param {string} url
|
|
72
|
+
* @param {PuppeteerWebBaseLoaderOptions} options
|
|
73
|
+
* @returns {Document} A document object containing the screenshot of the page encoded in base64.
|
|
74
|
+
*/
|
|
75
|
+
static async _screenshot(url, options) {
|
|
76
|
+
const { launch } = await PuppeteerWebBaseLoader.imports();
|
|
77
|
+
const browser = await launch({
|
|
78
|
+
headless: true,
|
|
79
|
+
defaultViewport: null,
|
|
80
|
+
ignoreDefaultArgs: ["--disable-extensions"],
|
|
81
|
+
...options?.launchOptions,
|
|
82
|
+
});
|
|
83
|
+
const page = await browser.newPage();
|
|
84
|
+
await page.goto(url, {
|
|
85
|
+
timeout: 180000,
|
|
86
|
+
waitUntil: "domcontentloaded",
|
|
87
|
+
...options?.gotoOptions,
|
|
88
|
+
});
|
|
89
|
+
const screenshot = await page.screenshot();
|
|
90
|
+
const base64 = screenshot.toString("base64");
|
|
91
|
+
const metadata = { source: url };
|
|
92
|
+
return new document_js_1.Document({ pageContent: base64, metadata });
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Screenshot a web page and return it as a {@link Document} object where
|
|
96
|
+
* the pageContent property is the screenshot encoded in base64.
|
|
97
|
+
*
|
|
98
|
+
* @returns {Promise<Document>} A document object containing the screenshot of the page encoded in base64.
|
|
99
|
+
*/
|
|
100
|
+
async screenshot() {
|
|
101
|
+
return PuppeteerWebBaseLoader._screenshot(this.webPath, this.options);
|
|
102
|
+
}
|
|
66
103
|
/**
|
|
67
104
|
* Static method that imports the necessary Puppeteer modules. It returns
|
|
68
105
|
* a Promise that resolves to an object containing the imported modules.
|
|
@@ -40,6 +40,23 @@ export declare class PuppeteerWebBaseLoader extends BaseDocumentLoader implement
|
|
|
40
40
|
* @returns Promise that resolves to an array of Document objects.
|
|
41
41
|
*/
|
|
42
42
|
load(): Promise<Document[]>;
|
|
43
|
+
/**
|
|
44
|
+
* Static class method used to screenshot a web page and return
|
|
45
|
+
* it as a {@link Document} object where the pageContent property
|
|
46
|
+
* is the screenshot encoded in base64.
|
|
47
|
+
*
|
|
48
|
+
* @param {string} url
|
|
49
|
+
* @param {PuppeteerWebBaseLoaderOptions} options
|
|
50
|
+
* @returns {Document} A document object containing the screenshot of the page encoded in base64.
|
|
51
|
+
*/
|
|
52
|
+
static _screenshot(url: string, options?: PuppeteerWebBaseLoaderOptions): Promise<Document>;
|
|
53
|
+
/**
|
|
54
|
+
* Screenshot a web page and return it as a {@link Document} object where
|
|
55
|
+
* the pageContent property is the screenshot encoded in base64.
|
|
56
|
+
*
|
|
57
|
+
* @returns {Promise<Document>} A document object containing the screenshot of the page encoded in base64.
|
|
58
|
+
*/
|
|
59
|
+
screenshot(): Promise<Document>;
|
|
43
60
|
/**
|
|
44
61
|
* Static method that imports the necessary Puppeteer modules. It returns
|
|
45
62
|
* a Promise that resolves to an object containing the imported modules.
|
|
@@ -60,6 +60,43 @@ export class PuppeteerWebBaseLoader extends BaseDocumentLoader {
|
|
|
60
60
|
const metadata = { source: this.webPath };
|
|
61
61
|
return [new Document({ pageContent: text, metadata })];
|
|
62
62
|
}
|
|
63
|
+
/**
|
|
64
|
+
* Static class method used to screenshot a web page and return
|
|
65
|
+
* it as a {@link Document} object where the pageContent property
|
|
66
|
+
* is the screenshot encoded in base64.
|
|
67
|
+
*
|
|
68
|
+
* @param {string} url
|
|
69
|
+
* @param {PuppeteerWebBaseLoaderOptions} options
|
|
70
|
+
* @returns {Document} A document object containing the screenshot of the page encoded in base64.
|
|
71
|
+
*/
|
|
72
|
+
static async _screenshot(url, options) {
|
|
73
|
+
const { launch } = await PuppeteerWebBaseLoader.imports();
|
|
74
|
+
const browser = await launch({
|
|
75
|
+
headless: true,
|
|
76
|
+
defaultViewport: null,
|
|
77
|
+
ignoreDefaultArgs: ["--disable-extensions"],
|
|
78
|
+
...options?.launchOptions,
|
|
79
|
+
});
|
|
80
|
+
const page = await browser.newPage();
|
|
81
|
+
await page.goto(url, {
|
|
82
|
+
timeout: 180000,
|
|
83
|
+
waitUntil: "domcontentloaded",
|
|
84
|
+
...options?.gotoOptions,
|
|
85
|
+
});
|
|
86
|
+
const screenshot = await page.screenshot();
|
|
87
|
+
const base64 = screenshot.toString("base64");
|
|
88
|
+
const metadata = { source: url };
|
|
89
|
+
return new Document({ pageContent: base64, metadata });
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Screenshot a web page and return it as a {@link Document} object where
|
|
93
|
+
* the pageContent property is the screenshot encoded in base64.
|
|
94
|
+
*
|
|
95
|
+
* @returns {Promise<Document>} A document object containing the screenshot of the page encoded in base64.
|
|
96
|
+
*/
|
|
97
|
+
async screenshot() {
|
|
98
|
+
return PuppeteerWebBaseLoader._screenshot(this.webPath, this.options);
|
|
99
|
+
}
|
|
63
100
|
/**
|
|
64
101
|
* Static method that imports the necessary Puppeteer modules. It returns
|
|
65
102
|
* a Promise that resolves to an object containing the imported modules.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { OllamaInput, OllamaRequestParams } from "../util/ollama.js";
|
|
2
2
|
import { Embeddings, EmbeddingsParams } from "./base.js";
|
|
3
|
-
type CamelCasedRequestOptions = Omit<OllamaInput, "baseUrl" | "model">;
|
|
3
|
+
type CamelCasedRequestOptions = Omit<OllamaInput, "baseUrl" | "model" | "format">;
|
|
4
4
|
/**
|
|
5
5
|
* Interface for OllamaEmbeddings parameters. Extends EmbeddingsParams and
|
|
6
6
|
* defines additional parameters specific to the OllamaEmbeddings class.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.PlanAndExecuteAgentExecutor = void 0;
|
|
3
|
+
exports.PlanAndExecuteAgentExecutor = exports.isDynamicStructuredTool = void 0;
|
|
4
4
|
const base_js_1 = require("../../chains/base.cjs");
|
|
5
5
|
const base_js_2 = require("./base.cjs");
|
|
6
6
|
const executor_js_1 = require("../../agents/executor.cjs");
|
|
@@ -8,6 +8,21 @@ const prompt_js_1 = require("./prompt.cjs");
|
|
|
8
8
|
const llm_chain_js_1 = require("../../chains/llm_chain.cjs");
|
|
9
9
|
const outputParser_js_1 = require("./outputParser.cjs");
|
|
10
10
|
const index_js_1 = require("../../agents/chat/index.cjs");
|
|
11
|
+
const index_js_2 = require("../../agents/index.cjs");
|
|
12
|
+
/**
|
|
13
|
+
* A utility function to distiguish a dynamicstructuredtool over other tools.
|
|
14
|
+
* @param tool the tool to test
|
|
15
|
+
* @returns bool
|
|
16
|
+
*/
|
|
17
|
+
function isDynamicStructuredTool(tool) {
|
|
18
|
+
// We check for the existence of the static lc_name method in the object's constructor
|
|
19
|
+
return (
|
|
20
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
21
|
+
typeof tool.constructor.lc_name === "function" &&
|
|
22
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
23
|
+
tool.constructor.lc_name() === "DynamicStructuredTool");
|
|
24
|
+
}
|
|
25
|
+
exports.isDynamicStructuredTool = isDynamicStructuredTool;
|
|
11
26
|
/**
|
|
12
27
|
* Class representing a plan-and-execute agent executor. This agent
|
|
13
28
|
* decides on the full sequence of actions upfront, then executes them all
|
|
@@ -86,7 +101,18 @@ class PlanAndExecuteAgentExecutor extends base_js_1.BaseChain {
|
|
|
86
101
|
* @returns A new ChainStepExecutor instance.
|
|
87
102
|
*/
|
|
88
103
|
static getDefaultStepExecutor({ llm, tools, humanMessageTemplate = prompt_js_1.DEFAULT_STEP_EXECUTOR_HUMAN_CHAT_MESSAGE_TEMPLATE, }) {
|
|
89
|
-
|
|
104
|
+
let agent;
|
|
105
|
+
if (isDynamicStructuredTool(tools[0])) {
|
|
106
|
+
agent = index_js_2.StructuredChatAgent.fromLLMAndTools(llm, tools, {
|
|
107
|
+
humanMessageTemplate,
|
|
108
|
+
inputVariables: ["previous_steps", "current_step", "agent_scratchpad"],
|
|
109
|
+
});
|
|
110
|
+
return new base_js_2.ChainStepExecutor(executor_js_1.AgentExecutor.fromAgentAndTools({
|
|
111
|
+
agent,
|
|
112
|
+
tools,
|
|
113
|
+
}));
|
|
114
|
+
}
|
|
115
|
+
agent = index_js_1.ChatAgent.fromLLMAndTools(llm, tools, {
|
|
90
116
|
humanMessageTemplate,
|
|
91
117
|
});
|
|
92
118
|
return new base_js_2.ChainStepExecutor(executor_js_1.AgentExecutor.fromAgentAndTools({
|
|
@@ -4,7 +4,14 @@ import { ChainValues } from "../../schema/index.js";
|
|
|
4
4
|
import { BaseLanguageModel } from "../../base_language/index.js";
|
|
5
5
|
import { CallbackManagerForChainRun } from "../../callbacks/manager.js";
|
|
6
6
|
import { Tool } from "../../tools/base.js";
|
|
7
|
+
import { DynamicStructuredTool } from "../../tools/dynamic.js";
|
|
7
8
|
import { SerializedLLMChain } from "../../chains/serde.js";
|
|
9
|
+
/**
|
|
10
|
+
* A utility function to distiguish a dynamicstructuredtool over other tools.
|
|
11
|
+
* @param tool the tool to test
|
|
12
|
+
* @returns bool
|
|
13
|
+
*/
|
|
14
|
+
export declare function isDynamicStructuredTool(tool: Tool | DynamicStructuredTool): tool is DynamicStructuredTool;
|
|
8
15
|
/**
|
|
9
16
|
* Interface for the input to the PlanAndExecuteAgentExecutor class. It
|
|
10
17
|
* extends ChainInputs and includes additional properties for the planner,
|
|
@@ -42,7 +49,7 @@ export declare class PlanAndExecuteAgentExecutor extends BaseChain {
|
|
|
42
49
|
*/
|
|
43
50
|
static getDefaultPlanner({ llm, tools, }: {
|
|
44
51
|
llm: BaseLanguageModel;
|
|
45
|
-
tools: Tool[];
|
|
52
|
+
tools: Tool[] | DynamicStructuredTool[];
|
|
46
53
|
}): Promise<LLMPlanner>;
|
|
47
54
|
/**
|
|
48
55
|
* Static method that returns a default step executor for the agent. It
|
|
@@ -55,7 +62,7 @@ export declare class PlanAndExecuteAgentExecutor extends BaseChain {
|
|
|
55
62
|
*/
|
|
56
63
|
static getDefaultStepExecutor({ llm, tools, humanMessageTemplate, }: {
|
|
57
64
|
llm: BaseLanguageModel;
|
|
58
|
-
tools: Tool[];
|
|
65
|
+
tools: Tool[] | DynamicStructuredTool[];
|
|
59
66
|
humanMessageTemplate?: string;
|
|
60
67
|
}): ChainStepExecutor;
|
|
61
68
|
/**
|
|
@@ -70,7 +77,7 @@ export declare class PlanAndExecuteAgentExecutor extends BaseChain {
|
|
|
70
77
|
*/
|
|
71
78
|
static fromLLMAndTools({ llm, tools, humanMessageTemplate, }: {
|
|
72
79
|
llm: BaseLanguageModel;
|
|
73
|
-
tools: Tool[];
|
|
80
|
+
tools: Tool[] | DynamicStructuredTool[];
|
|
74
81
|
humanMessageTemplate?: string;
|
|
75
82
|
} & Omit<PlanAndExecuteAgentExecutorInput, "planner" | "stepExecutor">): Promise<PlanAndExecuteAgentExecutor>;
|
|
76
83
|
/** @ignore */
|
|
@@ -5,6 +5,20 @@ import { DEFAULT_STEP_EXECUTOR_HUMAN_CHAT_MESSAGE_TEMPLATE, getPlannerChatPrompt
|
|
|
5
5
|
import { LLMChain } from "../../chains/llm_chain.js";
|
|
6
6
|
import { PlanOutputParser } from "./outputParser.js";
|
|
7
7
|
import { ChatAgent } from "../../agents/chat/index.js";
|
|
8
|
+
import { StructuredChatAgent } from "../../agents/index.js";
|
|
9
|
+
/**
|
|
10
|
+
* A utility function to distiguish a dynamicstructuredtool over other tools.
|
|
11
|
+
* @param tool the tool to test
|
|
12
|
+
* @returns bool
|
|
13
|
+
*/
|
|
14
|
+
export function isDynamicStructuredTool(tool) {
|
|
15
|
+
// We check for the existence of the static lc_name method in the object's constructor
|
|
16
|
+
return (
|
|
17
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
18
|
+
typeof tool.constructor.lc_name === "function" &&
|
|
19
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
20
|
+
tool.constructor.lc_name() === "DynamicStructuredTool");
|
|
21
|
+
}
|
|
8
22
|
/**
|
|
9
23
|
* Class representing a plan-and-execute agent executor. This agent
|
|
10
24
|
* decides on the full sequence of actions upfront, then executes them all
|
|
@@ -83,7 +97,18 @@ export class PlanAndExecuteAgentExecutor extends BaseChain {
|
|
|
83
97
|
* @returns A new ChainStepExecutor instance.
|
|
84
98
|
*/
|
|
85
99
|
static getDefaultStepExecutor({ llm, tools, humanMessageTemplate = DEFAULT_STEP_EXECUTOR_HUMAN_CHAT_MESSAGE_TEMPLATE, }) {
|
|
86
|
-
|
|
100
|
+
let agent;
|
|
101
|
+
if (isDynamicStructuredTool(tools[0])) {
|
|
102
|
+
agent = StructuredChatAgent.fromLLMAndTools(llm, tools, {
|
|
103
|
+
humanMessageTemplate,
|
|
104
|
+
inputVariables: ["previous_steps", "current_step", "agent_scratchpad"],
|
|
105
|
+
});
|
|
106
|
+
return new ChainStepExecutor(AgentExecutor.fromAgentAndTools({
|
|
107
|
+
agent,
|
|
108
|
+
tools,
|
|
109
|
+
}));
|
|
110
|
+
}
|
|
111
|
+
agent = ChatAgent.fromLLMAndTools(llm, tools, {
|
|
87
112
|
humanMessageTemplate,
|
|
88
113
|
});
|
|
89
114
|
return new ChainStepExecutor(AgentExecutor.fromAgentAndTools({
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ChatPromptTemplate } from "../../prompts/chat.js";
|
|
2
2
|
import { Tool } from "../../tools/base.js";
|
|
3
|
+
import { DynamicStructuredTool } from "../../tools/dynamic.js";
|
|
3
4
|
export declare const PLANNER_SYSTEM_PROMPT_MESSAGE_TEMPLATE: string;
|
|
4
5
|
export declare const DEFAULT_STEP_EXECUTOR_HUMAN_CHAT_MESSAGE_TEMPLATE = "Previous steps: {previous_steps}\n\nCurrent objective: {current_step}\n\n{agent_scratchpad}\n\nYou may extract and combine relevant data from your previous steps when responding to me.";
|
|
5
6
|
/**
|
|
@@ -9,4 +10,4 @@ export declare const DEFAULT_STEP_EXECUTOR_HUMAN_CHAT_MESSAGE_TEMPLATE = "Previo
|
|
|
9
10
|
* @param tools the tools available to the `planner`
|
|
10
11
|
* @returns
|
|
11
12
|
*/
|
|
12
|
-
export declare const getPlannerChatPrompt: (tools: Tool[]) => Promise<ChatPromptTemplate<import("../../schema/index.js").InputValues<string>, any>>;
|
|
13
|
+
export declare const getPlannerChatPrompt: (tools: Tool[] | DynamicStructuredTool[]) => Promise<ChatPromptTemplate<import("../../schema/index.js").InputValues<string>, any>>;
|
package/dist/llms/ollama.cjs
CHANGED
|
@@ -212,6 +212,12 @@ class Ollama extends base_js_1.LLM {
|
|
|
212
212
|
writable: true,
|
|
213
213
|
value: void 0
|
|
214
214
|
});
|
|
215
|
+
Object.defineProperty(this, "format", {
|
|
216
|
+
enumerable: true,
|
|
217
|
+
configurable: true,
|
|
218
|
+
writable: true,
|
|
219
|
+
value: void 0
|
|
220
|
+
});
|
|
215
221
|
this.model = fields.model ?? this.model;
|
|
216
222
|
this.baseUrl = fields.baseUrl?.endsWith("/")
|
|
217
223
|
? fields.baseUrl.slice(0, -1)
|
|
@@ -246,6 +252,7 @@ class Ollama extends base_js_1.LLM {
|
|
|
246
252
|
this.useMLock = fields.useMLock;
|
|
247
253
|
this.useMMap = fields.useMMap;
|
|
248
254
|
this.vocabOnly = fields.vocabOnly;
|
|
255
|
+
this.format = fields.format;
|
|
249
256
|
}
|
|
250
257
|
_llmType() {
|
|
251
258
|
return "ollama";
|
|
@@ -253,6 +260,7 @@ class Ollama extends base_js_1.LLM {
|
|
|
253
260
|
invocationParams(options) {
|
|
254
261
|
return {
|
|
255
262
|
model: this.model,
|
|
263
|
+
format: this.format,
|
|
256
264
|
options: {
|
|
257
265
|
embedding_only: this.embeddingOnly,
|
|
258
266
|
f16_kv: this.f16KV,
|