langchain 0.0.75 → 0.0.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/base_language/count_tokens.cjs +5 -21
- package/dist/base_language/count_tokens.d.ts +1 -6
- package/dist/base_language/count_tokens.js +4 -19
- package/dist/base_language/index.cjs +10 -24
- package/dist/base_language/index.d.ts +3 -2
- package/dist/base_language/index.js +11 -25
- package/dist/cache/redis.d.ts +3 -1
- package/dist/callbacks/base.d.ts +4 -4
- package/dist/callbacks/handlers/initialize.cjs +2 -9
- package/dist/callbacks/handlers/initialize.d.ts +1 -1
- package/dist/callbacks/handlers/initialize.js +2 -9
- package/dist/callbacks/handlers/tracer.cjs +11 -22
- package/dist/callbacks/handlers/tracer.d.ts +2 -2
- package/dist/callbacks/handlers/tracer.js +11 -22
- package/dist/callbacks/handlers/tracer_langchain.cjs +52 -91
- package/dist/callbacks/handlers/tracer_langchain.d.ts +12 -9
- package/dist/callbacks/handlers/tracer_langchain.js +52 -68
- package/dist/callbacks/handlers/tracer_langchain_v1.cjs +5 -1
- package/dist/callbacks/handlers/tracer_langchain_v1.js +5 -1
- package/dist/callbacks/manager.cjs +10 -10
- package/dist/callbacks/manager.d.ts +2 -2
- package/dist/callbacks/manager.js +10 -10
- package/dist/chains/question_answering/map_reduce_prompts.cjs +5 -5
- package/dist/chains/question_answering/map_reduce_prompts.d.ts +1 -1
- package/dist/chains/question_answering/map_reduce_prompts.js +1 -1
- package/dist/chains/question_answering/refine_prompts.cjs +5 -5
- package/dist/chains/question_answering/refine_prompts.d.ts +1 -1
- package/dist/chains/question_answering/refine_prompts.js +1 -1
- package/dist/chains/question_answering/stuff_prompts.cjs +2 -2
- package/dist/chains/question_answering/stuff_prompts.d.ts +1 -1
- package/dist/chains/question_answering/stuff_prompts.js +1 -1
- package/dist/chains/sequential_chain.cjs +2 -2
- package/dist/chains/sequential_chain.d.ts +2 -2
- package/dist/chains/sequential_chain.js +2 -2
- package/dist/chains/sql_db/sql_db_prompt.cjs +20 -1
- package/dist/chains/sql_db/sql_db_prompt.d.ts +1 -0
- package/dist/chains/sql_db/sql_db_prompt.js +19 -0
- package/dist/chat_models/base.cjs +17 -1
- package/dist/chat_models/base.d.ts +6 -0
- package/dist/chat_models/base.js +18 -2
- package/dist/chat_models/openai.cjs +2 -2
- package/dist/chat_models/openai.js +2 -2
- package/dist/client/langchainplus.cjs +90 -29
- package/dist/client/langchainplus.d.ts +7 -5
- package/dist/client/langchainplus.js +90 -29
- package/dist/document_loaders/fs/unstructured.cjs +8 -0
- package/dist/document_loaders/fs/unstructured.d.ts +3 -0
- package/dist/document_loaders/fs/unstructured.js +8 -0
- package/dist/document_loaders/web/apify_dataset.cjs +64 -0
- package/dist/document_loaders/web/apify_dataset.d.ts +28 -0
- package/dist/document_loaders/web/apify_dataset.js +60 -0
- package/dist/embeddings/openai.cjs +2 -2
- package/dist/embeddings/openai.js +2 -2
- package/dist/llms/base.cjs +18 -1
- package/dist/llms/base.d.ts +7 -1
- package/dist/llms/base.js +19 -2
- package/dist/llms/openai-chat.cjs +2 -2
- package/dist/llms/openai-chat.js +2 -2
- package/dist/llms/openai.cjs +2 -2
- package/dist/llms/openai.js +2 -2
- package/dist/memory/base.cjs +9 -1
- package/dist/memory/base.d.ts +1 -0
- package/dist/memory/base.js +7 -0
- package/dist/memory/entity_memory.cjs +151 -0
- package/dist/memory/entity_memory.d.ts +35 -0
- package/dist/memory/entity_memory.js +147 -0
- package/dist/memory/index.cjs +5 -1
- package/dist/memory/index.d.ts +2 -0
- package/dist/memory/index.js +2 -0
- package/dist/memory/prompt.cjs +84 -1
- package/dist/memory/prompt.d.ts +6 -0
- package/dist/memory/prompt.js +83 -0
- package/dist/memory/stores/entity/in_memory.cjs +32 -0
- package/dist/memory/stores/entity/in_memory.d.ts +10 -0
- package/dist/memory/stores/entity/in_memory.js +28 -0
- package/dist/prompts/index.cjs +6 -1
- package/dist/prompts/index.d.ts +1 -0
- package/dist/prompts/index.js +1 -0
- package/dist/{chains/prompt_selector.d.ts → prompts/selectors/conditional.d.ts} +4 -4
- package/dist/schema/index.cjs +13 -1
- package/dist/schema/index.d.ts +17 -0
- package/dist/schema/index.js +11 -0
- package/dist/stores/message/dynamodb.cjs +8 -6
- package/dist/stores/message/dynamodb.js +8 -6
- package/dist/stores/message/redis.cjs +5 -3
- package/dist/stores/message/redis.js +5 -3
- package/dist/stores/message/utils.cjs +30 -15
- package/dist/stores/message/utils.d.ts +4 -2
- package/dist/stores/message/utils.js +28 -14
- package/dist/text_splitter.cjs +3 -23
- package/dist/text_splitter.d.ts +1 -3
- package/dist/text_splitter.js +3 -23
- package/dist/tools/webbrowser.cjs +5 -7
- package/dist/tools/webbrowser.js +3 -5
- package/dist/util/env.cjs +39 -7
- package/dist/util/env.d.ts +19 -0
- package/dist/util/env.js +32 -6
- package/dist/util/sql_utils.cjs +18 -0
- package/dist/util/sql_utils.js +19 -1
- package/dist/util/tiktoken.cjs +26 -0
- package/dist/util/tiktoken.d.ts +9 -0
- package/dist/util/tiktoken.js +21 -0
- package/dist/vectorstores/redis.cjs +236 -0
- package/dist/vectorstores/redis.d.ts +80 -0
- package/dist/vectorstores/redis.js +232 -0
- package/document_loaders/web/apify_dataset.cjs +1 -0
- package/document_loaders/web/apify_dataset.d.ts +1 -0
- package/document_loaders/web/apify_dataset.js +1 -0
- package/package.json +25 -5
- package/vectorstores/redis.cjs +1 -0
- package/vectorstores/redis.d.ts +1 -0
- package/vectorstores/redis.js +1 -0
- /package/dist/{chains/prompt_selector.cjs → prompts/selectors/conditional.cjs} +0 -0
- /package/dist/{chains/prompt_selector.js → prompts/selectors/conditional.js} +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { LangChainTracer } from "../callbacks/handlers/tracer_langchain.js";
|
|
2
|
+
import { mapStoredMessagesToChatMessages } from "../stores/message/utils.js";
|
|
2
3
|
// utility functions
|
|
3
4
|
const isLocalhost = (url) => {
|
|
4
5
|
const strippedUrl = url.replace("http://", "").replace("https://", "");
|
|
@@ -50,10 +51,28 @@ export function isChatModel(llm) {
|
|
|
50
51
|
return (typeof blm?._modelType === "function" &&
|
|
51
52
|
blm?._modelType() === "base_chat_model");
|
|
52
53
|
}
|
|
53
|
-
export function isChain(llm) {
|
|
54
|
-
|
|
54
|
+
export async function isChain(llm) {
|
|
55
|
+
if (isLLM(llm)) {
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
const bchFactory = llm;
|
|
59
|
+
const bch = await bchFactory();
|
|
55
60
|
return (typeof bch?._chainType === "function" && bch?._chainType() !== undefined);
|
|
56
61
|
}
|
|
62
|
+
async function getModelOrFactoryType(llm) {
|
|
63
|
+
if (isLLM(llm)) {
|
|
64
|
+
return "llm";
|
|
65
|
+
}
|
|
66
|
+
if (isChatModel(llm)) {
|
|
67
|
+
return "chatModel";
|
|
68
|
+
}
|
|
69
|
+
const bchFactory = llm;
|
|
70
|
+
const bch = await bchFactory();
|
|
71
|
+
if (typeof bch?._chainType === "function") {
|
|
72
|
+
return "chainFactory";
|
|
73
|
+
}
|
|
74
|
+
throw new Error("Unknown model or factory type");
|
|
75
|
+
}
|
|
57
76
|
export class LangChainPlusClient {
|
|
58
77
|
constructor(apiUrl, tenantId, apiKey) {
|
|
59
78
|
Object.defineProperty(this, "apiKey", {
|
|
@@ -145,6 +164,26 @@ export class LangChainPlusClient {
|
|
|
145
164
|
const result = await response.json();
|
|
146
165
|
return result;
|
|
147
166
|
}
|
|
167
|
+
async createDataset(name, description) {
|
|
168
|
+
const response = await fetch(`${this.apiUrl}/datasets`, {
|
|
169
|
+
method: "POST",
|
|
170
|
+
headers: { ...this.headers, "Content-Type": "application/json" },
|
|
171
|
+
body: JSON.stringify({
|
|
172
|
+
name,
|
|
173
|
+
description,
|
|
174
|
+
tenant_id: this.tenantId,
|
|
175
|
+
}),
|
|
176
|
+
});
|
|
177
|
+
if (!response.ok) {
|
|
178
|
+
const result = await response.json();
|
|
179
|
+
if (result.detail && result.detail.includes("already exists")) {
|
|
180
|
+
throw new Error(`Dataset ${name} already exists`);
|
|
181
|
+
}
|
|
182
|
+
throw new Error(`Failed to create dataset ${response.status} ${response.statusText}`);
|
|
183
|
+
}
|
|
184
|
+
const result = await response.json();
|
|
185
|
+
return result;
|
|
186
|
+
}
|
|
148
187
|
async readDataset(datasetId, datasetName) {
|
|
149
188
|
let path = "/datasets";
|
|
150
189
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -280,58 +319,80 @@ export class LangChainPlusClient {
|
|
|
280
319
|
return result;
|
|
281
320
|
}
|
|
282
321
|
async runLLM(example, tracer, llm, numRepetitions = 1) {
|
|
283
|
-
const results =
|
|
284
|
-
for (let i = 0; i < numRepetitions; i += 1) {
|
|
322
|
+
const results = await Promise.all(Array.from({ length: numRepetitions }).map(async () => {
|
|
285
323
|
try {
|
|
286
|
-
const
|
|
287
|
-
|
|
324
|
+
const prompt = example.inputs.prompt;
|
|
325
|
+
return llm.generate([prompt], undefined, [tracer]);
|
|
288
326
|
}
|
|
289
327
|
catch (e) {
|
|
290
328
|
console.error(e);
|
|
291
|
-
|
|
329
|
+
return stringifyError(e);
|
|
292
330
|
}
|
|
293
|
-
}
|
|
331
|
+
}));
|
|
294
332
|
return results;
|
|
295
333
|
}
|
|
296
|
-
async runChain(example, tracer,
|
|
297
|
-
const results =
|
|
298
|
-
for (let i = 0; i < numRepetitions; i += 1) {
|
|
334
|
+
async runChain(example, tracer, chainFactory, numRepetitions = 1) {
|
|
335
|
+
const results = await Promise.all(Array.from({ length: numRepetitions }).map(async () => {
|
|
299
336
|
try {
|
|
300
|
-
|
|
337
|
+
const chain = await chainFactory();
|
|
338
|
+
return chain.call(example.inputs, [tracer]);
|
|
301
339
|
}
|
|
302
340
|
catch (e) {
|
|
303
341
|
console.error(e);
|
|
304
|
-
|
|
342
|
+
return stringifyError(e);
|
|
305
343
|
}
|
|
306
|
-
}
|
|
344
|
+
}));
|
|
345
|
+
return results;
|
|
346
|
+
}
|
|
347
|
+
async runChatModel(example, tracer, chatModel, numRepetitions = 1) {
|
|
348
|
+
const results = await Promise.all(Array.from({ length: numRepetitions }).map(async () => {
|
|
349
|
+
try {
|
|
350
|
+
const messages = example.inputs.messages;
|
|
351
|
+
return chatModel.generate([mapStoredMessagesToChatMessages(messages)], undefined, [tracer]);
|
|
352
|
+
}
|
|
353
|
+
catch (e) {
|
|
354
|
+
console.error(e);
|
|
355
|
+
return stringifyError(e);
|
|
356
|
+
}
|
|
357
|
+
}));
|
|
307
358
|
return results;
|
|
308
359
|
}
|
|
309
|
-
async runOnDataset(datasetName,
|
|
360
|
+
async runOnDataset(datasetName, llmOrChainFactory, numRepetitions = 1, sessionName = undefined) {
|
|
310
361
|
const examples = await this.listExamples(undefined, datasetName);
|
|
311
|
-
let sessionName_
|
|
362
|
+
let sessionName_;
|
|
312
363
|
if (sessionName === undefined) {
|
|
313
364
|
const currentTime = new Date().toISOString();
|
|
314
|
-
sessionName_ = `${datasetName}-${
|
|
365
|
+
sessionName_ = `${datasetName}-${llmOrChainFactory.constructor.name}-${currentTime}`;
|
|
366
|
+
}
|
|
367
|
+
else {
|
|
368
|
+
sessionName_ = sessionName;
|
|
315
369
|
}
|
|
316
370
|
const results = {};
|
|
317
|
-
const
|
|
318
|
-
await
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
371
|
+
const modelOrFactoryType = await getModelOrFactoryType(llmOrChainFactory);
|
|
372
|
+
await Promise.all(examples.map(async (example) => {
|
|
373
|
+
const tracer = new LangChainTracer({
|
|
374
|
+
exampleId: example.id,
|
|
375
|
+
sessionName: sessionName_,
|
|
376
|
+
});
|
|
377
|
+
if (modelOrFactoryType === "llm") {
|
|
378
|
+
const llm = llmOrChainFactory;
|
|
379
|
+
const llmResult = await this.runLLM(example, tracer, llm, numRepetitions);
|
|
322
380
|
results[example.id] = llmResult;
|
|
323
381
|
}
|
|
324
|
-
else if (
|
|
325
|
-
const
|
|
326
|
-
|
|
382
|
+
else if (modelOrFactoryType === "chainFactory") {
|
|
383
|
+
const chainFactory = llmOrChainFactory;
|
|
384
|
+
const chainResult = await this.runChain(example, tracer, chainFactory, numRepetitions);
|
|
385
|
+
results[example.id] = chainResult;
|
|
327
386
|
}
|
|
328
|
-
else if (
|
|
329
|
-
|
|
387
|
+
else if (modelOrFactoryType === "chatModel") {
|
|
388
|
+
const chatModel = llmOrChainFactory;
|
|
389
|
+
const chatModelResult = await this.runChatModel(example, tracer, chatModel, numRepetitions);
|
|
390
|
+
results[example.id] = chatModelResult;
|
|
330
391
|
}
|
|
331
392
|
else {
|
|
332
|
-
throw new Error(` llm or chain type: ${
|
|
393
|
+
throw new Error(` llm or chain type: ${llmOrChainFactory}`);
|
|
333
394
|
}
|
|
334
|
-
}
|
|
395
|
+
}));
|
|
335
396
|
return results;
|
|
336
397
|
}
|
|
337
398
|
}
|
|
@@ -42,6 +42,12 @@ class UnstructuredLoader extends base_js_1.BaseDocumentLoader {
|
|
|
42
42
|
writable: true,
|
|
43
43
|
value: void 0
|
|
44
44
|
});
|
|
45
|
+
Object.defineProperty(this, "strategy", {
|
|
46
|
+
enumerable: true,
|
|
47
|
+
configurable: true,
|
|
48
|
+
writable: true,
|
|
49
|
+
value: void 0
|
|
50
|
+
});
|
|
45
51
|
// Temporary shim to avoid breaking existing users
|
|
46
52
|
// Remove when API keys are enforced by Unstructured and existing code will break anyway
|
|
47
53
|
const isLegacySyntax = typeof optionsOrLegacyFilePath === "string";
|
|
@@ -53,6 +59,7 @@ class UnstructuredLoader extends base_js_1.BaseDocumentLoader {
|
|
|
53
59
|
this.filePath = filePathOrLegacyApiUrl;
|
|
54
60
|
this.apiKey = optionsOrLegacyFilePath.apiKey;
|
|
55
61
|
this.apiUrl = optionsOrLegacyFilePath.apiUrl ?? this.apiUrl;
|
|
62
|
+
this.strategy = optionsOrLegacyFilePath.strategy ?? "hi_res";
|
|
56
63
|
}
|
|
57
64
|
}
|
|
58
65
|
async _partition() {
|
|
@@ -66,6 +73,7 @@ class UnstructuredLoader extends base_js_1.BaseDocumentLoader {
|
|
|
66
73
|
formData.append("files", new Blob([buffer]), fileName);
|
|
67
74
|
const headers = {
|
|
68
75
|
"UNSTRUCTURED-API-KEY": this.apiKey ?? "",
|
|
76
|
+
strategy: this.strategy,
|
|
69
77
|
};
|
|
70
78
|
const response = await fetch(this.apiUrl, {
|
|
71
79
|
method: "POST",
|
|
@@ -15,15 +15,18 @@ type Element = {
|
|
|
15
15
|
type UnstructuredLoaderOptions = {
|
|
16
16
|
apiKey?: string;
|
|
17
17
|
apiUrl?: string;
|
|
18
|
+
strategy?: string;
|
|
18
19
|
};
|
|
19
20
|
type UnstructuredDirectoryLoaderOptions = UnstructuredLoaderOptions & {
|
|
20
21
|
recursive?: boolean;
|
|
21
22
|
unknown?: UnknownHandling;
|
|
23
|
+
strategy?: string;
|
|
22
24
|
};
|
|
23
25
|
export declare class UnstructuredLoader extends BaseDocumentLoader {
|
|
24
26
|
filePath: string;
|
|
25
27
|
private apiUrl;
|
|
26
28
|
private apiKey?;
|
|
29
|
+
private strategy;
|
|
27
30
|
constructor(filePathOrLegacyApiUrl: string, optionsOrLegacyFilePath?: UnstructuredLoaderOptions | string);
|
|
28
31
|
_partition(): Promise<Element[]>;
|
|
29
32
|
load(): Promise<Document[]>;
|
|
@@ -38,6 +38,12 @@ export class UnstructuredLoader extends BaseDocumentLoader {
|
|
|
38
38
|
writable: true,
|
|
39
39
|
value: void 0
|
|
40
40
|
});
|
|
41
|
+
Object.defineProperty(this, "strategy", {
|
|
42
|
+
enumerable: true,
|
|
43
|
+
configurable: true,
|
|
44
|
+
writable: true,
|
|
45
|
+
value: void 0
|
|
46
|
+
});
|
|
41
47
|
// Temporary shim to avoid breaking existing users
|
|
42
48
|
// Remove when API keys are enforced by Unstructured and existing code will break anyway
|
|
43
49
|
const isLegacySyntax = typeof optionsOrLegacyFilePath === "string";
|
|
@@ -49,6 +55,7 @@ export class UnstructuredLoader extends BaseDocumentLoader {
|
|
|
49
55
|
this.filePath = filePathOrLegacyApiUrl;
|
|
50
56
|
this.apiKey = optionsOrLegacyFilePath.apiKey;
|
|
51
57
|
this.apiUrl = optionsOrLegacyFilePath.apiUrl ?? this.apiUrl;
|
|
58
|
+
this.strategy = optionsOrLegacyFilePath.strategy ?? "hi_res";
|
|
52
59
|
}
|
|
53
60
|
}
|
|
54
61
|
async _partition() {
|
|
@@ -62,6 +69,7 @@ export class UnstructuredLoader extends BaseDocumentLoader {
|
|
|
62
69
|
formData.append("files", new Blob([buffer]), fileName);
|
|
63
70
|
const headers = {
|
|
64
71
|
"UNSTRUCTURED-API-KEY": this.apiKey ?? "",
|
|
72
|
+
strategy: this.strategy,
|
|
65
73
|
};
|
|
66
74
|
const response = await fetch(this.apiUrl, {
|
|
67
75
|
method: "POST",
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ApifyDatasetLoader = void 0;
|
|
4
|
+
const apify_client_1 = require("apify-client");
|
|
5
|
+
const base_js_1 = require("../base.cjs");
|
|
6
|
+
class ApifyDatasetLoader extends base_js_1.BaseDocumentLoader {
|
|
7
|
+
constructor(datasetId, config) {
|
|
8
|
+
super();
|
|
9
|
+
Object.defineProperty(this, "apifyClient", {
|
|
10
|
+
enumerable: true,
|
|
11
|
+
configurable: true,
|
|
12
|
+
writable: true,
|
|
13
|
+
value: void 0
|
|
14
|
+
});
|
|
15
|
+
Object.defineProperty(this, "datasetId", {
|
|
16
|
+
enumerable: true,
|
|
17
|
+
configurable: true,
|
|
18
|
+
writable: true,
|
|
19
|
+
value: void 0
|
|
20
|
+
});
|
|
21
|
+
Object.defineProperty(this, "datasetMappingFunction", {
|
|
22
|
+
enumerable: true,
|
|
23
|
+
configurable: true,
|
|
24
|
+
writable: true,
|
|
25
|
+
value: void 0
|
|
26
|
+
});
|
|
27
|
+
const apifyApiToken = ApifyDatasetLoader._getApifyApiToken(config.clientOptions);
|
|
28
|
+
this.apifyClient = new apify_client_1.ApifyClient({
|
|
29
|
+
...config.clientOptions,
|
|
30
|
+
token: apifyApiToken,
|
|
31
|
+
});
|
|
32
|
+
this.datasetId = datasetId;
|
|
33
|
+
this.datasetMappingFunction = config.datasetMappingFunction;
|
|
34
|
+
}
|
|
35
|
+
static _getApifyApiToken(config) {
|
|
36
|
+
return (config?.token ??
|
|
37
|
+
// eslint-disable-next-line no-process-env
|
|
38
|
+
(typeof process !== "undefined" ? process.env.APIFY_API_TOKEN : undefined));
|
|
39
|
+
}
|
|
40
|
+
async load() {
|
|
41
|
+
const datasetItems = (await this.apifyClient.dataset(this.datasetId).listItems({ clean: true })).items;
|
|
42
|
+
return datasetItems.map(this.datasetMappingFunction);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Create an ApifyDatasetLoader by calling an Actor on the Apify platform and waiting for its results to be ready.
|
|
46
|
+
* @param actorId The ID or name of the Actor on the Apify platform.
|
|
47
|
+
* @param input The input object of the Actor that you're trying to run.
|
|
48
|
+
* @param options Options specifying settings for the Actor run.
|
|
49
|
+
* @param options.datasetMappingFunction A function that takes a single object (an Apify dataset item) and converts it to an instance of the Document class.
|
|
50
|
+
* @returns An instance of `ApifyDatasetLoader` with the results from the Actor run.
|
|
51
|
+
*/
|
|
52
|
+
static async fromActorCall(actorId, input, config) {
|
|
53
|
+
const apifyApiToken = ApifyDatasetLoader._getApifyApiToken(config.clientOptions);
|
|
54
|
+
const apifyClient = new apify_client_1.ApifyClient({ token: apifyApiToken });
|
|
55
|
+
const actorCall = await apifyClient
|
|
56
|
+
.actor(actorId)
|
|
57
|
+
.call(input, config.callOptions ?? {});
|
|
58
|
+
return new ApifyDatasetLoader(actorCall.defaultDatasetId, {
|
|
59
|
+
datasetMappingFunction: config.datasetMappingFunction,
|
|
60
|
+
clientOptions: { ...config.clientOptions, token: apifyApiToken },
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
exports.ApifyDatasetLoader = ApifyDatasetLoader;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { ApifyClient, ApifyClientOptions, ActorCallOptions } from "apify-client";
|
|
2
|
+
import { Document } from "../../document.js";
|
|
3
|
+
import { BaseDocumentLoader, DocumentLoader } from "../base.js";
|
|
4
|
+
export type ApifyDatasetMappingFunction = (item: Record<string | number, unknown>) => Document;
|
|
5
|
+
export declare class ApifyDatasetLoader extends BaseDocumentLoader implements DocumentLoader {
|
|
6
|
+
protected apifyClient: ApifyClient;
|
|
7
|
+
protected datasetId: string;
|
|
8
|
+
protected datasetMappingFunction: (item: Record<string | number, unknown>) => Document;
|
|
9
|
+
constructor(datasetId: string, config: {
|
|
10
|
+
datasetMappingFunction: ApifyDatasetMappingFunction;
|
|
11
|
+
clientOptions?: ApifyClientOptions;
|
|
12
|
+
});
|
|
13
|
+
private static _getApifyApiToken;
|
|
14
|
+
load(): Promise<Document[]>;
|
|
15
|
+
/**
|
|
16
|
+
* Create an ApifyDatasetLoader by calling an Actor on the Apify platform and waiting for its results to be ready.
|
|
17
|
+
* @param actorId The ID or name of the Actor on the Apify platform.
|
|
18
|
+
* @param input The input object of the Actor that you're trying to run.
|
|
19
|
+
* @param options Options specifying settings for the Actor run.
|
|
20
|
+
* @param options.datasetMappingFunction A function that takes a single object (an Apify dataset item) and converts it to an instance of the Document class.
|
|
21
|
+
* @returns An instance of `ApifyDatasetLoader` with the results from the Actor run.
|
|
22
|
+
*/
|
|
23
|
+
static fromActorCall(actorId: string, input: Record<string | number, unknown>, config: {
|
|
24
|
+
callOptions?: ActorCallOptions;
|
|
25
|
+
clientOptions?: ApifyClientOptions;
|
|
26
|
+
datasetMappingFunction: ApifyDatasetMappingFunction;
|
|
27
|
+
}): Promise<ApifyDatasetLoader>;
|
|
28
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { ApifyClient, } from "apify-client";
|
|
2
|
+
import { BaseDocumentLoader } from "../base.js";
|
|
3
|
+
export class ApifyDatasetLoader extends BaseDocumentLoader {
|
|
4
|
+
constructor(datasetId, config) {
|
|
5
|
+
super();
|
|
6
|
+
Object.defineProperty(this, "apifyClient", {
|
|
7
|
+
enumerable: true,
|
|
8
|
+
configurable: true,
|
|
9
|
+
writable: true,
|
|
10
|
+
value: void 0
|
|
11
|
+
});
|
|
12
|
+
Object.defineProperty(this, "datasetId", {
|
|
13
|
+
enumerable: true,
|
|
14
|
+
configurable: true,
|
|
15
|
+
writable: true,
|
|
16
|
+
value: void 0
|
|
17
|
+
});
|
|
18
|
+
Object.defineProperty(this, "datasetMappingFunction", {
|
|
19
|
+
enumerable: true,
|
|
20
|
+
configurable: true,
|
|
21
|
+
writable: true,
|
|
22
|
+
value: void 0
|
|
23
|
+
});
|
|
24
|
+
const apifyApiToken = ApifyDatasetLoader._getApifyApiToken(config.clientOptions);
|
|
25
|
+
this.apifyClient = new ApifyClient({
|
|
26
|
+
...config.clientOptions,
|
|
27
|
+
token: apifyApiToken,
|
|
28
|
+
});
|
|
29
|
+
this.datasetId = datasetId;
|
|
30
|
+
this.datasetMappingFunction = config.datasetMappingFunction;
|
|
31
|
+
}
|
|
32
|
+
static _getApifyApiToken(config) {
|
|
33
|
+
return (config?.token ??
|
|
34
|
+
// eslint-disable-next-line no-process-env
|
|
35
|
+
(typeof process !== "undefined" ? process.env.APIFY_API_TOKEN : undefined));
|
|
36
|
+
}
|
|
37
|
+
async load() {
|
|
38
|
+
const datasetItems = (await this.apifyClient.dataset(this.datasetId).listItems({ clean: true })).items;
|
|
39
|
+
return datasetItems.map(this.datasetMappingFunction);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Create an ApifyDatasetLoader by calling an Actor on the Apify platform and waiting for its results to be ready.
|
|
43
|
+
* @param actorId The ID or name of the Actor on the Apify platform.
|
|
44
|
+
* @param input The input object of the Actor that you're trying to run.
|
|
45
|
+
* @param options Options specifying settings for the Actor run.
|
|
46
|
+
* @param options.datasetMappingFunction A function that takes a single object (an Apify dataset item) and converts it to an instance of the Document class.
|
|
47
|
+
* @returns An instance of `ApifyDatasetLoader` with the results from the Actor run.
|
|
48
|
+
*/
|
|
49
|
+
static async fromActorCall(actorId, input, config) {
|
|
50
|
+
const apifyApiToken = ApifyDatasetLoader._getApifyApiToken(config.clientOptions);
|
|
51
|
+
const apifyClient = new ApifyClient({ token: apifyApiToken });
|
|
52
|
+
const actorCall = await apifyClient
|
|
53
|
+
.actor(actorId)
|
|
54
|
+
.call(input, config.callOptions ?? {});
|
|
55
|
+
return new ApifyDatasetLoader(actorCall.defaultDatasetId, {
|
|
56
|
+
datasetMappingFunction: config.datasetMappingFunction,
|
|
57
|
+
clientOptions: { ...config.clientOptions, token: apifyApiToken },
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -5,7 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.OpenAIEmbeddings = void 0;
|
|
7
7
|
const openai_1 = require("openai");
|
|
8
|
-
const
|
|
8
|
+
const env_js_1 = require("../util/env.cjs");
|
|
9
9
|
const axios_fetch_adapter_js_1 = __importDefault(require("../util/axios-fetch-adapter.cjs"));
|
|
10
10
|
const chunk_js_1 = require("../util/chunk.cjs");
|
|
11
11
|
const base_js_1 = require("./base.cjs");
|
|
@@ -159,7 +159,7 @@ class OpenAIEmbeddings extends base_js_1.Embeddings {
|
|
|
159
159
|
basePath: endpoint,
|
|
160
160
|
baseOptions: {
|
|
161
161
|
timeout: this.timeout,
|
|
162
|
-
adapter:
|
|
162
|
+
adapter: (0, env_js_1.isNode)() ? undefined : axios_fetch_adapter_js_1.default,
|
|
163
163
|
...this.clientConfig.baseOptions,
|
|
164
164
|
},
|
|
165
165
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Configuration, OpenAIApi, } from "openai";
|
|
2
|
-
import { isNode } from "
|
|
2
|
+
import { isNode } from "../util/env.js";
|
|
3
3
|
import fetchAdapter from "../util/axios-fetch-adapter.js";
|
|
4
4
|
import { chunkArray } from "../util/chunk.js";
|
|
5
5
|
import { Embeddings } from "./base.js";
|
|
@@ -153,7 +153,7 @@ export class OpenAIEmbeddings extends Embeddings {
|
|
|
153
153
|
basePath: endpoint,
|
|
154
154
|
baseOptions: {
|
|
155
155
|
timeout: this.timeout,
|
|
156
|
-
adapter: isNode ? undefined : fetchAdapter,
|
|
156
|
+
adapter: isNode() ? undefined : fetchAdapter,
|
|
157
157
|
...this.clientConfig.baseOptions,
|
|
158
158
|
},
|
|
159
159
|
});
|
package/dist/llms/base.cjs
CHANGED
|
@@ -5,6 +5,7 @@ const index_js_1 = require("../cache/index.cjs");
|
|
|
5
5
|
const index_js_2 = require("../schema/index.cjs");
|
|
6
6
|
const index_js_3 = require("../base_language/index.cjs");
|
|
7
7
|
const manager_js_1 = require("../callbacks/manager.cjs");
|
|
8
|
+
const base_js_1 = require("../memory/base.cjs");
|
|
8
9
|
/**
|
|
9
10
|
* LLM Wrapper. Provides an {@link call} (an {@link generate}) function that takes in a prompt (or prompts) and returns a string.
|
|
10
11
|
*/
|
|
@@ -31,10 +32,18 @@ class BaseLLM extends index_js_3.BaseLanguageModel {
|
|
|
31
32
|
const prompts = promptValues.map((promptValue) => promptValue.toString());
|
|
32
33
|
return this.generate(prompts, options, callbacks);
|
|
33
34
|
}
|
|
35
|
+
/**
|
|
36
|
+
* Get the parameters used to invoke the model
|
|
37
|
+
*/
|
|
38
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
39
|
+
invocationParams() {
|
|
40
|
+
return {};
|
|
41
|
+
}
|
|
34
42
|
/** @ignore */
|
|
35
43
|
async _generateUncached(prompts, options, callbacks) {
|
|
36
44
|
const callbackManager_ = await manager_js_1.CallbackManager.configure(callbacks, this.callbacks, { verbose: this.verbose });
|
|
37
|
-
const
|
|
45
|
+
const invocationParams = { invocation_params: this?.invocationParams() };
|
|
46
|
+
const runManager = await callbackManager_?.handleLLMStart({ name: this._llmType() }, prompts, undefined, undefined, invocationParams);
|
|
38
47
|
let output;
|
|
39
48
|
try {
|
|
40
49
|
output = await this._generate(prompts, options, runManager);
|
|
@@ -107,6 +116,14 @@ class BaseLLM extends index_js_3.BaseLanguageModel {
|
|
|
107
116
|
const { generations } = await this.generate([prompt], options ?? {}, callbacks);
|
|
108
117
|
return generations[0][0].text;
|
|
109
118
|
}
|
|
119
|
+
async predict(text, options, callbacks) {
|
|
120
|
+
return this.call(text, options, callbacks);
|
|
121
|
+
}
|
|
122
|
+
async predictMessages(messages, options, callbacks) {
|
|
123
|
+
const text = (0, base_js_1.getBufferString)(messages);
|
|
124
|
+
const prediction = await this.call(text, options, callbacks);
|
|
125
|
+
return new index_js_2.AIChatMessage(prediction);
|
|
126
|
+
}
|
|
110
127
|
/**
|
|
111
128
|
* Get the identifying parameters of the LLM.
|
|
112
129
|
*/
|
package/dist/llms/base.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BaseCache, BasePromptValue, LLMResult } from "../schema/index.js";
|
|
1
|
+
import { BaseCache, BaseChatMessage, BasePromptValue, LLMResult } from "../schema/index.js";
|
|
2
2
|
import { BaseLanguageModel, BaseLanguageModelCallOptions, BaseLanguageModelParams } from "../base_language/index.js";
|
|
3
3
|
import { CallbackManagerForLLMRun, Callbacks } from "../callbacks/manager.js";
|
|
4
4
|
export type SerializedLLM = {
|
|
@@ -27,6 +27,10 @@ export declare abstract class BaseLLM extends BaseLanguageModel {
|
|
|
27
27
|
* Run the LLM on the given prompts and input.
|
|
28
28
|
*/
|
|
29
29
|
abstract _generate(prompts: string[], options: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun): Promise<LLMResult>;
|
|
30
|
+
/**
|
|
31
|
+
* Get the parameters used to invoke the model
|
|
32
|
+
*/
|
|
33
|
+
invocationParams(): any;
|
|
30
34
|
/** @ignore */
|
|
31
35
|
_generateUncached(prompts: string[], options: this["CallOptions"], callbacks?: Callbacks): Promise<LLMResult>;
|
|
32
36
|
/**
|
|
@@ -37,6 +41,8 @@ export declare abstract class BaseLLM extends BaseLanguageModel {
|
|
|
37
41
|
* Convenience wrapper for {@link generate} that takes in a single string prompt and returns a single string output.
|
|
38
42
|
*/
|
|
39
43
|
call(prompt: string, options?: string[] | this["CallOptions"], callbacks?: Callbacks): Promise<string>;
|
|
44
|
+
predict(text: string, options?: string[] | this["CallOptions"], callbacks?: Callbacks): Promise<string>;
|
|
45
|
+
predictMessages(messages: BaseChatMessage[], options?: string[] | this["CallOptions"], callbacks?: Callbacks): Promise<BaseChatMessage>;
|
|
40
46
|
/**
|
|
41
47
|
* Get the identifying parameters of the LLM.
|
|
42
48
|
*/
|
package/dist/llms/base.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { InMemoryCache } from "../cache/index.js";
|
|
2
|
-
import { RUN_KEY, } from "../schema/index.js";
|
|
2
|
+
import { AIChatMessage, RUN_KEY, } from "../schema/index.js";
|
|
3
3
|
import { BaseLanguageModel, } from "../base_language/index.js";
|
|
4
4
|
import { CallbackManager, } from "../callbacks/manager.js";
|
|
5
|
+
import { getBufferString } from "../memory/base.js";
|
|
5
6
|
/**
|
|
6
7
|
* LLM Wrapper. Provides an {@link call} (an {@link generate}) function that takes in a prompt (or prompts) and returns a string.
|
|
7
8
|
*/
|
|
@@ -28,10 +29,18 @@ export class BaseLLM extends BaseLanguageModel {
|
|
|
28
29
|
const prompts = promptValues.map((promptValue) => promptValue.toString());
|
|
29
30
|
return this.generate(prompts, options, callbacks);
|
|
30
31
|
}
|
|
32
|
+
/**
|
|
33
|
+
* Get the parameters used to invoke the model
|
|
34
|
+
*/
|
|
35
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
36
|
+
invocationParams() {
|
|
37
|
+
return {};
|
|
38
|
+
}
|
|
31
39
|
/** @ignore */
|
|
32
40
|
async _generateUncached(prompts, options, callbacks) {
|
|
33
41
|
const callbackManager_ = await CallbackManager.configure(callbacks, this.callbacks, { verbose: this.verbose });
|
|
34
|
-
const
|
|
42
|
+
const invocationParams = { invocation_params: this?.invocationParams() };
|
|
43
|
+
const runManager = await callbackManager_?.handleLLMStart({ name: this._llmType() }, prompts, undefined, undefined, invocationParams);
|
|
35
44
|
let output;
|
|
36
45
|
try {
|
|
37
46
|
output = await this._generate(prompts, options, runManager);
|
|
@@ -104,6 +113,14 @@ export class BaseLLM extends BaseLanguageModel {
|
|
|
104
113
|
const { generations } = await this.generate([prompt], options ?? {}, callbacks);
|
|
105
114
|
return generations[0][0].text;
|
|
106
115
|
}
|
|
116
|
+
async predict(text, options, callbacks) {
|
|
117
|
+
return this.call(text, options, callbacks);
|
|
118
|
+
}
|
|
119
|
+
async predictMessages(messages, options, callbacks) {
|
|
120
|
+
const text = getBufferString(messages);
|
|
121
|
+
const prediction = await this.call(text, options, callbacks);
|
|
122
|
+
return new AIChatMessage(prediction);
|
|
123
|
+
}
|
|
107
124
|
/**
|
|
108
125
|
* Get the identifying parameters of the LLM.
|
|
109
126
|
*/
|
|
@@ -4,8 +4,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.PromptLayerOpenAIChat = exports.OpenAIChat = void 0;
|
|
7
|
-
const browser_or_node_1 = require("browser-or-node");
|
|
8
7
|
const openai_1 = require("openai");
|
|
8
|
+
const env_js_1 = require("../util/env.cjs");
|
|
9
9
|
const axios_fetch_adapter_js_1 = __importDefault(require("../util/axios-fetch-adapter.cjs"));
|
|
10
10
|
const base_js_1 = require("./base.cjs");
|
|
11
11
|
/**
|
|
@@ -359,7 +359,7 @@ class OpenAIChat extends base_js_1.LLM {
|
|
|
359
359
|
this.client = new openai_1.OpenAIApi(clientConfig);
|
|
360
360
|
}
|
|
361
361
|
const axiosOptions = {
|
|
362
|
-
adapter:
|
|
362
|
+
adapter: (0, env_js_1.isNode)() ? undefined : axios_fetch_adapter_js_1.default,
|
|
363
363
|
...this.clientConfig.baseOptions,
|
|
364
364
|
...options,
|
|
365
365
|
};
|
package/dist/llms/openai-chat.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { isNode } from "browser-or-node";
|
|
2
1
|
import { Configuration, OpenAIApi, } from "openai";
|
|
2
|
+
import { isNode } from "../util/env.js";
|
|
3
3
|
import fetchAdapter from "../util/axios-fetch-adapter.js";
|
|
4
4
|
import { LLM } from "./base.js";
|
|
5
5
|
/**
|
|
@@ -353,7 +353,7 @@ export class OpenAIChat extends LLM {
|
|
|
353
353
|
this.client = new OpenAIApi(clientConfig);
|
|
354
354
|
}
|
|
355
355
|
const axiosOptions = {
|
|
356
|
-
adapter: isNode ? undefined : fetchAdapter,
|
|
356
|
+
adapter: isNode() ? undefined : fetchAdapter,
|
|
357
357
|
...this.clientConfig.baseOptions,
|
|
358
358
|
...options,
|
|
359
359
|
};
|
package/dist/llms/openai.cjs
CHANGED
|
@@ -4,8 +4,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.PromptLayerOpenAIChat = exports.OpenAIChat = exports.PromptLayerOpenAI = exports.OpenAI = void 0;
|
|
7
|
-
const browser_or_node_1 = require("browser-or-node");
|
|
8
7
|
const openai_1 = require("openai");
|
|
8
|
+
const env_js_1 = require("../util/env.cjs");
|
|
9
9
|
const axios_fetch_adapter_js_1 = __importDefault(require("../util/axios-fetch-adapter.cjs"));
|
|
10
10
|
const chunk_js_1 = require("../util/chunk.cjs");
|
|
11
11
|
const base_js_1 = require("./base.cjs");
|
|
@@ -419,7 +419,7 @@ class OpenAI extends base_js_1.BaseLLM {
|
|
|
419
419
|
this.client = new openai_1.OpenAIApi(clientConfig);
|
|
420
420
|
}
|
|
421
421
|
const axiosOptions = {
|
|
422
|
-
adapter:
|
|
422
|
+
adapter: (0, env_js_1.isNode)() ? undefined : axios_fetch_adapter_js_1.default,
|
|
423
423
|
...this.clientConfig.baseOptions,
|
|
424
424
|
...options,
|
|
425
425
|
};
|
package/dist/llms/openai.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { isNode } from "browser-or-node";
|
|
2
1
|
import { Configuration, OpenAIApi, } from "openai";
|
|
2
|
+
import { isNode } from "../util/env.js";
|
|
3
3
|
import fetchAdapter from "../util/axios-fetch-adapter.js";
|
|
4
4
|
import { chunkArray } from "../util/chunk.js";
|
|
5
5
|
import { BaseLLM } from "./base.js";
|
|
@@ -413,7 +413,7 @@ export class OpenAI extends BaseLLM {
|
|
|
413
413
|
this.client = new OpenAIApi(clientConfig);
|
|
414
414
|
}
|
|
415
415
|
const axiosOptions = {
|
|
416
|
-
adapter: isNode ? undefined : fetchAdapter,
|
|
416
|
+
adapter: isNode() ? undefined : fetchAdapter,
|
|
417
417
|
...this.clientConfig.baseOptions,
|
|
418
418
|
...options,
|
|
419
419
|
};
|
package/dist/memory/base.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.getBufferString = exports.getInputValue = exports.BaseMemory = void 0;
|
|
3
|
+
exports.getPromptInputKey = exports.getBufferString = exports.getInputValue = exports.BaseMemory = void 0;
|
|
4
4
|
class BaseMemory {
|
|
5
5
|
}
|
|
6
6
|
exports.BaseMemory = BaseMemory;
|
|
@@ -48,3 +48,11 @@ function getBufferString(messages, humanPrefix = "Human", aiPrefix = "AI") {
|
|
|
48
48
|
return string_messages.join("\n");
|
|
49
49
|
}
|
|
50
50
|
exports.getBufferString = getBufferString;
|
|
51
|
+
function getPromptInputKey(inputs, memoryVariables) {
|
|
52
|
+
const promptInputKeys = Object.keys(inputs).filter((key) => !memoryVariables.includes(key) && key !== "stop");
|
|
53
|
+
if (promptInputKeys.length !== 1) {
|
|
54
|
+
throw new Error(`One input key expected, but got ${promptInputKeys.length}`);
|
|
55
|
+
}
|
|
56
|
+
return promptInputKeys[0];
|
|
57
|
+
}
|
|
58
|
+
exports.getPromptInputKey = getPromptInputKey;
|
package/dist/memory/base.d.ts
CHANGED
|
@@ -18,3 +18,4 @@ export declare const getInputValue: (inputValues: InputValues, inputKey?: string
|
|
|
18
18
|
* of the chat message history, based on the message content and role.
|
|
19
19
|
*/
|
|
20
20
|
export declare function getBufferString(messages: BaseChatMessage[], humanPrefix?: string, aiPrefix?: string): string;
|
|
21
|
+
export declare function getPromptInputKey(inputs: Record<string, unknown>, memoryVariables: string[]): string;
|