langchain 0.0.129 → 0.0.131
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/chat/index.cjs +3 -0
- package/dist/agents/chat/index.d.ts +1 -0
- package/dist/agents/chat/index.js +3 -0
- package/dist/agents/chat_convo/index.cjs +3 -0
- package/dist/agents/chat_convo/index.d.ts +1 -0
- package/dist/agents/chat_convo/index.js +3 -0
- package/dist/agents/executor.cjs +3 -0
- package/dist/agents/executor.d.ts +1 -0
- package/dist/agents/executor.js +3 -0
- package/dist/agents/mrkl/index.cjs +3 -0
- package/dist/agents/mrkl/index.d.ts +1 -0
- package/dist/agents/mrkl/index.js +3 -0
- package/dist/agents/openai/index.cjs +3 -0
- package/dist/agents/openai/index.d.ts +1 -0
- package/dist/agents/openai/index.js +3 -0
- package/dist/agents/structured_chat/index.cjs +3 -0
- package/dist/agents/structured_chat/index.d.ts +1 -0
- package/dist/agents/structured_chat/index.js +3 -0
- package/dist/callbacks/base.cjs +18 -0
- package/dist/callbacks/base.d.ts +15 -0
- package/dist/callbacks/base.js +19 -1
- package/dist/callbacks/manager.cjs +1 -1
- package/dist/callbacks/manager.js +1 -1
- package/dist/chains/analyze_documents_chain.cjs +3 -0
- package/dist/chains/analyze_documents_chain.d.ts +1 -0
- package/dist/chains/analyze_documents_chain.js +3 -0
- package/dist/chains/combine_docs_chain.cjs +9 -0
- package/dist/chains/combine_docs_chain.d.ts +3 -0
- package/dist/chains/combine_docs_chain.js +9 -0
- package/dist/chains/constitutional_ai/constitutional_chain.cjs +3 -0
- package/dist/chains/constitutional_ai/constitutional_chain.d.ts +1 -0
- package/dist/chains/constitutional_ai/constitutional_chain.js +3 -0
- package/dist/chains/conversation.cjs +3 -0
- package/dist/chains/conversation.d.ts +1 -0
- package/dist/chains/conversation.js +3 -0
- package/dist/chains/conversational_retrieval_chain.cjs +3 -0
- package/dist/chains/conversational_retrieval_chain.d.ts +1 -0
- package/dist/chains/conversational_retrieval_chain.js +3 -0
- package/dist/chains/llm_chain.cjs +3 -0
- package/dist/chains/llm_chain.d.ts +1 -0
- package/dist/chains/llm_chain.js +3 -0
- package/dist/chains/openai_functions/openapi.cjs +3 -0
- package/dist/chains/openai_functions/openapi.js +3 -0
- package/dist/chains/openai_moderation.cjs +3 -0
- package/dist/chains/openai_moderation.d.ts +1 -0
- package/dist/chains/openai_moderation.js +3 -0
- package/dist/chains/retrieval_qa.cjs +3 -0
- package/dist/chains/retrieval_qa.d.ts +1 -0
- package/dist/chains/retrieval_qa.js +3 -0
- package/dist/chains/router/multi_route.cjs +3 -0
- package/dist/chains/router/multi_route.d.ts +1 -0
- package/dist/chains/router/multi_route.js +3 -0
- package/dist/chains/sequential_chain.cjs +6 -0
- package/dist/chains/sequential_chain.d.ts +2 -0
- package/dist/chains/sequential_chain.js +6 -0
- package/dist/chains/sql_db/sql_db_chain.cjs +3 -0
- package/dist/chains/sql_db/sql_db_chain.d.ts +1 -0
- package/dist/chains/sql_db/sql_db_chain.js +3 -0
- package/dist/chains/transform.cjs +3 -0
- package/dist/chains/transform.d.ts +1 -0
- package/dist/chains/transform.js +3 -0
- package/dist/chains/vector_db_qa.cjs +3 -0
- package/dist/chains/vector_db_qa.d.ts +1 -0
- package/dist/chains/vector_db_qa.js +3 -0
- package/dist/chat_models/anthropic.cjs +3 -0
- package/dist/chat_models/anthropic.d.ts +1 -0
- package/dist/chat_models/anthropic.js +3 -0
- package/dist/chat_models/baiduwenxin.cjs +3 -13
- package/dist/chat_models/baiduwenxin.d.ts +1 -3
- package/dist/chat_models/baiduwenxin.js +3 -13
- package/dist/chat_models/googlepalm.cjs +9 -0
- package/dist/chat_models/googlepalm.d.ts +2 -0
- package/dist/chat_models/googlepalm.js +9 -0
- package/dist/chat_models/googlevertexai.cjs +9 -0
- package/dist/chat_models/googlevertexai.d.ts +2 -0
- package/dist/chat_models/googlevertexai.js +9 -0
- package/dist/chat_models/ollama.cjs +3 -0
- package/dist/chat_models/ollama.d.ts +1 -0
- package/dist/chat_models/ollama.js +3 -0
- package/dist/chat_models/openai.cjs +19 -5
- package/dist/chat_models/openai.d.ts +7 -5
- package/dist/chat_models/openai.js +19 -5
- package/dist/document_loaders/web/youtube.cjs +77 -0
- package/dist/document_loaders/web/youtube.d.ts +17 -0
- package/dist/document_loaders/web/youtube.js +73 -0
- package/dist/document_transformers/html_to_text.cjs +3 -0
- package/dist/document_transformers/html_to_text.d.ts +1 -0
- package/dist/document_transformers/html_to_text.js +3 -0
- package/dist/document_transformers/mozilla_readability.cjs +3 -0
- package/dist/document_transformers/mozilla_readability.d.ts +1 -0
- package/dist/document_transformers/mozilla_readability.js +3 -0
- package/dist/document_transformers/openai_functions.cjs +3 -0
- package/dist/document_transformers/openai_functions.d.ts +1 -0
- package/dist/document_transformers/openai_functions.js +3 -0
- package/dist/embeddings/cohere.cjs +16 -13
- package/dist/embeddings/cohere.js +16 -13
- package/dist/embeddings/googlevertexai.cjs +1 -1
- package/dist/embeddings/googlevertexai.js +1 -1
- package/dist/embeddings/openai.cjs +27 -21
- package/dist/embeddings/openai.js +27 -21
- package/dist/evaluation/qa/eval_chain.cjs +3 -0
- package/dist/evaluation/qa/eval_chain.d.ts +1 -0
- package/dist/evaluation/qa/eval_chain.js +3 -0
- package/dist/experimental/babyagi/agent.cjs +3 -0
- package/dist/experimental/babyagi/agent.d.ts +1 -0
- package/dist/experimental/babyagi/agent.js +3 -0
- package/dist/experimental/babyagi/task_creation.cjs +3 -0
- package/dist/experimental/babyagi/task_creation.d.ts +1 -0
- package/dist/experimental/babyagi/task_creation.js +3 -0
- package/dist/experimental/babyagi/task_execution.cjs +3 -0
- package/dist/experimental/babyagi/task_execution.d.ts +1 -0
- package/dist/experimental/babyagi/task_execution.js +3 -0
- package/dist/experimental/babyagi/task_prioritization.cjs +3 -0
- package/dist/experimental/babyagi/task_prioritization.d.ts +1 -0
- package/dist/experimental/babyagi/task_prioritization.js +3 -0
- package/dist/experimental/generative_agents/generative_agent.cjs +3 -0
- package/dist/experimental/generative_agents/generative_agent.d.ts +1 -0
- package/dist/experimental/generative_agents/generative_agent.js +3 -0
- package/dist/experimental/generative_agents/generative_agent_memory.cjs +3 -0
- package/dist/experimental/generative_agents/generative_agent_memory.d.ts +1 -0
- package/dist/experimental/generative_agents/generative_agent_memory.js +3 -0
- package/dist/experimental/plan_and_execute/agent_executor.cjs +3 -0
- package/dist/experimental/plan_and_execute/agent_executor.d.ts +1 -0
- package/dist/experimental/plan_and_execute/agent_executor.js +3 -0
- package/dist/llms/cohere.cjs +3 -0
- package/dist/llms/cohere.d.ts +1 -0
- package/dist/llms/cohere.js +3 -0
- package/dist/llms/ollama.cjs +3 -0
- package/dist/llms/ollama.d.ts +1 -0
- package/dist/llms/ollama.js +3 -0
- package/dist/llms/openai-chat.cjs +18 -4
- package/dist/llms/openai-chat.d.ts +6 -4
- package/dist/llms/openai-chat.js +18 -4
- package/dist/llms/openai.cjs +20 -6
- package/dist/llms/openai.d.ts +5 -3
- package/dist/llms/openai.js +20 -6
- package/dist/llms/replicate.cjs +3 -0
- package/dist/llms/replicate.d.ts +1 -0
- package/dist/llms/replicate.js +3 -0
- package/dist/load/import_constants.cjs +1 -0
- package/dist/load/import_constants.js +1 -0
- package/dist/load/index.cjs +7 -1
- package/dist/load/index.js +7 -1
- package/dist/load/serializable.cjs +42 -3
- package/dist/load/serializable.d.ts +16 -0
- package/dist/load/serializable.js +40 -2
- package/dist/output_parsers/combining.cjs +3 -0
- package/dist/output_parsers/combining.d.ts +1 -0
- package/dist/output_parsers/combining.js +3 -0
- package/dist/output_parsers/fix.cjs +3 -0
- package/dist/output_parsers/fix.d.ts +1 -0
- package/dist/output_parsers/fix.js +3 -0
- package/dist/output_parsers/list.cjs +3 -0
- package/dist/output_parsers/list.d.ts +1 -0
- package/dist/output_parsers/list.js +3 -0
- package/dist/output_parsers/noop.cjs +3 -0
- package/dist/output_parsers/noop.d.ts +1 -0
- package/dist/output_parsers/noop.js +3 -0
- package/dist/output_parsers/openai_functions.cjs +9 -0
- package/dist/output_parsers/openai_functions.d.ts +3 -0
- package/dist/output_parsers/openai_functions.js +9 -0
- package/dist/output_parsers/regex.cjs +3 -0
- package/dist/output_parsers/regex.d.ts +1 -0
- package/dist/output_parsers/regex.js +3 -0
- package/dist/output_parsers/structured.cjs +6 -0
- package/dist/output_parsers/structured.d.ts +2 -0
- package/dist/output_parsers/structured.js +6 -0
- package/dist/prompts/chat.cjs +21 -0
- package/dist/prompts/chat.d.ts +7 -0
- package/dist/prompts/chat.js +21 -0
- package/dist/prompts/pipeline.cjs +3 -0
- package/dist/prompts/pipeline.d.ts +1 -0
- package/dist/prompts/pipeline.js +3 -0
- package/dist/prompts/prompt.cjs +3 -0
- package/dist/prompts/prompt.d.ts +1 -0
- package/dist/prompts/prompt.js +3 -0
- package/dist/retrievers/amazon_kendra.cjs +3 -0
- package/dist/retrievers/amazon_kendra.d.ts +1 -0
- package/dist/retrievers/amazon_kendra.js +3 -0
- package/dist/retrievers/contextual_compression.cjs +3 -0
- package/dist/retrievers/contextual_compression.d.ts +1 -0
- package/dist/retrievers/contextual_compression.js +3 -0
- package/dist/retrievers/databerry.cjs +3 -0
- package/dist/retrievers/databerry.d.ts +1 -0
- package/dist/retrievers/databerry.js +3 -0
- package/dist/retrievers/hyde.cjs +3 -0
- package/dist/retrievers/hyde.d.ts +1 -0
- package/dist/retrievers/hyde.js +3 -0
- package/dist/retrievers/metal.cjs +3 -0
- package/dist/retrievers/metal.d.ts +1 -0
- package/dist/retrievers/metal.js +3 -0
- package/dist/retrievers/parent_document.cjs +3 -0
- package/dist/retrievers/parent_document.d.ts +1 -0
- package/dist/retrievers/parent_document.js +3 -0
- package/dist/retrievers/self_query/index.cjs +3 -0
- package/dist/retrievers/self_query/index.d.ts +1 -0
- package/dist/retrievers/self_query/index.js +3 -0
- package/dist/retrievers/self_query/supabase.cjs +28 -7
- package/dist/retrievers/self_query/supabase.d.ts +2 -2
- package/dist/retrievers/self_query/supabase.js +29 -8
- package/dist/retrievers/self_query/supabase_utils.cjs +177 -0
- package/dist/retrievers/self_query/supabase_utils.d.ts +24 -0
- package/dist/retrievers/self_query/supabase_utils.js +172 -0
- package/dist/retrievers/self_query/utils.cjs +19 -3
- package/dist/retrievers/self_query/utils.d.ts +3 -0
- package/dist/retrievers/self_query/utils.js +15 -2
- package/dist/retrievers/self_query/weaviate.cjs +3 -14
- package/dist/retrievers/self_query/weaviate.js +1 -12
- package/dist/retrievers/supabase.cjs +3 -0
- package/dist/retrievers/supabase.d.ts +1 -0
- package/dist/retrievers/supabase.js +3 -0
- package/dist/retrievers/time_weighted.cjs +3 -0
- package/dist/retrievers/time_weighted.d.ts +1 -0
- package/dist/retrievers/time_weighted.js +3 -0
- package/dist/retrievers/vespa.cjs +3 -0
- package/dist/retrievers/vespa.d.ts +1 -0
- package/dist/retrievers/vespa.js +3 -0
- package/dist/retrievers/zep.cjs +3 -0
- package/dist/retrievers/zep.d.ts +1 -0
- package/dist/retrievers/zep.js +3 -0
- package/dist/schema/index.cjs +32 -2
- package/dist/schema/index.d.ts +14 -0
- package/dist/schema/index.js +32 -2
- package/dist/schema/output_parser.cjs +6 -0
- package/dist/schema/output_parser.d.ts +2 -0
- package/dist/schema/output_parser.js +6 -0
- package/dist/schema/runnable.cjs +25 -7
- package/dist/schema/runnable.d.ts +6 -0
- package/dist/schema/runnable.js +25 -7
- package/dist/stores/message/utils.cjs +5 -0
- package/dist/stores/message/utils.js +6 -1
- package/dist/text_splitter.cjs +9 -0
- package/dist/text_splitter.d.ts +3 -0
- package/dist/text_splitter.js +9 -0
- package/dist/tools/IFTTTWebhook.cjs +3 -0
- package/dist/tools/IFTTTWebhook.d.ts +1 -0
- package/dist/tools/IFTTTWebhook.js +3 -0
- package/dist/tools/aiplugin.cjs +3 -0
- package/dist/tools/aiplugin.d.ts +1 -0
- package/dist/tools/aiplugin.js +3 -0
- package/dist/tools/aws_sfn.cjs +9 -0
- package/dist/tools/aws_sfn.d.ts +3 -0
- package/dist/tools/aws_sfn.js +9 -0
- package/dist/tools/bingserpapi.cjs +3 -0
- package/dist/tools/bingserpapi.d.ts +1 -0
- package/dist/tools/bingserpapi.js +3 -0
- package/dist/tools/brave_search.cjs +3 -0
- package/dist/tools/brave_search.d.ts +1 -0
- package/dist/tools/brave_search.js +3 -0
- package/dist/tools/calculator.cjs +3 -0
- package/dist/tools/calculator.d.ts +1 -0
- package/dist/tools/calculator.js +3 -0
- package/dist/tools/chain.cjs +3 -0
- package/dist/tools/chain.d.ts +1 -0
- package/dist/tools/chain.js +3 -0
- package/dist/tools/dadjokeapi.cjs +3 -0
- package/dist/tools/dadjokeapi.d.ts +1 -0
- package/dist/tools/dadjokeapi.js +3 -0
- package/dist/tools/dataforseo_api_search.cjs +3 -0
- package/dist/tools/dataforseo_api_search.d.ts +1 -0
- package/dist/tools/dataforseo_api_search.js +3 -0
- package/dist/tools/dynamic.cjs +6 -0
- package/dist/tools/dynamic.d.ts +2 -0
- package/dist/tools/dynamic.js +6 -0
- package/dist/tools/fs.cjs +6 -0
- package/dist/tools/fs.d.ts +2 -0
- package/dist/tools/fs.js +6 -0
- package/dist/tools/google_custom_search.cjs +3 -0
- package/dist/tools/google_custom_search.d.ts +1 -0
- package/dist/tools/google_custom_search.js +3 -0
- package/dist/tools/json.cjs +7 -1
- package/dist/tools/json.d.ts +2 -0
- package/dist/tools/json.js +7 -1
- package/dist/tools/requests.cjs +8 -2
- package/dist/tools/requests.d.ts +2 -0
- package/dist/tools/requests.js +8 -2
- package/dist/tools/searxng_search.cjs +3 -0
- package/dist/tools/searxng_search.d.ts +1 -0
- package/dist/tools/searxng_search.js +3 -0
- package/dist/tools/serpapi.cjs +3 -0
- package/dist/tools/serpapi.d.ts +1 -0
- package/dist/tools/serpapi.js +3 -0
- package/dist/tools/serper.cjs +3 -0
- package/dist/tools/serper.d.ts +1 -0
- package/dist/tools/serper.js +3 -0
- package/dist/tools/sql.cjs +12 -0
- package/dist/tools/sql.d.ts +4 -0
- package/dist/tools/sql.js +12 -0
- package/dist/tools/vectorstore.cjs +3 -0
- package/dist/tools/vectorstore.d.ts +1 -0
- package/dist/tools/vectorstore.js +3 -0
- package/dist/tools/webbrowser.cjs +3 -0
- package/dist/tools/webbrowser.d.ts +1 -0
- package/dist/tools/webbrowser.js +3 -0
- package/dist/tools/wikipedia_query_run.cjs +3 -0
- package/dist/tools/wikipedia_query_run.d.ts +1 -0
- package/dist/tools/wikipedia_query_run.js +3 -0
- package/dist/tools/zapier.cjs +3 -0
- package/dist/tools/zapier.d.ts +1 -0
- package/dist/tools/zapier.js +3 -0
- package/dist/vectorstores/base.cjs +3 -0
- package/dist/vectorstores/base.d.ts +1 -0
- package/dist/vectorstores/base.js +3 -0
- package/dist/vectorstores/elasticsearch.cjs +1 -0
- package/dist/vectorstores/elasticsearch.js +1 -0
- package/dist/vectorstores/hnswlib.cjs +19 -0
- package/dist/vectorstores/hnswlib.d.ts +3 -0
- package/dist/vectorstores/hnswlib.js +19 -0
- package/dist/vectorstores/myscale.cjs +1 -1
- package/dist/vectorstores/myscale.js +1 -1
- package/dist/vectorstores/supabase.d.ts +1 -2
- package/document_loaders/web/youtube.cjs +1 -0
- package/document_loaders/web/youtube.d.ts +1 -0
- package/document_loaders/web/youtube.js +1 -0
- package/package.json +27 -3
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.YoutubeLoader = void 0;
|
|
4
|
+
const youtube_transcript_1 = require("youtube-transcript");
|
|
5
|
+
const youtubei_js_1 = require("youtubei.js");
|
|
6
|
+
const document_js_1 = require("../../document.cjs");
|
|
7
|
+
const base_js_1 = require("../base.cjs");
|
|
8
|
+
class YoutubeLoader extends base_js_1.BaseDocumentLoader {
|
|
9
|
+
constructor(config) {
|
|
10
|
+
super();
|
|
11
|
+
Object.defineProperty(this, "videoId", {
|
|
12
|
+
enumerable: true,
|
|
13
|
+
configurable: true,
|
|
14
|
+
writable: true,
|
|
15
|
+
value: void 0
|
|
16
|
+
});
|
|
17
|
+
Object.defineProperty(this, "language", {
|
|
18
|
+
enumerable: true,
|
|
19
|
+
configurable: true,
|
|
20
|
+
writable: true,
|
|
21
|
+
value: void 0
|
|
22
|
+
});
|
|
23
|
+
Object.defineProperty(this, "addVideoInfo", {
|
|
24
|
+
enumerable: true,
|
|
25
|
+
configurable: true,
|
|
26
|
+
writable: true,
|
|
27
|
+
value: void 0
|
|
28
|
+
});
|
|
29
|
+
this.videoId = config.videoId;
|
|
30
|
+
this.language = config?.language;
|
|
31
|
+
this.addVideoInfo = config?.addVideoInfo ?? false;
|
|
32
|
+
}
|
|
33
|
+
static getVideoID(url) {
|
|
34
|
+
const match = url.match(/.*(?:youtu.be\/|v\/|u\/\w\/|embed\/|watch\?v=)([^#&?]*).*/);
|
|
35
|
+
if (match !== null && match[1].length === 11) {
|
|
36
|
+
return match[1];
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
throw new Error("Failed to get youtube video id from the url");
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
static createFromUrl(url, config) {
|
|
43
|
+
const videoId = YoutubeLoader.getVideoID(url);
|
|
44
|
+
return new YoutubeLoader({ ...config, videoId });
|
|
45
|
+
}
|
|
46
|
+
async load() {
|
|
47
|
+
let transcript;
|
|
48
|
+
const metadata = {
|
|
49
|
+
source: this.videoId,
|
|
50
|
+
};
|
|
51
|
+
try {
|
|
52
|
+
transcript = await youtube_transcript_1.YoutubeTranscript.fetchTranscript(this.videoId, {
|
|
53
|
+
lang: this.language,
|
|
54
|
+
});
|
|
55
|
+
if (transcript === undefined) {
|
|
56
|
+
throw new Error("Transcription not found");
|
|
57
|
+
}
|
|
58
|
+
if (this.addVideoInfo) {
|
|
59
|
+
const youtube = await youtubei_js_1.Innertube.create();
|
|
60
|
+
const info = (await youtube.getBasicInfo(this.videoId)).basic_info;
|
|
61
|
+
metadata.description = info.short_description;
|
|
62
|
+
metadata.title = info.title;
|
|
63
|
+
metadata.view_count = info.view_count;
|
|
64
|
+
metadata.author = info.author;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
catch (e) {
|
|
68
|
+
throw new Error(`Failed to get YouTube video transcription: ${e.message}`);
|
|
69
|
+
}
|
|
70
|
+
const document = new document_js_1.Document({
|
|
71
|
+
pageContent: transcript.map((item) => item.text).join(" "),
|
|
72
|
+
metadata,
|
|
73
|
+
});
|
|
74
|
+
return [document];
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
exports.YoutubeLoader = YoutubeLoader;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Document } from "../../document.js";
|
|
2
|
+
import { BaseDocumentLoader } from "../base.js";
|
|
3
|
+
interface YoutubeConfig {
|
|
4
|
+
videoId: string;
|
|
5
|
+
language?: string;
|
|
6
|
+
addVideoInfo?: boolean;
|
|
7
|
+
}
|
|
8
|
+
export declare class YoutubeLoader extends BaseDocumentLoader {
|
|
9
|
+
private videoId;
|
|
10
|
+
private language?;
|
|
11
|
+
private addVideoInfo;
|
|
12
|
+
constructor(config: YoutubeConfig);
|
|
13
|
+
private static getVideoID;
|
|
14
|
+
static createFromUrl(url: string, config?: Omit<YoutubeConfig, "videoId">): YoutubeLoader;
|
|
15
|
+
load(): Promise<Document[]>;
|
|
16
|
+
}
|
|
17
|
+
export {};
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { YoutubeTranscript } from "youtube-transcript";
|
|
2
|
+
import { Innertube } from "youtubei.js";
|
|
3
|
+
import { Document } from "../../document.js";
|
|
4
|
+
import { BaseDocumentLoader } from "../base.js";
|
|
5
|
+
export class YoutubeLoader extends BaseDocumentLoader {
|
|
6
|
+
constructor(config) {
|
|
7
|
+
super();
|
|
8
|
+
Object.defineProperty(this, "videoId", {
|
|
9
|
+
enumerable: true,
|
|
10
|
+
configurable: true,
|
|
11
|
+
writable: true,
|
|
12
|
+
value: void 0
|
|
13
|
+
});
|
|
14
|
+
Object.defineProperty(this, "language", {
|
|
15
|
+
enumerable: true,
|
|
16
|
+
configurable: true,
|
|
17
|
+
writable: true,
|
|
18
|
+
value: void 0
|
|
19
|
+
});
|
|
20
|
+
Object.defineProperty(this, "addVideoInfo", {
|
|
21
|
+
enumerable: true,
|
|
22
|
+
configurable: true,
|
|
23
|
+
writable: true,
|
|
24
|
+
value: void 0
|
|
25
|
+
});
|
|
26
|
+
this.videoId = config.videoId;
|
|
27
|
+
this.language = config?.language;
|
|
28
|
+
this.addVideoInfo = config?.addVideoInfo ?? false;
|
|
29
|
+
}
|
|
30
|
+
static getVideoID(url) {
|
|
31
|
+
const match = url.match(/.*(?:youtu.be\/|v\/|u\/\w\/|embed\/|watch\?v=)([^#&?]*).*/);
|
|
32
|
+
if (match !== null && match[1].length === 11) {
|
|
33
|
+
return match[1];
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
throw new Error("Failed to get youtube video id from the url");
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
static createFromUrl(url, config) {
|
|
40
|
+
const videoId = YoutubeLoader.getVideoID(url);
|
|
41
|
+
return new YoutubeLoader({ ...config, videoId });
|
|
42
|
+
}
|
|
43
|
+
async load() {
|
|
44
|
+
let transcript;
|
|
45
|
+
const metadata = {
|
|
46
|
+
source: this.videoId,
|
|
47
|
+
};
|
|
48
|
+
try {
|
|
49
|
+
transcript = await YoutubeTranscript.fetchTranscript(this.videoId, {
|
|
50
|
+
lang: this.language,
|
|
51
|
+
});
|
|
52
|
+
if (transcript === undefined) {
|
|
53
|
+
throw new Error("Transcription not found");
|
|
54
|
+
}
|
|
55
|
+
if (this.addVideoInfo) {
|
|
56
|
+
const youtube = await Innertube.create();
|
|
57
|
+
const info = (await youtube.getBasicInfo(this.videoId)).basic_info;
|
|
58
|
+
metadata.description = info.short_description;
|
|
59
|
+
metadata.title = info.title;
|
|
60
|
+
metadata.view_count = info.view_count;
|
|
61
|
+
metadata.author = info.author;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
catch (e) {
|
|
65
|
+
throw new Error(`Failed to get YouTube video transcription: ${e.message}`);
|
|
66
|
+
}
|
|
67
|
+
const document = new Document({
|
|
68
|
+
pageContent: transcript.map((item) => item.text).join(" "),
|
|
69
|
+
metadata,
|
|
70
|
+
});
|
|
71
|
+
return [document];
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -5,6 +5,9 @@ const html_to_text_1 = require("html-to-text");
|
|
|
5
5
|
const document_js_1 = require("../document.cjs");
|
|
6
6
|
const document_js_2 = require("../schema/document.cjs");
|
|
7
7
|
class HtmlToTextTransformer extends document_js_2.MappingDocumentTransformer {
|
|
8
|
+
static lc_name() {
|
|
9
|
+
return "HtmlToTextTransformer";
|
|
10
|
+
}
|
|
8
11
|
constructor(options = {}) {
|
|
9
12
|
super(options);
|
|
10
13
|
Object.defineProperty(this, "options", {
|
|
@@ -3,6 +3,7 @@ import { Document } from "../document.js";
|
|
|
3
3
|
import { MappingDocumentTransformer } from "../schema/document.js";
|
|
4
4
|
export declare class HtmlToTextTransformer extends MappingDocumentTransformer {
|
|
5
5
|
protected options: HtmlToTextOptions;
|
|
6
|
+
static lc_name(): string;
|
|
6
7
|
constructor(options?: HtmlToTextOptions);
|
|
7
8
|
_transformDocument(document: Document): Promise<Document>;
|
|
8
9
|
}
|
|
@@ -2,6 +2,9 @@ import { htmlToText } from "html-to-text";
|
|
|
2
2
|
import { Document } from "../document.js";
|
|
3
3
|
import { MappingDocumentTransformer } from "../schema/document.js";
|
|
4
4
|
export class HtmlToTextTransformer extends MappingDocumentTransformer {
|
|
5
|
+
static lc_name() {
|
|
6
|
+
return "HtmlToTextTransformer";
|
|
7
|
+
}
|
|
5
8
|
constructor(options = {}) {
|
|
6
9
|
super(options);
|
|
7
10
|
Object.defineProperty(this, "options", {
|
|
@@ -6,6 +6,9 @@ const jsdom_1 = require("jsdom");
|
|
|
6
6
|
const document_js_1 = require("../document.cjs");
|
|
7
7
|
const document_js_2 = require("../schema/document.cjs");
|
|
8
8
|
class MozillaReadabilityTransformer extends document_js_2.MappingDocumentTransformer {
|
|
9
|
+
static lc_name() {
|
|
10
|
+
return "MozillaReadabilityTransformer";
|
|
11
|
+
}
|
|
9
12
|
constructor(options = {}) {
|
|
10
13
|
super(options);
|
|
11
14
|
Object.defineProperty(this, "options", {
|
|
@@ -3,6 +3,7 @@ import { Document } from "../document.js";
|
|
|
3
3
|
import { MappingDocumentTransformer } from "../schema/document.js";
|
|
4
4
|
export declare class MozillaReadabilityTransformer extends MappingDocumentTransformer {
|
|
5
5
|
protected options: Options;
|
|
6
|
+
static lc_name(): string;
|
|
6
7
|
constructor(options?: Options);
|
|
7
8
|
_transformDocument(document: Document): Promise<Document>;
|
|
8
9
|
}
|
|
@@ -3,6 +3,9 @@ import { JSDOM } from "jsdom";
|
|
|
3
3
|
import { Document } from "../document.js";
|
|
4
4
|
import { MappingDocumentTransformer } from "../schema/document.js";
|
|
5
5
|
export class MozillaReadabilityTransformer extends MappingDocumentTransformer {
|
|
6
|
+
static lc_name() {
|
|
7
|
+
return "MozillaReadabilityTransformer";
|
|
8
|
+
}
|
|
6
9
|
constructor(options = {}) {
|
|
7
10
|
super(options);
|
|
8
11
|
Object.defineProperty(this, "options", {
|
|
@@ -7,6 +7,9 @@ const document_js_2 = require("../schema/document.cjs");
|
|
|
7
7
|
const index_js_1 = require("../chains/openai_functions/index.cjs");
|
|
8
8
|
const openai_js_1 = require("../chat_models/openai.cjs");
|
|
9
9
|
class MetadataTagger extends document_js_2.MappingDocumentTransformer {
|
|
10
|
+
static lc_name() {
|
|
11
|
+
return "MetadataTagger";
|
|
12
|
+
}
|
|
10
13
|
constructor(fields) {
|
|
11
14
|
super();
|
|
12
15
|
Object.defineProperty(this, "taggingChain", {
|
|
@@ -6,6 +6,7 @@ import { MappingDocumentTransformer } from "../schema/document.js";
|
|
|
6
6
|
import { TaggingChainOptions } from "../chains/openai_functions/index.js";
|
|
7
7
|
import { ChatOpenAI } from "../chat_models/openai.js";
|
|
8
8
|
export declare class MetadataTagger extends MappingDocumentTransformer {
|
|
9
|
+
static lc_name(): string;
|
|
9
10
|
protected taggingChain: BaseChain;
|
|
10
11
|
constructor(fields: {
|
|
11
12
|
taggingChain: BaseChain;
|
|
@@ -4,6 +4,9 @@ import { MappingDocumentTransformer } from "../schema/document.js";
|
|
|
4
4
|
import { createTaggingChain, } from "../chains/openai_functions/index.js";
|
|
5
5
|
import { ChatOpenAI } from "../chat_models/openai.js";
|
|
6
6
|
export class MetadataTagger extends MappingDocumentTransformer {
|
|
7
|
+
static lc_name() {
|
|
8
|
+
return "MetadataTagger";
|
|
9
|
+
}
|
|
7
10
|
constructor(fields) {
|
|
8
11
|
super();
|
|
9
12
|
Object.defineProperty(this, "taggingChain", {
|
|
@@ -13,7 +13,8 @@ class CohereEmbeddings extends base_js_1.Embeddings {
|
|
|
13
13
|
* @param fields - An optional object with properties to configure the instance.
|
|
14
14
|
*/
|
|
15
15
|
constructor(fields) {
|
|
16
|
-
|
|
16
|
+
const fieldsWithDefaults = { maxConcurrency: 2, ...fields };
|
|
17
|
+
super(fieldsWithDefaults);
|
|
17
18
|
Object.defineProperty(this, "modelName", {
|
|
18
19
|
enumerable: true,
|
|
19
20
|
configurable: true,
|
|
@@ -38,12 +39,12 @@ class CohereEmbeddings extends base_js_1.Embeddings {
|
|
|
38
39
|
writable: true,
|
|
39
40
|
value: void 0
|
|
40
41
|
});
|
|
41
|
-
const apiKey =
|
|
42
|
+
const apiKey = fieldsWithDefaults?.apiKey || (0, env_js_1.getEnvironmentVariable)("COHERE_API_KEY");
|
|
42
43
|
if (!apiKey) {
|
|
43
44
|
throw new Error("Cohere API key not found");
|
|
44
45
|
}
|
|
45
|
-
this.modelName =
|
|
46
|
-
this.batchSize =
|
|
46
|
+
this.modelName = fieldsWithDefaults?.modelName ?? this.modelName;
|
|
47
|
+
this.batchSize = fieldsWithDefaults?.batchSize ?? this.batchSize;
|
|
47
48
|
this.apiKey = apiKey;
|
|
48
49
|
}
|
|
49
50
|
/**
|
|
@@ -53,16 +54,18 @@ class CohereEmbeddings extends base_js_1.Embeddings {
|
|
|
53
54
|
*/
|
|
54
55
|
async embedDocuments(texts) {
|
|
55
56
|
await this.maybeInitClient();
|
|
56
|
-
const
|
|
57
|
+
const batches = (0, chunk_js_1.chunkArray)(texts, this.batchSize);
|
|
58
|
+
const batchRequests = batches.map((batch) => this.embeddingWithRetry({
|
|
59
|
+
model: this.modelName,
|
|
60
|
+
texts: batch,
|
|
61
|
+
}));
|
|
62
|
+
const batchResponses = await Promise.all(batchRequests);
|
|
57
63
|
const embeddings = [];
|
|
58
|
-
for (let i = 0; i <
|
|
59
|
-
const
|
|
60
|
-
const { body } =
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
});
|
|
64
|
-
for (let j = 0; j < input.length; j += 1) {
|
|
65
|
-
embeddings.push(body.embeddings[j]);
|
|
64
|
+
for (let i = 0; i < batchResponses.length; i += 1) {
|
|
65
|
+
const batch = batches[i];
|
|
66
|
+
const { body: batchResponse } = batchResponses[i];
|
|
67
|
+
for (let j = 0; j < batch.length; j += 1) {
|
|
68
|
+
embeddings.push(batchResponse.embeddings[j]);
|
|
66
69
|
}
|
|
67
70
|
}
|
|
68
71
|
return embeddings;
|
|
@@ -10,7 +10,8 @@ export class CohereEmbeddings extends Embeddings {
|
|
|
10
10
|
* @param fields - An optional object with properties to configure the instance.
|
|
11
11
|
*/
|
|
12
12
|
constructor(fields) {
|
|
13
|
-
|
|
13
|
+
const fieldsWithDefaults = { maxConcurrency: 2, ...fields };
|
|
14
|
+
super(fieldsWithDefaults);
|
|
14
15
|
Object.defineProperty(this, "modelName", {
|
|
15
16
|
enumerable: true,
|
|
16
17
|
configurable: true,
|
|
@@ -35,12 +36,12 @@ export class CohereEmbeddings extends Embeddings {
|
|
|
35
36
|
writable: true,
|
|
36
37
|
value: void 0
|
|
37
38
|
});
|
|
38
|
-
const apiKey =
|
|
39
|
+
const apiKey = fieldsWithDefaults?.apiKey || getEnvironmentVariable("COHERE_API_KEY");
|
|
39
40
|
if (!apiKey) {
|
|
40
41
|
throw new Error("Cohere API key not found");
|
|
41
42
|
}
|
|
42
|
-
this.modelName =
|
|
43
|
-
this.batchSize =
|
|
43
|
+
this.modelName = fieldsWithDefaults?.modelName ?? this.modelName;
|
|
44
|
+
this.batchSize = fieldsWithDefaults?.batchSize ?? this.batchSize;
|
|
44
45
|
this.apiKey = apiKey;
|
|
45
46
|
}
|
|
46
47
|
/**
|
|
@@ -50,16 +51,18 @@ export class CohereEmbeddings extends Embeddings {
|
|
|
50
51
|
*/
|
|
51
52
|
async embedDocuments(texts) {
|
|
52
53
|
await this.maybeInitClient();
|
|
53
|
-
const
|
|
54
|
+
const batches = chunkArray(texts, this.batchSize);
|
|
55
|
+
const batchRequests = batches.map((batch) => this.embeddingWithRetry({
|
|
56
|
+
model: this.modelName,
|
|
57
|
+
texts: batch,
|
|
58
|
+
}));
|
|
59
|
+
const batchResponses = await Promise.all(batchRequests);
|
|
54
60
|
const embeddings = [];
|
|
55
|
-
for (let i = 0; i <
|
|
56
|
-
const
|
|
57
|
-
const { body } =
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
});
|
|
61
|
-
for (let j = 0; j < input.length; j += 1) {
|
|
62
|
-
embeddings.push(body.embeddings[j]);
|
|
61
|
+
for (let i = 0; i < batchResponses.length; i += 1) {
|
|
62
|
+
const batch = batches[i];
|
|
63
|
+
const { body: batchResponse } = batchResponses[i];
|
|
64
|
+
for (let j = 0; j < batch.length; j += 1) {
|
|
65
|
+
embeddings.push(batchResponse.embeddings[j]);
|
|
63
66
|
}
|
|
64
67
|
}
|
|
65
68
|
return embeddings;
|
|
@@ -39,7 +39,7 @@ class GoogleVertexAIEmbeddings extends base_js_1.Embeddings {
|
|
|
39
39
|
async embedDocuments(documents) {
|
|
40
40
|
const instanceChunks = (0, chunk_js_1.chunkArray)(documents.map((document) => ({
|
|
41
41
|
content: document,
|
|
42
|
-
})), 5); // Vertex AI accepts max 5
|
|
42
|
+
})), 5); // Vertex AI accepts max 5 instances per prediction
|
|
43
43
|
const parameters = {};
|
|
44
44
|
const options = {};
|
|
45
45
|
const responses = await Promise.all(instanceChunks.map((instances) => this.connection.request(instances, parameters, options)));
|
|
@@ -36,7 +36,7 @@ export class GoogleVertexAIEmbeddings extends Embeddings {
|
|
|
36
36
|
async embedDocuments(documents) {
|
|
37
37
|
const instanceChunks = chunkArray(documents.map((document) => ({
|
|
38
38
|
content: document,
|
|
39
|
-
})), 5); // Vertex AI accepts max 5
|
|
39
|
+
})), 5); // Vertex AI accepts max 5 instances per prediction
|
|
40
40
|
const parameters = {};
|
|
41
41
|
const options = {};
|
|
42
42
|
const responses = await Promise.all(instanceChunks.map((instances) => this.connection.request(instances, parameters, options)));
|
|
@@ -12,7 +12,8 @@ const base_js_1 = require("./base.cjs");
|
|
|
12
12
|
const azure_js_1 = require("../util/azure.cjs");
|
|
13
13
|
class OpenAIEmbeddings extends base_js_1.Embeddings {
|
|
14
14
|
constructor(fields, configuration) {
|
|
15
|
-
|
|
15
|
+
const fieldsWithDefaults = { maxConcurrency: 2, ...fields };
|
|
16
|
+
super(fieldsWithDefaults);
|
|
16
17
|
Object.defineProperty(this, "modelName", {
|
|
17
18
|
enumerable: true,
|
|
18
19
|
configurable: true,
|
|
@@ -79,27 +80,30 @@ class OpenAIEmbeddings extends base_js_1.Embeddings {
|
|
|
79
80
|
writable: true,
|
|
80
81
|
value: void 0
|
|
81
82
|
});
|
|
82
|
-
const apiKey =
|
|
83
|
-
|
|
83
|
+
const apiKey = fieldsWithDefaults?.openAIApiKey ??
|
|
84
|
+
(0, env_js_1.getEnvironmentVariable)("OPENAI_API_KEY");
|
|
85
|
+
const azureApiKey = fieldsWithDefaults?.azureOpenAIApiKey ??
|
|
84
86
|
(0, env_js_1.getEnvironmentVariable)("AZURE_OPENAI_API_KEY");
|
|
85
87
|
if (!azureApiKey && !apiKey) {
|
|
86
88
|
throw new Error("OpenAI or Azure OpenAI API key not found");
|
|
87
89
|
}
|
|
88
|
-
const azureApiInstanceName =
|
|
90
|
+
const azureApiInstanceName = fieldsWithDefaults?.azureOpenAIApiInstanceName ??
|
|
89
91
|
(0, env_js_1.getEnvironmentVariable)("AZURE_OPENAI_API_INSTANCE_NAME");
|
|
90
|
-
const azureApiDeploymentName = (
|
|
91
|
-
|
|
92
|
+
const azureApiDeploymentName = (fieldsWithDefaults?.azureOpenAIApiEmbeddingsDeploymentName ||
|
|
93
|
+
fieldsWithDefaults?.azureOpenAIApiDeploymentName) ??
|
|
92
94
|
((0, env_js_1.getEnvironmentVariable)("AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME") ||
|
|
93
95
|
(0, env_js_1.getEnvironmentVariable)("AZURE_OPENAI_API_DEPLOYMENT_NAME"));
|
|
94
|
-
const azureApiVersion =
|
|
96
|
+
const azureApiVersion = fieldsWithDefaults?.azureOpenAIApiVersion ??
|
|
95
97
|
(0, env_js_1.getEnvironmentVariable)("AZURE_OPENAI_API_VERSION");
|
|
96
98
|
this.azureOpenAIBasePath =
|
|
97
|
-
|
|
99
|
+
fieldsWithDefaults?.azureOpenAIBasePath ??
|
|
98
100
|
(0, env_js_1.getEnvironmentVariable)("AZURE_OPENAI_BASE_PATH");
|
|
99
|
-
this.modelName =
|
|
100
|
-
this.batchSize =
|
|
101
|
-
|
|
102
|
-
this.
|
|
101
|
+
this.modelName = fieldsWithDefaults?.modelName ?? this.modelName;
|
|
102
|
+
this.batchSize =
|
|
103
|
+
fieldsWithDefaults?.batchSize ?? (azureApiKey ? 1 : this.batchSize);
|
|
104
|
+
this.stripNewLines =
|
|
105
|
+
fieldsWithDefaults?.stripNewLines ?? this.stripNewLines;
|
|
106
|
+
this.timeout = fieldsWithDefaults?.timeout;
|
|
103
107
|
this.azureOpenAIApiVersion = azureApiVersion;
|
|
104
108
|
this.azureOpenAIApiKey = azureApiKey;
|
|
105
109
|
this.azureOpenAIApiInstanceName = azureApiInstanceName;
|
|
@@ -121,16 +125,18 @@ class OpenAIEmbeddings extends base_js_1.Embeddings {
|
|
|
121
125
|
};
|
|
122
126
|
}
|
|
123
127
|
async embedDocuments(texts) {
|
|
124
|
-
const
|
|
128
|
+
const batches = (0, chunk_js_1.chunkArray)(this.stripNewLines ? texts.map((t) => t.replace(/\n/g, " ")) : texts, this.batchSize);
|
|
129
|
+
const batchRequests = batches.map((batch) => this.embeddingWithRetry({
|
|
130
|
+
model: this.modelName,
|
|
131
|
+
input: batch,
|
|
132
|
+
}));
|
|
133
|
+
const batchResponses = await Promise.all(batchRequests);
|
|
125
134
|
const embeddings = [];
|
|
126
|
-
for (let i = 0; i <
|
|
127
|
-
const
|
|
128
|
-
const { data } =
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
});
|
|
132
|
-
for (let j = 0; j < input.length; j += 1) {
|
|
133
|
-
embeddings.push(data.data[j].embedding);
|
|
135
|
+
for (let i = 0; i < batchResponses.length; i += 1) {
|
|
136
|
+
const batch = batches[i];
|
|
137
|
+
const { data: batchResponse } = batchResponses[i];
|
|
138
|
+
for (let j = 0; j < batch.length; j += 1) {
|
|
139
|
+
embeddings.push(batchResponse.data[j].embedding);
|
|
134
140
|
}
|
|
135
141
|
}
|
|
136
142
|
return embeddings;
|
|
@@ -6,7 +6,8 @@ import { Embeddings } from "./base.js";
|
|
|
6
6
|
import { getEndpoint } from "../util/azure.js";
|
|
7
7
|
export class OpenAIEmbeddings extends Embeddings {
|
|
8
8
|
constructor(fields, configuration) {
|
|
9
|
-
|
|
9
|
+
const fieldsWithDefaults = { maxConcurrency: 2, ...fields };
|
|
10
|
+
super(fieldsWithDefaults);
|
|
10
11
|
Object.defineProperty(this, "modelName", {
|
|
11
12
|
enumerable: true,
|
|
12
13
|
configurable: true,
|
|
@@ -73,27 +74,30 @@ export class OpenAIEmbeddings extends Embeddings {
|
|
|
73
74
|
writable: true,
|
|
74
75
|
value: void 0
|
|
75
76
|
});
|
|
76
|
-
const apiKey =
|
|
77
|
-
|
|
77
|
+
const apiKey = fieldsWithDefaults?.openAIApiKey ??
|
|
78
|
+
getEnvironmentVariable("OPENAI_API_KEY");
|
|
79
|
+
const azureApiKey = fieldsWithDefaults?.azureOpenAIApiKey ??
|
|
78
80
|
getEnvironmentVariable("AZURE_OPENAI_API_KEY");
|
|
79
81
|
if (!azureApiKey && !apiKey) {
|
|
80
82
|
throw new Error("OpenAI or Azure OpenAI API key not found");
|
|
81
83
|
}
|
|
82
|
-
const azureApiInstanceName =
|
|
84
|
+
const azureApiInstanceName = fieldsWithDefaults?.azureOpenAIApiInstanceName ??
|
|
83
85
|
getEnvironmentVariable("AZURE_OPENAI_API_INSTANCE_NAME");
|
|
84
|
-
const azureApiDeploymentName = (
|
|
85
|
-
|
|
86
|
+
const azureApiDeploymentName = (fieldsWithDefaults?.azureOpenAIApiEmbeddingsDeploymentName ||
|
|
87
|
+
fieldsWithDefaults?.azureOpenAIApiDeploymentName) ??
|
|
86
88
|
(getEnvironmentVariable("AZURE_OPENAI_API_EMBEDDINGS_DEPLOYMENT_NAME") ||
|
|
87
89
|
getEnvironmentVariable("AZURE_OPENAI_API_DEPLOYMENT_NAME"));
|
|
88
|
-
const azureApiVersion =
|
|
90
|
+
const azureApiVersion = fieldsWithDefaults?.azureOpenAIApiVersion ??
|
|
89
91
|
getEnvironmentVariable("AZURE_OPENAI_API_VERSION");
|
|
90
92
|
this.azureOpenAIBasePath =
|
|
91
|
-
|
|
93
|
+
fieldsWithDefaults?.azureOpenAIBasePath ??
|
|
92
94
|
getEnvironmentVariable("AZURE_OPENAI_BASE_PATH");
|
|
93
|
-
this.modelName =
|
|
94
|
-
this.batchSize =
|
|
95
|
-
|
|
96
|
-
this.
|
|
95
|
+
this.modelName = fieldsWithDefaults?.modelName ?? this.modelName;
|
|
96
|
+
this.batchSize =
|
|
97
|
+
fieldsWithDefaults?.batchSize ?? (azureApiKey ? 1 : this.batchSize);
|
|
98
|
+
this.stripNewLines =
|
|
99
|
+
fieldsWithDefaults?.stripNewLines ?? this.stripNewLines;
|
|
100
|
+
this.timeout = fieldsWithDefaults?.timeout;
|
|
97
101
|
this.azureOpenAIApiVersion = azureApiVersion;
|
|
98
102
|
this.azureOpenAIApiKey = azureApiKey;
|
|
99
103
|
this.azureOpenAIApiInstanceName = azureApiInstanceName;
|
|
@@ -115,16 +119,18 @@ export class OpenAIEmbeddings extends Embeddings {
|
|
|
115
119
|
};
|
|
116
120
|
}
|
|
117
121
|
async embedDocuments(texts) {
|
|
118
|
-
const
|
|
122
|
+
const batches = chunkArray(this.stripNewLines ? texts.map((t) => t.replace(/\n/g, " ")) : texts, this.batchSize);
|
|
123
|
+
const batchRequests = batches.map((batch) => this.embeddingWithRetry({
|
|
124
|
+
model: this.modelName,
|
|
125
|
+
input: batch,
|
|
126
|
+
}));
|
|
127
|
+
const batchResponses = await Promise.all(batchRequests);
|
|
119
128
|
const embeddings = [];
|
|
120
|
-
for (let i = 0; i <
|
|
121
|
-
const
|
|
122
|
-
const { data } =
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
});
|
|
126
|
-
for (let j = 0; j < input.length; j += 1) {
|
|
127
|
-
embeddings.push(data.data[j].embedding);
|
|
129
|
+
for (let i = 0; i < batchResponses.length; i += 1) {
|
|
130
|
+
const batch = batches[i];
|
|
131
|
+
const { data: batchResponse } = batchResponses[i];
|
|
132
|
+
for (let j = 0; j < batch.length; j += 1) {
|
|
133
|
+
embeddings.push(batchResponse.data[j].embedding);
|
|
128
134
|
}
|
|
129
135
|
}
|
|
130
136
|
return embeddings;
|
|
@@ -5,6 +5,9 @@ const prompt_js_1 = require("./prompt.cjs");
|
|
|
5
5
|
const llm_chain_js_1 = require("../../chains/llm_chain.cjs");
|
|
6
6
|
const eqSet = (xs, ys) => xs.size === ys.size && [...xs].every((x) => ys.has(x));
|
|
7
7
|
class QAEvalChain extends llm_chain_js_1.LLMChain {
|
|
8
|
+
static lc_name() {
|
|
9
|
+
return "QAEvalChain";
|
|
10
|
+
}
|
|
8
11
|
static fromLlm(llm, options = {}) {
|
|
9
12
|
const prompt = options.prompt || prompt_js_1.QA_PROMPT;
|
|
10
13
|
const expectedInputVars = new Set([
|
|
@@ -8,6 +8,7 @@ export interface EvaluateArgs {
|
|
|
8
8
|
predictionKey: string;
|
|
9
9
|
}
|
|
10
10
|
export declare class QAEvalChain extends LLMChain {
|
|
11
|
+
static lc_name(): string;
|
|
11
12
|
static fromLlm(llm: BaseLanguageModel, options?: {
|
|
12
13
|
prompt?: PromptTemplate;
|
|
13
14
|
chainInput?: Omit<LLMChainInput, "llm">;
|
|
@@ -2,6 +2,9 @@ import { QA_PROMPT } from "./prompt.js";
|
|
|
2
2
|
import { LLMChain } from "../../chains/llm_chain.js";
|
|
3
3
|
const eqSet = (xs, ys) => xs.size === ys.size && [...xs].every((x) => ys.has(x));
|
|
4
4
|
export class QAEvalChain extends LLMChain {
|
|
5
|
+
static lc_name() {
|
|
6
|
+
return "QAEvalChain";
|
|
7
|
+
}
|
|
5
8
|
static fromLlm(llm, options = {}) {
|
|
6
9
|
const prompt = options.prompt || QA_PROMPT;
|
|
7
10
|
const expectedInputVars = new Set([
|
|
@@ -7,6 +7,9 @@ const task_creation_js_1 = require("./task_creation.cjs");
|
|
|
7
7
|
const task_execution_js_1 = require("./task_execution.cjs");
|
|
8
8
|
const task_prioritization_js_1 = require("./task_prioritization.cjs");
|
|
9
9
|
class BabyAGI extends base_js_1.BaseChain {
|
|
10
|
+
static lc_name() {
|
|
11
|
+
return "BabyAGI";
|
|
12
|
+
}
|
|
10
13
|
constructor({ creationChain, prioritizationChain, executionChain, vectorstore, maxIterations = 100, verbose, callbacks, }) {
|
|
11
14
|
super(undefined, verbose, callbacks);
|
|
12
15
|
Object.defineProperty(this, "taskList", {
|
|
@@ -17,6 +17,7 @@ export interface BabyAGIInputs extends Omit<ChainInputs, "memory" | "callbackMan
|
|
|
17
17
|
maxIterations?: number;
|
|
18
18
|
}
|
|
19
19
|
export declare class BabyAGI extends BaseChain implements BabyAGIInputs {
|
|
20
|
+
static lc_name(): string;
|
|
20
21
|
taskList: Task[];
|
|
21
22
|
creationChain: BaseChain;
|
|
22
23
|
prioritizationChain: BaseChain;
|
|
@@ -4,6 +4,9 @@ import { TaskCreationChain } from "./task_creation.js";
|
|
|
4
4
|
import { TaskExecutionChain } from "./task_execution.js";
|
|
5
5
|
import { TaskPrioritizationChain } from "./task_prioritization.js";
|
|
6
6
|
export class BabyAGI extends BaseChain {
|
|
7
|
+
static lc_name() {
|
|
8
|
+
return "BabyAGI";
|
|
9
|
+
}
|
|
7
10
|
constructor({ creationChain, prioritizationChain, executionChain, vectorstore, maxIterations = 100, verbose, callbacks, }) {
|
|
8
11
|
super(undefined, verbose, callbacks);
|
|
9
12
|
Object.defineProperty(this, "taskList", {
|
|
@@ -5,6 +5,9 @@ const llm_chain_js_1 = require("../../chains/llm_chain.cjs");
|
|
|
5
5
|
const prompt_js_1 = require("../../prompts/prompt.cjs");
|
|
6
6
|
/** Chain to generate tasks. */
|
|
7
7
|
class TaskCreationChain extends llm_chain_js_1.LLMChain {
|
|
8
|
+
static lc_name() {
|
|
9
|
+
return "TaskCreationChain";
|
|
10
|
+
}
|
|
8
11
|
static fromLLM(fields) {
|
|
9
12
|
const taskCreationTemplate = `You are an task creation AI that uses the result of an execution agent` +
|
|
10
13
|
` to create new tasks with the following objective: {objective},` +
|
|
@@ -2,6 +2,9 @@ import { LLMChain } from "../../chains/llm_chain.js";
|
|
|
2
2
|
import { PromptTemplate } from "../../prompts/prompt.js";
|
|
3
3
|
/** Chain to generate tasks. */
|
|
4
4
|
export class TaskCreationChain extends LLMChain {
|
|
5
|
+
static lc_name() {
|
|
6
|
+
return "TaskCreationChain";
|
|
7
|
+
}
|
|
5
8
|
static fromLLM(fields) {
|
|
6
9
|
const taskCreationTemplate = `You are an task creation AI that uses the result of an execution agent` +
|
|
7
10
|
` to create new tasks with the following objective: {objective},` +
|