langchain 0.0.127 → 0.0.128

package/dist/agents/toolkits/conversational_retrieval/tool.cjs

@@ -1,12 +1,18 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createRetrieverTool = void 0;
+const zod_1 = require("zod");
 const dynamic_js_1 = require("../../../tools/dynamic.cjs");
 function createRetrieverTool(retriever, input) {
-    const func = async (input, runManager) => {
+    const func = async ({ input }, runManager) => {
         const docs = await retriever.getRelevantDocuments(input, runManager?.getChild("retriever"));
         return docs.map((doc) => doc.pageContent).join("\n");
     };
-    return new dynamic_js_1.DynamicTool({ ...input, func });
+    const schema = zod_1.z.object({
+        input: zod_1.z
+            .string()
+            .describe("Natural language query used as input to the retriever"),
+    });
+    return new dynamic_js_1.DynamicStructuredTool({ ...input, func, schema });
 }
 exports.createRetrieverTool = createRetrieverTool;

package/dist/agents/toolkits/conversational_retrieval/tool.d.ts

@@ -1,3 +1,10 @@
+import { z } from "zod";
 import { BaseRetriever } from "../../../schema/retriever.js";
-import { DynamicTool, DynamicToolInput } from "../../../tools/dynamic.js";
-export declare function createRetrieverTool(retriever: BaseRetriever, input: Omit<DynamicToolInput, "func">): DynamicTool;
+import { DynamicStructuredTool, DynamicStructuredToolInput } from "../../../tools/dynamic.js";
+export declare function createRetrieverTool(retriever: BaseRetriever, input: Omit<DynamicStructuredToolInput, "func" | "schema">): DynamicStructuredTool<z.ZodObject<{
+    input: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    input: string;
+}, {
+    input: string;
+}>>;

package/dist/agents/toolkits/conversational_retrieval/tool.js

@@ -1,8 +1,14 @@
-import { DynamicTool } from "../../../tools/dynamic.js";
+import { z } from "zod";
+import { DynamicStructuredTool, } from "../../../tools/dynamic.js";
 export function createRetrieverTool(retriever, input) {
-    const func = async (input, runManager) => {
+    const func = async ({ input }, runManager) => {
         const docs = await retriever.getRelevantDocuments(input, runManager?.getChild("retriever"));
         return docs.map((doc) => doc.pageContent).join("\n");
     };
-    return new DynamicTool({ ...input, func });
+    const schema = z.object({
+        input: z
+            .string()
+            .describe("Natural language query used as input to the retriever"),
+    });
+    return new DynamicStructuredTool({ ...input, func, schema });
 }

package/dist/chains/query_constructor/ir.d.ts

@@ -1,3 +1,4 @@
+import { VectorStore } from "../../vectorstores/base.js";
 export type AND = "and";
 export type OR = "or";
 export type NOT = "not";
@@ -15,7 +16,7 @@ export declare const Operators: {
 export declare const Comparators: {
     [key: string]: Comparator;
 };
-export type VisitorResult = VisitorOperationResult | VisitorComparisonResult | VisitorStructuredQueryResult;
+export type VisitorResult = VisitorOperationResult | VisitorComparisonResult;
 export type VisitorOperationResult = {
     [operator: string]: VisitorResult[];
 };
@@ -25,13 +26,13 @@ export type VisitorComparisonResult = {
     };
 };
 export type VisitorStructuredQueryResult = {
-    filter?: VisitorStructuredQueryResult | VisitorComparisonResult | VisitorOperationResult;
+    filter?: VisitorComparisonResult | VisitorOperationResult;
 };
-export declare abstract class Visitor {
+export declare abstract class Visitor<T extends VectorStore = VectorStore> {
     VisitOperationOutput: object;
     VisitComparisonOutput: object;
     VisitStructuredQueryOutput: {
-        filter?: object;
+        filter?: T["FilterType"];
     };
     abstract allowedOperators: Operator[];
     abstract allowedComparators: Comparator[];

package/dist/chains/query_constructor/prompt.cjs

@@ -100,6 +100,8 @@ statements): one or more statements to apply the operation to
 Make sure that you only use the comparators and logical operators listed above and \
 no others.
 Make sure that filters only refer to attributes that exist in the data source.
+Make sure that filters only use the attributed names with its function names if there are functions applied on them.
+Make sure that filters only use format \`YYYY-MM-DD\` when handling timestamp data typed values.
 Make sure that filters take into account the descriptions of attributes and only make \
 comparisons that are feasible given the type of data being stored.
 Make sure that filters are only used as needed. If there are no filters that should be \
@@ -115,7 +117,7 @@ exports.DEFAULT_SUFFIX = `\
 Data Source:
 \`\`\`json
 {{{{
-    "content": {content},
+    "content": "{content}",
     "attributes": {attributes}
 }}}}
 \`\`\`

package/dist/chains/query_constructor/prompt.d.ts

@@ -15,6 +15,6 @@ export declare const EXAMPLE_PROMPT: PromptTemplate<{
     user_query: any;
     structured_request: any;
 }, any>;
-export declare const DEFAULT_SCHEMA = "<< Structured Request Schema >>\nWhen responding use a markdown code snippet with a JSON object formatted in the following schema:\n\n```json\n{{{{\n    \"query\": string \\ text string to compare to document contents\n    \"filter\": string \\ logical condition statement for filtering documents\n}}}}\n```\n\nThe query string should contain only text that is expected to match the contents of documents. Any conditions in the filter should not be mentioned in the query as well.\n\nA logical condition statement is composed of one or more comparison and logical operation statements.\n\nA comparison statement takes the form: `comp(attr, val)`:\n- `comp` ({allowed_comparators}): comparator\n- `attr` (string):  name of attribute to apply the comparison to\n- `val` (string): is the comparison value\n\nA logical operation statement takes the form `op(statement1, statement2, ...)`:\n- `op` ({allowed_operators}): logical operator\n- `statement1`, `statement2`, ... (comparison statements or logical operation statements): one or more statements to apply the operation to\n\nMake sure that you only use the comparators and logical operators listed above and no others.\nMake sure that filters only refer to attributes that exist in the data source.\nMake sure that filters take into account the descriptions of attributes and only make comparisons that are feasible given the type of data being stored.\nMake sure that filters are only used as needed. If there are no filters that should be applied return \"NO_FILTER\" for the filter value.";
+export declare const DEFAULT_SCHEMA = "<< Structured Request Schema >>\nWhen responding use a markdown code snippet with a JSON object formatted in the following schema:\n\n```json\n{{{{\n    \"query\": string \\ text string to compare to document contents\n    \"filter\": string \\ logical condition statement for filtering documents\n}}}}\n```\n\nThe query string should contain only text that is expected to match the contents of documents. Any conditions in the filter should not be mentioned in the query as well.\n\nA logical condition statement is composed of one or more comparison and logical operation statements.\n\nA comparison statement takes the form: `comp(attr, val)`:\n- `comp` ({allowed_comparators}): comparator\n- `attr` (string):  name of attribute to apply the comparison to\n- `val` (string): is the comparison value\n\nA logical operation statement takes the form `op(statement1, statement2, ...)`:\n- `op` ({allowed_operators}): logical operator\n- `statement1`, `statement2`, ... (comparison statements or logical operation statements): one or more statements to apply the operation to\n\nMake sure that you only use the comparators and logical operators listed above and no others.\nMake sure that filters only refer to attributes that exist in the data source.\nMake sure that filters only use the attributed names with its function names if there are functions applied on them.\nMake sure that filters only use format `YYYY-MM-DD` when handling timestamp data typed values.\nMake sure that filters take into account the descriptions of attributes and only make comparisons that are feasible given the type of data being stored.\nMake sure that filters are only used as needed. If there are no filters that should be applied return \"NO_FILTER\" for the filter value.";
 export declare const DEFAULT_PREFIX = "Your goal is to structure the user's query to match the request schema provided below.\n\n{schema}";
-export declare const DEFAULT_SUFFIX = "<< Example {i}. >>\nData Source:\n```json\n{{{{\n    \"content\": {content},\n    \"attributes\": {attributes}\n}}}}\n```\n\nUser Query:\n{{query}}\n\nStructured Request:\n";
+export declare const DEFAULT_SUFFIX = "<< Example {i}. >>\nData Source:\n```json\n{{{{\n    \"content\": \"{content}\",\n    \"attributes\": {attributes}\n}}}}\n```\n\nUser Query:\n{{query}}\n\nStructured Request:\n";

package/dist/chains/query_constructor/prompt.js

@@ -97,6 +97,8 @@ statements): one or more statements to apply the operation to
 Make sure that you only use the comparators and logical operators listed above and \
 no others.
 Make sure that filters only refer to attributes that exist in the data source.
+Make sure that filters only use the attributed names with its function names if there are functions applied on them.
+Make sure that filters only use format \`YYYY-MM-DD\` when handling timestamp data typed values.
 Make sure that filters take into account the descriptions of attributes and only make \
 comparisons that are feasible given the type of data being stored.
 Make sure that filters are only used as needed. If there are no filters that should be \
@@ -112,7 +114,7 @@ export const DEFAULT_SUFFIX = `\
 Data Source:
 \`\`\`json
 {{{{
-    "content": {content},
+    "content": "{content}",
     "attributes": {attributes}
 }}}}
 \`\`\`

package/dist/chains/question_answering/refine_prompts.cjs

@@ -41,7 +41,7 @@ exports.DEFAULT_TEXT_QA_PROMPT_TMPL = `Context information is below.
 ---------------------
 {context}
 ---------------------
-Given the context information and not prior knowledge, answer the question: {question}`;
+Given the context information and no prior knowledge, answer the question: {question}`;
 exports.DEFAULT_TEXT_QA_PROMPT = new index_js_1.PromptTemplate({
     inputVariables: ["context", "question"],
     template: exports.DEFAULT_TEXT_QA_PROMPT_TMPL,
@@ -50,7 +50,7 @@ const chat_qa_prompt_template = `Context information is below.
 ---------------------
 {context}
 ---------------------
-Given the context information and not prior knowledge, answer any questions`;
+Given the context information and no prior knowledge, answer any questions`;
 const chat_messages = [
     /*#__PURE__*/ index_js_1.SystemMessagePromptTemplate.fromTemplate(chat_qa_prompt_template),
     /*#__PURE__*/ index_js_1.HumanMessagePromptTemplate.fromTemplate("{question}"),

package/dist/chains/question_answering/refine_prompts.d.ts

@@ -8,7 +8,7 @@ export declare const DEFAULT_REFINE_PROMPT: PromptTemplate<{
 }, any>;
 export declare const CHAT_REFINE_PROMPT: ChatPromptTemplate<any, any>;
 export declare const REFINE_PROMPT_SELECTOR: ConditionalPromptSelector;
-export declare const DEFAULT_TEXT_QA_PROMPT_TMPL = "Context information is below. \n---------------------\n{context}\n---------------------\nGiven the context information and not prior knowledge, answer the question: {question}";
+export declare const DEFAULT_TEXT_QA_PROMPT_TMPL = "Context information is below. \n---------------------\n{context}\n---------------------\nGiven the context information and no prior knowledge, answer the question: {question}";
 export declare const DEFAULT_TEXT_QA_PROMPT: PromptTemplate<{
     context: any;
     question: any;

package/dist/chains/question_answering/refine_prompts.js

@@ -38,7 +38,7 @@ export const DEFAULT_TEXT_QA_PROMPT_TMPL = `Context information is below.
 ---------------------
 {context}
 ---------------------
-Given the context information and not prior knowledge, answer the question: {question}`;
+Given the context information and no prior knowledge, answer the question: {question}`;
 export const DEFAULT_TEXT_QA_PROMPT = /*#__PURE__*/ new PromptTemplate({
     inputVariables: ["context", "question"],
     template: DEFAULT_TEXT_QA_PROMPT_TMPL,
@@ -47,7 +47,7 @@ const chat_qa_prompt_template = `Context information is below.
 ---------------------
 {context}
 ---------------------
-Given the context information and not prior knowledge, answer any questions`;
+Given the context information and no prior knowledge, answer any questions`;
 const chat_messages = [
     /*#__PURE__*/ SystemMessagePromptTemplate.fromTemplate(chat_qa_prompt_template),
     /*#__PURE__*/ HumanMessagePromptTemplate.fromTemplate("{question}"),

package/dist/chains/sql_db/index.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SQL_MYSQL_PROMPT = exports.SQL_MSSQL_PROMPT = exports.SQL_SQLITE_PROMPT = exports.SQL_POSTGRES_PROMPT = exports.DEFAULT_SQL_DATABASE_PROMPT = exports.SqlDatabaseChain = void 0;
+exports.SQL_SAP_HANA_PROMPT = exports.SQL_MYSQL_PROMPT = exports.SQL_MSSQL_PROMPT = exports.SQL_SQLITE_PROMPT = exports.SQL_POSTGRES_PROMPT = exports.DEFAULT_SQL_DATABASE_PROMPT = exports.SqlDatabaseChain = void 0;
 var sql_db_chain_js_1 = require("./sql_db_chain.cjs");
 Object.defineProperty(exports, "SqlDatabaseChain", { enumerable: true, get: function () { return sql_db_chain_js_1.SqlDatabaseChain; } });
 var sql_db_prompt_js_1 = require("./sql_db_prompt.cjs");
@@ -9,3 +9,4 @@ Object.defineProperty(exports, "SQL_POSTGRES_PROMPT", { enumerable: true, get: f
 Object.defineProperty(exports, "SQL_SQLITE_PROMPT", { enumerable: true, get: function () { return sql_db_prompt_js_1.SQL_SQLITE_PROMPT; } });
 Object.defineProperty(exports, "SQL_MSSQL_PROMPT", { enumerable: true, get: function () { return sql_db_prompt_js_1.SQL_MSSQL_PROMPT; } });
 Object.defineProperty(exports, "SQL_MYSQL_PROMPT", { enumerable: true, get: function () { return sql_db_prompt_js_1.SQL_MYSQL_PROMPT; } });
+Object.defineProperty(exports, "SQL_SAP_HANA_PROMPT", { enumerable: true, get: function () { return sql_db_prompt_js_1.SQL_SAP_HANA_PROMPT; } });

package/dist/chains/sql_db/index.d.ts

@@ -1,2 +1,2 @@
 export { SqlDatabaseChain, SqlDatabaseChainInput } from "./sql_db_chain.js";
-export { DEFAULT_SQL_DATABASE_PROMPT, SQL_POSTGRES_PROMPT, SQL_SQLITE_PROMPT, SQL_MSSQL_PROMPT, SQL_MYSQL_PROMPT, } from "./sql_db_prompt.js";
+export { DEFAULT_SQL_DATABASE_PROMPT, SQL_POSTGRES_PROMPT, SQL_SQLITE_PROMPT, SQL_MSSQL_PROMPT, SQL_MYSQL_PROMPT, SQL_SAP_HANA_PROMPT, } from "./sql_db_prompt.js";

package/dist/chains/sql_db/index.js

@@ -1,2 +1,2 @@
 export { SqlDatabaseChain } from "./sql_db_chain.js";
-export { DEFAULT_SQL_DATABASE_PROMPT, SQL_POSTGRES_PROMPT, SQL_SQLITE_PROMPT, SQL_MSSQL_PROMPT, SQL_MYSQL_PROMPT, } from "./sql_db_prompt.js";
+export { DEFAULT_SQL_DATABASE_PROMPT, SQL_POSTGRES_PROMPT, SQL_SQLITE_PROMPT, SQL_MSSQL_PROMPT, SQL_MYSQL_PROMPT, SQL_SAP_HANA_PROMPT, } from "./sql_db_prompt.js";

package/dist/chains/sql_db/sql_db_prompt.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SQL_MSSQL_PROMPT = exports.SQL_MYSQL_PROMPT = exports.SQL_SQLITE_PROMPT = exports.SQL_POSTGRES_PROMPT = exports.DEFAULT_SQL_DATABASE_PROMPT = void 0;
+exports.SQL_SAP_HANA_PROMPT = exports.SQL_MSSQL_PROMPT = exports.SQL_MYSQL_PROMPT = exports.SQL_SQLITE_PROMPT = exports.SQL_POSTGRES_PROMPT = exports.DEFAULT_SQL_DATABASE_PROMPT = void 0;
 /* eslint-disable spaced-comment */
 const prompt_js_1 = require("../../prompts/prompt.cjs");
 exports.DEFAULT_SQL_DATABASE_PROMPT = new prompt_js_1.PromptTemplate({
@@ -100,3 +100,23 @@ Only use the following tables:
 Question: {input}`,
     inputVariables: ["dialect", "table_info", "input", "top_k"],
 });
+exports.SQL_SAP_HANA_PROMPT = new prompt_js_1.PromptTemplate({
+    template: `You are a SAP HANA expert. Given an input question, first create a syntactically correct SAP HANA query to run, then look at the results of the query and return the answer to the input question.
+Unless the user specifies in the question a specific number of examples to obtain, query for at most {top_k} results using the LIMIT clause as per SAP HANA. You can order the results to return the most informative data in the database.
+Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in double quotes (") to denote them as delimited identifiers.
+Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.
+Always use a schema name when executing a query.
+
+Use the following format:
+
+Question: "Question here"
+SQLQuery: "SQL Query to run"
+SQLResult: "Result of the SQLQuery"
+Answer: "Final answer here"
+
+Only use the following tables:
+{table_info}
+
+Question: {input}`,
+    inputVariables: ["dialect", "table_info", "input", "top_k"],
+});

package/dist/chains/sql_db/sql_db_prompt.d.ts

@@ -29,3 +29,9 @@ export declare const SQL_MSSQL_PROMPT: PromptTemplate<{
     table_info: any;
     top_k: any;
 }, any>;
+export declare const SQL_SAP_HANA_PROMPT: PromptTemplate<{
+    input: any;
+    dialect: any;
+    table_info: any;
+    top_k: any;
+}, any>;

package/dist/chains/sql_db/sql_db_prompt.js

@@ -97,3 +97,23 @@ Only use the following tables:
 Question: {input}`,
     inputVariables: ["dialect", "table_info", "input", "top_k"],
 });
+export const SQL_SAP_HANA_PROMPT = /*#__PURE__*/ new PromptTemplate({
+    template: `You are a SAP HANA expert. Given an input question, first create a syntactically correct SAP HANA query to run, then look at the results of the query and return the answer to the input question.
+Unless the user specifies in the question a specific number of examples to obtain, query for at most {top_k} results using the LIMIT clause as per SAP HANA. You can order the results to return the most informative data in the database.
+Never query for all columns from a table. You must query only the columns that are needed to answer the question. Wrap each column name in double quotes (") to denote them as delimited identifiers.
+Pay attention to use only the column names you can see in the tables below. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.
+Always use a schema name when executing a query.
+
+Use the following format:
+
+Question: "Question here"
+SQLQuery: "SQL Query to run"
+SQLResult: "Result of the SQLQuery"
+Answer: "Final answer here"
+
+Only use the following tables:
+{table_info}
+
+Question: {input}`,
+    inputVariables: ["dialect", "table_info", "input", "top_k"],
+});

package/dist/chat_models/openai.cjs

@@ -201,6 +201,12 @@ class ChatOpenAI extends base_js_1.BaseChatModel {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "user", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "timeout", {
             enumerable: true,
             configurable: true,
@@ -298,6 +304,7 @@ class ChatOpenAI extends base_js_1.BaseChatModel {
         this.n = fields?.n ?? this.n;
         this.logitBias = fields?.logitBias;
         this.stop = fields?.stop;
+        this.user = fields?.user;
         this.streaming = fields?.streaming ?? false;
         if (this.azureOpenAIApiKey) {
             if (!this.azureOpenAIApiInstanceName && !this.azureOpenAIBasePath) {
@@ -330,6 +337,7 @@ class ChatOpenAI extends base_js_1.BaseChatModel {
             n: this.n,
             logit_bias: this.logitBias,
             stop: options?.stop ?? this.stop,
+            user: this.user,
             stream: this.streaming,
             functions: options?.functions ??
                 (options?.tools

package/dist/chat_models/openai.d.ts

@@ -55,6 +55,7 @@ export declare class ChatOpenAI extends BaseChatModel<ChatOpenAICallOptions> imp
     modelName: string;
     modelKwargs?: OpenAIChatInput["modelKwargs"];
     stop?: string[];
+    user?: string;
     timeout?: number;
     streaming: boolean;
     maxTokens?: number;

package/dist/chat_models/openai.js

@@ -195,6 +195,12 @@ export class ChatOpenAI extends BaseChatModel {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "user", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "timeout", {
             enumerable: true,
             configurable: true,
@@ -292,6 +298,7 @@ export class ChatOpenAI extends BaseChatModel {
         this.n = fields?.n ?? this.n;
         this.logitBias = fields?.logitBias;
         this.stop = fields?.stop;
+        this.user = fields?.user;
         this.streaming = fields?.streaming ?? false;
         if (this.azureOpenAIApiKey) {
             if (!this.azureOpenAIApiInstanceName && !this.azureOpenAIBasePath) {
@@ -324,6 +331,7 @@ export class ChatOpenAI extends BaseChatModel {
             n: this.n,
             logit_bias: this.logitBias,
             stop: options?.stop ?? this.stop,
+            user: this.user,
             stream: this.streaming,
             functions: options?.functions ??
                 (options?.tools

package/dist/document_loaders/web/github.cjs

@@ -11,12 +11,13 @@ const base_js_1 = require("../base.cjs");
 const directory_js_1 = require("../fs/directory.cjs");
 const extname_js_1 = require("../../util/extname.cjs");
 const env_js_1 = require("../../util/env.cjs");
+const async_caller_js_1 = require("../../util/async_caller.cjs");
 const extensions = new Set(binary_extensions_1.default);
 function isBinaryPath(name) {
     return extensions.has((0, extname_js_1.extname)(name).slice(1).toLowerCase());
 }
 class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
-    constructor(githubUrl, { accessToken = (0, env_js_1.getEnvironmentVariable)("GITHUB_ACCESS_TOKEN"), branch = "main", recursive = true, unknown = directory_js_1.UnknownHandling.Warn, ignoreFiles = [], ignorePaths, } = {}) {
+    constructor(githubUrl, { accessToken = (0, env_js_1.getEnvironmentVariable)("GITHUB_ACCESS_TOKEN"), branch = "main", recursive = true, unknown = directory_js_1.UnknownHandling.Warn, ignoreFiles = [], ignorePaths, verbose = false, maxConcurrency = 2, maxRetries = 2, ...rest } = {}) {
         super();
         Object.defineProperty(this, "owner", {
             enumerable: true,
@@ -78,6 +79,18 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "verbose", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "caller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         const { owner, repo, path } = this.extractOwnerAndRepoAndPath(githubUrl);
         this.owner = owner;
         this.repo = repo;
@@ -87,6 +100,12 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
         this.unknown = unknown;
         this.accessToken = accessToken;
         this.ignoreFiles = ignoreFiles;
+        this.verbose = verbose;
+        this.caller = new async_caller_js_1.AsyncCaller({
+            maxConcurrency,
+            maxRetries,
+            ...rest,
+        });
         if (ignorePaths) {
             this.ignore = ignore_1.default.default().add(ignorePaths);
         }
@@ -104,11 +123,12 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
         return { owner: match[1], repo: match[2], path: match[4] || "" };
     }
     async load() {
-        const documents = [];
-        await this.processDirectory(this.initialPath, documents);
-        return documents;
+        return (await this.processRepo()).map((fileResponse) => new document_js_1.Document({
+            pageContent: fileResponse.contents,
+            metadata: fileResponse.metadata,
+        }));
     }
-    async shouldIgnore(path, fileType) {
+    shouldIgnore(path, fileType) {
         if (fileType !== "dir" && isBinaryPath(path)) {
             return true;
         }
@@ -128,46 +148,98 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
                 }
             }));
     }
-    async processDirectory(path, documents) {
-        try {
-            const files = await this.fetchRepoFiles(path);
-            for (const file of files) {
-                if (!(await this.shouldIgnore(file.path, file.type))) {
-                    if (file.type !== "dir") {
-                        try {
-                            const fileContent = await this.fetchFileContent(file);
-                            const metadata = { source: file.path };
-                            documents.push(new document_js_1.Document({ pageContent: fileContent, metadata }));
-                        }
-                        catch (e) {
-                            this.handleError(`Failed to fetch file content: ${file.path}, ${e}`);
-                        }
+    /**
+     * Takes the file info and wrap it in a promise that will resolve to the file content and metadata
+     * @param file
+     * @returns
+     */
+    async fetchFileContentWrapper(file) {
+        const fileContent = await this.fetchFileContent(file).catch((error) => {
+            this.handleError(`Failed wrap file content: ${file}, ${error}`);
+        });
+        return {
+            contents: fileContent || "",
+            metadata: { source: file.path },
+        };
+    }
+    /**
+     * Maps a list of files / directories to a list of promises that will fetch the file / directory contents
+     */
+    async getCurrentDirectoryFilePromises(files) {
+        const currentDirectoryFilePromises = [];
+        // Directories have nested files / directories, which is why this is a list of promises of promises
+        const currentDirectoryDirectoryPromises = [];
+        for (const file of files) {
+            if (!this.shouldIgnore(file.path, file.type)) {
+                if (file.type !== "dir") {
+                    try {
+                        currentDirectoryFilePromises.push(this.fetchFileContentWrapper(file));
                     }
-                    else if (this.recursive) {
-                        await this.processDirectory(file.path, documents);
+                    catch (e) {
+                        this.handleError(`Failed to fetch file content: ${file.path}, ${e}`);
                     }
                 }
+                else if (this.recursive) {
+                    currentDirectoryDirectoryPromises.push(this.processDirectory(file.path));
+                }
             }
         }
+        const curDirDirectories = await Promise.all(currentDirectoryDirectoryPromises);
+        return [...currentDirectoryFilePromises, ...curDirDirectories.flat()];
+    }
+    /**
+     * Begins the process of fetching the contents of the repository
+     */
+    async processRepo() {
+        try {
+            // Get the list of file / directory names in the root directory
+            const files = await this.fetchRepoFiles(this.initialPath);
+            // Map the file / directory paths to promises that will fetch the file / directory contents
+            const currentDirectoryFilePromises = await this.getCurrentDirectoryFilePromises(files);
+            return Promise.all(currentDirectoryFilePromises);
+        }
+        catch (error) {
+            this.handleError(`Failed to process directory: ${this.initialPath}, ${error}`);
+            return Promise.reject(error);
+        }
+    }
+    async processDirectory(path) {
+        try {
+            const files = await this.fetchRepoFiles(path);
+            return this.getCurrentDirectoryFilePromises(files);
+        }
         catch (error) {
             this.handleError(`Failed to process directory: ${path}, ${error}`);
+            return Promise.reject(error);
         }
     }
     async fetchRepoFiles(path) {
         const url = `https://api.github.com/repos/${this.owner}/${this.repo}/contents/${path}?ref=${this.branch}`;
-        const response = await fetch(url, { headers: this.headers });
-        const data = await response.json();
-        if (!response.ok) {
-            throw new Error(`Unable to fetch repository files: ${response.status} ${JSON.stringify(data)}`);
-        }
-        if (!Array.isArray(data)) {
-            throw new Error("Unable to fetch repository files.");
-        }
-        return data;
+        return this.caller.call(async () => {
+            if (this.verbose) {
+                console.log("Fetching", url);
+            }
+            const response = await fetch(url, { headers: this.headers });
+            const data = await response.json();
+            if (!response.ok) {
+                throw new Error(`Unable to fetch repository files: ${response.status} ${JSON.stringify(data)}`);
+            }
+            if (!Array.isArray(data)) {
+                throw new Error("Unable to fetch repository files.");
+            }
+            return data;
+        });
     }
     async fetchFileContent(file) {
-        const response = await fetch(file.download_url, { headers: this.headers });
-        return response.text();
+        return this.caller.call(async () => {
+            if (this.verbose) {
+                console.log("Fetching", file.download_url);
+            }
+            const response = await fetch(file.download_url, {
+                headers: this.headers,
+            });
+            return response.text();
+        });
     }
     handleError(message) {
         switch (this.unknown) {

package/dist/document_loaders/web/github.d.ts

@@ -2,13 +2,40 @@ import { Ignore } from "ignore";
 import { Document } from "../../document.js";
 import { BaseDocumentLoader } from "../base.js";
 import { UnknownHandling } from "../fs/directory.js";
-export interface GithubRepoLoaderParams {
+import { AsyncCaller, AsyncCallerParams } from "../../util/async_caller.js";
+export interface GithubFile {
+    name: string;
+    path: string;
+    sha: string;
+    size: number;
+    url: string;
+    html_url: string;
+    git_url: string;
+    download_url: string;
+    type: string;
+    _links: {
+        self: string;
+        git: string;
+        html: string;
+    };
+}
+export interface GithubRepoLoaderParams extends AsyncCallerParams {
     branch?: string;
     recursive?: boolean;
     unknown?: UnknownHandling;
     accessToken?: string;
     ignoreFiles?: (string | RegExp)[];
     ignorePaths?: string[];
+    verbose?: boolean;
+    /**
+     * The maximum number of concurrent calls that can be made. Defaults to 2.
+     */
+    maxConcurrency?: number;
+    /**
+     * The maximum number of retries that can be made for a single call,
+     * with an exponential backoff between each attempt. Defaults to 2.
+     */
+    maxRetries?: number;
 }
 export declare class GithubRepoLoader extends BaseDocumentLoader implements GithubRepoLoaderParams {
     private readonly owner;
@@ -21,10 +48,26 @@ export declare class GithubRepoLoader extends BaseDocumentLoader implements Gith
     accessToken?: string;
     ignoreFiles: (string | RegExp)[];
     ignore?: Ignore;
-    constructor(githubUrl: string, { accessToken, branch, recursive, unknown, ignoreFiles, ignorePaths, }?: GithubRepoLoaderParams);
+    verbose?: boolean;
+    protected caller: AsyncCaller;
+    constructor(githubUrl: string, { accessToken, branch, recursive, unknown, ignoreFiles, ignorePaths, verbose, maxConcurrency, maxRetries, ...rest }?: GithubRepoLoaderParams);
     private extractOwnerAndRepoAndPath;
     load(): Promise<Document[]>;
-    protected shouldIgnore(path: string, fileType: string): Promise<boolean>;
+    protected shouldIgnore(path: string, fileType: string): boolean;
+    /**
+     * Takes the file info and wrap it in a promise that will resolve to the file content and metadata
+     * @param file
+     * @returns
+     */
+    private fetchFileContentWrapper;
+    /**
+     * Maps a list of files / directories to a list of promises that will fetch the file / directory contents
+     */
+    private getCurrentDirectoryFilePromises;
+    /**
+     * Begins the process of fetching the contents of the repository
+     */
+    private processRepo;
     private processDirectory;
     private fetchRepoFiles;
     private fetchFileContent;