langchain 0.0.142 → 0.0.144

package/dist/chains/openai_moderation.js

@@ -114,6 +114,7 @@ export class OpenAIModerationChain extends BaseChain {
         const output = this._moderate(text, mod.results[0]);
         return {
             [this.outputKey]: output,
+            results: mod.results,
         };
     }
     _chainType() {

package/dist/chat_models/base.cjs

@@ -110,6 +110,7 @@ class BaseChatModel extends index_js_2.BaseLanguageModel {
         else {
             parsedOptions = options;
         }
+        const baseMessages = messages.map((messageList) => messageList.map(index_js_1.coerceMessageLikeToMessage));
         const [runnableConfig, callOptions] = this._separateRunnableConfigFromCallOptions(parsedOptions);
         // create callback manager and start run
         const callbackManager_ = await manager_js_1.CallbackManager.configure(runnableConfig.callbacks ?? callbacks, this.callbacks, runnableConfig.tags, this.tags, runnableConfig.metadata, this.metadata, { verbose: this.verbose });
@@ -117,9 +118,9 @@ class BaseChatModel extends index_js_2.BaseLanguageModel {
             options: callOptions,
             invocation_params: this?.invocationParams(parsedOptions),
         };
-        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), messages, undefined, undefined, extra);
+        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages, undefined, undefined, extra);
         // generate results
-        const results = await Promise.allSettled(messages.map((messageList, i) => this._generate(messageList, { ...callOptions, promptIndex: i }, runManagers?.[i])));
+        const results = await Promise.allSettled(baseMessages.map((messageList, i) => this._generate(messageList, { ...callOptions, promptIndex: i }, runManagers?.[i])));
         // handle results
         const generations = [];
         const llmOutputs = [];
@@ -183,7 +184,7 @@ class BaseChatModel extends index_js_2.BaseLanguageModel {
      * @returns A Promise that resolves to a BaseMessage.
      */
     async call(messages, options, callbacks) {
-        const result = await this.generate([messages], options, callbacks);
+        const result = await this.generate([messages.map(index_js_1.coerceMessageLikeToMessage)], options, callbacks);
         const generations = result.generations;
         return generations[0][0].message;
     }

package/dist/chat_models/base.d.ts

@@ -1,4 +1,4 @@
-import { BaseMessage, BasePromptValue, ChatResult, BaseMessageChunk, LLMResult, ChatGenerationChunk } from "../schema/index.js";
+import { BaseMessage, BasePromptValue, ChatResult, BaseMessageChunk, LLMResult, ChatGenerationChunk, BaseMessageLike } from "../schema/index.js";
 import { BaseLanguageModel, BaseLanguageModelCallOptions, BaseLanguageModelInput, BaseLanguageModelParams } from "../base_language/index.js";
 import { CallbackManagerForLLMRun, Callbacks } from "../callbacks/manager.js";
 import { RunnableConfig } from "../schema/runnable.js";
@@ -56,7 +56,7 @@ export declare abstract class BaseChatModel<CallOptions extends BaseChatModelCal
      * @param callbacks The callbacks for the language model.
      * @returns A Promise that resolves to an LLMResult.
      */
-    generate(messages: BaseMessage[][], options?: string[] | CallOptions, callbacks?: Callbacks): Promise<LLMResult>;
+    generate(messages: BaseMessageLike[][], options?: string[] | CallOptions, callbacks?: Callbacks): Promise<LLMResult>;
     /**
      * Get the parameters used to invoke the model
      */
@@ -79,7 +79,7 @@ export declare abstract class BaseChatModel<CallOptions extends BaseChatModelCal
      * @param callbacks The callbacks for the language model.
      * @returns A Promise that resolves to a BaseMessage.
      */
-    call(messages: BaseMessage[], options?: string[] | CallOptions, callbacks?: Callbacks): Promise<BaseMessage>;
+    call(messages: BaseMessageLike[], options?: string[] | CallOptions, callbacks?: Callbacks): Promise<BaseMessage>;
     /**
      * Makes a single call to the chat model with a prompt value.
      * @param promptValue The value of the prompt.

package/dist/chat_models/base.js

@@ -1,4 +1,4 @@
-import { AIMessage, HumanMessage, RUN_KEY, } from "../schema/index.js";
+import { AIMessage, HumanMessage, RUN_KEY, coerceMessageLikeToMessage, } from "../schema/index.js";
 import { BaseLanguageModel, } from "../base_language/index.js";
 import { CallbackManager, } from "../callbacks/manager.js";
 /**
@@ -106,6 +106,7 @@ export class BaseChatModel extends BaseLanguageModel {
         else {
             parsedOptions = options;
         }
+        const baseMessages = messages.map((messageList) => messageList.map(coerceMessageLikeToMessage));
         const [runnableConfig, callOptions] = this._separateRunnableConfigFromCallOptions(parsedOptions);
         // create callback manager and start run
         const callbackManager_ = await CallbackManager.configure(runnableConfig.callbacks ?? callbacks, this.callbacks, runnableConfig.tags, this.tags, runnableConfig.metadata, this.metadata, { verbose: this.verbose });
@@ -113,9 +114,9 @@ export class BaseChatModel extends BaseLanguageModel {
             options: callOptions,
             invocation_params: this?.invocationParams(parsedOptions),
         };
-        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), messages, undefined, undefined, extra);
+        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages, undefined, undefined, extra);
         // generate results
-        const results = await Promise.allSettled(messages.map((messageList, i) => this._generate(messageList, { ...callOptions, promptIndex: i }, runManagers?.[i])));
+        const results = await Promise.allSettled(baseMessages.map((messageList, i) => this._generate(messageList, { ...callOptions, promptIndex: i }, runManagers?.[i])));
         // handle results
         const generations = [];
         const llmOutputs = [];
@@ -179,7 +180,7 @@ export class BaseChatModel extends BaseLanguageModel {
      * @returns A Promise that resolves to a BaseMessage.
      */
     async call(messages, options, callbacks) {
-        const result = await this.generate([messages], options, callbacks);
+        const result = await this.generate([messages.map(coerceMessageLikeToMessage)], options, callbacks);
         const generations = result.generations;
         return generations[0][0].message;
     }

package/dist/chat_models/minimax.d.ts

@@ -1,8 +1,9 @@
+import type { OpenAI as OpenAIClient } from "openai";
 import { BaseChatModel, BaseChatModelParams } from "./base.js";
 import { BaseMessage, ChatResult } from "../schema/index.js";
 import { CallbackManagerForLLMRun } from "../callbacks/manager.js";
 import { StructuredTool } from "../tools/index.js";
-import { BaseLanguageModelCallOptions } from "../base_language/index.js";
+import { BaseFunctionCallOptions } from "../base_language/index.js";
 /**
  * Type representing the sender_type of a message in the Minimax chat model.
  */
@@ -15,28 +16,6 @@ interface MinimaxChatCompletionRequestMessage {
     sender_name?: string;
     text: string;
 }
-export interface MinimaxChatCompletionRequestFunctions {
-    /**
-     * The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64.
-     * @type {string}
-     * @memberof MinimaxChatCompletionRequestFunctions
-     */
-    name: string;
-    /**
-     * The description of what the function does.
-     * @type {string}
-     * @memberof MinimaxChatCompletionRequestFunctions
-     */
-    description?: string;
-    /**
-     * The parameters the functions accepts, described as a JSON Schema object.
-     * @type {{ [key: string]: any; }}
-     * @memberof MinimaxChatCompletionRequestFunctions
-     */
-    parameters?: {
-        [key: string]: any;
-    };
-}
 /**
  * Interface representing a request for a chat completion.
  */
@@ -58,9 +37,9 @@ interface MinimaxChatCompletionRequest {
     sample_messages?: MinimaxChatCompletionRequestMessage[];
     /**
      * A list of functions the model may generate JSON inputs for.
-     * @type {Array<MinimaxChatCompletionRequestFunctions>}
+     * @type {Array<OpenAIClient.Chat.ChatCompletionCreateParams.Function[]>}
      */
-    functions?: Array<MinimaxChatCompletionRequestFunctions>;
+    functions?: OpenAIClient.Chat.ChatCompletionCreateParams.Function[];
     plugins?: string[];
 }
 interface RoleMeta {
@@ -191,8 +170,7 @@ declare interface MinimaxChatInputPro extends MinimaxChatInputBase {
     replyConstraints?: ReplyConstraints;
 }
 type MinimaxChatInput = MinimaxChatInputNormal & MinimaxChatInputPro;
-export interface ChatMinimaxCallOptions extends BaseLanguageModelCallOptions {
-    functions?: MinimaxChatCompletionRequestFunctions[];
+export interface ChatMinimaxCallOptions extends BaseFunctionCallOptions {
     tools?: StructuredTool[];
     defaultUserName?: string;
     defaultBotName?: string;
@@ -252,7 +230,7 @@ export declare class ChatMinimax extends BaseChatModel<ChatMinimaxCallOptions> i
     identifyingParams(): {
         prompt?: string | undefined;
         stream?: boolean | undefined;
-        functions?: MinimaxChatCompletionRequestFunctions[] | undefined;
+        functions?: OpenAIClient.Chat.Completions.ChatCompletionCreateParams.Function[] | undefined;
         model: string;
         temperature?: number | undefined;
         top_p?: number | undefined;

package/dist/chat_models/openai.cjs

@@ -325,6 +325,7 @@ class ChatOpenAI extends base_js_1.BaseChatModel {
             if (!this.azureOpenAIApiVersion) {
                 throw new Error("Azure OpenAI API version not found");
             }
+            this.openAIApiKey = this.openAIApiKey ?? "";
         }
         this.clientConfig = {
             apiKey: this.openAIApiKey,

package/dist/chat_models/openai.d.ts

@@ -4,6 +4,7 @@ import { BaseMessage, ChatGenerationChunk, ChatResult } from "../schema/index.js
 import { StructuredTool } from "../tools/base.js";
 import { AzureOpenAIInput, OpenAICallOptions, OpenAIChatInput, OpenAICoreRequestOptions, LegacyOpenAIInput } from "../types/openai-types.js";
 import { BaseChatModel, BaseChatModelParams } from "./base.js";
+import { BaseFunctionCallOptions } from "../base_language/index.js";
 export { AzureOpenAIInput, OpenAICallOptions, OpenAIChatInput };
 interface TokenUsage {
     completionTokens?: number;
@@ -13,9 +14,7 @@ interface TokenUsage {
 interface OpenAILLMOutput {
     tokenUsage: TokenUsage;
 }
-export interface ChatOpenAICallOptions extends OpenAICallOptions {
-    function_call?: OpenAIClient.Chat.ChatCompletionCreateParams.FunctionCallOption;
-    functions?: OpenAIClient.Chat.ChatCompletionCreateParams.Function[];
+export interface ChatOpenAICallOptions extends OpenAICallOptions, BaseFunctionCallOptions {
     tools?: StructuredTool[];
     promptIndex?: number;
 }

package/dist/chat_models/openai.js

@@ -322,6 +322,7 @@ export class ChatOpenAI extends BaseChatModel {
             if (!this.azureOpenAIApiVersion) {
                 throw new Error("Azure OpenAI API version not found");
             }
+            this.openAIApiKey = this.openAIApiKey ?? "";
         }
         this.clientConfig = {
             apiKey: this.openAIApiKey,

package/dist/document_loaders/fs/openai_whisper_audio.cjs

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIWhisperAudio = void 0;
+const openai_1 = require("openai");
+const document_js_1 = require("../../document.cjs");
+const buffer_js_1 = require("./buffer.cjs");
+const MODEL_NAME = "whisper-1";
+class OpenAIWhisperAudio extends buffer_js_1.BufferLoader {
+    constructor(filePathOrBlob, fields) {
+        super(filePathOrBlob);
+        Object.defineProperty(this, "openAIClient", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.openAIClient = new openai_1.OpenAI(fields?.clientOptions);
+    }
+    async parse(raw, metadata) {
+        const fileName = metadata.source === "blob" ? metadata.blobType : metadata.source;
+        const transcriptionResponse = await this.openAIClient.audio.transcriptions.create({
+            file: await (0, openai_1.toFile)(raw, fileName),
+            model: MODEL_NAME,
+        });
+        const document = new document_js_1.Document({
+            pageContent: transcriptionResponse.text,
+            metadata,
+        });
+        return [document];
+    }
+}
+exports.OpenAIWhisperAudio = OpenAIWhisperAudio;

package/dist/document_loaders/fs/openai_whisper_audio.d.ts

@@ -0,0 +1,11 @@
+/// <reference types="node" resolution-mode="require"/>
+import { type ClientOptions } from "openai";
+import { Document } from "../../document.js";
+import { BufferLoader } from "./buffer.js";
+export declare class OpenAIWhisperAudio extends BufferLoader {
+    private readonly openAIClient;
+    constructor(filePathOrBlob: string | Blob, fields?: {
+        clientOptions?: ClientOptions;
+    });
+    protected parse(raw: Buffer, metadata: Record<string, string>): Promise<Document[]>;
+}

package/dist/document_loaders/fs/openai_whisper_audio.js

@@ -0,0 +1,28 @@
+import { OpenAI as OpenAIClient, toFile } from "openai";
+import { Document } from "../../document.js";
+import { BufferLoader } from "./buffer.js";
+const MODEL_NAME = "whisper-1";
+export class OpenAIWhisperAudio extends BufferLoader {
+    constructor(filePathOrBlob, fields) {
+        super(filePathOrBlob);
+        Object.defineProperty(this, "openAIClient", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.openAIClient = new OpenAIClient(fields?.clientOptions);
+    }
+    async parse(raw, metadata) {
+        const fileName = metadata.source === "blob" ? metadata.blobType : metadata.source;
+        const transcriptionResponse = await this.openAIClient.audio.transcriptions.create({
+            file: await toFile(raw, fileName),
+            model: MODEL_NAME,
+        });
+        const document = new Document({
+            pageContent: transcriptionResponse.text,
+            metadata,
+        });
+        return [document];
+    }
+}

package/dist/document_loaders/web/github.cjs

@@ -28,8 +28,20 @@ function isBinaryPath(name) {
  * loading files from a GitHub repository.
  */
 class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
-    constructor(githubUrl, { accessToken = (0, env_js_1.getEnvironmentVariable)("GITHUB_ACCESS_TOKEN"), branch = "main", recursive = true, unknown = directory_js_1.UnknownHandling.Warn, ignoreFiles = [], ignorePaths, verbose = false, maxConcurrency = 2, maxRetries = 2, ...rest } = {}) {
+    constructor(githubUrl, { accessToken = (0, env_js_1.getEnvironmentVariable)("GITHUB_ACCESS_TOKEN"), baseUrl = "https://github.com", apiUrl = "https://api.github.com", branch = "main", recursive = true, processSubmodules = false, unknown = directory_js_1.UnknownHandling.Warn, ignoreFiles = [], ignorePaths, verbose = false, maxConcurrency = 2, maxRetries = 2, ...rest } = {}) {
         super();
+        Object.defineProperty(this, "baseUrl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "apiUrl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "owner", {
             enumerable: true,
             configurable: true,
@@ -66,6 +78,12 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "processSubmodules", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "unknown", {
             enumerable: true,
             configurable: true,
@@ -96,22 +114,55 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "maxConcurrency", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxRetries", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "caller", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "ignorePaths", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "submoduleInfos", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.baseUrl = baseUrl;
+        this.apiUrl = apiUrl;
         const { owner, repo, path } = this.extractOwnerAndRepoAndPath(githubUrl);
         this.owner = owner;
         this.repo = repo;
         this.initialPath = path;
         this.branch = branch;
         this.recursive = recursive;
+        // processing submodules without processing contents of other directories makes no sense
+        if (processSubmodules && !recursive) {
+            throw new Error(`Input property "recursive" must be true if "processSubmodules" is true.`);
+        }
+        this.processSubmodules = processSubmodules;
         this.unknown = unknown;
         this.accessToken = accessToken;
         this.ignoreFiles = ignoreFiles;
         this.verbose = verbose;
+        this.maxConcurrency = maxConcurrency;
+        this.maxRetries = maxRetries;
         this.headers = {
             "User-Agent": "langchain",
         };
@@ -120,6 +171,7 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             maxRetries,
             ...rest,
         });
+        this.ignorePaths = ignorePaths;
         if (ignorePaths) {
             this.ignore = ignore_1.default.default().add(ignorePaths);
         }
@@ -136,7 +188,7 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
      * @returns An object containing the owner, repository, and path extracted from the GitHub URL.
      */
     extractOwnerAndRepoAndPath(url) {
-        const match = url.match(/https:\/\/github.com\/([^/]+)\/([^/]+)(\/tree\/[^/]+\/(.+))?/i);
+        const match = url.match(new RegExp(`${this.baseUrl}/([^/]+)/([^/]+)(/tree/[^/]+/(.+))?`, "i"));
         if (!match) {
             throw new Error("Invalid GitHub URL format.");
         }
@@ -149,10 +201,127 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
      * @returns A promise that resolves to an array of Document instances.
      */
     async load() {
-        return (await this.processRepo()).map((fileResponse) => new document_js_1.Document({
+        this.log(`Loading documents from ${this.baseUrl}/${this.owner}/${this.repo}/${this.initialPath}...`);
+        // process repository without submodules
+        const documents = (await this.processRepo()).map((fileResponse) => new document_js_1.Document({
             pageContent: fileResponse.contents,
             metadata: fileResponse.metadata,
         }));
+        if (this.processSubmodules) {
+            // process submodules
+            await this.getSubmoduleInfo();
+            for (const submoduleInfo of this.submoduleInfos) {
+                documents.push(...(await this.loadSubmodule(submoduleInfo)));
+            }
+        }
+        return documents;
+    }
+    /**
+     * Loads the information about Git submodules from the repository, if available.
+     */
+    async getSubmoduleInfo() {
+        this.log("Loading info about submodules...");
+        // we have to fetch the files of the root directory to get the download url of the .gitmodules file
+        // however, we cannot reuse the files retrieved in processRepo() as initialPath may be != ""
+        // so it may be that we end up fetching this file list twice
+        const repoFiles = await this.fetchRepoFiles("");
+        const gitmodulesFile = repoFiles.filter(({ name }) => name === ".gitmodules")?.[0];
+        if (gitmodulesFile) {
+            const gitmodulesContent = await this.fetchFileContent({
+                download_url: gitmodulesFile.download_url,
+            });
+            this.submoduleInfos = await this.parseGitmodules(gitmodulesContent);
+        }
+        else {
+            this.submoduleInfos = [];
+        }
+        this.log(`Found ${this.submoduleInfos.length} submodules:`);
+        for (const submoduleInfo of this.submoduleInfos) {
+            this.log(JSON.stringify(submoduleInfo));
+        }
+    }
+    /**
+     * Parses the given content of a .gitmodules file. Furthermore, queries the current SHA ref of all submodules.
+     * Returns the submodule information as array.
+     * @param gitmodulesContent the content of a .gitmodules file
+     */
+    async parseGitmodules(gitmodulesContent) {
+        // catches the initial line of submodule entries
+        const submodulePattern = /\[submodule "(.*?)"]\n((\s+.*?\s*=\s*.*?\n)*)/g;
+        // catches the properties of a submodule
+        const keyValuePattern = /\s+(.*?)\s*=\s*(.*?)\s/g;
+        const submoduleInfos = [];
+        for (const [, name, propertyLines] of gitmodulesContent.matchAll(submodulePattern)) {
+            if (!name || !propertyLines) {
+                throw new Error("Could not parse submodule entry");
+            }
+            const submodulePropertyLines = propertyLines.matchAll(keyValuePattern);
+            let path;
+            let url;
+            for (const [, key, value] of submodulePropertyLines) {
+                if (!key || !value) {
+                    throw new Error(`Could not parse key/value pairs for submodule ${name}`);
+                }
+                switch (key) {
+                    case "path":
+                        path = value;
+                        break;
+                    case "url":
+                        url = value;
+                        if (url.endsWith(".git")) {
+                            url = url.substring(0, url.length - 4);
+                        }
+                        break;
+                    default:
+                    // ignoring unused keys
+                }
+            }
+            if (!path || !url) {
+                throw new Error(`Missing properties for submodule ${name}`);
+            }
+            // fetch the current ref of the submodule
+            const files = await this.fetchRepoFiles(path);
+            const submoduleInfo = {
+                name,
+                path,
+                url,
+                ref: files[0].sha,
+            };
+            submoduleInfos.push(submoduleInfo);
+        }
+        return submoduleInfos;
+    }
+    /**
+     * Loads the documents of the given submodule. Uses the same parameters as for the current repository.
+     * External submodules, i.e. submodules pointing to another GitHub instance, are ignored.
+     * @param submoduleInfo the info about the submodule to be loaded
+     */
+    async loadSubmodule(submoduleInfo) {
+        if (!submoduleInfo.url.startsWith(this.baseUrl)) {
+            this.log(`Ignoring external submodule ${submoduleInfo.url}.`);
+            return [];
+        }
+        else if (!submoduleInfo.path.startsWith(this.initialPath)) {
+            this.log(`Ignoring submodule ${submoduleInfo.url}, as it is not on initial path.`);
+            return [];
+        }
+        else {
+            this.log(`Accessing submodule ${submoduleInfo.name} (${submoduleInfo.url})...`);
+            return new GithubRepoLoader(submoduleInfo.url, {
+                accessToken: this.accessToken,
+                apiUrl: this.apiUrl,
+                baseUrl: this.baseUrl,
+                branch: submoduleInfo.ref,
+                recursive: this.recursive,
+                processSubmodules: this.processSubmodules,
+                unknown: this.unknown,
+                ignoreFiles: this.ignoreFiles,
+                ignorePaths: this.ignorePaths,
+                verbose: this.verbose,
+                maxConcurrency: this.maxConcurrency,
+                maxRetries: this.maxRetries,
+            }).load();
+        }
     }
     /**
      * Determines whether a file or directory should be ignored based on its
@@ -192,7 +361,11 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
         });
         return {
             contents: fileContent || "",
-            metadata: { source: file.path },
+            metadata: {
+                source: file.path,
+                repository: `${this.baseUrl}/${this.owner}/${this.repo}`,
+                branch: this.branch,
+            },
         };
     }
     /**
@@ -203,19 +376,24 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
         // Directories have nested files / directories, which is why this is a list of promises of promises
         const currentDirectoryDirectoryPromises = [];
         for (const file of files) {
-            if (!this.shouldIgnore(file.path, file.type)) {
-                if (file.type !== "dir") {
-                    try {
-                        currentDirectoryFilePromises.push(this.fetchFileContentWrapper(file));
-                    }
-                    catch (e) {
-                        this.handleError(`Failed to fetch file content: ${file.path}, ${e}`);
-                    }
+            if (this.shouldIgnore(file.path, file.type)) {
+                continue;
+            }
+            if (file.type === "file" && file.size === 0) {
+                // this is a submodule. ignoring for the moment. submodule processing is done separately
+                continue;
+            }
+            if (file.type !== "dir") {
+                try {
+                    currentDirectoryFilePromises.push(this.fetchFileContentWrapper(file));
                 }
-                else if (this.recursive) {
-                    currentDirectoryDirectoryPromises.push(this.processDirectory(file.path));
+                catch (e) {
+                    this.handleError(`Failed to fetch file content: ${file.path}, ${e}`);
                 }
             }
+            else if (this.recursive) {
+                currentDirectoryDirectoryPromises.push(this.processDirectory(file.path));
+            }
         }
         const curDirDirectories = await Promise.all(currentDirectoryDirectoryPromises);
         return [...currentDirectoryFilePromises, ...curDirDirectories.flat()];
@@ -254,24 +432,25 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
     }
     /**
      * Fetches the files from a GitHub repository.
+     * If the path denotes a single file, the resulting array contains only one element.
      * @param path The path of the repository to fetch the files from.
      * @returns A promise that resolves to an array of GithubFile instances.
      */
     async fetchRepoFiles(path) {
-        const url = `https://api.github.com/repos/${this.owner}/${this.repo}/contents/${path}?ref=${this.branch}`;
+        const url = `${this.apiUrl}/repos/${this.owner}/${this.repo}/contents/${path}?ref=${this.branch}`;
         return this.caller.call(async () => {
-            if (this.verbose) {
-                console.log("Fetching", url);
-            }
+            this.log(`Fetching ${url}`);
             const response = await fetch(url, { headers: this.headers });
             const data = await response.json();
             if (!response.ok) {
                 throw new Error(`Unable to fetch repository files: ${response.status} ${JSON.stringify(data)}`);
             }
-            if (!Array.isArray(data)) {
-                throw new Error("Unable to fetch repository files.");
+            if (Array.isArray(data)) {
+                return data;
+            }
+            else {
+                return [data];
             }
-            return data;
         });
     }
     /**
@@ -281,9 +460,7 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
      */
     async fetchFileContent(file) {
         return this.caller.call(async () => {
-            if (this.verbose) {
-                console.log("Fetching", file.download_url);
-            }
+            this.log(`Fetching ${file.download_url}`);
             const response = await fetch(file.download_url, {
                 headers: this.headers,
             });
@@ -308,5 +485,14 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
                 throw new Error(`Unknown unknown handling: ${this.unknown}`);
         }
     }
+    /**
+     * Logs the given message to the console, if parameter 'verbose' is set to true.
+     * @param message the message to be logged.
+     */
+    log(message) {
+        if (this.verbose) {
+            console.log(message);
+        }
+    }
 }
 exports.GithubRepoLoader = GithubRepoLoader;