langchain 0.2.17 → 0.3.0-rc.0

package/dist/document_loaders/web/github.cjs

@@ -1,615 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.GithubRepoLoader = void 0;
-const ignore_1 = __importDefault(require("ignore"));
-const binary_extensions_1 = __importDefault(require("binary-extensions"));
-const documents_1 = require("@langchain/core/documents");
-const env_1 = require("@langchain/core/utils/env");
-const async_caller_1 = require("@langchain/core/utils/async_caller");
-const base_js_1 = require("../base.cjs");
-const directory_js_1 = require("../fs/directory.cjs");
-const extname_js_1 = require("../../util/extname.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/github",
-    newPackageName: "@langchain/community",
-});
-const extensions = /* #__PURE__ */ new Set(binary_extensions_1.default);
-/**
- * A function that checks if a file path is a binary file based on its
- * extension.
- * @param name The file path to check.
- * @returns A boolean indicating whether the file path is a binary file.
- */
-function isBinaryPath(name) {
-    return extensions.has((0, extname_js_1.extname)(name).slice(1).toLowerCase());
-}
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/github" instead. This entrypoint will be removed in 0.3.0.
- *
- * A class that extends the BaseDocumentLoader and implements the
- * GithubRepoLoaderParams interface. It represents a document loader for
- * loading files from a GitHub repository.
- */
-class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
-    constructor(githubUrl, { accessToken = (0, env_1.getEnvironmentVariable)("GITHUB_ACCESS_TOKEN"), baseUrl = "https://github.com", apiUrl = "https://api.github.com", branch = "main", recursive = true, processSubmodules = false, unknown = directory_js_1.UnknownHandling.Warn, ignoreFiles = [], ignorePaths, verbose = false, maxConcurrency = 2, maxRetries = 2, ...rest } = {}) {
-        super();
-        Object.defineProperty(this, "baseUrl", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "apiUrl", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "owner", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "repo", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "initialPath", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "headers", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: {}
-        });
-        Object.defineProperty(this, "branch", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "recursive", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "processSubmodules", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "unknown", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "accessToken", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "ignoreFiles", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "ignore", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "verbose", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "maxConcurrency", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "maxRetries", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "caller", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "ignorePaths", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "submoduleInfos", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.baseUrl = baseUrl;
-        this.apiUrl = apiUrl;
-        const { owner, repo, path } = this.extractOwnerAndRepoAndPath(githubUrl);
-        this.owner = owner;
-        this.repo = repo;
-        this.initialPath = path;
-        this.branch = branch;
-        this.recursive = recursive;
-        // processing submodules without processing contents of other directories makes no sense
-        if (processSubmodules && !recursive) {
-            throw new Error(`Input property "recursive" must be true if "processSubmodules" is true.`);
-        }
-        this.processSubmodules = processSubmodules;
-        this.unknown = unknown;
-        this.accessToken = accessToken;
-        this.ignoreFiles = ignoreFiles;
-        this.verbose = verbose;
-        this.maxConcurrency = maxConcurrency;
-        this.maxRetries = maxRetries;
-        this.headers = {
-            "User-Agent": "langchain",
-        };
-        this.caller = new async_caller_1.AsyncCaller({
-            maxConcurrency,
-            maxRetries,
-            ...rest,
-        });
-        this.ignorePaths = ignorePaths;
-        if (ignorePaths) {
-            this.ignore = ignore_1.default.default().add(ignorePaths);
-        }
-        if (this.accessToken) {
-            this.headers = {
-                ...this.headers,
-                Authorization: `Bearer ${this.accessToken}`,
-            };
-        }
-    }
-    /**
-     * Extracts the owner, repository, and path from a GitHub URL.
-     * @param url The GitHub URL to extract information from.
-     * @returns An object containing the owner, repository, and path extracted from the GitHub URL.
-     */
-    extractOwnerAndRepoAndPath(url) {
-        const match = url.match(new RegExp(`${this.baseUrl}/([^/]+)/([^/]+)(/tree/[^/]+/(.+))?`, "i"));
-        if (!match) {
-            throw new Error("Invalid GitHub URL format.");
-        }
-        return { owner: match[1], repo: match[2], path: match[4] || "" };
-    }
-    /**
-     * Fetches the files from the GitHub repository and creates Document
-     * instances for each file. It also handles error handling based on the
-     * unknown handling option.
-     * @returns A promise that resolves to an array of Document instances.
-     */
-    async load() {
-        this.log(`Loading documents from ${this.baseUrl}/${this.owner}/${this.repo}/${this.initialPath}...`);
-        // process repository without submodules
-        const documents = (await this.processRepo()).map((fileResponse) => new documents_1.Document({
-            pageContent: fileResponse.contents,
-            metadata: fileResponse.metadata,
-        }));
-        if (this.processSubmodules) {
-            // process submodules
-            await this.getSubmoduleInfo();
-            for (const submoduleInfo of this.submoduleInfos) {
-                documents.push(...(await this.loadSubmodule(submoduleInfo)));
-            }
-        }
-        return documents;
-    }
-    /**
-     * Asynchronously streams documents from the entire GitHub repository.
-     * It is suitable for situations where processing large repositories in a memory-efficient manner is required.
-     * @yields Yields a Promise that resolves to a Document object for each file or submodule content found in the repository.
-     */
-    async *loadAsStream() {
-        this.log(`Loading documents from ${this.baseUrl}/${this.owner}/${this.repo}/${this.initialPath}...`);
-        yield* await this.processRepoAsStream(this.initialPath);
-        if (!this.processSubmodules) {
-            return;
-        }
-        await this.getSubmoduleInfo();
-        for (const submoduleInfo of this.submoduleInfos) {
-            yield* await this.loadSubmoduleAsStream(submoduleInfo);
-        }
-    }
-    /**
-     * Loads the information about Git submodules from the repository, if available.
-     */
-    async getSubmoduleInfo() {
-        this.log("Loading info about submodules...");
-        // we have to fetch the files of the root directory to get the download url of the .gitmodules file
-        // however, we cannot reuse the files retrieved in processRepo() as initialPath may be != ""
-        // so it may be that we end up fetching this file list twice
-        const repoFiles = await this.fetchRepoFiles("");
-        const gitmodulesFile = repoFiles.filter(({ name }) => name === ".gitmodules")?.[0];
-        if (gitmodulesFile) {
-            const gitmodulesContent = await this.fetchFileContent({
-                download_url: gitmodulesFile.download_url,
-            });
-            this.submoduleInfos = await this.parseGitmodules(gitmodulesContent);
-        }
-        else {
-            this.submoduleInfos = [];
-        }
-        this.log(`Found ${this.submoduleInfos.length} submodules:`);
-        for (const submoduleInfo of this.submoduleInfos) {
-            this.log(JSON.stringify(submoduleInfo));
-        }
-    }
-    /**
-     * Parses the given content of a .gitmodules file. Furthermore, queries the current SHA ref of all submodules.
-     * Returns the submodule information as array.
-     * @param gitmodulesContent the content of a .gitmodules file
-     */
-    async parseGitmodules(gitmodulesContent) {
-        let validGitmodulesContent = gitmodulesContent;
-        // in case the .gitmodules file does not end with a newline, we add one to make the regex work
-        if (!validGitmodulesContent.endsWith("\n")) {
-            validGitmodulesContent += "\n";
-        }
-        // catches the initial line of submodule entries
-        const submodulePattern = /\[submodule "(.*?)"]\n((\s+.*?\s*=\s*.*?\n)*)/g;
-        // catches the properties of a submodule
-        const keyValuePattern = /\s+(.*?)\s*=\s*(.*?)\s/g;
-        const submoduleInfos = [];
-        for (const [, name, propertyLines] of validGitmodulesContent.matchAll(submodulePattern)) {
-            if (!name || !propertyLines) {
-                throw new Error("Could not parse submodule entry");
-            }
-            const submodulePropertyLines = propertyLines.matchAll(keyValuePattern);
-            let path;
-            let url;
-            for (const [, key, value] of submodulePropertyLines) {
-                if (!key || !value) {
-                    throw new Error(`Could not parse key/value pairs for submodule ${name}`);
-                }
-                switch (key) {
-                    case "path":
-                        path = value;
-                        break;
-                    case "url":
-                        url = value;
-                        if (url.endsWith(".git")) {
-                            url = url.substring(0, url.length - 4);
-                        }
-                        break;
-                    default:
-                    // ignoring unused keys
-                }
-            }
-            if (!path || !url) {
-                throw new Error(`Missing properties for submodule ${name}`);
-            }
-            // fetch the current ref of the submodule
-            const files = await this.fetchRepoFiles(path);
-            const submoduleInfo = {
-                name,
-                path,
-                url,
-                ref: files[0].sha,
-            };
-            submoduleInfos.push(submoduleInfo);
-        }
-        return submoduleInfos;
-    }
-    /**
-     * Loads the documents of the given submodule. Uses the same parameters as for the current repository.
-     * External submodules, i.e. submodules pointing to another GitHub instance, are ignored.
-     * @param submoduleInfo the info about the submodule to be loaded
-     */
-    async loadSubmodule(submoduleInfo) {
-        if (!submoduleInfo.url.startsWith(this.baseUrl)) {
-            this.log(`Ignoring external submodule ${submoduleInfo.url}.`);
-            return [];
-        }
-        else if (!submoduleInfo.path.startsWith(this.initialPath)) {
-            this.log(`Ignoring submodule ${submoduleInfo.url}, as it is not on initial path.`);
-            return [];
-        }
-        else {
-            this.log(`Accessing submodule ${submoduleInfo.name} (${submoduleInfo.url})...`);
-            return new GithubRepoLoader(submoduleInfo.url, {
-                accessToken: this.accessToken,
-                apiUrl: this.apiUrl,
-                baseUrl: this.baseUrl,
-                branch: submoduleInfo.ref,
-                recursive: this.recursive,
-                processSubmodules: this.processSubmodules,
-                unknown: this.unknown,
-                ignoreFiles: this.ignoreFiles,
-                ignorePaths: this.ignorePaths,
-                verbose: this.verbose,
-                maxConcurrency: this.maxConcurrency,
-                maxRetries: this.maxRetries,
-            }).load();
-        }
-    }
-    /**
-     * Asynchronously processes and streams the contents of a specified submodule in the GitHub repository.
-     * @param submoduleInfo the info about the submodule to be loaded
-     * @yields Yields a Promise that resolves to a Document object for each file found in the submodule.
-     */
-    async *loadSubmoduleAsStream(submoduleInfo) {
-        if (!submoduleInfo.url.startsWith(this.baseUrl)) {
-            this.log(`Ignoring external submodule ${submoduleInfo.url}.`);
-            yield* [];
-        }
-        if (!submoduleInfo.path.startsWith(this.initialPath)) {
-            this.log(`Ignoring submodule ${submoduleInfo.url}, as it is not on initial path.`);
-            yield* [];
-        }
-        this.log(`Accessing submodule ${submoduleInfo.name} (${submoduleInfo.url})...`);
-        const submoduleLoader = new GithubRepoLoader(submoduleInfo.url, {
-            accessToken: this.accessToken,
-            baseUrl: this.baseUrl,
-            apiUrl: this.apiUrl,
-            branch: submoduleInfo.ref,
-            recursive: this.recursive,
-            processSubmodules: this.processSubmodules,
-            unknown: this.unknown,
-            ignoreFiles: this.ignoreFiles,
-            ignorePaths: this.ignorePaths,
-            verbose: this.verbose,
-            maxConcurrency: this.maxConcurrency,
-            maxRetries: this.maxRetries,
-        });
-        yield* await submoduleLoader.processRepoAsStream(submoduleInfo.path);
-    }
-    /**
-     * Determines whether a file or directory should be ignored based on its
-     * path and type.
-     * @param path The path of the file or directory.
-     * @param fileType The type of the file or directory.
-     * @returns A boolean indicating whether the file or directory should be ignored.
-     */
-    shouldIgnore(path, fileType) {
-        if (fileType !== "dir" && isBinaryPath(path)) {
-            return true;
-        }
-        if (this.ignore !== undefined) {
-            return this.ignore.ignores(path);
-        }
-        return (fileType !== "dir" &&
-            this.ignoreFiles.some((pattern) => {
-                if (typeof pattern === "string") {
-                    return path === pattern;
-                }
-                try {
-                    return pattern.test(path);
-                }
-                catch {
-                    throw new Error(`Unknown ignore file pattern: ${pattern}`);
-                }
-            }));
-    }
-    /**
-     * Takes the file info and wrap it in a promise that will resolve to the file content and metadata
-     * @param file
-     * @returns
-     */
-    async fetchFileContentWrapper(file) {
-        const fileContent = await this.fetchFileContent(file).catch((error) => {
-            this.handleError(`Failed wrap file content: ${file}, ${error}`);
-        });
-        return {
-            contents: fileContent || "",
-            metadata: {
-                source: file.path,
-                repository: `${this.baseUrl}/${this.owner}/${this.repo}`,
-                branch: this.branch,
-            },
-        };
-    }
-    /**
-     * Maps a list of files / directories to a list of promises that will fetch the file / directory contents
-     */
-    async getCurrentDirectoryFilePromises(files) {
-        const currentDirectoryFilePromises = [];
-        // Directories have nested files / directories, which is why this is a list of promises of promises
-        const currentDirectoryDirectoryPromises = [];
-        for (const file of files) {
-            if (this.shouldIgnore(file.path, file.type)) {
-                continue;
-            }
-            if (file.type === "file" && file.size === 0) {
-                // this is a submodule. ignoring for the moment. submodule processing is done separately
-                continue;
-            }
-            if (file.type !== "dir") {
-                try {
-                    currentDirectoryFilePromises.push(this.fetchFileContentWrapper(file));
-                }
-                catch (e) {
-                    this.handleError(`Failed to fetch file content: ${file.path}, ${e}`);
-                }
-            }
-            else if (this.recursive) {
-                currentDirectoryDirectoryPromises.push(this.processDirectory(file.path));
-            }
-        }
-        const curDirDirectories = await Promise.all(currentDirectoryDirectoryPromises);
-        return [...currentDirectoryFilePromises, ...curDirDirectories.flat()];
-    }
-    /**
-     * Begins the process of fetching the contents of the repository
-     */
-    async processRepo() {
-        try {
-            // Get the list of file / directory names in the root directory
-            const files = await this.fetchRepoFiles(this.initialPath);
-            // Map the file / directory paths to promises that will fetch the file / directory contents
-            const currentDirectoryFilePromises = await this.getCurrentDirectoryFilePromises(files);
-            return Promise.all(currentDirectoryFilePromises);
-        }
-        catch (error) {
-            this.handleError(`Failed to process directory: ${this.initialPath}, ${error}`);
-            return Promise.reject(error);
-        }
-    }
-    /**
-     * Asynchronously processes the contents of the entire GitHub repository,
-     * streaming each file as a Document object.
-     * @param path The path of the directory to process.
-     * @yields Yields a Promise that resolves to a Document object for each file found in the repository.
-     */
-    async *processRepoAsStream(path) {
-        const files = await this.fetchRepoFiles(path);
-        for (const file of files) {
-            if (this.shouldIgnore(file.path, file.type)) {
-                continue;
-            }
-            if (file.type === "file") {
-                try {
-                    const fileResponse = await this.fetchFileContentWrapper(file);
-                    yield new documents_1.Document({
-                        pageContent: fileResponse.contents,
-                        metadata: fileResponse.metadata,
-                    });
-                }
-                catch (error) {
-                    this.handleError(`Failed to fetch file content: ${file.path}, ${error}`);
-                }
-            }
-            else if (this.recursive) {
-                yield* await this.processDirectoryAsStream(file.path);
-            }
-        }
-    }
-    /**
-     * Fetches the contents of a directory and maps the file / directory paths
-     * to promises that will fetch the file / directory contents.
-     * @param path The path of the directory to process.
-     * @returns A promise that resolves to an array of promises that will fetch the file / directory contents.
-     */
-    async processDirectory(path) {
-        try {
-            const files = await this.fetchRepoFiles(path);
-            return this.getCurrentDirectoryFilePromises(files);
-        }
-        catch (error) {
-            this.handleError(`Failed to process directory: ${path}, ${error}`);
-            return Promise.reject(error);
-        }
-    }
-    /**
-     * Asynchronously processes the contents of a given directory in the GitHub repository,
-     * streaming each file as a Document object.
-     * @param path The path of the directory to process.
-     * @yields Yields a Promise that resolves to a Document object for each file in the directory.
-     */
-    async *processDirectoryAsStream(path) {
-        const files = await this.fetchRepoFiles(path);
-        for (const file of files) {
-            if (this.shouldIgnore(file.path, file.type)) {
-                continue;
-            }
-            if (file.type === "file") {
-                try {
-                    const fileResponse = await this.fetchFileContentWrapper(file);
-                    yield new documents_1.Document({
-                        pageContent: fileResponse.contents,
-                        metadata: fileResponse.metadata,
-                    });
-                }
-                catch {
-                    this.handleError(`Failed to fetch file content: ${file.path}`);
-                }
-            }
-            else if (this.recursive) {
-                yield* await this.processDirectoryAsStream(file.path);
-            }
-        }
-    }
-    /**
-     * Fetches the files from a GitHub repository.
-     * If the path denotes a single file, the resulting array contains only one element.
-     * @param path The path of the repository to fetch the files from.
-     * @returns A promise that resolves to an array of GithubFile instances.
-     */
-    async fetchRepoFiles(path) {
-        const url = `${this.apiUrl}/repos/${this.owner}/${this.repo}/contents/${path}?ref=${this.branch}`;
-        return this.caller.call(async () => {
-            this.log(`Fetching ${url}`);
-            const response = await fetch(url, { headers: this.headers });
-            const data = await response.json();
-            if (!response.ok) {
-                throw new Error(`Unable to fetch repository files: ${response.status} ${JSON.stringify(data)}`);
-            }
-            if (Array.isArray(data)) {
-                return data;
-            }
-            else {
-                return [data];
-            }
-        });
-    }
-    /**
-     * Fetches the content of a file from a GitHub repository.
-     * @param file The file to fetch the content from.
-     * @returns A promise that resolves to the content of the file.
-     */
-    async fetchFileContent(file) {
-        return this.caller.call(async () => {
-            this.log(`Fetching ${file.download_url}`);
-            const response = await fetch(file.download_url, {
-                headers: this.headers,
-            });
-            return response.text();
-        });
-    }
-    /**
-     * Handles errors based on the unknown handling option.
-     * @param message The error message.
-     * @returns void
-     */
-    handleError(message) {
-        switch (this.unknown) {
-            case directory_js_1.UnknownHandling.Ignore:
-                break;
-            case directory_js_1.UnknownHandling.Warn:
-                console.warn(message);
-                break;
-            case directory_js_1.UnknownHandling.Error:
-                throw new Error(message);
-            default:
-                throw new Error(`Unknown unknown handling: ${this.unknown}`);
-        }
-    }
-    /**
-     * Logs the given message to the console, if parameter 'verbose' is set to true.
-     * @param message the message to be logged.
-     */
-    log(message) {
-        if (this.verbose) {
-            console.log(message);
-        }
-    }
-}
-exports.GithubRepoLoader = GithubRepoLoader;