langchain 0.0.130 → 0.0.132

package/dist/document_loaders/web/gitbook.js

@@ -1,5 +1,9 @@
 import { Document } from "../../document.js";
 import { CheerioWebBaseLoader } from "./cheerio.js";
+/**
+ * Class representing a document loader specifically designed for loading
+ * documents from Gitbook. It extends the CheerioWebBaseLoader.
+ */
 export class GitbookLoader extends CheerioWebBaseLoader {
     constructor(webPath, params = {}) {
         const path = params.shouldLoadAllPaths === true ? `${webPath}/sitemap.xml` : webPath;
@@ -20,6 +24,13 @@ export class GitbookLoader extends CheerioWebBaseLoader {
         this.shouldLoadAllPaths =
             params.shouldLoadAllPaths ?? this.shouldLoadAllPaths;
     }
+    /**
+     * Method that scrapes the web document using Cheerio and loads the
+     * content based on the value of shouldLoadAllPaths. If shouldLoadAllPaths
+     * is true, it calls the loadAllPaths() method to load all paths.
+     * Otherwise, it calls the loadPath() method to load a single path.
+     * @returns Promise resolving to an array of Document instances.
+     */
     async load() {
         const $ = await this.scrape();
         if (this.shouldLoadAllPaths === true) {
@@ -27,6 +38,18 @@ export class GitbookLoader extends CheerioWebBaseLoader {
         }
         return this.loadPath($);
     }
+    /**
+     * Private method that loads the content of a single path from the Gitbook
+     * web document. It extracts the page content by selecting all elements
+     * inside the "main" element, filters out empty text nodes, and joins the
+     * remaining text nodes with line breaks. It extracts the title by
+     * selecting the first "h1" element inside the "main" element. It creates
+     * a Document instance with the extracted page content and metadata
+     * containing the source URL and title.
+     * @param $ CheerioAPI instance representing the loaded web document.
+     * @param url Optional string representing the URL of the web document.
+     * @returns Array of Document instances.
+     */
     loadPath($, url) {
         const pageContent = $("main *")
             .contents()
@@ -42,6 +65,16 @@ export class GitbookLoader extends CheerioWebBaseLoader {
             }),
         ];
     }
+    /**
+     * Private method that loads the content of all paths from the Gitbook web
+     * document. It extracts the URLs of all paths from the "loc" elements in
+     * the sitemap.xml. It iterates over each URL, scrapes the web document
+     * using the _scrape() method, and calls the loadPath() method to load the
+     * content of each path. It collects all the loaded documents and returns
+     * them as an array.
+     * @param $ CheerioAPI instance representing the loaded web document.
+     * @returns Promise resolving to an array of Document instances.
+     */
     async loadAllPaths($) {
         const urls = $("loc")
             .toArray()

package/dist/document_loaders/web/github.cjs

@@ -13,9 +13,20 @@ const extname_js_1 = require("../../util/extname.cjs");
 const env_js_1 = require("../../util/env.cjs");
 const async_caller_js_1 = require("../../util/async_caller.cjs");
 const extensions = new Set(binary_extensions_1.default);
+/**
+ * A function that checks if a file path is a binary file based on its
+ * extension.
+ * @param name The file path to check.
+ * @returns A boolean indicating whether the file path is a binary file.
+ */
 function isBinaryPath(name) {
     return extensions.has((0, extname_js_1.extname)(name).slice(1).toLowerCase());
 }
+/**
+ * A class that extends the BaseDocumentLoader and implements the
+ * GithubRepoLoaderParams interface. It represents a document loader for
+ * loading files from a GitHub repository.
+ */
 class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
     constructor(githubUrl, { accessToken = (0, env_js_1.getEnvironmentVariable)("GITHUB_ACCESS_TOKEN"), branch = "main", recursive = true, unknown = directory_js_1.UnknownHandling.Warn, ignoreFiles = [], ignorePaths, verbose = false, maxConcurrency = 2, maxRetries = 2, ...rest } = {}) {
         super();
@@ -115,6 +126,11 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             };
         }
     }
+    /**
+     * Extracts the owner, repository, and path from a GitHub URL.
+     * @param url The GitHub URL to extract information from.
+     * @returns An object containing the owner, repository, and path extracted from the GitHub URL.
+     */
     extractOwnerAndRepoAndPath(url) {
         const match = url.match(/https:\/\/github.com\/([^/]+)\/([^/]+)(\/tree\/[^/]+\/(.+))?/i);
         if (!match) {
@@ -122,12 +138,25 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
         }
         return { owner: match[1], repo: match[2], path: match[4] || "" };
     }
+    /**
+     * Fetches the files from the GitHub repository and creates Document
+     * instances for each file. It also handles error handling based on the
+     * unknown handling option.
+     * @returns A promise that resolves to an array of Document instances.
+     */
     async load() {
         return (await this.processRepo()).map((fileResponse) => new document_js_1.Document({
             pageContent: fileResponse.contents,
             metadata: fileResponse.metadata,
         }));
     }
+    /**
+     * Determines whether a file or directory should be ignored based on its
+     * path and type.
+     * @param path The path of the file or directory.
+     * @param fileType The type of the file or directory.
+     * @returns A boolean indicating whether the file or directory should be ignored.
+     */
     shouldIgnore(path, fileType) {
         if (fileType !== "dir" && isBinaryPath(path)) {
             return true;
@@ -203,6 +232,12 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             return Promise.reject(error);
         }
     }
+    /**
+     * Fetches the contents of a directory and maps the file / directory paths
+     * to promises that will fetch the file / directory contents.
+     * @param path The path of the directory to process.
+     * @returns A promise that resolves to an array of promises that will fetch the file / directory contents.
+     */
     async processDirectory(path) {
         try {
             const files = await this.fetchRepoFiles(path);
@@ -213,6 +248,11 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             return Promise.reject(error);
         }
     }
+    /**
+     * Fetches the files from a GitHub repository.
+     * @param path The path of the repository to fetch the files from.
+     * @returns A promise that resolves to an array of GithubFile instances.
+     */
     async fetchRepoFiles(path) {
         const url = `https://api.github.com/repos/${this.owner}/${this.repo}/contents/${path}?ref=${this.branch}`;
         return this.caller.call(async () => {
@@ -230,6 +270,11 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             return data;
         });
     }
+    /**
+     * Fetches the content of a file from a GitHub repository.
+     * @param file The file to fetch the content from.
+     * @returns A promise that resolves to the content of the file.
+     */
     async fetchFileContent(file) {
         return this.caller.call(async () => {
             if (this.verbose) {
@@ -241,6 +286,11 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             return response.text();
         });
     }
+    /**
+     * Handles errors based on the unknown handling option.
+     * @param message The error message.
+     * @returns void
+     */
     handleError(message) {
         switch (this.unknown) {
             case directory_js_1.UnknownHandling.Ignore:

package/dist/document_loaders/web/github.d.ts

@@ -3,6 +3,10 @@ import { Document } from "../../document.js";
 import { BaseDocumentLoader } from "../base.js";
 import { UnknownHandling } from "../fs/directory.js";
 import { AsyncCaller, AsyncCallerParams } from "../../util/async_caller.js";
+/**
+ * An interface that represents a file in a GitHub repository. It has
+ * properties for the file name, path, SHA, size, URLs, type, and links.
+ */
 export interface GithubFile {
     name: string;
     path: string;
@@ -19,6 +23,11 @@ export interface GithubFile {
         html: string;
     };
 }
+/**
+ * An interface that represents the parameters for the GithubRepoLoader
+ * class. It extends the AsyncCallerParams interface and adds additional
+ * properties specific to the GitHub repository loader.
+ */
 export interface GithubRepoLoaderParams extends AsyncCallerParams {
     branch?: string;
     recursive?: boolean;
@@ -37,6 +46,11 @@ export interface GithubRepoLoaderParams extends AsyncCallerParams {
      */
     maxRetries?: number;
 }
+/**
+ * A class that extends the BaseDocumentLoader and implements the
+ * GithubRepoLoaderParams interface. It represents a document loader for
+ * loading files from a GitHub repository.
+ */
 export declare class GithubRepoLoader extends BaseDocumentLoader implements GithubRepoLoaderParams {
     private readonly owner;
     private readonly repo;
@@ -51,8 +65,26 @@ export declare class GithubRepoLoader extends BaseDocumentLoader implements Gith
     verbose?: boolean;
     protected caller: AsyncCaller;
     constructor(githubUrl: string, { accessToken, branch, recursive, unknown, ignoreFiles, ignorePaths, verbose, maxConcurrency, maxRetries, ...rest }?: GithubRepoLoaderParams);
+    /**
+     * Extracts the owner, repository, and path from a GitHub URL.
+     * @param url The GitHub URL to extract information from.
+     * @returns An object containing the owner, repository, and path extracted from the GitHub URL.
+     */
     private extractOwnerAndRepoAndPath;
+    /**
+     * Fetches the files from the GitHub repository and creates Document
+     * instances for each file. It also handles error handling based on the
+     * unknown handling option.
+     * @returns A promise that resolves to an array of Document instances.
+     */
     load(): Promise<Document[]>;
+    /**
+     * Determines whether a file or directory should be ignored based on its
+     * path and type.
+     * @param path The path of the file or directory.
+     * @param fileType The type of the file or directory.
+     * @returns A boolean indicating whether the file or directory should be ignored.
+     */
     protected shouldIgnore(path: string, fileType: string): boolean;
     /**
      * Takes the file info and wrap it in a promise that will resolve to the file content and metadata
@@ -68,8 +100,29 @@ export declare class GithubRepoLoader extends BaseDocumentLoader implements Gith
      * Begins the process of fetching the contents of the repository
      */
     private processRepo;
+    /**
+     * Fetches the contents of a directory and maps the file / directory paths
+     * to promises that will fetch the file / directory contents.
+     * @param path The path of the directory to process.
+     * @returns A promise that resolves to an array of promises that will fetch the file / directory contents.
+     */
     private processDirectory;
+    /**
+     * Fetches the files from a GitHub repository.
+     * @param path The path of the repository to fetch the files from.
+     * @returns A promise that resolves to an array of GithubFile instances.
+     */
     private fetchRepoFiles;
+    /**
+     * Fetches the content of a file from a GitHub repository.
+     * @param file The file to fetch the content from.
+     * @returns A promise that resolves to the content of the file.
+     */
     private fetchFileContent;
+    /**
+     * Handles errors based on the unknown handling option.
+     * @param message The error message.
+     * @returns void
+     */
     private handleError;
 }

package/dist/document_loaders/web/github.js

@@ -7,9 +7,20 @@ import { extname } from "../../util/extname.js";
 import { getEnvironmentVariable } from "../../util/env.js";
 import { AsyncCaller } from "../../util/async_caller.js";
 const extensions = new Set(binaryExtensions);
+/**
+ * A function that checks if a file path is a binary file based on its
+ * extension.
+ * @param name The file path to check.
+ * @returns A boolean indicating whether the file path is a binary file.
+ */
 function isBinaryPath(name) {
     return extensions.has(extname(name).slice(1).toLowerCase());
 }
+/**
+ * A class that extends the BaseDocumentLoader and implements the
+ * GithubRepoLoaderParams interface. It represents a document loader for
+ * loading files from a GitHub repository.
+ */
 export class GithubRepoLoader extends BaseDocumentLoader {
     constructor(githubUrl, { accessToken = getEnvironmentVariable("GITHUB_ACCESS_TOKEN"), branch = "main", recursive = true, unknown = UnknownHandling.Warn, ignoreFiles = [], ignorePaths, verbose = false, maxConcurrency = 2, maxRetries = 2, ...rest } = {}) {
         super();
@@ -109,6 +120,11 @@ export class GithubRepoLoader extends BaseDocumentLoader {
             };
         }
     }
+    /**
+     * Extracts the owner, repository, and path from a GitHub URL.
+     * @param url The GitHub URL to extract information from.
+     * @returns An object containing the owner, repository, and path extracted from the GitHub URL.
+     */
     extractOwnerAndRepoAndPath(url) {
         const match = url.match(/https:\/\/github.com\/([^/]+)\/([^/]+)(\/tree\/[^/]+\/(.+))?/i);
         if (!match) {
@@ -116,12 +132,25 @@ export class GithubRepoLoader extends BaseDocumentLoader {
         }
         return { owner: match[1], repo: match[2], path: match[4] || "" };
     }
+    /**
+     * Fetches the files from the GitHub repository and creates Document
+     * instances for each file. It also handles error handling based on the
+     * unknown handling option.
+     * @returns A promise that resolves to an array of Document instances.
+     */
     async load() {
         return (await this.processRepo()).map((fileResponse) => new Document({
             pageContent: fileResponse.contents,
             metadata: fileResponse.metadata,
         }));
     }
+    /**
+     * Determines whether a file or directory should be ignored based on its
+     * path and type.
+     * @param path The path of the file or directory.
+     * @param fileType The type of the file or directory.
+     * @returns A boolean indicating whether the file or directory should be ignored.
+     */
     shouldIgnore(path, fileType) {
         if (fileType !== "dir" && isBinaryPath(path)) {
             return true;
@@ -197,6 +226,12 @@ export class GithubRepoLoader extends BaseDocumentLoader {
             return Promise.reject(error);
         }
     }
+    /**
+     * Fetches the contents of a directory and maps the file / directory paths
+     * to promises that will fetch the file / directory contents.
+     * @param path The path of the directory to process.
+     * @returns A promise that resolves to an array of promises that will fetch the file / directory contents.
+     */
     async processDirectory(path) {
         try {
             const files = await this.fetchRepoFiles(path);
@@ -207,6 +242,11 @@ export class GithubRepoLoader extends BaseDocumentLoader {
             return Promise.reject(error);
         }
     }
+    /**
+     * Fetches the files from a GitHub repository.
+     * @param path The path of the repository to fetch the files from.
+     * @returns A promise that resolves to an array of GithubFile instances.
+     */
     async fetchRepoFiles(path) {
         const url = `https://api.github.com/repos/${this.owner}/${this.repo}/contents/${path}?ref=${this.branch}`;
         return this.caller.call(async () => {
@@ -224,6 +264,11 @@ export class GithubRepoLoader extends BaseDocumentLoader {
             return data;
         });
     }
+    /**
+     * Fetches the content of a file from a GitHub repository.
+     * @param file The file to fetch the content from.
+     * @returns A promise that resolves to the content of the file.
+     */
     async fetchFileContent(file) {
         return this.caller.call(async () => {
             if (this.verbose) {
@@ -235,6 +280,11 @@ export class GithubRepoLoader extends BaseDocumentLoader {
             return response.text();
         });
     }
+    /**
+     * Handles errors based on the unknown handling option.
+     * @param message The error message.
+     * @returns void
+     */
     handleError(message) {
         switch (this.unknown) {
             case UnknownHandling.Ignore:

package/dist/document_loaders/web/hn.cjs

@@ -3,6 +3,10 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.HNLoader = void 0;
 const document_js_1 = require("../../document.cjs");
 const cheerio_js_1 = require("./cheerio.cjs");
+/**
+ * A class that extends the CheerioWebBaseLoader class. It represents a
+ * loader for loading web pages from the Hacker News website.
+ */
 class HNLoader extends cheerio_js_1.CheerioWebBaseLoader {
     constructor(webPath) {
         super(webPath);
@@ -13,6 +17,13 @@ class HNLoader extends cheerio_js_1.CheerioWebBaseLoader {
             value: webPath
         });
     }
+    /**
+     * An asynchronous method that loads the web page. If the webPath includes
+     * "item", it calls the loadComments() method to load the comments from
+     * the web page. Otherwise, it calls the loadResults() method to load the
+     * results from the web page.
+     * @returns A Promise that resolves to an array of Document instances.
+     */
     async load() {
         const $ = await this.scrape();
         if (this.webPath.includes("item")) {
@@ -20,6 +31,16 @@ class HNLoader extends cheerio_js_1.CheerioWebBaseLoader {
         }
         return this.loadResults($);
     }
+    /**
+     * A private method that loads the comments from the web page. It selects
+     * the elements with the class "athing comtr" using the $ function
+     * provided by Cheerio. It also extracts the title of the web page from
+     * the element with the id "pagespace". It creates Document instances for
+     * each comment, with the comment text as the page content and the source
+     * and title as metadata.
+     * @param $ A CheerioAPI instance.
+     * @returns An array of Document instances.
+     */
     loadComments($) {
         const comments = $("tr[class='athing comtr']");
         const title = $("tr[id='pagespace']").attr("title");
@@ -31,6 +52,16 @@ class HNLoader extends cheerio_js_1.CheerioWebBaseLoader {
         });
         return documents;
     }
+    /**
+     * A private method that loads the results from the web page. It selects
+     * the elements with the class "athing" using the $ function provided by
+     * Cheerio. It extracts the ranking, link, title, and other metadata from
+     * each result item. It creates Document instances for each result item,
+     * with the title as the page content and the source, title, link, and
+     * ranking as metadata.
+     * @param $ A CheerioAPI instance.
+     * @returns An array of Document instances.
+     */
     loadResults($) {
         const items = $("tr[class='athing']");
         const documents = [];

package/dist/document_loaders/web/hn.d.ts

@@ -1,9 +1,40 @@
 import { Document } from "../../document.js";
 import { CheerioWebBaseLoader } from "./cheerio.js";
+/**
+ * A class that extends the CheerioWebBaseLoader class. It represents a
+ * loader for loading web pages from the Hacker News website.
+ */
 export declare class HNLoader extends CheerioWebBaseLoader {
     webPath: string;
     constructor(webPath: string);
+    /**
+     * An asynchronous method that loads the web page. If the webPath includes
+     * "item", it calls the loadComments() method to load the comments from
+     * the web page. Otherwise, it calls the loadResults() method to load the
+     * results from the web page.
+     * @returns A Promise that resolves to an array of Document instances.
+     */
     load(): Promise<Document[]>;
+    /**
+     * A private method that loads the comments from the web page. It selects
+     * the elements with the class "athing comtr" using the $ function
+     * provided by Cheerio. It also extracts the title of the web page from
+     * the element with the id "pagespace". It creates Document instances for
+     * each comment, with the comment text as the page content and the source
+     * and title as metadata.
+     * @param $ A CheerioAPI instance.
+     * @returns An array of Document instances.
+     */
     private loadComments;
+    /**
+     * A private method that loads the results from the web page. It selects
+     * the elements with the class "athing" using the $ function provided by
+     * Cheerio. It extracts the ranking, link, title, and other metadata from
+     * each result item. It creates Document instances for each result item,
+     * with the title as the page content and the source, title, link, and
+     * ranking as metadata.
+     * @param $ A CheerioAPI instance.
+     * @returns An array of Document instances.
+     */
     private loadResults;
 }

package/dist/document_loaders/web/hn.js

@@ -1,5 +1,9 @@
 import { Document } from "../../document.js";
 import { CheerioWebBaseLoader } from "./cheerio.js";
+/**
+ * A class that extends the CheerioWebBaseLoader class. It represents a
+ * loader for loading web pages from the Hacker News website.
+ */
 export class HNLoader extends CheerioWebBaseLoader {
     constructor(webPath) {
         super(webPath);
@@ -10,6 +14,13 @@ export class HNLoader extends CheerioWebBaseLoader {
             value: webPath
         });
     }
+    /**
+     * An asynchronous method that loads the web page. If the webPath includes
+     * "item", it calls the loadComments() method to load the comments from
+     * the web page. Otherwise, it calls the loadResults() method to load the
+     * results from the web page.
+     * @returns A Promise that resolves to an array of Document instances.
+     */
     async load() {
         const $ = await this.scrape();
         if (this.webPath.includes("item")) {
@@ -17,6 +28,16 @@ export class HNLoader extends CheerioWebBaseLoader {
         }
         return this.loadResults($);
     }
+    /**
+     * A private method that loads the comments from the web page. It selects
+     * the elements with the class "athing comtr" using the $ function
+     * provided by Cheerio. It also extracts the title of the web page from
+     * the element with the id "pagespace". It creates Document instances for
+     * each comment, with the comment text as the page content and the source
+     * and title as metadata.
+     * @param $ A CheerioAPI instance.
+     * @returns An array of Document instances.
+     */
     loadComments($) {
         const comments = $("tr[class='athing comtr']");
         const title = $("tr[id='pagespace']").attr("title");
@@ -28,6 +49,16 @@ export class HNLoader extends CheerioWebBaseLoader {
         });
         return documents;
     }
+    /**
+     * A private method that loads the results from the web page. It selects
+     * the elements with the class "athing" using the $ function provided by
+     * Cheerio. It extracts the ranking, link, title, and other metadata from
+     * each result item. It creates Document instances for each result item,
+     * with the title as the page content and the source, title, link, and
+     * ranking as metadata.
+     * @param $ A CheerioAPI instance.
+     * @returns An array of Document instances.
+     */
     loadResults($) {
         const items = $("tr[class='athing']");
         const documents = [];

package/dist/document_loaders/web/imsdb.cjs

@@ -3,6 +3,11 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.IMSDBLoader = void 0;
 const document_js_1 = require("../../document.cjs");
 const cheerio_js_1 = require("./cheerio.cjs");
+/**
+ * A class that extends the CheerioWebBaseLoader class. It represents a
+ * loader for loading web pages from the IMSDB (Internet Movie Script
+ * Database) website.
+ */
 class IMSDBLoader extends cheerio_js_1.CheerioWebBaseLoader {
     constructor(webPath) {
         super(webPath);
@@ -13,6 +18,15 @@ class IMSDBLoader extends cheerio_js_1.CheerioWebBaseLoader {
             value: webPath
         });
     }
+    /**
+     * An asynchronous method that loads the web page using the scrape()
+     * method inherited from the base class. It selects the element with the
+     * class 'scrtext' using the $ function provided by Cheerio and extracts
+     * the text content. It creates a Document instance with the text content
+     * as the page content and the source as metadata. It returns an array
+     * containing the Document instance.
+     * @returns An array containing a Document instance.
+     */
     async load() {
         const $ = await this.scrape();
         const text = $("td[class='scrtext']").text().trim();

package/dist/document_loaders/web/imsdb.d.ts

@@ -1,7 +1,21 @@
 import { Document } from "../../document.js";
 import { CheerioWebBaseLoader } from "./cheerio.js";
+/**
+ * A class that extends the CheerioWebBaseLoader class. It represents a
+ * loader for loading web pages from the IMSDB (Internet Movie Script
+ * Database) website.
+ */
 export declare class IMSDBLoader extends CheerioWebBaseLoader {
     webPath: string;
     constructor(webPath: string);
+    /**
+     * An asynchronous method that loads the web page using the scrape()
+     * method inherited from the base class. It selects the element with the
+     * class 'scrtext' using the $ function provided by Cheerio and extracts
+     * the text content. It creates a Document instance with the text content
+     * as the page content and the source as metadata. It returns an array
+     * containing the Document instance.
+     * @returns An array containing a Document instance.
+     */
     load(): Promise<Document[]>;
 }

package/dist/document_loaders/web/imsdb.js

@@ -1,5 +1,10 @@
 import { Document } from "../../document.js";
 import { CheerioWebBaseLoader } from "./cheerio.js";
+/**
+ * A class that extends the CheerioWebBaseLoader class. It represents a
+ * loader for loading web pages from the IMSDB (Internet Movie Script
+ * Database) website.
+ */
 export class IMSDBLoader extends CheerioWebBaseLoader {
     constructor(webPath) {
         super(webPath);
@@ -10,6 +15,15 @@ export class IMSDBLoader extends CheerioWebBaseLoader {
             value: webPath
         });
     }
+    /**
+     * An asynchronous method that loads the web page using the scrape()
+     * method inherited from the base class. It selects the element with the
+     * class 'scrtext' using the $ function provided by Cheerio and extracts
+     * the text content. It creates a Document instance with the text content
+     * as the page content and the source as metadata. It returns an array
+     * containing the Document instance.
+     * @returns An array containing a Document instance.
+     */
     async load() {
         const $ = await this.scrape();
         const text = $("td[class='scrtext']").text().trim();