langchain 0.2.18 → 0.3.0-rc.0

package/dist/document_loaders/web/firecrawl.cjs

@@ -1,95 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.FireCrawlLoader = void 0;
-const firecrawl_js_1 = __importDefault(require("@mendable/firecrawl-js"));
-const documents_1 = require("@langchain/core/documents");
-const env_1 = require("@langchain/core/utils/env");
-const base_js_1 = require("../base.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/firecrawl",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/firecrawl" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for loading data from
- * Firecrawl (firecrawl.dev). It extends the BaseDocumentLoader class.
- * @example
- * ```typescript
- * const loader = new FireCrawlLoader({
- *   url: "{url}",
- *   apiKey: "{apiKey}",
- *   mode: "crawl"
- * });
- * const docs = await loader.load();
- * ```
- */
-class FireCrawlLoader extends base_js_1.BaseDocumentLoader {
-    constructor(loaderParams) {
-        super();
-        Object.defineProperty(this, "apiKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "url", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "mode", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "params", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        const { apiKey = (0, env_1.getEnvironmentVariable)("FIRECRAWL_API_KEY"), url, mode = "crawl", params, } = loaderParams;
-        if (!apiKey) {
-            throw new Error("Firecrawl API key not set. You can set it as FIRECRAWL_API_KEY in your .env file, or pass it to Firecrawl.");
-        }
-        this.apiKey = apiKey;
-        this.url = url;
-        this.mode = mode;
-        this.params = params;
-    }
-    /**
-     * Loads the data from the Firecrawl.
-     * @returns An array of Documents representing the retrieved data.
-     * @throws An error if the data could not be loaded.
-     */
-    async load() {
-        const app = new firecrawl_js_1.default({ apiKey: this.apiKey });
-        let firecrawlDocs;
-        if (this.mode === "scrape") {
-            const response = await app.scrapeUrl(this.url, this.params);
-            if (!response.success) {
-                throw new Error(`Firecrawl: Failed to scrape URL. Error: ${response.error}`);
-            }
-            firecrawlDocs = [response.data];
-        }
-        else if (this.mode === "crawl") {
-            const response = await app.crawlUrl(this.url, this.params, true);
-            firecrawlDocs = response;
-        }
-        else {
-            throw new Error(`Unrecognized mode '${this.mode}'. Expected one of 'crawl', 'scrape'.`);
-        }
-        return firecrawlDocs.map((doc) => new documents_1.Document({
-            pageContent: doc.markdown || "",
-            metadata: doc.metadata || {},
-        }));
-    }
-}
-exports.FireCrawlLoader = FireCrawlLoader;

package/dist/document_loaders/web/firecrawl.d.ts

@@ -1,50 +0,0 @@
-import { type DocumentInterface } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
-/**
- * Interface representing the parameters for the Firecrawl loader. It
- * includes properties such as the URL to scrape or crawl and the API key.
- */
-interface FirecrawlLoaderParameters {
-    /**
-     * URL to scrape or crawl
-     */
-    url: string;
-    /**
-     * API key for Firecrawl. If not provided, the default value is the value of the FIRECRAWL_API_KEY environment variable.
-     */
-    apiKey?: string;
-    /**
-     * Mode of operation. Can be either "crawl" or "scrape". If not provided, the default value is "crawl".
-     */
-    mode?: "crawl" | "scrape";
-    params?: Record<string, unknown>;
-}
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/firecrawl" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for loading data from
- * Firecrawl (firecrawl.dev). It extends the BaseDocumentLoader class.
- * @example
- * ```typescript
- * const loader = new FireCrawlLoader({
- *   url: "{url}",
- *   apiKey: "{apiKey}",
- *   mode: "crawl"
- * });
- * const docs = await loader.load();
- * ```
- */
-export declare class FireCrawlLoader extends BaseDocumentLoader {
-    private apiKey;
-    private url;
-    private mode;
-    private params?;
-    constructor(loaderParams: FirecrawlLoaderParameters);
-    /**
-     * Loads the data from the Firecrawl.
-     * @returns An array of Documents representing the retrieved data.
-     * @throws An error if the data could not be loaded.
-     */
-    load(): Promise<DocumentInterface[]>;
-}
-export {};

package/dist/document_loaders/web/firecrawl.js

@@ -1,88 +0,0 @@
-import FirecrawlApp from "@mendable/firecrawl-js";
-import { Document } from "@langchain/core/documents";
-import { getEnvironmentVariable } from "@langchain/core/utils/env";
-import { BaseDocumentLoader } from "../base.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/firecrawl",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/firecrawl" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for loading data from
- * Firecrawl (firecrawl.dev). It extends the BaseDocumentLoader class.
- * @example
- * ```typescript
- * const loader = new FireCrawlLoader({
- *   url: "{url}",
- *   apiKey: "{apiKey}",
- *   mode: "crawl"
- * });
- * const docs = await loader.load();
- * ```
- */
-export class FireCrawlLoader extends BaseDocumentLoader {
-    constructor(loaderParams) {
-        super();
-        Object.defineProperty(this, "apiKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "url", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "mode", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "params", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        const { apiKey = getEnvironmentVariable("FIRECRAWL_API_KEY"), url, mode = "crawl", params, } = loaderParams;
-        if (!apiKey) {
-            throw new Error("Firecrawl API key not set. You can set it as FIRECRAWL_API_KEY in your .env file, or pass it to Firecrawl.");
-        }
-        this.apiKey = apiKey;
-        this.url = url;
-        this.mode = mode;
-        this.params = params;
-    }
-    /**
-     * Loads the data from the Firecrawl.
-     * @returns An array of Documents representing the retrieved data.
-     * @throws An error if the data could not be loaded.
-     */
-    async load() {
-        const app = new FirecrawlApp({ apiKey: this.apiKey });
-        let firecrawlDocs;
-        if (this.mode === "scrape") {
-            const response = await app.scrapeUrl(this.url, this.params);
-            if (!response.success) {
-                throw new Error(`Firecrawl: Failed to scrape URL. Error: ${response.error}`);
-            }
-            firecrawlDocs = [response.data];
-        }
-        else if (this.mode === "crawl") {
-            const response = await app.crawlUrl(this.url, this.params, true);
-            firecrawlDocs = response;
-        }
-        else {
-            throw new Error(`Unrecognized mode '${this.mode}'. Expected one of 'crawl', 'scrape'.`);
-        }
-        return firecrawlDocs.map((doc) => new Document({
-            pageContent: doc.markdown || "",
-            metadata: doc.metadata || {},
-        }));
-    }
-}

package/dist/document_loaders/web/gitbook.cjs

@@ -1,110 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.GitbookLoader = void 0;
-const documents_1 = require("@langchain/core/documents");
-const cheerio_js_1 = require("./cheerio.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/gitbook",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/gitbook" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader specifically designed for loading
- * documents from Gitbook. It extends the CheerioWebBaseLoader.
- */
-class GitbookLoader extends cheerio_js_1.CheerioWebBaseLoader {
-    constructor(webPath, params = {}) {
-        const path = params.shouldLoadAllPaths === true ? `${webPath}/sitemap.xml` : webPath;
-        super(path);
-        Object.defineProperty(this, "webPath", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: webPath
-        });
-        Object.defineProperty(this, "shouldLoadAllPaths", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        Object.defineProperty(this, "baseUrl", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.baseUrl = webPath;
-        this.webPath = path;
-        this.shouldLoadAllPaths =
-            params.shouldLoadAllPaths ?? this.shouldLoadAllPaths;
-    }
-    /**
-     * Method that scrapes the web document using Cheerio and loads the
-     * content based on the value of shouldLoadAllPaths. If shouldLoadAllPaths
-     * is true, it calls the loadAllPaths() method to load all paths.
-     * Otherwise, it calls the loadPath() method to load a single path.
-     * @returns Promise resolving to an array of Document instances.
-     */
-    async load() {
-        const $ = await this.scrape();
-        if (this.shouldLoadAllPaths === true) {
-            return this.loadAllPaths($);
-        }
-        return this.loadPath($);
-    }
-    /**
-     * Private method that loads the content of a single path from the Gitbook
-     * web document. It extracts the page content by selecting all elements
-     * inside the "main" element, filters out empty text nodes, and joins the
-     * remaining text nodes with line breaks. It extracts the title by
-     * selecting the first "h1" element inside the "main" element. It creates
-     * a Document instance with the extracted page content and metadata
-     * containing the source URL and title.
-     * @param $ CheerioAPI instance representing the loaded web document.
-     * @param url Optional string representing the URL of the web document.
-     * @returns Array of Document instances.
-     */
-    loadPath($, url) {
-        const pageContent = $("main *")
-            .contents()
-            .toArray()
-            .map((element) => element.type === "text" ? $(element).text().trim() : null)
-            .filter((text) => text)
-            .join("\n");
-        const title = $("main h1").first().text().trim();
-        return [
-            new documents_1.Document({
-                pageContent,
-                metadata: { source: url ?? this.webPath, title },
-            }),
-        ];
-    }
-    /**
-     * Private method that loads the content of all paths from the Gitbook web
-     * document. It extracts the URLs of all paths from the "loc" elements in
-     * the sitemap.xml. It iterates over each URL, scrapes the web document
-     * using the _scrape() method, and calls the loadPath() method to load the
-     * content of each path. It collects all the loaded documents and returns
-     * them as an array.
-     * @param $ CheerioAPI instance representing the loaded web document.
-     * @returns Promise resolving to an array of Document instances.
-     */
-    async loadAllPaths($) {
-        const urls = $("loc")
-            .toArray()
-            .map((element) => $(element).text());
-        const documents = [];
-        for (const url of urls) {
-            const buildUrl = url.includes(this.baseUrl) ? url : this.baseUrl + url;
-            console.log(`Fetching text from ${buildUrl}`);
-            const html = await GitbookLoader._scrape(buildUrl, this.caller, this.timeout);
-            documents.push(...this.loadPath(html, buildUrl));
-        }
-        console.log(`Fetched ${documents.length} documents.`);
-        return documents;
-    }
-}
-exports.GitbookLoader = GitbookLoader;

package/dist/document_loaders/web/gitbook.d.ts

@@ -1,55 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { CheerioWebBaseLoader } from "./cheerio.js";
-/**
- * Interface representing the parameters for configuring the
- * GitbookLoader. It has an optional property shouldLoadAllPaths, which
- * indicates whether all paths should be loaded.
- */
-interface GitbookLoaderParams {
-    shouldLoadAllPaths?: boolean;
-}
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/gitbook" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader specifically designed for loading
- * documents from Gitbook. It extends the CheerioWebBaseLoader.
- */
-export declare class GitbookLoader extends CheerioWebBaseLoader {
-    webPath: string;
-    shouldLoadAllPaths: boolean;
-    private readonly baseUrl;
-    constructor(webPath: string, params?: GitbookLoaderParams);
-    /**
-     * Method that scrapes the web document using Cheerio and loads the
-     * content based on the value of shouldLoadAllPaths. If shouldLoadAllPaths
-     * is true, it calls the loadAllPaths() method to load all paths.
-     * Otherwise, it calls the loadPath() method to load a single path.
-     * @returns Promise resolving to an array of Document instances.
-     */
-    load(): Promise<Document[]>;
-    /**
-     * Private method that loads the content of a single path from the Gitbook
-     * web document. It extracts the page content by selecting all elements
-     * inside the "main" element, filters out empty text nodes, and joins the
-     * remaining text nodes with line breaks. It extracts the title by
-     * selecting the first "h1" element inside the "main" element. It creates
-     * a Document instance with the extracted page content and metadata
-     * containing the source URL and title.
-     * @param $ CheerioAPI instance representing the loaded web document.
-     * @param url Optional string representing the URL of the web document.
-     * @returns Array of Document instances.
-     */
-    private loadPath;
-    /**
-     * Private method that loads the content of all paths from the Gitbook web
-     * document. It extracts the URLs of all paths from the "loc" elements in
-     * the sitemap.xml. It iterates over each URL, scrapes the web document
-     * using the _scrape() method, and calls the loadPath() method to load the
-     * content of each path. It collects all the loaded documents and returns
-     * them as an array.
-     * @param $ CheerioAPI instance representing the loaded web document.
-     * @returns Promise resolving to an array of Document instances.
-     */
-    private loadAllPaths;
-}
-export {};

package/dist/document_loaders/web/gitbook.js

@@ -1,106 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { CheerioWebBaseLoader } from "./cheerio.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/gitbook",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/gitbook" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader specifically designed for loading
- * documents from Gitbook. It extends the CheerioWebBaseLoader.
- */
-export class GitbookLoader extends CheerioWebBaseLoader {
-    constructor(webPath, params = {}) {
-        const path = params.shouldLoadAllPaths === true ? `${webPath}/sitemap.xml` : webPath;
-        super(path);
-        Object.defineProperty(this, "webPath", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: webPath
-        });
-        Object.defineProperty(this, "shouldLoadAllPaths", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: false
-        });
-        Object.defineProperty(this, "baseUrl", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.baseUrl = webPath;
-        this.webPath = path;
-        this.shouldLoadAllPaths =
-            params.shouldLoadAllPaths ?? this.shouldLoadAllPaths;
-    }
-    /**
-     * Method that scrapes the web document using Cheerio and loads the
-     * content based on the value of shouldLoadAllPaths. If shouldLoadAllPaths
-     * is true, it calls the loadAllPaths() method to load all paths.
-     * Otherwise, it calls the loadPath() method to load a single path.
-     * @returns Promise resolving to an array of Document instances.
-     */
-    async load() {
-        const $ = await this.scrape();
-        if (this.shouldLoadAllPaths === true) {
-            return this.loadAllPaths($);
-        }
-        return this.loadPath($);
-    }
-    /**
-     * Private method that loads the content of a single path from the Gitbook
-     * web document. It extracts the page content by selecting all elements
-     * inside the "main" element, filters out empty text nodes, and joins the
-     * remaining text nodes with line breaks. It extracts the title by
-     * selecting the first "h1" element inside the "main" element. It creates
-     * a Document instance with the extracted page content and metadata
-     * containing the source URL and title.
-     * @param $ CheerioAPI instance representing the loaded web document.
-     * @param url Optional string representing the URL of the web document.
-     * @returns Array of Document instances.
-     */
-    loadPath($, url) {
-        const pageContent = $("main *")
-            .contents()
-            .toArray()
-            .map((element) => element.type === "text" ? $(element).text().trim() : null)
-            .filter((text) => text)
-            .join("\n");
-        const title = $("main h1").first().text().trim();
-        return [
-            new Document({
-                pageContent,
-                metadata: { source: url ?? this.webPath, title },
-            }),
-        ];
-    }
-    /**
-     * Private method that loads the content of all paths from the Gitbook web
-     * document. It extracts the URLs of all paths from the "loc" elements in
-     * the sitemap.xml. It iterates over each URL, scrapes the web document
-     * using the _scrape() method, and calls the loadPath() method to load the
-     * content of each path. It collects all the loaded documents and returns
-     * them as an array.
-     * @param $ CheerioAPI instance representing the loaded web document.
-     * @returns Promise resolving to an array of Document instances.
-     */
-    async loadAllPaths($) {
-        const urls = $("loc")
-            .toArray()
-            .map((element) => $(element).text());
-        const documents = [];
-        for (const url of urls) {
-            const buildUrl = url.includes(this.baseUrl) ? url : this.baseUrl + url;
-            console.log(`Fetching text from ${buildUrl}`);
-            const html = await GitbookLoader._scrape(buildUrl, this.caller, this.timeout);
-            documents.push(...this.loadPath(html, buildUrl));
-        }
-        console.log(`Fetched ${documents.length} documents.`);
-        return documents;
-    }
-}