langchain 0.2.18 → 0.3.0-rc.0

package/dist/document_loaders/web/puppeteer.js

@@ -1,135 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/puppeteer",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/puppeteer" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class that extends the BaseDocumentLoader class and implements the
- * DocumentLoader interface. It represents a document loader for scraping
- * web pages using Puppeteer.
- * @example
- * ```typescript
- * const loader = new PuppeteerWebBaseLoader("https:exampleurl.com", {
- *   launchOptions: {
- *     headless: true,
- *   },
- *   gotoOptions: {
- *     waitUntil: "domcontentloaded",
- *   },
- * });
- * const screenshot = await loader.screenshot();
- * ```
- */
-export class PuppeteerWebBaseLoader extends BaseDocumentLoader {
-    constructor(webPath, options) {
-        super();
-        Object.defineProperty(this, "webPath", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: webPath
-        });
-        Object.defineProperty(this, "options", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.options = options ?? undefined;
-    }
-    static async _scrape(url, options) {
-        const { launch } = await PuppeteerWebBaseLoader.imports();
-        const browser = await launch({
-            headless: true,
-            defaultViewport: null,
-            ignoreDefaultArgs: ["--disable-extensions"],
-            ...options?.launchOptions,
-        });
-        const page = await browser.newPage();
-        await page.goto(url, {
-            timeout: 180000,
-            waitUntil: "domcontentloaded",
-            ...options?.gotoOptions,
-        });
-        const bodyHTML = options?.evaluate
-            ? await options?.evaluate(page, browser)
-            : await page.evaluate(() => document.body.innerHTML);
-        await browser.close();
-        return bodyHTML;
-    }
-    /**
-     * Method that calls the _scrape method to perform the scraping of the web
-     * page specified by the webPath property.
-     * @returns Promise that resolves to the scraped HTML content of the web page.
-     */
-    async scrape() {
-        return PuppeteerWebBaseLoader._scrape(this.webPath, this.options);
-    }
-    /**
-     * Method that calls the scrape method and returns the scraped HTML
-     * content as a Document object.
-     * @returns Promise that resolves to an array of Document objects.
-     */
-    async load() {
-        const text = await this.scrape();
-        const metadata = { source: this.webPath };
-        return [new Document({ pageContent: text, metadata })];
-    }
-    /**
-     * Static class method used to screenshot a web page and return
-     * it as a {@link Document} object where  the pageContent property
-     * is the screenshot encoded in base64.
-     *
-     * @param {string} url
-     * @param {PuppeteerWebBaseLoaderOptions} options
-     * @returns {Document} A document object containing the screenshot of the page encoded in base64.
-     */
-    static async _screenshot(url, options) {
-        const { launch } = await PuppeteerWebBaseLoader.imports();
-        const browser = await launch({
-            headless: true,
-            defaultViewport: null,
-            ignoreDefaultArgs: ["--disable-extensions"],
-            ...options?.launchOptions,
-        });
-        const page = await browser.newPage();
-        await page.goto(url, {
-            timeout: 180000,
-            waitUntil: "domcontentloaded",
-            ...options?.gotoOptions,
-        });
-        const screenshot = await page.screenshot();
-        const base64 = screenshot.toString("base64");
-        const metadata = { source: url };
-        return new Document({ pageContent: base64, metadata });
-    }
-    /**
-     * Screenshot a web page and return it as a {@link Document} object where
-     * the pageContent property is the screenshot encoded in base64.
-     *
-     * @returns {Promise<Document>} A document object containing the screenshot of the page encoded in base64.
-     */
-    async screenshot() {
-        return PuppeteerWebBaseLoader._screenshot(this.webPath, this.options);
-    }
-    /**
-     * Static method that imports the necessary Puppeteer modules. It returns
-     * a Promise that resolves to an object containing the imported modules.
-     * @returns Promise that resolves to an object containing the imported Puppeteer modules.
-     */
-    static async imports() {
-        try {
-            // eslint-disable-next-line import/no-extraneous-dependencies
-            const { launch } = await import("puppeteer");
-            return { launch };
-        }
-        catch (e) {
-            console.error(e);
-            throw new Error("Please install puppeteer as a dependency with, e.g. `yarn add puppeteer`");
-        }
-    }
-}

package/dist/document_loaders/web/recursive_url.cjs

@@ -1,198 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RecursiveUrlLoader = void 0;
-const jsdom_1 = require("jsdom");
-const async_caller_1 = require("@langchain/core/utils/async_caller");
-const base_js_1 = require("../base.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/recursive_url",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/recursive_url" instead. This entrypoint will be removed in 0.3.0.
- */
-class RecursiveUrlLoader extends base_js_1.BaseDocumentLoader {
-    constructor(url, options) {
-        super();
-        Object.defineProperty(this, "caller", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "url", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "excludeDirs", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "extractor", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "maxDepth", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "timeout", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "preventOutside", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.caller = new async_caller_1.AsyncCaller({
-            maxConcurrency: 64,
-            maxRetries: 0,
-            ...options.callerOptions,
-        });
-        this.url = url;
-        this.excludeDirs = options.excludeDirs ?? [];
-        this.extractor = options.extractor ?? ((s) => s);
-        this.maxDepth = options.maxDepth ?? 2;
-        this.timeout = options.timeout ?? 10000;
-        this.preventOutside = options.preventOutside ?? true;
-    }
-    async fetchWithTimeout(resource, options) {
-        const { timeout, ...rest } = options;
-        return this.caller.call(() => fetch(resource, { ...rest, signal: AbortSignal.timeout(timeout) }));
-    }
-    getChildLinks(html, baseUrl) {
-        const allLinks = Array.from(new jsdom_1.JSDOM(html).window.document.querySelectorAll("a")).map((a) => a.href);
-        const absolutePaths = [];
-        // eslint-disable-next-line no-script-url
-        const invalidPrefixes = ["javascript:", "mailto:", "#"];
-        const invalidSuffixes = [
-            ".css",
-            ".js",
-            ".ico",
-            ".png",
-            ".jpg",
-            ".jpeg",
-            ".gif",
-            ".svg",
-        ];
-        for (const link of allLinks) {
-            if (invalidPrefixes.some((prefix) => link.startsWith(prefix)) ||
-                invalidSuffixes.some((suffix) => link.endsWith(suffix)))
-                continue;
-            let standardizedLink;
-            if (link.startsWith("http")) {
-                standardizedLink = link;
-            }
-            else if (link.startsWith("//")) {
-                const base = new URL(baseUrl);
-                standardizedLink = base.protocol + link;
-            }
-            else {
-                standardizedLink = new URL(link, baseUrl).href;
-            }
-            if (this.excludeDirs.some((exDir) => standardizedLink.startsWith(exDir)))
-                continue;
-            if (link.startsWith("http")) {
-                const isAllowed = !this.preventOutside || link.startsWith(baseUrl);
-                if (isAllowed)
-                    absolutePaths.push(link);
-            }
-            else if (link.startsWith("//")) {
-                const base = new URL(baseUrl);
-                absolutePaths.push(base.protocol + link);
-            }
-            else {
-                const newLink = new URL(link, baseUrl).href;
-                absolutePaths.push(newLink);
-            }
-        }
-        return Array.from(new Set(absolutePaths));
-    }
-    extractMetadata(rawHtml, url) {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const metadata = { source: url };
-        const { document } = new jsdom_1.JSDOM(rawHtml).window;
-        const title = document.getElementsByTagName("title")[0];
-        if (title) {
-            metadata.title = title.textContent;
-        }
-        const description = document.querySelector("meta[name=description]");
-        if (description) {
-            metadata.description = description.getAttribute("content");
-        }
-        const html = document.getElementsByTagName("html")[0];
-        if (html) {
-            metadata.language = html.getAttribute("lang");
-        }
-        return metadata;
-    }
-    async getUrlAsDoc(url) {
-        let res;
-        try {
-            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
-            res = await res.text();
-        }
-        catch (e) {
-            return null;
-        }
-        return {
-            pageContent: this.extractor(res),
-            metadata: this.extractMetadata(res, url),
-        };
-    }
-    async getChildUrlsRecursive(inputUrl, visited = new Set(), depth = 0) {
-        if (depth >= this.maxDepth)
-            return [];
-        let url = inputUrl;
-        if (!inputUrl.endsWith("/"))
-            url += "/";
-        const isExcluded = this.excludeDirs.some((exDir) => url.startsWith(exDir));
-        if (isExcluded)
-            return [];
-        let res;
-        try {
-            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
-            res = await res.text();
-        }
-        catch (e) {
-            return [];
-        }
-        const childUrls = this.getChildLinks(res, url);
-        const results = await Promise.all(childUrls.map((childUrl) => (async () => {
-            if (visited.has(childUrl))
-                return null;
-            visited.add(childUrl);
-            const childDoc = await this.getUrlAsDoc(childUrl);
-            if (!childDoc)
-                return null;
-            if (childUrl.endsWith("/")) {
-                const childUrlResponses = await this.getChildUrlsRecursive(childUrl, visited, depth + 1);
-                return [childDoc, ...childUrlResponses];
-            }
-            return [childDoc];
-        })()));
-        return results.flat().filter((docs) => docs !== null);
-    }
-    async load() {
-        const rootDoc = await this.getUrlAsDoc(this.url);
-        if (!rootDoc)
-            return [];
-        const docs = [rootDoc];
-        docs.push(...(await this.getChildUrlsRecursive(this.url, new Set([this.url]))));
-        return docs;
-    }
-}
-exports.RecursiveUrlLoader = RecursiveUrlLoader;

package/dist/document_loaders/web/recursive_url.d.ts

@@ -1,33 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { AsyncCaller } from "@langchain/core/utils/async_caller";
-import { BaseDocumentLoader, DocumentLoader } from "../base.js";
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/recursive_url" instead. This entrypoint will be removed in 0.3.0.
- */
-export interface RecursiveUrlLoaderOptions {
-    excludeDirs?: string[];
-    extractor?: (text: string) => string;
-    maxDepth?: number;
-    timeout?: number;
-    preventOutside?: boolean;
-    callerOptions?: ConstructorParameters<typeof AsyncCaller>[0];
-}
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/recursive_url" instead. This entrypoint will be removed in 0.3.0.
- */
-export declare class RecursiveUrlLoader extends BaseDocumentLoader implements DocumentLoader {
-    private caller;
-    private url;
-    private excludeDirs;
-    private extractor;
-    private maxDepth;
-    private timeout;
-    private preventOutside;
-    constructor(url: string, options: RecursiveUrlLoaderOptions);
-    private fetchWithTimeout;
-    private getChildLinks;
-    private extractMetadata;
-    private getUrlAsDoc;
-    private getChildUrlsRecursive;
-    load(): Promise<Document[]>;
-}

package/dist/document_loaders/web/recursive_url.js

@@ -1,194 +0,0 @@
-import { JSDOM } from "jsdom";
-import { AsyncCaller } from "@langchain/core/utils/async_caller";
-import { BaseDocumentLoader } from "../base.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/recursive_url",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/recursive_url" instead. This entrypoint will be removed in 0.3.0.
- */
-export class RecursiveUrlLoader extends BaseDocumentLoader {
-    constructor(url, options) {
-        super();
-        Object.defineProperty(this, "caller", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "url", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "excludeDirs", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "extractor", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "maxDepth", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "timeout", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "preventOutside", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.caller = new AsyncCaller({
-            maxConcurrency: 64,
-            maxRetries: 0,
-            ...options.callerOptions,
-        });
-        this.url = url;
-        this.excludeDirs = options.excludeDirs ?? [];
-        this.extractor = options.extractor ?? ((s) => s);
-        this.maxDepth = options.maxDepth ?? 2;
-        this.timeout = options.timeout ?? 10000;
-        this.preventOutside = options.preventOutside ?? true;
-    }
-    async fetchWithTimeout(resource, options) {
-        const { timeout, ...rest } = options;
-        return this.caller.call(() => fetch(resource, { ...rest, signal: AbortSignal.timeout(timeout) }));
-    }
-    getChildLinks(html, baseUrl) {
-        const allLinks = Array.from(new JSDOM(html).window.document.querySelectorAll("a")).map((a) => a.href);
-        const absolutePaths = [];
-        // eslint-disable-next-line no-script-url
-        const invalidPrefixes = ["javascript:", "mailto:", "#"];
-        const invalidSuffixes = [
-            ".css",
-            ".js",
-            ".ico",
-            ".png",
-            ".jpg",
-            ".jpeg",
-            ".gif",
-            ".svg",
-        ];
-        for (const link of allLinks) {
-            if (invalidPrefixes.some((prefix) => link.startsWith(prefix)) ||
-                invalidSuffixes.some((suffix) => link.endsWith(suffix)))
-                continue;
-            let standardizedLink;
-            if (link.startsWith("http")) {
-                standardizedLink = link;
-            }
-            else if (link.startsWith("//")) {
-                const base = new URL(baseUrl);
-                standardizedLink = base.protocol + link;
-            }
-            else {
-                standardizedLink = new URL(link, baseUrl).href;
-            }
-            if (this.excludeDirs.some((exDir) => standardizedLink.startsWith(exDir)))
-                continue;
-            if (link.startsWith("http")) {
-                const isAllowed = !this.preventOutside || link.startsWith(baseUrl);
-                if (isAllowed)
-                    absolutePaths.push(link);
-            }
-            else if (link.startsWith("//")) {
-                const base = new URL(baseUrl);
-                absolutePaths.push(base.protocol + link);
-            }
-            else {
-                const newLink = new URL(link, baseUrl).href;
-                absolutePaths.push(newLink);
-            }
-        }
-        return Array.from(new Set(absolutePaths));
-    }
-    extractMetadata(rawHtml, url) {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const metadata = { source: url };
-        const { document } = new JSDOM(rawHtml).window;
-        const title = document.getElementsByTagName("title")[0];
-        if (title) {
-            metadata.title = title.textContent;
-        }
-        const description = document.querySelector("meta[name=description]");
-        if (description) {
-            metadata.description = description.getAttribute("content");
-        }
-        const html = document.getElementsByTagName("html")[0];
-        if (html) {
-            metadata.language = html.getAttribute("lang");
-        }
-        return metadata;
-    }
-    async getUrlAsDoc(url) {
-        let res;
-        try {
-            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
-            res = await res.text();
-        }
-        catch (e) {
-            return null;
-        }
-        return {
-            pageContent: this.extractor(res),
-            metadata: this.extractMetadata(res, url),
-        };
-    }
-    async getChildUrlsRecursive(inputUrl, visited = new Set(), depth = 0) {
-        if (depth >= this.maxDepth)
-            return [];
-        let url = inputUrl;
-        if (!inputUrl.endsWith("/"))
-            url += "/";
-        const isExcluded = this.excludeDirs.some((exDir) => url.startsWith(exDir));
-        if (isExcluded)
-            return [];
-        let res;
-        try {
-            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
-            res = await res.text();
-        }
-        catch (e) {
-            return [];
-        }
-        const childUrls = this.getChildLinks(res, url);
-        const results = await Promise.all(childUrls.map((childUrl) => (async () => {
-            if (visited.has(childUrl))
-                return null;
-            visited.add(childUrl);
-            const childDoc = await this.getUrlAsDoc(childUrl);
-            if (!childDoc)
-                return null;
-            if (childUrl.endsWith("/")) {
-                const childUrlResponses = await this.getChildUrlsRecursive(childUrl, visited, depth + 1);
-                return [childDoc, ...childUrlResponses];
-            }
-            return [childDoc];
-        })()));
-        return results.flat().filter((docs) => docs !== null);
-    }
-    async load() {
-        const rootDoc = await this.getUrlAsDoc(this.url);
-        if (!rootDoc)
-            return [];
-        const docs = [rootDoc];
-        docs.push(...(await this.getChildUrlsRecursive(this.url, new Set([this.url]))));
-        return docs;
-    }
-}