langchain 0.2.17 → 0.3.0-rc.0

package/dist/document_loaders/web/pdf.js

@@ -1,136 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
-import { formatDocumentsAsString } from "../../util/document.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/pdf",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/pdf" instead. This entrypoint will be removed in 0.3.0.
- *
- * A document loader for loading data from PDFs.
- * @example
- * ```typescript
- * const loader = new WebPDFLoader(new Blob());
- * const docs = await loader.load();
- * console.log({ docs });
- * ```
- */
-export class WebPDFLoader extends BaseDocumentLoader {
-    constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "", } = {}) {
-        super();
-        Object.defineProperty(this, "blob", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "splitPages", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: true
-        });
-        Object.defineProperty(this, "pdfjs", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "parsedItemSeparator", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.blob = blob;
-        this.splitPages = splitPages ?? this.splitPages;
-        this.pdfjs = pdfjs;
-        this.parsedItemSeparator = parsedItemSeparator;
-    }
-    /**
-     * Loads the contents of the PDF as documents.
-     * @returns An array of Documents representing the retrieved data.
-     */
-    async load() {
-        const { getDocument, version } = await this.pdfjs();
-        const parsedPdf = await getDocument({
-            data: new Uint8Array(await this.blob.arrayBuffer()),
-            useWorkerFetch: false,
-            isEvalSupported: false,
-            useSystemFonts: true,
-        }).promise;
-        const meta = await parsedPdf.getMetadata().catch(() => null);
-        const documents = [];
-        for (let i = 1; i <= parsedPdf.numPages; i += 1) {
-            const page = await parsedPdf.getPage(i);
-            const content = await page.getTextContent();
-            if (content.items.length === 0) {
-                continue;
-            }
-            // Eliminate excessive newlines
-            // Source: https://github.com/albertcui/pdf-parse/blob/7086fc1cc9058545cdf41dd0646d6ae5832c7107/lib/pdf-parse.js#L16
-            let lastY;
-            const textItems = [];
-            for (const item of content.items) {
-                if ("str" in item) {
-                    if (lastY === item.transform[5] || !lastY) {
-                        textItems.push(item.str);
-                    }
-                    else {
-                        textItems.push(`\n${item.str}`);
-                    }
-                    // eslint-disable-next-line prefer-destructuring
-                    lastY = item.transform[5];
-                }
-            }
-            const text = textItems.join(this.parsedItemSeparator);
-            documents.push(new Document({
-                pageContent: text,
-                metadata: {
-                    pdf: {
-                        version,
-                        info: meta?.info,
-                        metadata: meta?.metadata,
-                        totalPages: parsedPdf.numPages,
-                    },
-                    loc: {
-                        pageNumber: i,
-                    },
-                },
-            }));
-        }
-        if (this.splitPages) {
-            return documents;
-        }
-        if (documents.length === 0) {
-            return [];
-        }
-        return [
-            new Document({
-                pageContent: formatDocumentsAsString(documents),
-                metadata: {
-                    pdf: {
-                        version,
-                        info: meta?.info,
-                        metadata: meta?.metadata,
-                        totalPages: parsedPdf.numPages,
-                    },
-                },
-            }),
-        ];
-        return documents;
-    }
-}
-async function PDFLoaderImports() {
-    try {
-        const { default: mod } = await import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js");
-        const { getDocument, version } = mod;
-        return { getDocument, version };
-    }
-    catch (e) {
-        console.error(e);
-        throw new Error("Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`.");
-    }
-}

package/dist/document_loaders/web/playwright.cjs

@@ -1,89 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PlaywrightWebBaseLoader = void 0;
-const documents_1 = require("@langchain/core/documents");
-const base_js_1 = require("../base.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/playwright",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/pplaywrightdf" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for scraping web pages using
- * Playwright. Extends the BaseDocumentLoader class and implements the
- * DocumentLoader interface.
- */
-class PlaywrightWebBaseLoader extends base_js_1.BaseDocumentLoader {
-    constructor(webPath, options) {
-        super();
-        Object.defineProperty(this, "webPath", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: webPath
-        });
-        Object.defineProperty(this, "options", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.options = options ?? undefined;
-    }
-    static async _scrape(url, options) {
-        const { chromium } = await PlaywrightWebBaseLoader.imports();
-        const browser = await chromium.launch({
-            headless: true,
-            ...options?.launchOptions,
-        });
-        const page = await browser.newPage();
-        const response = await page.goto(url, {
-            timeout: 180000,
-            waitUntil: "domcontentloaded",
-            ...options?.gotoOptions,
-        });
-        const bodyHTML = options?.evaluate
-            ? await options?.evaluate(page, browser, response)
-            : await page.content();
-        await browser.close();
-        return bodyHTML;
-    }
-    /**
-     * Method that calls the _scrape method to perform the scraping of the web
-     * page specified by the webPath property. Returns a Promise that resolves
-     * to the scraped HTML content of the web page.
-     * @returns Promise that resolves to the scraped HTML content of the web page.
-     */
-    async scrape() {
-        return PlaywrightWebBaseLoader._scrape(this.webPath, this.options);
-    }
-    /**
-     * Method that calls the scrape method and returns the scraped HTML
-     * content as a Document object. Returns a Promise that resolves to an
-     * array of Document objects.
-     * @returns Promise that resolves to an array of Document objects.
-     */
-    async load() {
-        const text = await this.scrape();
-        const metadata = { source: this.webPath };
-        return [new documents_1.Document({ pageContent: text, metadata })];
-    }
-    /**
-     * Static method that imports the necessary Playwright modules. Returns a
-     * Promise that resolves to an object containing the imported modules.
-     * @returns Promise that resolves to an object containing the imported modules.
-     */
-    static async imports() {
-        try {
-            const { chromium } = await import("playwright");
-            return { chromium };
-        }
-        catch (e) {
-            console.error(e);
-            throw new Error("Please install playwright as a dependency with, e.g. `yarn add playwright`");
-        }
-    }
-}
-exports.PlaywrightWebBaseLoader = PlaywrightWebBaseLoader;

package/dist/document_loaders/web/playwright.d.ts

@@ -1,58 +0,0 @@
-import type { LaunchOptions, Page, Browser, Response } from "playwright";
-import { Document } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
-import type { DocumentLoader } from "../base.js";
-export { Page, Browser, Response };
-export type PlaywrightGotoOptions = {
-    referer?: string;
-    timeout?: number;
-    waitUntil?: "load" | "domcontentloaded" | "networkidle" | "commit";
-};
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/playwright" instead. This entrypoint will be removed in 0.3.0.
- *
- * Type representing a function for evaluating JavaScript code on a web
- * page using Playwright. Takes a Page, Browser, and Response object as
- * parameters and returns a Promise that resolves to a string.
- */
-export type PlaywrightEvaluate = (page: Page, browser: Browser, response: Response | null) => Promise<string>;
-export type PlaywrightWebBaseLoaderOptions = {
-    launchOptions?: LaunchOptions;
-    gotoOptions?: PlaywrightGotoOptions;
-    evaluate?: PlaywrightEvaluate;
-};
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/pplaywrightdf" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for scraping web pages using
- * Playwright. Extends the BaseDocumentLoader class and implements the
- * DocumentLoader interface.
- */
-export declare class PlaywrightWebBaseLoader extends BaseDocumentLoader implements DocumentLoader {
-    webPath: string;
-    options: PlaywrightWebBaseLoaderOptions | undefined;
-    constructor(webPath: string, options?: PlaywrightWebBaseLoaderOptions);
-    static _scrape(url: string, options?: PlaywrightWebBaseLoaderOptions): Promise<string>;
-    /**
-     * Method that calls the _scrape method to perform the scraping of the web
-     * page specified by the webPath property. Returns a Promise that resolves
-     * to the scraped HTML content of the web page.
-     * @returns Promise that resolves to the scraped HTML content of the web page.
-     */
-    scrape(): Promise<string>;
-    /**
-     * Method that calls the scrape method and returns the scraped HTML
-     * content as a Document object. Returns a Promise that resolves to an
-     * array of Document objects.
-     * @returns Promise that resolves to an array of Document objects.
-     */
-    load(): Promise<Document[]>;
-    /**
-     * Static method that imports the necessary Playwright modules. Returns a
-     * Promise that resolves to an object containing the imported modules.
-     * @returns Promise that resolves to an object containing the imported modules.
-     */
-    static imports(): Promise<{
-        chromium: typeof import("playwright").chromium;
-    }>;
-}

package/dist/document_loaders/web/playwright.js

@@ -1,85 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/playwright",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/pplaywrightdf" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for scraping web pages using
- * Playwright. Extends the BaseDocumentLoader class and implements the
- * DocumentLoader interface.
- */
-export class PlaywrightWebBaseLoader extends BaseDocumentLoader {
-    constructor(webPath, options) {
-        super();
-        Object.defineProperty(this, "webPath", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: webPath
-        });
-        Object.defineProperty(this, "options", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.options = options ?? undefined;
-    }
-    static async _scrape(url, options) {
-        const { chromium } = await PlaywrightWebBaseLoader.imports();
-        const browser = await chromium.launch({
-            headless: true,
-            ...options?.launchOptions,
-        });
-        const page = await browser.newPage();
-        const response = await page.goto(url, {
-            timeout: 180000,
-            waitUntil: "domcontentloaded",
-            ...options?.gotoOptions,
-        });
-        const bodyHTML = options?.evaluate
-            ? await options?.evaluate(page, browser, response)
-            : await page.content();
-        await browser.close();
-        return bodyHTML;
-    }
-    /**
-     * Method that calls the _scrape method to perform the scraping of the web
-     * page specified by the webPath property. Returns a Promise that resolves
-     * to the scraped HTML content of the web page.
-     * @returns Promise that resolves to the scraped HTML content of the web page.
-     */
-    async scrape() {
-        return PlaywrightWebBaseLoader._scrape(this.webPath, this.options);
-    }
-    /**
-     * Method that calls the scrape method and returns the scraped HTML
-     * content as a Document object. Returns a Promise that resolves to an
-     * array of Document objects.
-     * @returns Promise that resolves to an array of Document objects.
-     */
-    async load() {
-        const text = await this.scrape();
-        const metadata = { source: this.webPath };
-        return [new Document({ pageContent: text, metadata })];
-    }
-    /**
-     * Static method that imports the necessary Playwright modules. Returns a
-     * Promise that resolves to an object containing the imported modules.
-     * @returns Promise that resolves to an object containing the imported modules.
-     */
-    static async imports() {
-        try {
-            const { chromium } = await import("playwright");
-            return { chromium };
-        }
-        catch (e) {
-            console.error(e);
-            throw new Error("Please install playwright as a dependency with, e.g. `yarn add playwright`");
-        }
-    }
-}

package/dist/document_loaders/web/puppeteer.cjs

@@ -1,139 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PuppeteerWebBaseLoader = void 0;
-const documents_1 = require("@langchain/core/documents");
-const base_js_1 = require("../base.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/puppeteer",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/puppeteer" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class that extends the BaseDocumentLoader class and implements the
- * DocumentLoader interface. It represents a document loader for scraping
- * web pages using Puppeteer.
- * @example
- * ```typescript
- * const loader = new PuppeteerWebBaseLoader("https:exampleurl.com", {
- *   launchOptions: {
- *     headless: true,
- *   },
- *   gotoOptions: {
- *     waitUntil: "domcontentloaded",
- *   },
- * });
- * const screenshot = await loader.screenshot();
- * ```
- */
-class PuppeteerWebBaseLoader extends base_js_1.BaseDocumentLoader {
-    constructor(webPath, options) {
-        super();
-        Object.defineProperty(this, "webPath", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: webPath
-        });
-        Object.defineProperty(this, "options", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.options = options ?? undefined;
-    }
-    static async _scrape(url, options) {
-        const { launch } = await PuppeteerWebBaseLoader.imports();
-        const browser = await launch({
-            headless: true,
-            defaultViewport: null,
-            ignoreDefaultArgs: ["--disable-extensions"],
-            ...options?.launchOptions,
-        });
-        const page = await browser.newPage();
-        await page.goto(url, {
-            timeout: 180000,
-            waitUntil: "domcontentloaded",
-            ...options?.gotoOptions,
-        });
-        const bodyHTML = options?.evaluate
-            ? await options?.evaluate(page, browser)
-            : await page.evaluate(() => document.body.innerHTML);
-        await browser.close();
-        return bodyHTML;
-    }
-    /**
-     * Method that calls the _scrape method to perform the scraping of the web
-     * page specified by the webPath property.
-     * @returns Promise that resolves to the scraped HTML content of the web page.
-     */
-    async scrape() {
-        return PuppeteerWebBaseLoader._scrape(this.webPath, this.options);
-    }
-    /**
-     * Method that calls the scrape method and returns the scraped HTML
-     * content as a Document object.
-     * @returns Promise that resolves to an array of Document objects.
-     */
-    async load() {
-        const text = await this.scrape();
-        const metadata = { source: this.webPath };
-        return [new documents_1.Document({ pageContent: text, metadata })];
-    }
-    /**
-     * Static class method used to screenshot a web page and return
-     * it as a {@link Document} object where  the pageContent property
-     * is the screenshot encoded in base64.
-     *
-     * @param {string} url
-     * @param {PuppeteerWebBaseLoaderOptions} options
-     * @returns {Document} A document object containing the screenshot of the page encoded in base64.
-     */
-    static async _screenshot(url, options) {
-        const { launch } = await PuppeteerWebBaseLoader.imports();
-        const browser = await launch({
-            headless: true,
-            defaultViewport: null,
-            ignoreDefaultArgs: ["--disable-extensions"],
-            ...options?.launchOptions,
-        });
-        const page = await browser.newPage();
-        await page.goto(url, {
-            timeout: 180000,
-            waitUntil: "domcontentloaded",
-            ...options?.gotoOptions,
-        });
-        const screenshot = await page.screenshot();
-        const base64 = screenshot.toString("base64");
-        const metadata = { source: url };
-        return new documents_1.Document({ pageContent: base64, metadata });
-    }
-    /**
-     * Screenshot a web page and return it as a {@link Document} object where
-     * the pageContent property is the screenshot encoded in base64.
-     *
-     * @returns {Promise<Document>} A document object containing the screenshot of the page encoded in base64.
-     */
-    async screenshot() {
-        return PuppeteerWebBaseLoader._screenshot(this.webPath, this.options);
-    }
-    /**
-     * Static method that imports the necessary Puppeteer modules. It returns
-     * a Promise that resolves to an object containing the imported modules.
-     * @returns Promise that resolves to an object containing the imported Puppeteer modules.
-     */
-    static async imports() {
-        try {
-            // eslint-disable-next-line import/no-extraneous-dependencies
-            const { launch } = await import("puppeteer");
-            return { launch };
-        }
-        catch (e) {
-            console.error(e);
-            throw new Error("Please install puppeteer as a dependency with, e.g. `yarn add puppeteer`");
-        }
-    }
-}
-exports.PuppeteerWebBaseLoader = PuppeteerWebBaseLoader;

package/dist/document_loaders/web/puppeteer.d.ts

@@ -1,82 +0,0 @@
-import type { launch, WaitForOptions, Page, Browser, PuppeteerLaunchOptions } from "puppeteer";
-import { Document } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
-import type { DocumentLoader } from "../base.js";
-export { Page, Browser };
-export type PuppeteerGotoOptions = WaitForOptions & {
-    referer?: string;
-    referrerPolicy?: string;
-};
-/**
- * Type representing a function for evaluating JavaScript code on a web
- * page using Puppeteer. It takes a Page and Browser object as parameters
- * and returns a Promise that resolves to a string.
- */
-export type PuppeteerEvaluate = (page: Page, browser: Browser) => Promise<string>;
-export type PuppeteerWebBaseLoaderOptions = {
-    launchOptions?: PuppeteerLaunchOptions;
-    gotoOptions?: PuppeteerGotoOptions;
-    evaluate?: PuppeteerEvaluate;
-};
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/puppeteer" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class that extends the BaseDocumentLoader class and implements the
- * DocumentLoader interface. It represents a document loader for scraping
- * web pages using Puppeteer.
- * @example
- * ```typescript
- * const loader = new PuppeteerWebBaseLoader("https:exampleurl.com", {
- *   launchOptions: {
- *     headless: true,
- *   },
- *   gotoOptions: {
- *     waitUntil: "domcontentloaded",
- *   },
- * });
- * const screenshot = await loader.screenshot();
- * ```
- */
-export declare class PuppeteerWebBaseLoader extends BaseDocumentLoader implements DocumentLoader {
-    webPath: string;
-    options: PuppeteerWebBaseLoaderOptions | undefined;
-    constructor(webPath: string, options?: PuppeteerWebBaseLoaderOptions);
-    static _scrape(url: string, options?: PuppeteerWebBaseLoaderOptions): Promise<string>;
-    /**
-     * Method that calls the _scrape method to perform the scraping of the web
-     * page specified by the webPath property.
-     * @returns Promise that resolves to the scraped HTML content of the web page.
-     */
-    scrape(): Promise<string>;
-    /**
-     * Method that calls the scrape method and returns the scraped HTML
-     * content as a Document object.
-     * @returns Promise that resolves to an array of Document objects.
-     */
-    load(): Promise<Document[]>;
-    /**
-     * Static class method used to screenshot a web page and return
-     * it as a {@link Document} object where  the pageContent property
-     * is the screenshot encoded in base64.
-     *
-     * @param {string} url
-     * @param {PuppeteerWebBaseLoaderOptions} options
-     * @returns {Document} A document object containing the screenshot of the page encoded in base64.
-     */
-    static _screenshot(url: string, options?: PuppeteerWebBaseLoaderOptions): Promise<Document>;
-    /**
-     * Screenshot a web page and return it as a {@link Document} object where
-     * the pageContent property is the screenshot encoded in base64.
-     *
-     * @returns {Promise<Document>} A document object containing the screenshot of the page encoded in base64.
-     */
-    screenshot(): Promise<Document>;
-    /**
-     * Static method that imports the necessary Puppeteer modules. It returns
-     * a Promise that resolves to an object containing the imported modules.
-     * @returns Promise that resolves to an object containing the imported Puppeteer modules.
-     */
-    static imports(): Promise<{
-        launch: typeof launch;
-    }>;
-}