langchain 0.2.18 → 0.3.0-rc.0

package/dist/document_loaders/web/cheerio.d.ts

@@ -1,77 +0,0 @@
-import type { CheerioAPI, CheerioOptions, load as LoadT, SelectorType } from "cheerio";
-import { Document } from "@langchain/core/documents";
-import { AsyncCaller, AsyncCallerParams } from "@langchain/core/utils/async_caller";
-import { BaseDocumentLoader } from "../base.js";
-import type { DocumentLoader } from "../base.js";
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/cheerio" instead. This entrypoint will be removed in 0.3.0.
- *
- * Represents the parameters for configuring the CheerioWebBaseLoader. It
- * extends the AsyncCallerParams interface and adds additional parameters
- * specific to web-based loaders.
- */
-export interface WebBaseLoaderParams extends AsyncCallerParams {
-    /**
-     * The timeout in milliseconds for the fetch request. Defaults to 10s.
-     */
-    timeout?: number;
-    /**
-     * The selector to use to extract the text from the document. Defaults to
-     * "body".
-     */
-    selector?: SelectorType;
-    /**
-     * The text decoder to use to decode the response. Defaults to UTF-8.
-     */
-    textDecoder?: TextDecoder;
-}
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/cheerio" instead. This entrypoint will be removed in 0.3.0.
- *
- * A class that extends the BaseDocumentLoader and implements the
- * DocumentLoader interface. It represents a document loader for loading
- * web-based documents using Cheerio.
- * @example
- * ```typescript
- * const loader = new CheerioWebBaseLoader("https:exampleurl.com");
- * const docs = await loader.load();
- * console.log({ docs });
- * ```
- */
-export declare class CheerioWebBaseLoader extends BaseDocumentLoader implements DocumentLoader {
-    webPath: string;
-    timeout: number;
-    caller: AsyncCaller;
-    selector?: SelectorType;
-    textDecoder?: TextDecoder;
-    constructor(webPath: string, fields?: WebBaseLoaderParams);
-    /**
-     * Fetches web documents from the given array of URLs and loads them using Cheerio.
-     * It returns an array of CheerioAPI instances.
-     * @param urls An array of URLs to fetch and load.
-     * @returns A Promise that resolves to an array of CheerioAPI instances.
-     */
-    static scrapeAll(urls: string[], caller: AsyncCaller, timeout: number | undefined, textDecoder?: TextDecoder, options?: CheerioOptions): Promise<CheerioAPI[]>;
-    static _scrape(url: string, caller: AsyncCaller, timeout: number | undefined, textDecoder?: TextDecoder, options?: CheerioOptions): Promise<CheerioAPI>;
-    /**
-     * Fetches the web document from the webPath and loads it using Cheerio.
-     * It returns a CheerioAPI instance.
-     * @returns A Promise that resolves to a CheerioAPI instance.
-     */
-    scrape(): Promise<CheerioAPI>;
-    /**
-     * Extracts the text content from the loaded document using the selector
-     * and creates a Document instance with the extracted text and metadata.
-     * It returns an array of Document instances.
-     * @returns A Promise that resolves to an array of Document instances.
-     */
-    load(): Promise<Document[]>;
-    /**
-     * A static method that dynamically imports the Cheerio library and
-     * returns the load function. If the import fails, it throws an error.
-     * @returns A Promise that resolves to an object containing the load function from the Cheerio library.
-     */
-    static imports(): Promise<{
-        load: typeof LoadT;
-    }>;
-}

package/dist/document_loaders/web/cheerio.js

@@ -1,114 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { AsyncCaller, } from "@langchain/core/utils/async_caller";
-import { BaseDocumentLoader } from "../base.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/cheerio",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/cheerio" instead. This entrypoint will be removed in 0.3.0.
- *
- * A class that extends the BaseDocumentLoader and implements the
- * DocumentLoader interface. It represents a document loader for loading
- * web-based documents using Cheerio.
- * @example
- * ```typescript
- * const loader = new CheerioWebBaseLoader("https:exampleurl.com");
- * const docs = await loader.load();
- * console.log({ docs });
- * ```
- */
-export class CheerioWebBaseLoader extends BaseDocumentLoader {
-    constructor(webPath, fields) {
-        super();
-        Object.defineProperty(this, "webPath", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: webPath
-        });
-        Object.defineProperty(this, "timeout", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "caller", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "selector", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "textDecoder", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        const { timeout, selector, textDecoder, ...rest } = fields ?? {};
-        this.timeout = timeout ?? 10000;
-        this.caller = new AsyncCaller(rest);
-        this.selector = selector ?? "body";
-        this.textDecoder = textDecoder;
-    }
-    /**
-     * Fetches web documents from the given array of URLs and loads them using Cheerio.
-     * It returns an array of CheerioAPI instances.
-     * @param urls An array of URLs to fetch and load.
-     * @returns A Promise that resolves to an array of CheerioAPI instances.
-     */
-    static async scrapeAll(urls, caller, timeout, textDecoder, options) {
-        return Promise.all(urls.map((url) => CheerioWebBaseLoader._scrape(url, caller, timeout, textDecoder, options)));
-    }
-    static async _scrape(url, caller, timeout, textDecoder, options) {
-        const { load } = await CheerioWebBaseLoader.imports();
-        const response = await caller.call(fetch, url, {
-            signal: timeout ? AbortSignal.timeout(timeout) : undefined,
-        });
-        const html = textDecoder?.decode(await response.arrayBuffer()) ??
-            (await response.text());
-        return load(html, options);
-    }
-    /**
-     * Fetches the web document from the webPath and loads it using Cheerio.
-     * It returns a CheerioAPI instance.
-     * @returns A Promise that resolves to a CheerioAPI instance.
-     */
-    async scrape() {
-        return CheerioWebBaseLoader._scrape(this.webPath, this.caller, this.timeout, this.textDecoder);
-    }
-    /**
-     * Extracts the text content from the loaded document using the selector
-     * and creates a Document instance with the extracted text and metadata.
-     * It returns an array of Document instances.
-     * @returns A Promise that resolves to an array of Document instances.
-     */
-    async load() {
-        const $ = await this.scrape();
-        const text = $(this.selector).text();
-        const metadata = { source: this.webPath };
-        return [new Document({ pageContent: text, metadata })];
-    }
-    /**
-     * A static method that dynamically imports the Cheerio library and
-     * returns the load function. If the import fails, it throws an error.
-     * @returns A Promise that resolves to an object containing the load function from the Cheerio library.
-     */
-    static async imports() {
-        try {
-            const { load } = await import("cheerio");
-            return { load };
-        }
-        catch (e) {
-            console.error(e);
-            throw new Error("Please install cheerio as a dependency with, e.g. `yarn add cheerio`");
-        }
-    }
-}

package/dist/document_loaders/web/college_confidential.cjs

@@ -1,41 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CollegeConfidentialLoader = void 0;
-const documents_1 = require("@langchain/core/documents");
-const cheerio_js_1 = require("./cheerio.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/college_confidential",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/college_confidential" instead. This entrypoint will be removed in 0.3.0.
- *
- * A document loader specifically designed for loading documents from the
- * College Confidential website. It extends the CheerioWebBaseLoader.
- * @example
- * ```typescript
- * const loader = new CollegeConfidentialLoader("https:exampleurl.com");
- * const docs = await loader.load();
- * console.log({ docs });
- * ```
- */
-class CollegeConfidentialLoader extends cheerio_js_1.CheerioWebBaseLoader {
-    constructor(webPath) {
-        super(webPath);
-    }
-    /**
-     * Overrides the base load() method to extract the text content from the
-     * loaded document using a specific selector for the College Confidential
-     * website. It creates a Document instance with the extracted text and
-     * metadata, and returns an array containing the Document instance.
-     * @returns An array containing a Document instance with the extracted text and metadata from the loaded College Confidential web document.
-     */
-    async load() {
-        const $ = await this.scrape();
-        const text = $("main[class='skin-handler']").text();
-        const metadata = { source: this.webPath };
-        return [new documents_1.Document({ pageContent: text, metadata })];
-    }
-}
-exports.CollegeConfidentialLoader = CollegeConfidentialLoader;

package/dist/document_loaders/web/college_confidential.d.ts

@@ -1,25 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { CheerioWebBaseLoader } from "./cheerio.js";
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/college_confidential" instead. This entrypoint will be removed in 0.3.0.
- *
- * A document loader specifically designed for loading documents from the
- * College Confidential website. It extends the CheerioWebBaseLoader.
- * @example
- * ```typescript
- * const loader = new CollegeConfidentialLoader("https:exampleurl.com");
- * const docs = await loader.load();
- * console.log({ docs });
- * ```
- */
-export declare class CollegeConfidentialLoader extends CheerioWebBaseLoader {
-    constructor(webPath: string);
-    /**
-     * Overrides the base load() method to extract the text content from the
-     * loaded document using a specific selector for the College Confidential
-     * website. It creates a Document instance with the extracted text and
-     * metadata, and returns an array containing the Document instance.
-     * @returns An array containing a Document instance with the extracted text and metadata from the loaded College Confidential web document.
-     */
-    load(): Promise<Document[]>;
-}

package/dist/document_loaders/web/college_confidential.js

@@ -1,37 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { CheerioWebBaseLoader } from "./cheerio.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/college_confidential",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/college_confidential" instead. This entrypoint will be removed in 0.3.0.
- *
- * A document loader specifically designed for loading documents from the
- * College Confidential website. It extends the CheerioWebBaseLoader.
- * @example
- * ```typescript
- * const loader = new CollegeConfidentialLoader("https:exampleurl.com");
- * const docs = await loader.load();
- * console.log({ docs });
- * ```
- */
-export class CollegeConfidentialLoader extends CheerioWebBaseLoader {
-    constructor(webPath) {
-        super(webPath);
-    }
-    /**
-     * Overrides the base load() method to extract the text content from the
-     * loaded document using a specific selector for the College Confidential
-     * website. It creates a Document instance with the extracted text and
-     * metadata, and returns an array containing the Document instance.
-     * @returns An array containing a Document instance with the extracted text and metadata from the loaded College Confidential web document.
-     */
-    async load() {
-        const $ = await this.scrape();
-        const text = $("main[class='skin-handler']").text();
-        const metadata = { source: this.webPath };
-        return [new Document({ pageContent: text, metadata })];
-    }
-}

package/dist/document_loaders/web/confluence.cjs

@@ -1,190 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ConfluencePagesLoader = void 0;
-const html_to_text_1 = require("html-to-text");
-const documents_1 = require("@langchain/core/documents");
-const base_js_1 = require("../base.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/confluence",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/confluence" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for loading pages from Confluence.
- * @example
- * ```typescript
- * const loader = new ConfluencePagesLoader({
- *   baseUrl: "https:
- *   spaceKey: "~EXAMPLE362906de5d343d49dcdbae5dEXAMPLE",
- *   username: "your-username",
- *   accessToken: "your-access-token",
- * });
- * const documents = await loader.load();
- * console.log(documents);
- * ```
- */
-class ConfluencePagesLoader extends base_js_1.BaseDocumentLoader {
-    constructor({ baseUrl, spaceKey, username, accessToken, limit = 25, expand = "body.storage,version", personalAccessToken, }) {
-        super();
-        Object.defineProperty(this, "baseUrl", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "spaceKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "username", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "accessToken", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "limit", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        /**
-         * expand parameter for confluence rest api
-         * description can be found at https://developer.atlassian.com/server/confluence/expansions-in-the-rest-api/
-         */
-        Object.defineProperty(this, "expand", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "personalAccessToken", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.baseUrl = baseUrl;
-        this.spaceKey = spaceKey;
-        this.username = username;
-        this.accessToken = accessToken;
-        this.limit = limit;
-        this.expand = expand;
-        this.personalAccessToken = personalAccessToken;
-    }
-    /**
-     * Returns the authorization header for the request.
-     * @returns The authorization header as a string, or undefined if no credentials were provided.
-     */
-    get authorizationHeader() {
-        if (this.personalAccessToken) {
-            return `Bearer ${this.personalAccessToken}`;
-        }
-        else if (this.username && this.accessToken) {
-            const authToken = Buffer.from(`${this.username}:${this.accessToken}`).toString("base64");
-            return `Basic ${authToken}`;
-        }
-        return undefined;
-    }
-    /**
-     * Fetches all the pages in the specified space and converts each page to
-     * a Document instance.
-     * @param options the extra options of the load function
-     * @param options.limit The limit parameter to overwrite the size to fetch pages.
-     * @param options.start The start parameter to set inital offset to fetch pages.
-     * @returns Promise resolving to an array of Document instances.
-     */
-    async load(options) {
-        try {
-            const pages = await this.fetchAllPagesInSpace(options?.start, options?.limit);
-            return pages.map((page) => this.createDocumentFromPage(page));
-        }
-        catch (error) {
-            console.error("Error:", error);
-            return [];
-        }
-    }
-    /**
-     * Fetches data from the Confluence API using the provided URL.
-     * @param url The URL to fetch data from.
-     * @returns Promise resolving to the JSON response from the API.
-     */
-    async fetchConfluenceData(url) {
-        try {
-            const initialHeaders = {
-                "Content-Type": "application/json",
-                Accept: "application/json",
-            };
-            const authHeader = this.authorizationHeader;
-            if (authHeader) {
-                initialHeaders.Authorization = authHeader;
-            }
-            const response = await fetch(url, {
-                headers: initialHeaders,
-            });
-            if (!response.ok) {
-                throw new Error(`Failed to fetch ${url} from Confluence: ${response.status}`);
-            }
-            return await response.json();
-        }
-        catch (error) {
-            throw new Error(`Failed to fetch ${url} from Confluence: ${error}`);
-        }
-    }
-    /**
-     * Recursively fetches all the pages in the specified space.
-     * @param start The start parameter to paginate through the results.
-     * @returns Promise resolving to an array of ConfluencePage objects.
-     */
-    async fetchAllPagesInSpace(start = 0, limit = this.limit) {
-        const url = `${this.baseUrl}/rest/api/content?spaceKey=${this.spaceKey}&limit=${limit}&start=${start}&expand=${this.expand}`;
-        const data = await this.fetchConfluenceData(url);
-        if (data.size === 0) {
-            return [];
-        }
-        const nextPageStart = start + data.size;
-        const nextPageResults = await this.fetchAllPagesInSpace(nextPageStart, limit);
-        return data.results.concat(nextPageResults);
-    }
-    /**
-     * Creates a Document instance from a ConfluencePage object.
-     * @param page The ConfluencePage object to convert.
-     * @returns A Document instance.
-     */
-    createDocumentFromPage(page) {
-        // Convert the HTML content to plain text
-        const plainTextContent = (0, html_to_text_1.htmlToText)(page.body.storage.value, {
-            wordwrap: false,
-            preserveNewlines: false,
-        });
-        // Remove empty lines
-        const textWithoutEmptyLines = plainTextContent.replace(/^\s*[\r\n]/gm, "");
-        // Generate the URL
-        const pageUrl = `${this.baseUrl}/spaces/${this.spaceKey}/pages/${page.id}`;
-        // Return a langchain document
-        return new documents_1.Document({
-            pageContent: textWithoutEmptyLines,
-            metadata: {
-                id: page.id,
-                status: page.status,
-                title: page.title,
-                type: page.type,
-                url: pageUrl,
-                version: page.version?.number,
-                updated_by: page.version?.by?.displayName,
-                updated_at: page.version?.when,
-            },
-        });
-    }
-}
-exports.ConfluencePagesLoader = ConfluencePagesLoader;

package/dist/document_loaders/web/confluence.d.ts

@@ -1,114 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/confluence" instead. This entrypoint will be removed in 0.3.0.
- *
- * Interface representing the parameters for configuring the
- * ConfluencePagesLoader.
- */
-export interface ConfluencePagesLoaderParams {
-    baseUrl: string;
-    spaceKey: string;
-    username?: string;
-    accessToken?: string;
-    personalAccessToken?: string;
-    limit?: number;
-    expand?: string;
-}
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/confluence" instead. This entrypoint will be removed in 0.3.0.
- *
- * Interface representing a Confluence page.
- */
-export interface ConfluencePage {
-    id: string;
-    title: string;
-    type: string;
-    body: {
-        storage: {
-            value: string;
-        };
-    };
-    status: string;
-    version?: {
-        number: number;
-        when: string;
-        by: {
-            displayName: string;
-        };
-    };
-}
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/confluence" instead. This entrypoint will be removed in 0.3.0.
- *
- * Interface representing the response from the Confluence API.
- */
-export interface ConfluenceAPIResponse {
-    size: number;
-    results: ConfluencePage[];
-}
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/confluence" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for loading pages from Confluence.
- * @example
- * ```typescript
- * const loader = new ConfluencePagesLoader({
- *   baseUrl: "https:
- *   spaceKey: "~EXAMPLE362906de5d343d49dcdbae5dEXAMPLE",
- *   username: "your-username",
- *   accessToken: "your-access-token",
- * });
- * const documents = await loader.load();
- * console.log(documents);
- * ```
- */
-export declare class ConfluencePagesLoader extends BaseDocumentLoader {
-    readonly baseUrl: string;
-    readonly spaceKey: string;
-    readonly username?: string;
-    readonly accessToken?: string;
-    readonly limit: number;
-    /**
-     * expand parameter for confluence rest api
-     * description can be found at https://developer.atlassian.com/server/confluence/expansions-in-the-rest-api/
-     */
-    readonly expand?: string;
-    readonly personalAccessToken?: string;
-    constructor({ baseUrl, spaceKey, username, accessToken, limit, expand, personalAccessToken, }: ConfluencePagesLoaderParams);
-    /**
-     * Returns the authorization header for the request.
-     * @returns The authorization header as a string, or undefined if no credentials were provided.
-     */
-    private get authorizationHeader();
-    /**
-     * Fetches all the pages in the specified space and converts each page to
-     * a Document instance.
-     * @param options the extra options of the load function
-     * @param options.limit The limit parameter to overwrite the size to fetch pages.
-     * @param options.start The start parameter to set inital offset to fetch pages.
-     * @returns Promise resolving to an array of Document instances.
-     */
-    load(options?: {
-        start?: number;
-        limit?: number;
-    }): Promise<Document[]>;
-    /**
-     * Fetches data from the Confluence API using the provided URL.
-     * @param url The URL to fetch data from.
-     * @returns Promise resolving to the JSON response from the API.
-     */
-    protected fetchConfluenceData(url: string): Promise<ConfluenceAPIResponse>;
-    /**
-     * Recursively fetches all the pages in the specified space.
-     * @param start The start parameter to paginate through the results.
-     * @returns Promise resolving to an array of ConfluencePage objects.
-     */
-    private fetchAllPagesInSpace;
-    /**
-     * Creates a Document instance from a ConfluencePage object.
-     * @param page The ConfluencePage object to convert.
-     * @returns A Document instance.
-     */
-    private createDocumentFromPage;
-}