langchain 0.2.17 → 0.3.0-rc.0

package/dist/document_loaders/web/azure_blob_storage_file.cjs

@@ -1,124 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.AzureBlobStorageFileLoader = void 0;
-const fs = __importStar(require("node:fs"));
-const path = __importStar(require("node:path"));
-const os = __importStar(require("node:os"));
-const storage_blob_1 = require("@azure/storage-blob");
-const base_js_1 = require("../base.cjs");
-const unstructured_js_1 = require("../fs/unstructured.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/azure_blog_storage_file",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/azure_blog_storage_file" instead. This entrypoint will be removed in 0.3.0.
- * Class representing a document loader that loads a specific file from
- * Azure Blob Storage. It extends the BaseDocumentLoader class and
- * implements the DocumentLoader interface.
- * @example
- * ```typescript
- * const loader = new AzureBlobStorageFileLoader({
- *   azureConfig: {
- *     connectionString: "{connectionString}",
- *     container: "{containerName}",
- *     blobName: "{blobName}",
- *   },
- * });
- * const docs = await loader.load();
- * ```
- */
-class AzureBlobStorageFileLoader extends base_js_1.BaseDocumentLoader {
-    constructor({ azureConfig, unstructuredConfig, }) {
-        super();
-        Object.defineProperty(this, "connectionString", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "container", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "blobName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "unstructuredConfig", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.connectionString = azureConfig.connectionString;
-        this.container = azureConfig.container;
-        this.blobName = azureConfig.blobName;
-        this.unstructuredConfig = unstructuredConfig;
-    }
-    /**
-     * Method to load a specific file from Azure Blob Storage. It creates a
-     * temporary directory, constructs the file path, downloads the file, and
-     * loads the documents using the UnstructuredLoader. The loaded documents
-     * are returned, and the temporary directory is deleted.
-     * @returns An array of documents loaded from the file in Azure Blob Storage.
-     */
-    async load() {
-        const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "azureblobfileloader-"));
-        const filePath = path.join(tempDir, this.blobName);
-        try {
-            const blobServiceClient = storage_blob_1.BlobServiceClient.fromConnectionString(this.connectionString, {
-                userAgentOptions: {
-                    userAgentPrefix: "langchainjs-blob-storage-file",
-                },
-            });
-            const containerClient = blobServiceClient.getContainerClient(this.container);
-            const blobClient = containerClient.getBlobClient(this.blobName);
-            fs.mkdirSync(path.dirname(filePath), { recursive: true });
-            await blobClient.downloadToFile(filePath);
-        }
-        catch (e) {
-            throw new Error(`Failed to download file ${this.blobName} from Azure Blob Storage container ${this.container}: ${e.message}`);
-        }
-        try {
-            const unstructuredLoader = new unstructured_js_1.UnstructuredLoader(filePath, this.unstructuredConfig);
-            const docs = await unstructuredLoader.load();
-            return docs;
-        }
-        catch {
-            throw new Error(`Failed to load file ${filePath} using unstructured loader.`);
-        }
-        finally {
-            fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
-        }
-    }
-}
-exports.AzureBlobStorageFileLoader = AzureBlobStorageFileLoader;

package/dist/document_loaders/web/azure_blob_storage_file.d.ts

@@ -1,53 +0,0 @@
-import { BaseDocumentLoader } from "../base.js";
-import { UnstructuredLoaderOptions } from "../fs/unstructured.js";
-/**
- * Interface representing the configuration for accessing a specific file
- * in Azure Blob Storage.
- */
-interface AzureBlobStorageFileConfig {
-    connectionString: string;
-    container: string;
-    blobName: string;
-}
-/**
- * Interface representing the configuration for the
- * AzureBlobStorageFileLoader. It contains the Azure Blob Storage file
- * configuration and the options for the UnstructuredLoader.
- */
-interface AzureBlobStorageFileLoaderConfig {
-    azureConfig: AzureBlobStorageFileConfig;
-    unstructuredConfig?: UnstructuredLoaderOptions;
-}
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/azure_blog_storage_file" instead. This entrypoint will be removed in 0.3.0.
- * Class representing a document loader that loads a specific file from
- * Azure Blob Storage. It extends the BaseDocumentLoader class and
- * implements the DocumentLoader interface.
- * @example
- * ```typescript
- * const loader = new AzureBlobStorageFileLoader({
- *   azureConfig: {
- *     connectionString: "{connectionString}",
- *     container: "{containerName}",
- *     blobName: "{blobName}",
- *   },
- * });
- * const docs = await loader.load();
- * ```
- */
-export declare class AzureBlobStorageFileLoader extends BaseDocumentLoader {
-    private readonly connectionString;
-    private readonly container;
-    private readonly blobName;
-    private readonly unstructuredConfig?;
-    constructor({ azureConfig, unstructuredConfig, }: AzureBlobStorageFileLoaderConfig);
-    /**
-     * Method to load a specific file from Azure Blob Storage. It creates a
-     * temporary directory, constructs the file path, downloads the file, and
-     * loads the documents using the UnstructuredLoader. The loaded documents
-     * are returned, and the temporary directory is deleted.
-     * @returns An array of documents loaded from the file in Azure Blob Storage.
-     */
-    load(): Promise<import("@langchain/core/documents").Document<Record<string, any>>[]>;
-}
-export {};

package/dist/document_loaders/web/azure_blob_storage_file.js

@@ -1,97 +0,0 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-import * as os from "node:os";
-import { BlobServiceClient } from "@azure/storage-blob";
-import { BaseDocumentLoader } from "../base.js";
-import { UnstructuredLoader, } from "../fs/unstructured.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/azure_blog_storage_file",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/azure_blog_storage_file" instead. This entrypoint will be removed in 0.3.0.
- * Class representing a document loader that loads a specific file from
- * Azure Blob Storage. It extends the BaseDocumentLoader class and
- * implements the DocumentLoader interface.
- * @example
- * ```typescript
- * const loader = new AzureBlobStorageFileLoader({
- *   azureConfig: {
- *     connectionString: "{connectionString}",
- *     container: "{containerName}",
- *     blobName: "{blobName}",
- *   },
- * });
- * const docs = await loader.load();
- * ```
- */
-export class AzureBlobStorageFileLoader extends BaseDocumentLoader {
-    constructor({ azureConfig, unstructuredConfig, }) {
-        super();
-        Object.defineProperty(this, "connectionString", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "container", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "blobName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "unstructuredConfig", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.connectionString = azureConfig.connectionString;
-        this.container = azureConfig.container;
-        this.blobName = azureConfig.blobName;
-        this.unstructuredConfig = unstructuredConfig;
-    }
-    /**
-     * Method to load a specific file from Azure Blob Storage. It creates a
-     * temporary directory, constructs the file path, downloads the file, and
-     * loads the documents using the UnstructuredLoader. The loaded documents
-     * are returned, and the temporary directory is deleted.
-     * @returns An array of documents loaded from the file in Azure Blob Storage.
-     */
-    async load() {
-        const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "azureblobfileloader-"));
-        const filePath = path.join(tempDir, this.blobName);
-        try {
-            const blobServiceClient = BlobServiceClient.fromConnectionString(this.connectionString, {
-                userAgentOptions: {
-                    userAgentPrefix: "langchainjs-blob-storage-file",
-                },
-            });
-            const containerClient = blobServiceClient.getContainerClient(this.container);
-            const blobClient = containerClient.getBlobClient(this.blobName);
-            fs.mkdirSync(path.dirname(filePath), { recursive: true });
-            await blobClient.downloadToFile(filePath);
-        }
-        catch (e) {
-            throw new Error(`Failed to download file ${this.blobName} from Azure Blob Storage container ${this.container}: ${e.message}`);
-        }
-        try {
-            const unstructuredLoader = new UnstructuredLoader(filePath, this.unstructuredConfig);
-            const docs = await unstructuredLoader.load();
-            return docs;
-        }
-        catch {
-            throw new Error(`Failed to load file ${filePath} using unstructured loader.`);
-        }
-        finally {
-            fs.rmSync(path.dirname(filePath), { recursive: true, force: true });
-        }
-    }
-}

package/dist/document_loaders/web/browserbase.cjs

@@ -1,93 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.BrowserbaseLoader = void 0;
-const documents_1 = require("@langchain/core/documents");
-const sdk_1 = __importDefault(require("@browserbasehq/sdk"));
-const base_js_1 = require("../base.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/browserbase",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated Import from "@langchain/community/document_loaders/web/browserbase" instead. This entrypoint will be removed in 0.3.0.
- * Load pre-rendered web pages using a headless browser hosted on Browserbase.
- *
- * Depends on `@browserbasehq/sdk` package.
- * Get your API key from https://browserbase.com
- *
- * @example
- * ```typescript
- * import { BrowserbaseLoader } from "langchain/document_loaders/web/browserbase";
- *
- * const loader = new BrowserbaseLoader(["https://example.com"], {
- *   apiKey: process.env.BROWSERBASE_API_KEY,
- *   textContent: true,
- * });
- *
- * const docs = await loader.load();
- * ```
- *
- * @param {string[]} urls - The URLs of the web pages to load.
- * @param {BrowserbaseLoaderOptions} [options] - Browserbase client options.
- */
-class BrowserbaseLoader extends base_js_1.BaseDocumentLoader {
-    constructor(urls, options = {}) {
-        super();
-        Object.defineProperty(this, "urls", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "options", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "browserbase", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.urls = urls;
-        this.options = options;
-        this.browserbase = new sdk_1.default(options);
-    }
-    /**
-     * Load pages from URLs.
-     *
-     * @returns {Promise<DocumentInterface[]>} - A promise which resolves to a list of documents.
-     */
-    async load() {
-        const documents = [];
-        for await (const doc of this.lazyLoad()) {
-            documents.push(doc);
-        }
-        return documents;
-    }
-    /**
-     * Load pages from URLs.
-     *
-     * @returns {Generator<DocumentInterface>} - A generator that yields documents.
-     */
-    async *lazyLoad() {
-        const pages = await this.browserbase.loadURLs(this.urls, this.options);
-        let index = 0;
-        for await (const page of pages) {
-            yield new documents_1.Document({
-                pageContent: page,
-                metadata: {
-                    url: this.urls[index],
-                },
-            });
-            index += index + 1;
-        }
-    }
-}
-exports.BrowserbaseLoader = BrowserbaseLoader;

package/dist/document_loaders/web/browserbase.d.ts

@@ -1,48 +0,0 @@
-import { Document, type DocumentInterface } from "@langchain/core/documents";
-import Browserbase, { LoadOptions, ClientOptions } from "@browserbasehq/sdk";
-import { BaseDocumentLoader } from "../base.js";
-import type { DocumentLoader } from "../base.js";
-type BrowserbaseLoaderOptions = ClientOptions & LoadOptions;
-/**
- * @deprecated Import from "@langchain/community/document_loaders/web/browserbase" instead. This entrypoint will be removed in 0.3.0.
- * Load pre-rendered web pages using a headless browser hosted on Browserbase.
- *
- * Depends on `@browserbasehq/sdk` package.
- * Get your API key from https://browserbase.com
- *
- * @example
- * ```typescript
- * import { BrowserbaseLoader } from "langchain/document_loaders/web/browserbase";
- *
- * const loader = new BrowserbaseLoader(["https://example.com"], {
- *   apiKey: process.env.BROWSERBASE_API_KEY,
- *   textContent: true,
- * });
- *
- * const docs = await loader.load();
- * ```
- *
- * @param {string[]} urls - The URLs of the web pages to load.
- * @param {BrowserbaseLoaderOptions} [options] - Browserbase client options.
- */
-export declare class BrowserbaseLoader extends BaseDocumentLoader implements DocumentLoader {
-    urls: string[];
-    options: BrowserbaseLoaderOptions;
-    browserbase: Browserbase;
-    constructor(urls: string[], options?: BrowserbaseLoaderOptions);
-    /**
-     * Load pages from URLs.
-     *
-     * @returns {Promise<DocumentInterface[]>} - A promise which resolves to a list of documents.
-     */
-    load(): Promise<DocumentInterface[]>;
-    /**
-     * Load pages from URLs.
-     *
-     * @returns {Generator<DocumentInterface>} - A generator that yields documents.
-     */
-    lazyLoad(): AsyncGenerator<Document<{
-        url: string;
-    }>, void, unknown>;
-}
-export {};

package/dist/document_loaders/web/browserbase.js

@@ -1,86 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import Browserbase from "@browserbasehq/sdk";
-import { BaseDocumentLoader } from "../base.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/browserbase",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated Import from "@langchain/community/document_loaders/web/browserbase" instead. This entrypoint will be removed in 0.3.0.
- * Load pre-rendered web pages using a headless browser hosted on Browserbase.
- *
- * Depends on `@browserbasehq/sdk` package.
- * Get your API key from https://browserbase.com
- *
- * @example
- * ```typescript
- * import { BrowserbaseLoader } from "langchain/document_loaders/web/browserbase";
- *
- * const loader = new BrowserbaseLoader(["https://example.com"], {
- *   apiKey: process.env.BROWSERBASE_API_KEY,
- *   textContent: true,
- * });
- *
- * const docs = await loader.load();
- * ```
- *
- * @param {string[]} urls - The URLs of the web pages to load.
- * @param {BrowserbaseLoaderOptions} [options] - Browserbase client options.
- */
-export class BrowserbaseLoader extends BaseDocumentLoader {
-    constructor(urls, options = {}) {
-        super();
-        Object.defineProperty(this, "urls", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "options", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "browserbase", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.urls = urls;
-        this.options = options;
-        this.browserbase = new Browserbase(options);
-    }
-    /**
-     * Load pages from URLs.
-     *
-     * @returns {Promise<DocumentInterface[]>} - A promise which resolves to a list of documents.
-     */
-    async load() {
-        const documents = [];
-        for await (const doc of this.lazyLoad()) {
-            documents.push(doc);
-        }
-        return documents;
-    }
-    /**
-     * Load pages from URLs.
-     *
-     * @returns {Generator<DocumentInterface>} - A generator that yields documents.
-     */
-    async *lazyLoad() {
-        const pages = await this.browserbase.loadURLs(this.urls, this.options);
-        let index = 0;
-        for await (const page of pages) {
-            yield new Document({
-                pageContent: page,
-                metadata: {
-                    url: this.urls[index],
-                },
-            });
-            index += index + 1;
-        }
-    }
-}

package/dist/document_loaders/web/cheerio.cjs

@@ -1,118 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CheerioWebBaseLoader = void 0;
-const documents_1 = require("@langchain/core/documents");
-const async_caller_1 = require("@langchain/core/utils/async_caller");
-const base_js_1 = require("../base.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/cheerio",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/cheerio" instead. This entrypoint will be removed in 0.3.0.
- *
- * A class that extends the BaseDocumentLoader and implements the
- * DocumentLoader interface. It represents a document loader for loading
- * web-based documents using Cheerio.
- * @example
- * ```typescript
- * const loader = new CheerioWebBaseLoader("https:exampleurl.com");
- * const docs = await loader.load();
- * console.log({ docs });
- * ```
- */
-class CheerioWebBaseLoader extends base_js_1.BaseDocumentLoader {
-    constructor(webPath, fields) {
-        super();
-        Object.defineProperty(this, "webPath", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: webPath
-        });
-        Object.defineProperty(this, "timeout", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "caller", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "selector", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "textDecoder", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        const { timeout, selector, textDecoder, ...rest } = fields ?? {};
-        this.timeout = timeout ?? 10000;
-        this.caller = new async_caller_1.AsyncCaller(rest);
-        this.selector = selector ?? "body";
-        this.textDecoder = textDecoder;
-    }
-    /**
-     * Fetches web documents from the given array of URLs and loads them using Cheerio.
-     * It returns an array of CheerioAPI instances.
-     * @param urls An array of URLs to fetch and load.
-     * @returns A Promise that resolves to an array of CheerioAPI instances.
-     */
-    static async scrapeAll(urls, caller, timeout, textDecoder, options) {
-        return Promise.all(urls.map((url) => CheerioWebBaseLoader._scrape(url, caller, timeout, textDecoder, options)));
-    }
-    static async _scrape(url, caller, timeout, textDecoder, options) {
-        const { load } = await CheerioWebBaseLoader.imports();
-        const response = await caller.call(fetch, url, {
-            signal: timeout ? AbortSignal.timeout(timeout) : undefined,
-        });
-        const html = textDecoder?.decode(await response.arrayBuffer()) ??
-            (await response.text());
-        return load(html, options);
-    }
-    /**
-     * Fetches the web document from the webPath and loads it using Cheerio.
-     * It returns a CheerioAPI instance.
-     * @returns A Promise that resolves to a CheerioAPI instance.
-     */
-    async scrape() {
-        return CheerioWebBaseLoader._scrape(this.webPath, this.caller, this.timeout, this.textDecoder);
-    }
-    /**
-     * Extracts the text content from the loaded document using the selector
-     * and creates a Document instance with the extracted text and metadata.
-     * It returns an array of Document instances.
-     * @returns A Promise that resolves to an array of Document instances.
-     */
-    async load() {
-        const $ = await this.scrape();
-        const text = $(this.selector).text();
-        const metadata = { source: this.webPath };
-        return [new documents_1.Document({ pageContent: text, metadata })];
-    }
-    /**
-     * A static method that dynamically imports the Cheerio library and
-     * returns the load function. If the import fails, it throws an error.
-     * @returns A Promise that resolves to an object containing the load function from the Cheerio library.
-     */
-    static async imports() {
-        try {
-            const { load } = await import("cheerio");
-            return { load };
-        }
-        catch (e) {
-            console.error(e);
-            throw new Error("Please install cheerio as a dependency with, e.g. `yarn add cheerio`");
-        }
-    }
-}
-exports.CheerioWebBaseLoader = CheerioWebBaseLoader;