langchain 0.2.18 → 0.3.0-rc.0

package/dist/document_loaders/web/s3.cjs

@@ -1,164 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.S3Loader = void 0;
-const fsDefault = __importStar(require("node:fs"));
-const path = __importStar(require("node:path"));
-const os = __importStar(require("node:os"));
-const node_stream_1 = require("node:stream");
-const client_s3_1 = require("@aws-sdk/client-s3");
-const base_js_1 = require("../base.cjs");
-const unstructured_js_1 = require("../fs/unstructured.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/s3",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/s3" instead. This entrypoint will be removed in 0.3.0.
- *
- * A class that extends the BaseDocumentLoader class. It represents a
- * document loader for loading files from an S3 bucket.
- * @example
- * ```typescript
- * const loader = new S3Loader({
- *   bucket: "my-document-bucket-123",
- *   key: "AccountingOverview.pdf",
- *   s3Config: {
- *     region: "us-east-1",
- *     credentials: {
- *       accessKeyId: "<YourAccessKeyId>",
- *       secretAccessKey: "<YourSecretAccessKey>",
- *     },
- *   },
- *   unstructuredAPIURL: "<YourUnstructuredAPIURL>",
- *   unstructuredAPIKey: "<YourUnstructuredAPIKey>",
- * });
- * const docs = await loader.load();
- * ```
- */
-class S3Loader extends base_js_1.BaseDocumentLoader {
-    constructor({ bucket, key, unstructuredAPIURL, unstructuredAPIKey, s3Config = {}, fs = fsDefault, UnstructuredLoader = unstructured_js_1.UnstructuredLoader, }) {
-        super();
-        Object.defineProperty(this, "bucket", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "key", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "unstructuredAPIURL", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "unstructuredAPIKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "s3Config", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "_fs", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "_UnstructuredLoader", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.bucket = bucket;
-        this.key = key;
-        this.unstructuredAPIURL = unstructuredAPIURL;
-        this.unstructuredAPIKey = unstructuredAPIKey;
-        this.s3Config = s3Config;
-        this._fs = fs;
-        this._UnstructuredLoader = UnstructuredLoader;
-    }
-    /**
-     * Loads the file from the S3 bucket, saves it to a temporary directory,
-     * and then uses the UnstructuredLoader to load the file as a document.
-     * @returns An array of Document objects representing the loaded documents.
-     */
-    async load() {
-        const tempDir = this._fs.mkdtempSync(path.join(os.tmpdir(), "s3fileloader-"));
-        const filePath = path.join(tempDir, this.key);
-        try {
-            const s3Client = new client_s3_1.S3Client(this.s3Config);
-            const getObjectCommand = new client_s3_1.GetObjectCommand({
-                Bucket: this.bucket,
-                Key: this.key,
-            });
-            const response = await s3Client.send(getObjectCommand);
-            const objectData = await new Promise((resolve, reject) => {
-                const chunks = [];
-                // eslint-disable-next-line no-instanceof/no-instanceof
-                if (response.Body instanceof node_stream_1.Readable) {
-                    response.Body.on("data", (chunk) => chunks.push(chunk));
-                    response.Body.on("end", () => resolve(Buffer.concat(chunks)));
-                    response.Body.on("error", reject);
-                }
-                else {
-                    reject(new Error("Response body is not a readable stream."));
-                }
-            });
-            this._fs.mkdirSync(path.dirname(filePath), { recursive: true });
-            this._fs.writeFileSync(filePath, objectData);
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        }
-        catch (e) {
-            throw new Error(`Failed to download file ${this.key} from S3 bucket ${this.bucket}: ${e.message}`);
-        }
-        try {
-            const options = {
-                apiUrl: this.unstructuredAPIURL,
-                apiKey: this.unstructuredAPIKey,
-            };
-            const unstructuredLoader = new this._UnstructuredLoader(filePath, options);
-            const docs = await unstructuredLoader.load();
-            return docs;
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        }
-        catch (e) {
-            throw new Error(`Failed to load file ${filePath} using unstructured loader: ${e.message}`);
-        }
-    }
-}
-exports.S3Loader = S3Loader;

package/dist/document_loaders/web/s3.d.ts

@@ -1,78 +0,0 @@
-/// <reference types="node" resolution-mode="require"/>
-import * as fsDefault from "node:fs";
-import { S3ClientConfig } from "@aws-sdk/client-s3";
-import { BaseDocumentLoader } from "../base.js";
-import { UnstructuredLoader as UnstructuredLoaderDefault } from "../fs/unstructured.js";
-/**
- * Represents the configuration options for the S3 client. It extends the
- * S3ClientConfig interface from the "@aws-sdk/client-s3" package and
- * includes additional deprecated properties for access key ID and secret
- * access key.
- */
-export type S3Config = S3ClientConfig & {
-    /** @deprecated Use the credentials object instead */
-    accessKeyId?: string;
-    /** @deprecated Use the credentials object instead */
-    secretAccessKey?: string;
-};
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/s3" instead. This entrypoint will be removed in 0.3.0.
- *
- * Represents the parameters for the S3Loader class. It includes
- * properties such as the S3 bucket, key, unstructured API URL,
- * unstructured API key, S3 configuration, file system module, and
- * UnstructuredLoader module.
- */
-export interface S3LoaderParams {
-    bucket: string;
-    key: string;
-    unstructuredAPIURL: string;
-    unstructuredAPIKey: string;
-    s3Config?: S3Config & {
-        /** @deprecated Use the credentials object instead */
-        accessKeyId?: string;
-        /** @deprecated Use the credentials object instead */
-        secretAccessKey?: string;
-    };
-    fs?: typeof fsDefault;
-    UnstructuredLoader?: typeof UnstructuredLoaderDefault;
-}
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/s3" instead. This entrypoint will be removed in 0.3.0.
- *
- * A class that extends the BaseDocumentLoader class. It represents a
- * document loader for loading files from an S3 bucket.
- * @example
- * ```typescript
- * const loader = new S3Loader({
- *   bucket: "my-document-bucket-123",
- *   key: "AccountingOverview.pdf",
- *   s3Config: {
- *     region: "us-east-1",
- *     credentials: {
- *       accessKeyId: "<YourAccessKeyId>",
- *       secretAccessKey: "<YourSecretAccessKey>",
- *     },
- *   },
- *   unstructuredAPIURL: "<YourUnstructuredAPIURL>",
- *   unstructuredAPIKey: "<YourUnstructuredAPIKey>",
- * });
- * const docs = await loader.load();
- * ```
- */
-export declare class S3Loader extends BaseDocumentLoader {
-    private bucket;
-    private key;
-    private unstructuredAPIURL;
-    private unstructuredAPIKey;
-    private s3Config;
-    private _fs;
-    private _UnstructuredLoader;
-    constructor({ bucket, key, unstructuredAPIURL, unstructuredAPIKey, s3Config, fs, UnstructuredLoader, }: S3LoaderParams);
-    /**
-     * Loads the file from the S3 bucket, saves it to a temporary directory,
-     * and then uses the UnstructuredLoader to load the file as a document.
-     * @returns An array of Document objects representing the loaded documents.
-     */
-    load(): Promise<import("@langchain/core/documents").Document<Record<string, any>>[]>;
-}

package/dist/document_loaders/web/s3.js

@@ -1,137 +0,0 @@
-import * as fsDefault from "node:fs";
-import * as path from "node:path";
-import * as os from "node:os";
-import { Readable } from "node:stream";
-import { S3Client, GetObjectCommand } from "@aws-sdk/client-s3";
-import { BaseDocumentLoader } from "../base.js";
-import { UnstructuredLoader as UnstructuredLoaderDefault } from "../fs/unstructured.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/s3",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/s3" instead. This entrypoint will be removed in 0.3.0.
- *
- * A class that extends the BaseDocumentLoader class. It represents a
- * document loader for loading files from an S3 bucket.
- * @example
- * ```typescript
- * const loader = new S3Loader({
- *   bucket: "my-document-bucket-123",
- *   key: "AccountingOverview.pdf",
- *   s3Config: {
- *     region: "us-east-1",
- *     credentials: {
- *       accessKeyId: "<YourAccessKeyId>",
- *       secretAccessKey: "<YourSecretAccessKey>",
- *     },
- *   },
- *   unstructuredAPIURL: "<YourUnstructuredAPIURL>",
- *   unstructuredAPIKey: "<YourUnstructuredAPIKey>",
- * });
- * const docs = await loader.load();
- * ```
- */
-export class S3Loader extends BaseDocumentLoader {
-    constructor({ bucket, key, unstructuredAPIURL, unstructuredAPIKey, s3Config = {}, fs = fsDefault, UnstructuredLoader = UnstructuredLoaderDefault, }) {
-        super();
-        Object.defineProperty(this, "bucket", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "key", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "unstructuredAPIURL", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "unstructuredAPIKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "s3Config", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "_fs", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "_UnstructuredLoader", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.bucket = bucket;
-        this.key = key;
-        this.unstructuredAPIURL = unstructuredAPIURL;
-        this.unstructuredAPIKey = unstructuredAPIKey;
-        this.s3Config = s3Config;
-        this._fs = fs;
-        this._UnstructuredLoader = UnstructuredLoader;
-    }
-    /**
-     * Loads the file from the S3 bucket, saves it to a temporary directory,
-     * and then uses the UnstructuredLoader to load the file as a document.
-     * @returns An array of Document objects representing the loaded documents.
-     */
-    async load() {
-        const tempDir = this._fs.mkdtempSync(path.join(os.tmpdir(), "s3fileloader-"));
-        const filePath = path.join(tempDir, this.key);
-        try {
-            const s3Client = new S3Client(this.s3Config);
-            const getObjectCommand = new GetObjectCommand({
-                Bucket: this.bucket,
-                Key: this.key,
-            });
-            const response = await s3Client.send(getObjectCommand);
-            const objectData = await new Promise((resolve, reject) => {
-                const chunks = [];
-                // eslint-disable-next-line no-instanceof/no-instanceof
-                if (response.Body instanceof Readable) {
-                    response.Body.on("data", (chunk) => chunks.push(chunk));
-                    response.Body.on("end", () => resolve(Buffer.concat(chunks)));
-                    response.Body.on("error", reject);
-                }
-                else {
-                    reject(new Error("Response body is not a readable stream."));
-                }
-            });
-            this._fs.mkdirSync(path.dirname(filePath), { recursive: true });
-            this._fs.writeFileSync(filePath, objectData);
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        }
-        catch (e) {
-            throw new Error(`Failed to download file ${this.key} from S3 bucket ${this.bucket}: ${e.message}`);
-        }
-        try {
-            const options = {
-                apiUrl: this.unstructuredAPIURL,
-                apiKey: this.unstructuredAPIKey,
-            };
-            const unstructuredLoader = new this._UnstructuredLoader(filePath, options);
-            const docs = await unstructuredLoader.load();
-            return docs;
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        }
-        catch (e) {
-            throw new Error(`Failed to load file ${filePath} using unstructured loader: ${e.message}`);
-        }
-    }
-}

package/dist/document_loaders/web/searchapi.cjs

@@ -1,150 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SearchApiLoader = void 0;
-const documents_1 = require("@langchain/core/documents");
-const env_1 = require("@langchain/core/utils/env");
-const base_js_1 = require("../base.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/searchapi",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/searchapi" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for loading search results from
- * the SearchApi. It extends the BaseDocumentLoader class.
- * @example
- * ```typescript
- * const loader = new SearchApiLoader({
- *   q: "{query}",
- *   apiKey: "{apiKey}",
- *   engine: "google",
- * });
- * const docs = await loader.load();
- * ```
- */
-class SearchApiLoader extends base_js_1.BaseDocumentLoader {
-    constructor(params) {
-        super();
-        Object.defineProperty(this, "apiKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "parameters", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        const { apiKey = (0, env_1.getEnvironmentVariable)("SEARCHAPI_API_KEY") } = params;
-        if (typeof apiKey !== "string") {
-            throw new Error("Invalid type for apiKey. Expected string.");
-        }
-        if (!apiKey) {
-            throw new Error("SearchApi API key not set. You can set it as SEARCHAPI_API_KEY in your .env file, or pass it to SearchApi.");
-        }
-        this.apiKey = apiKey;
-        this.parameters = { ...params };
-    }
-    /**
-     * Builds the URL for the SearchApi search request.
-     * @returns The URL for the search request.
-     */
-    buildUrl() {
-        this.parameters = {
-            engine: "google",
-            api_key: this.apiKey,
-            ...this.parameters,
-        };
-        const preparedParams = Object.entries(this.parameters)
-            .filter(([key, value]) => value !== undefined && value !== null && key !== "apiKey")
-            .map(([key, value]) => [key, `${value}`]);
-        const searchParams = new URLSearchParams(preparedParams);
-        return `https://www.searchapi.io/api/v1/search?${searchParams}`;
-    }
-    /**
-     * Extracts documents from the provided output.
-     * @param output - The output to extract documents from.
-     * @param responseType - The type of the response to extract documents from.
-     * @returns An array of Documents.
-     */
-    extractDocuments(output, responseType) {
-        const documents = [];
-        const results = Array.isArray(output) ? output : [output];
-        if (responseType === "transcripts") {
-            const pageContent = results.map((result) => result.text).join("\n");
-            const metadata = {
-                source: "SearchApi",
-                responseType,
-            };
-            documents.push(new documents_1.Document({ pageContent, metadata }));
-        }
-        else {
-            for (const result of results) {
-                const pageContent = JSON.stringify(result);
-                const metadata = {
-                    source: "SearchApi",
-                    responseType,
-                };
-                documents.push(new documents_1.Document({ pageContent, metadata }));
-            }
-        }
-        return documents;
-    }
-    /**
-     * Processes the response data from the SearchApi search request and converts it into an array of Documents.
-     * @param data - The response data from the SearchApi search request.
-     * @returns An array of Documents.
-     */
-    processResponseData(data) {
-        const documents = [];
-        const responseTypes = [
-            "answer_box",
-            "shopping_results",
-            "knowledge_graph",
-            "organic_results",
-            "transcripts",
-        ];
-        for (const responseType of responseTypes) {
-            if (responseType in data) {
-                documents.push(...this.extractDocuments(data[responseType], responseType));
-            }
-        }
-        return documents;
-    }
-    /**
-     * Fetches the data from the provided URL and returns it as a JSON object.
-     * If an error occurs during the fetch operation, an exception is thrown with the error message.
-     * @param url - The URL to fetch data from.
-     * @returns A promise that resolves to the fetched data as a JSON object.
-     * @throws An error if the fetch operation fails.
-     */
-    async fetchData(url) {
-        const response = await fetch(url);
-        const data = await response.json();
-        if (data.error) {
-            throw new Error(`Failed to load search results from SearchApi due to: ${data.error}`);
-        }
-        return data;
-    }
-    /**
-     * Loads the search results from the SearchApi.
-     * @returns An array of Documents representing the search results.
-     * @throws An error if the search results could not be loaded.
-     */
-    async load() {
-        const url = this.buildUrl();
-        const data = await this.fetchData(url);
-        try {
-            return this.processResponseData(data);
-        }
-        catch (error) {
-            console.error(error);
-            throw new Error(`Failed to process search results from SearchApi: ${error}`);
-        }
-    }
-}
-exports.SearchApiLoader = SearchApiLoader;

package/dist/document_loaders/web/searchapi.d.ts

@@ -1,76 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
-type JSONPrimitive = string | number | boolean | null;
-type JSONValue = JSONPrimitive | JSONObject | JSONArray;
-interface JSONObject {
-    [key: string]: JSONValue;
-}
-interface JSONArray extends Array<JSONValue> {
-}
-/**
- * SearchApiParameters Type Definition.
- *
- * For more parameters and supported search engines, refer specific engine documentation:
- * Google - https://www.searchapi.io/docs/google
- * Google News - https://www.searchapi.io/docs/google-news
- * Google Scholar - https://www.searchapi.io/docs/google-scholar
- * YouTube Transcripts - https://www.searchapi.io/docs/youtube-transcripts
- * and others.
- *
- */
-type SearchApiParameters = {
-    [key: string]: JSONValue;
-};
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/searchapi" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for loading search results from
- * the SearchApi. It extends the BaseDocumentLoader class.
- * @example
- * ```typescript
- * const loader = new SearchApiLoader({
- *   q: "{query}",
- *   apiKey: "{apiKey}",
- *   engine: "google",
- * });
- * const docs = await loader.load();
- * ```
- */
-export declare class SearchApiLoader extends BaseDocumentLoader {
-    private apiKey;
-    private parameters;
-    constructor(params: SearchApiParameters);
-    /**
-     * Builds the URL for the SearchApi search request.
-     * @returns The URL for the search request.
-     */
-    buildUrl(): string;
-    /**
-     * Extracts documents from the provided output.
-     * @param output - The output to extract documents from.
-     * @param responseType - The type of the response to extract documents from.
-     * @returns An array of Documents.
-     */
-    private extractDocuments;
-    /**
-     * Processes the response data from the SearchApi search request and converts it into an array of Documents.
-     * @param data - The response data from the SearchApi search request.
-     * @returns An array of Documents.
-     */
-    processResponseData(data: Record<string, unknown>): Document[];
-    /**
-     * Fetches the data from the provided URL and returns it as a JSON object.
-     * If an error occurs during the fetch operation, an exception is thrown with the error message.
-     * @param url - The URL to fetch data from.
-     * @returns A promise that resolves to the fetched data as a JSON object.
-     * @throws An error if the fetch operation fails.
-     */
-    private fetchData;
-    /**
-     * Loads the search results from the SearchApi.
-     * @returns An array of Documents representing the search results.
-     * @throws An error if the search results could not be loaded.
-     */
-    load(): Promise<Document[]>;
-}
-export {};