langchain 0.2.18 → 0.3.0-rc.0

package/dist/document_loaders/web/searchapi.js

@@ -1,146 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { getEnvironmentVariable } from "@langchain/core/utils/env";
-import { BaseDocumentLoader } from "../base.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/searchapi",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/searchapi" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for loading search results from
- * the SearchApi. It extends the BaseDocumentLoader class.
- * @example
- * ```typescript
- * const loader = new SearchApiLoader({
- *   q: "{query}",
- *   apiKey: "{apiKey}",
- *   engine: "google",
- * });
- * const docs = await loader.load();
- * ```
- */
-export class SearchApiLoader extends BaseDocumentLoader {
-    constructor(params) {
-        super();
-        Object.defineProperty(this, "apiKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "parameters", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        const { apiKey = getEnvironmentVariable("SEARCHAPI_API_KEY") } = params;
-        if (typeof apiKey !== "string") {
-            throw new Error("Invalid type for apiKey. Expected string.");
-        }
-        if (!apiKey) {
-            throw new Error("SearchApi API key not set. You can set it as SEARCHAPI_API_KEY in your .env file, or pass it to SearchApi.");
-        }
-        this.apiKey = apiKey;
-        this.parameters = { ...params };
-    }
-    /**
-     * Builds the URL for the SearchApi search request.
-     * @returns The URL for the search request.
-     */
-    buildUrl() {
-        this.parameters = {
-            engine: "google",
-            api_key: this.apiKey,
-            ...this.parameters,
-        };
-        const preparedParams = Object.entries(this.parameters)
-            .filter(([key, value]) => value !== undefined && value !== null && key !== "apiKey")
-            .map(([key, value]) => [key, `${value}`]);
-        const searchParams = new URLSearchParams(preparedParams);
-        return `https://www.searchapi.io/api/v1/search?${searchParams}`;
-    }
-    /**
-     * Extracts documents from the provided output.
-     * @param output - The output to extract documents from.
-     * @param responseType - The type of the response to extract documents from.
-     * @returns An array of Documents.
-     */
-    extractDocuments(output, responseType) {
-        const documents = [];
-        const results = Array.isArray(output) ? output : [output];
-        if (responseType === "transcripts") {
-            const pageContent = results.map((result) => result.text).join("\n");
-            const metadata = {
-                source: "SearchApi",
-                responseType,
-            };
-            documents.push(new Document({ pageContent, metadata }));
-        }
-        else {
-            for (const result of results) {
-                const pageContent = JSON.stringify(result);
-                const metadata = {
-                    source: "SearchApi",
-                    responseType,
-                };
-                documents.push(new Document({ pageContent, metadata }));
-            }
-        }
-        return documents;
-    }
-    /**
-     * Processes the response data from the SearchApi search request and converts it into an array of Documents.
-     * @param data - The response data from the SearchApi search request.
-     * @returns An array of Documents.
-     */
-    processResponseData(data) {
-        const documents = [];
-        const responseTypes = [
-            "answer_box",
-            "shopping_results",
-            "knowledge_graph",
-            "organic_results",
-            "transcripts",
-        ];
-        for (const responseType of responseTypes) {
-            if (responseType in data) {
-                documents.push(...this.extractDocuments(data[responseType], responseType));
-            }
-        }
-        return documents;
-    }
-    /**
-     * Fetches the data from the provided URL and returns it as a JSON object.
-     * If an error occurs during the fetch operation, an exception is thrown with the error message.
-     * @param url - The URL to fetch data from.
-     * @returns A promise that resolves to the fetched data as a JSON object.
-     * @throws An error if the fetch operation fails.
-     */
-    async fetchData(url) {
-        const response = await fetch(url);
-        const data = await response.json();
-        if (data.error) {
-            throw new Error(`Failed to load search results from SearchApi due to: ${data.error}`);
-        }
-        return data;
-    }
-    /**
-     * Loads the search results from the SearchApi.
-     * @returns An array of Documents representing the search results.
-     * @throws An error if the search results could not be loaded.
-     */
-    async load() {
-        const url = this.buildUrl();
-        const data = await this.fetchData(url);
-        try {
-            return this.processResponseData(data);
-        }
-        catch (error) {
-            console.error(error);
-            throw new Error(`Failed to process search results from SearchApi: ${error}`);
-        }
-    }
-}

package/dist/document_loaders/web/serpapi.cjs

@@ -1,127 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SerpAPILoader = void 0;
-const documents_1 = require("@langchain/core/documents");
-const env_1 = require("@langchain/core/utils/env");
-const base_js_1 = require("../base.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/serpapi",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/serpapi" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for loading search results from
- * the SerpAPI. It extends the BaseDocumentLoader class.
- * @example
- * ```typescript
- * const loader = new SerpAPILoader({ q: "{query}", apiKey: "{apiKey}" });
- * const docs = await loader.load();
- * ```
- */
-class SerpAPILoader extends base_js_1.BaseDocumentLoader {
-    constructor(params) {
-        super();
-        Object.defineProperty(this, "apiKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "searchQuery", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        const { apiKey = (0, env_1.getEnvironmentVariable)("SERPAPI_API_KEY"), q } = params;
-        if (!apiKey) {
-            throw new Error("SerpAPI API key not set. You can set it as SERPAPI_API_KEY in your .env file, or pass it to SerpAPI.");
-        }
-        this.apiKey = apiKey;
-        this.searchQuery = q;
-    }
-    /**
-     * Builds the URL for the SerpAPI search request.
-     * @returns The URL for the search request.
-     */
-    buildUrl() {
-        const params = new URLSearchParams();
-        params.append("api_key", this.apiKey);
-        params.append("q", this.searchQuery);
-        return `https://serpapi.com/search?${params.toString()}`;
-    }
-    /**
-     * Extracts documents from the provided output.
-     * @param output - The output to extract documents from.
-     * @param responseType - The type of the response to extract documents from.
-     * @returns An array of Documents.
-     */
-    extractDocuments(output, responseType) {
-        const documents = [];
-        const results = Array.isArray(output) ? output : [output];
-        for (const result of results) {
-            const pageContent = JSON.stringify(result);
-            const metadata = {
-                source: "SerpAPI",
-                responseType,
-            };
-            documents.push(new documents_1.Document({ pageContent, metadata }));
-        }
-        return documents;
-    }
-    /**
-     * Processes the response data from the SerpAPI search request and converts it into an array of Documents.
-     * @param data - The response data from the SerpAPI search request.
-     * @returns An array of Documents.
-     */
-    processResponseData(data) {
-        const documents = [];
-        const responseTypes = [
-            "answer_box",
-            "sports_results",
-            "shopping_results",
-            "knowledge_graph",
-            "organic_results",
-        ];
-        for (const responseType of responseTypes) {
-            if (responseType in data) {
-                documents.push(...this.extractDocuments(data[responseType], responseType));
-            }
-        }
-        return documents;
-    }
-    /**
-     * Fetches the data from the provided URL and returns it as a JSON object.
-     * If an error occurs during the fetch operation, an exception is thrown with the error message.
-     * @param url - The URL to fetch data from.
-     * @returns A promise that resolves to the fetched data as a JSON object.
-     * @throws An error if the fetch operation fails.
-     */
-    async fetchData(url) {
-        const response = await fetch(url);
-        const data = await response.json();
-        if (data.error) {
-            throw new Error(`Failed to load search results from SerpAPI due to: ${data.error}`);
-        }
-        return data;
-    }
-    /**
-     * Loads the search results from the SerpAPI.
-     * @returns An array of Documents representing the search results.
-     * @throws An error if the search results could not be loaded.
-     */
-    async load() {
-        const url = this.buildUrl();
-        const data = await this.fetchData(url);
-        try {
-            return this.processResponseData(data);
-        }
-        catch (error) {
-            console.error(error);
-            throw new Error(`Failed to process search results from SerpAPI: ${error}`);
-        }
-    }
-}
-exports.SerpAPILoader = SerpAPILoader;

package/dist/document_loaders/web/serpapi.d.ts

@@ -1,62 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
-/**
- * Interface representing the parameters for the SerpAPI loader. It
- * includes properties such as the search query and the API key.
- */
-interface SerpAPIParameters {
-    /**
-     * Search Query
-     */
-    q: string;
-    apiKey?: string;
-}
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/serpapi" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for loading search results from
- * the SerpAPI. It extends the BaseDocumentLoader class.
- * @example
- * ```typescript
- * const loader = new SerpAPILoader({ q: "{query}", apiKey: "{apiKey}" });
- * const docs = await loader.load();
- * ```
- */
-export declare class SerpAPILoader extends BaseDocumentLoader {
-    private apiKey;
-    private searchQuery;
-    constructor(params: SerpAPIParameters);
-    /**
-     * Builds the URL for the SerpAPI search request.
-     * @returns The URL for the search request.
-     */
-    buildUrl(): string;
-    /**
-     * Extracts documents from the provided output.
-     * @param output - The output to extract documents from.
-     * @param responseType - The type of the response to extract documents from.
-     * @returns An array of Documents.
-     */
-    private extractDocuments;
-    /**
-     * Processes the response data from the SerpAPI search request and converts it into an array of Documents.
-     * @param data - The response data from the SerpAPI search request.
-     * @returns An array of Documents.
-     */
-    processResponseData(data: Record<string, unknown>): Document[];
-    /**
-     * Fetches the data from the provided URL and returns it as a JSON object.
-     * If an error occurs during the fetch operation, an exception is thrown with the error message.
-     * @param url - The URL to fetch data from.
-     * @returns A promise that resolves to the fetched data as a JSON object.
-     * @throws An error if the fetch operation fails.
-     */
-    private fetchData;
-    /**
-     * Loads the search results from the SerpAPI.
-     * @returns An array of Documents representing the search results.
-     * @throws An error if the search results could not be loaded.
-     */
-    load(): Promise<Document[]>;
-}
-export {};

package/dist/document_loaders/web/serpapi.js

@@ -1,123 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { getEnvironmentVariable } from "@langchain/core/utils/env";
-import { BaseDocumentLoader } from "../base.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/web/serpapi",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/serpapi" instead. This entrypoint will be removed in 0.3.0.
- *
- * Class representing a document loader for loading search results from
- * the SerpAPI. It extends the BaseDocumentLoader class.
- * @example
- * ```typescript
- * const loader = new SerpAPILoader({ q: "{query}", apiKey: "{apiKey}" });
- * const docs = await loader.load();
- * ```
- */
-export class SerpAPILoader extends BaseDocumentLoader {
-    constructor(params) {
-        super();
-        Object.defineProperty(this, "apiKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "searchQuery", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        const { apiKey = getEnvironmentVariable("SERPAPI_API_KEY"), q } = params;
-        if (!apiKey) {
-            throw new Error("SerpAPI API key not set. You can set it as SERPAPI_API_KEY in your .env file, or pass it to SerpAPI.");
-        }
-        this.apiKey = apiKey;
-        this.searchQuery = q;
-    }
-    /**
-     * Builds the URL for the SerpAPI search request.
-     * @returns The URL for the search request.
-     */
-    buildUrl() {
-        const params = new URLSearchParams();
-        params.append("api_key", this.apiKey);
-        params.append("q", this.searchQuery);
-        return `https://serpapi.com/search?${params.toString()}`;
-    }
-    /**
-     * Extracts documents from the provided output.
-     * @param output - The output to extract documents from.
-     * @param responseType - The type of the response to extract documents from.
-     * @returns An array of Documents.
-     */
-    extractDocuments(output, responseType) {
-        const documents = [];
-        const results = Array.isArray(output) ? output : [output];
-        for (const result of results) {
-            const pageContent = JSON.stringify(result);
-            const metadata = {
-                source: "SerpAPI",
-                responseType,
-            };
-            documents.push(new Document({ pageContent, metadata }));
-        }
-        return documents;
-    }
-    /**
-     * Processes the response data from the SerpAPI search request and converts it into an array of Documents.
-     * @param data - The response data from the SerpAPI search request.
-     * @returns An array of Documents.
-     */
-    processResponseData(data) {
-        const documents = [];
-        const responseTypes = [
-            "answer_box",
-            "sports_results",
-            "shopping_results",
-            "knowledge_graph",
-            "organic_results",
-        ];
-        for (const responseType of responseTypes) {
-            if (responseType in data) {
-                documents.push(...this.extractDocuments(data[responseType], responseType));
-            }
-        }
-        return documents;
-    }
-    /**
-     * Fetches the data from the provided URL and returns it as a JSON object.
-     * If an error occurs during the fetch operation, an exception is thrown with the error message.
-     * @param url - The URL to fetch data from.
-     * @returns A promise that resolves to the fetched data as a JSON object.
-     * @throws An error if the fetch operation fails.
-     */
-    async fetchData(url) {
-        const response = await fetch(url);
-        const data = await response.json();
-        if (data.error) {
-            throw new Error(`Failed to load search results from SerpAPI due to: ${data.error}`);
-        }
-        return data;
-    }
-    /**
-     * Loads the search results from the SerpAPI.
-     * @returns An array of Documents representing the search results.
-     * @throws An error if the search results could not be loaded.
-     */
-    async load() {
-        const url = this.buildUrl();
-        const data = await this.fetchData(url);
-        try {
-            return this.processResponseData(data);
-        }
-        catch (error) {
-            console.error(error);
-            throw new Error(`Failed to process search results from SerpAPI: ${error}`);
-        }
-    }
-}

package/dist/document_loaders/web/sitemap.cjs

@@ -1,118 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.SitemapLoader = void 0;
-const documents_1 = require("@langchain/core/documents");
-const chunk_array_1 = require("@langchain/core/utils/chunk_array");
-const cheerio_js_1 = require("./cheerio.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/sitemap",
-    newPackageName: "@langchain/community",
-});
-const DEFAULT_CHUNK_SIZE = 300;
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/sitemap" instead. This entrypoint will be removed in 0.3.0.
- */
-class SitemapLoader extends cheerio_js_1.CheerioWebBaseLoader {
-    constructor(webPath, params = {}) {
-        const paramsWithDefaults = { chunkSize: DEFAULT_CHUNK_SIZE, ...params };
-        let path = webPath.endsWith("/") ? webPath.slice(0, -1) : webPath;
-        // Allow for custom sitemap paths to be passed in with the url.
-        path = path.endsWith(".xml") ? path : `${path}/sitemap.xml`;
-        super(path, paramsWithDefaults);
-        Object.defineProperty(this, "webPath", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: webPath
-        });
-        Object.defineProperty(this, "allowUrlPatterns", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "chunkSize", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.webPath = path;
-        this.allowUrlPatterns = paramsWithDefaults.filterUrls;
-        this.chunkSize = paramsWithDefaults.chunkSize;
-    }
-    _checkUrlPatterns(url) {
-        if (!this.allowUrlPatterns) {
-            return false;
-        }
-        return !this.allowUrlPatterns.some((pattern) => !new RegExp(pattern).test(url));
-    }
-    async parseSitemap() {
-        const $ = await cheerio_js_1.CheerioWebBaseLoader._scrape(this.webPath, this.caller, this.timeout, this.textDecoder, {
-            xmlMode: true,
-            xml: true,
-        });
-        const elements = [];
-        $("url").each((_, element) => {
-            const loc = $(element).find("loc").text();
-            if (!loc) {
-                return;
-            }
-            if (this._checkUrlPatterns(loc)) {
-                return;
-            }
-            const changefreq = $(element).find("changefreq").text();
-            const lastmod = $(element).find("lastmod").text();
-            const priority = $(element).find("priority").text();
-            elements.push({ loc, changefreq, lastmod, priority });
-        });
-        $("sitemap").each((_, element) => {
-            const loc = $(element).find("loc").text();
-            if (!loc) {
-                return;
-            }
-            const changefreq = $(element).find("changefreq").text();
-            const lastmod = $(element).find("lastmod").text();
-            const priority = $(element).find("priority").text();
-            elements.push({ loc, changefreq, lastmod, priority });
-        });
-        return elements;
-    }
-    async _loadSitemapUrls(elements) {
-        const all = await cheerio_js_1.CheerioWebBaseLoader.scrapeAll(elements.map((ele) => ele.loc), this.caller, this.timeout, this.textDecoder);
-        const documents = all.map(($, i) => {
-            if (!elements[i]) {
-                throw new Error("Scraped docs and elements not in sync");
-            }
-            const text = $(this.selector).text();
-            const { loc: source, ...metadata } = elements[i];
-            // extract page metadata
-            const description = $("meta[name='description']").attr("content");
-            const title = $("meta[property='og:title']").attr("content");
-            const lang = $("meta[property='og:locale']").attr("content");
-            return new documents_1.Document({
-                pageContent: text,
-                metadata: {
-                    ...metadata,
-                    description,
-                    title,
-                    lang,
-                    source: source.trim(),
-                },
-            });
-        });
-        return documents;
-    }
-    async load() {
-        const elements = await this.parseSitemap();
-        const chunks = (0, chunk_array_1.chunkArray)(elements, this.chunkSize);
-        const documents = [];
-        for await (const chunk of chunks) {
-            const chunkedDocuments = await this._loadSitemapUrls(chunk);
-            documents.push(...chunkedDocuments);
-        }
-        return documents;
-    }
-}
-exports.SitemapLoader = SitemapLoader;

package/dist/document_loaders/web/sitemap.d.ts

@@ -1,41 +0,0 @@
-import { Document, DocumentInterface } from "@langchain/core/documents";
-import { CheerioWebBaseLoader, WebBaseLoaderParams } from "./cheerio.js";
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/sitemap" instead. This entrypoint will be removed in 0.3.0.
- *
- * Interface representing the parameters for initializing a SitemapLoader.
- * @interface SitemapLoaderParams
- * @extends WebBaseLoaderParams
- */
-export interface SitemapLoaderParams extends WebBaseLoaderParams {
-    /**
-     * @property {(string | RegExp)[] | undefined} filterUrls - A list of regexes. Only URLs that match one of the filter URLs will be loaded.
-     * WARNING: The filter URLs are interpreted as regular expressions. Escape special characters if needed.
-     */
-    filterUrls?: (string | RegExp)[];
-    /**
-     * The size to chunk the sitemap URLs into for scraping.
-     * @default {300}
-     */
-    chunkSize?: number;
-}
-type SiteMapElement = {
-    loc: string;
-    changefreq?: string;
-    lastmod?: string;
-    priority?: string;
-};
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/sitemap" instead. This entrypoint will be removed in 0.3.0.
- */
-export declare class SitemapLoader extends CheerioWebBaseLoader implements SitemapLoaderParams {
-    webPath: string;
-    allowUrlPatterns: (string | RegExp)[] | undefined;
-    chunkSize: number;
-    constructor(webPath: string, params?: SitemapLoaderParams);
-    _checkUrlPatterns(url: string): boolean;
-    parseSitemap(): Promise<SiteMapElement[]>;
-    _loadSitemapUrls(elements: Array<SiteMapElement>): Promise<DocumentInterface[]>;
-    load(): Promise<Document[]>;
-}
-export {};