langchain 0.1.35 → 0.1.37

package/dist/document_loaders/web/browserbase.js

@@ -0,0 +1,80 @@
+import { Document } from "@langchain/core/documents";
+import Browserbase from "@browserbasehq/sdk";
+import { BaseDocumentLoader } from "../base.js";
+/**
+ * Load pre-rendered web pages using a headless browser hosted on Browserbase.
+ *
+ * Depends on `@browserbasehq/sdk` package.
+ * Get your API key from https://browserbase.com
+ *
+ * @example
+ * ```typescript
+ * import { BrowserbaseLoader } from "langchain/document_loaders/web/browserbase";
+ *
+ * const loader = new BrowserbaseLoader(["https://example.com"], {
+ *   apiKey: process.env.BROWSERBASE_API_KEY,
+ *   textContent: true,
+ * });
+ *
+ * const docs = await loader.load();
+ * ```
+ *
+ * @param {string[]} urls - The URLs of the web pages to load.
+ * @param {BrowserbaseLoaderOptions} [options] - Browserbase client options.
+ */
+export class BrowserbaseLoader extends BaseDocumentLoader {
+    constructor(urls, options = {}) {
+        super();
+        Object.defineProperty(this, "urls", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "browserbase", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.urls = urls;
+        this.options = options;
+        this.browserbase = new Browserbase(options.apiKey);
+    }
+    /**
+     * Load pages from URLs.
+     *
+     * @returns {Promise<DocumentInterface[]>} - A promise which resolves to a list of documents.
+     */
+    async load() {
+        const documents = [];
+        for await (const doc of this.lazyLoad()) {
+            documents.push(doc);
+        }
+        return documents;
+    }
+    /**
+     * Load pages from URLs.
+     *
+     * @returns {Generator<DocumentInterface>} - A generator that yields documents.
+     */
+    async *lazyLoad() {
+        const pages = await this.browserbase.loadURLs(this.urls, this.options);
+        let index = 0;
+        for await (const page of pages) {
+            yield new Document({
+                pageContent: page,
+                metadata: {
+                    url: this.urls[index],
+                },
+            });
+            index += index + 1;
+        }
+    }
+}

package/dist/document_loaders/web/firecrawl.cjs

@@ -0,0 +1,88 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FireCrawlLoader = void 0;
+const firecrawl_js_1 = __importDefault(require("@mendable/firecrawl-js"));
+const documents_1 = require("@langchain/core/documents");
+const env_1 = require("@langchain/core/utils/env");
+const base_js_1 = require("../base.cjs");
+/**
+ * Class representing a document loader for loading data from
+ * Firecrawl (firecrawl.dev). It extends the BaseDocumentLoader class.
+ * @example
+ * ```typescript
+ * const loader = new FireCrawlLoader({
+ *   url: "{url}",
+ *   apiKey: "{apiKey}",
+ *   mode: "crawl"
+ * });
+ * const docs = await loader.load();
+ * ```
+ */
+class FireCrawlLoader extends base_js_1.BaseDocumentLoader {
+    constructor(loaderParams) {
+        super();
+        Object.defineProperty(this, "apiKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "mode", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "params", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { apiKey = (0, env_1.getEnvironmentVariable)("FIRECRAWL_API_KEY"), url, mode = "crawl", params, } = loaderParams;
+        if (!apiKey) {
+            throw new Error("Firecrawl API key not set. You can set it as FIRECRAWL_API_KEY in your .env file, or pass it to Firecrawl.");
+        }
+        this.apiKey = apiKey;
+        this.url = url;
+        this.mode = mode;
+        this.params = params;
+    }
+    /**
+     * Loads the data from the Firecrawl.
+     * @returns An array of Documents representing the retrieved data.
+     * @throws An error if the data could not be loaded.
+     */
+    async load() {
+        const app = new firecrawl_js_1.default({ apiKey: this.apiKey });
+        let firecrawlDocs;
+        if (this.mode === "scrape") {
+            const response = await app.scrapeUrl(this.url, this.params);
+            if (!response.success) {
+                throw new Error(`Firecrawl: Failed to scrape URL. Error: ${response.error}`);
+            }
+            firecrawlDocs = [response.data];
+        }
+        else if (this.mode === "crawl") {
+            const response = await app.crawlUrl(this.url, this.params, true);
+            firecrawlDocs = response;
+        }
+        else {
+            throw new Error(`Unrecognized mode '${this.mode}'. Expected one of 'crawl', 'scrape'.`);
+        }
+        return firecrawlDocs.map((doc) => new documents_1.Document({
+            pageContent: doc.markdown || "",
+            metadata: doc.metadata || {},
+        }));
+    }
+}
+exports.FireCrawlLoader = FireCrawlLoader;

package/dist/document_loaders/web/firecrawl.d.ts

@@ -0,0 +1,48 @@
+import { type DocumentInterface } from "@langchain/core/documents";
+import { BaseDocumentLoader } from "../base.js";
+/**
+ * Interface representing the parameters for the Firecrawl loader. It
+ * includes properties such as the URL to scrape or crawl and the API key.
+ */
+interface FirecrawlLoaderParameters {
+    /**
+     * URL to scrape or crawl
+     */
+    url: string;
+    /**
+     * API key for Firecrawl. If not provided, the default value is the value of the FIRECRAWL_API_KEY environment variable.
+     */
+    apiKey?: string;
+    /**
+     * Mode of operation. Can be either "crawl" or "scrape". If not provided, the default value is "crawl".
+     */
+    mode?: "crawl" | "scrape";
+    params?: Record<string, unknown>;
+}
+/**
+ * Class representing a document loader for loading data from
+ * Firecrawl (firecrawl.dev). It extends the BaseDocumentLoader class.
+ * @example
+ * ```typescript
+ * const loader = new FireCrawlLoader({
+ *   url: "{url}",
+ *   apiKey: "{apiKey}",
+ *   mode: "crawl"
+ * });
+ * const docs = await loader.load();
+ * ```
+ */
+export declare class FireCrawlLoader extends BaseDocumentLoader {
+    private apiKey;
+    private url;
+    private mode;
+    private params?;
+    constructor(loaderParams: FirecrawlLoaderParameters);
+    /**
+     * Loads the data from the Firecrawl.
+     * @returns An array of Documents representing the retrieved data.
+     * @throws An error if the data could not be loaded.
+     */
+    load(): Promise<DocumentInterface[]>;
+}
+export {};

package/dist/document_loaders/web/firecrawl.js

@@ -0,0 +1,81 @@
+import FirecrawlApp from "@mendable/firecrawl-js";
+import { Document } from "@langchain/core/documents";
+import { getEnvironmentVariable } from "@langchain/core/utils/env";
+import { BaseDocumentLoader } from "../base.js";
+/**
+ * Class representing a document loader for loading data from
+ * Firecrawl (firecrawl.dev). It extends the BaseDocumentLoader class.
+ * @example
+ * ```typescript
+ * const loader = new FireCrawlLoader({
+ *   url: "{url}",
+ *   apiKey: "{apiKey}",
+ *   mode: "crawl"
+ * });
+ * const docs = await loader.load();
+ * ```
+ */
+export class FireCrawlLoader extends BaseDocumentLoader {
+    constructor(loaderParams) {
+        super();
+        Object.defineProperty(this, "apiKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "mode", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "params", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { apiKey = getEnvironmentVariable("FIRECRAWL_API_KEY"), url, mode = "crawl", params, } = loaderParams;
+        if (!apiKey) {
+            throw new Error("Firecrawl API key not set. You can set it as FIRECRAWL_API_KEY in your .env file, or pass it to Firecrawl.");
+        }
+        this.apiKey = apiKey;
+        this.url = url;
+        this.mode = mode;
+        this.params = params;
+    }
+    /**
+     * Loads the data from the Firecrawl.
+     * @returns An array of Documents representing the retrieved data.
+     * @throws An error if the data could not be loaded.
+     */
+    async load() {
+        const app = new FirecrawlApp({ apiKey: this.apiKey });
+        let firecrawlDocs;
+        if (this.mode === "scrape") {
+            const response = await app.scrapeUrl(this.url, this.params);
+            if (!response.success) {
+                throw new Error(`Firecrawl: Failed to scrape URL. Error: ${response.error}`);
+            }
+            firecrawlDocs = [response.data];
+        }
+        else if (this.mode === "crawl") {
+            const response = await app.crawlUrl(this.url, this.params, true);
+            firecrawlDocs = response;
+        }
+        else {
+            throw new Error(`Unrecognized mode '${this.mode}'. Expected one of 'crawl', 'scrape'.`);
+        }
+        return firecrawlDocs.map((doc) => new Document({
+            pageContent: doc.markdown || "",
+            metadata: doc.metadata || {},
+        }));
+    }
+}

package/dist/document_loaders/web/s3.cjs

@@ -148,8 +148,8 @@ class S3Loader extends base_js_1.BaseDocumentLoader {
             const docs = await unstructuredLoader.load();
             return docs;
         }
-        catch {
-            throw new Error(`Failed to load file ${filePath} using unstructured loader.`);
+        catch (e) {
+            throw new Error(`Failed to load file ${filePath} using unstructured loader: ${e.message}`);
         }
     }
 }

package/dist/document_loaders/web/s3.js

@@ -122,8 +122,8 @@ export class S3Loader extends BaseDocumentLoader {
             const docs = await unstructuredLoader.load();
             return docs;
         }
-        catch {
-            throw new Error(`Failed to load file ${filePath} using unstructured loader.`);
+        catch (e) {
+            throw new Error(`Failed to load file ${filePath} using unstructured loader: ${e.message}`);
         }
     }
 }

package/dist/load/import_constants.cjs

@@ -85,6 +85,7 @@ exports.optionalImportEntrypoints = [
     "langchain/document_loaders/web/assemblyai",
     "langchain/document_loaders/web/azure_blob_storage_container",
     "langchain/document_loaders/web/azure_blob_storage_file",
+    "langchain/document_loaders/web/browserbase",
     "langchain/document_loaders/web/cheerio",
     "langchain/document_loaders/web/puppeteer",
     "langchain/document_loaders/web/playwright",
@@ -93,6 +94,7 @@ exports.optionalImportEntrypoints = [
     "langchain/document_loaders/web/hn",
     "langchain/document_loaders/web/imsdb",
     "langchain/document_loaders/web/figma",
+    "langchain/document_loaders/web/firecrawl",
     "langchain/document_loaders/web/github",
     "langchain/document_loaders/web/notiondb",
     "langchain/document_loaders/web/notionapi",

package/dist/load/import_constants.js

@@ -82,6 +82,7 @@ export const optionalImportEntrypoints = [
     "langchain/document_loaders/web/assemblyai",
     "langchain/document_loaders/web/azure_blob_storage_container",
     "langchain/document_loaders/web/azure_blob_storage_file",
+    "langchain/document_loaders/web/browserbase",
     "langchain/document_loaders/web/cheerio",
     "langchain/document_loaders/web/puppeteer",
     "langchain/document_loaders/web/playwright",
@@ -90,6 +91,7 @@ export const optionalImportEntrypoints = [
     "langchain/document_loaders/web/hn",
     "langchain/document_loaders/web/imsdb",
     "langchain/document_loaders/web/figma",
+    "langchain/document_loaders/web/firecrawl",
     "langchain/document_loaders/web/github",
     "langchain/document_loaders/web/notiondb",
     "langchain/document_loaders/web/notionapi",

package/dist/output_parsers/expression.cjs

@@ -19,7 +19,7 @@ const output_parsers_1 = require("@langchain/core/output_parsers");
 const factory_js_1 = require("./expression_type_handlers/factory.cjs");
 const base_js_1 = require("./expression_type_handlers/base.cjs");
 /**
- * okay so we need to be able to handle the following cases:
+ * We need to be able to handle the following cases:
  * ExpressionStatement
  *  CallExpression
  *      Identifier | MemberExpression

package/dist/output_parsers/expression.d.ts

@@ -1,7 +1,7 @@
 import { BaseOutputParser } from "@langchain/core/output_parsers";
 import { ParsedType } from "./expression_type_handlers/types.js";
 /**
- * okay so we need to be able to handle the following cases:
+ * We need to be able to handle the following cases:
  * ExpressionStatement
  *  CallExpression
  *      Identifier | MemberExpression

package/dist/output_parsers/expression.js

@@ -2,7 +2,7 @@ import { BaseOutputParser } from "@langchain/core/output_parsers";
 import { MasterHandler } from "./expression_type_handlers/factory.js";
 import { ASTParser } from "./expression_type_handlers/base.js";
 /**
- * okay so we need to be able to handle the following cases:
+ * We need to be able to handle the following cases:
  * ExpressionStatement
  *  CallExpression
  *      Identifier | MemberExpression

package/dist/retrievers/self_query/base.cjs

@@ -1,139 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BasicTranslator = exports.BaseTranslator = void 0;
-const ir_js_1 = require("../../chains/query_constructor/ir.cjs");
-const utils_js_1 = require("./utils.cjs");
-/**
- * Abstract class that provides a blueprint for creating specific
- * translator classes. Defines two abstract methods: formatFunction and
- * mergeFilters.
- */
-class BaseTranslator extends ir_js_1.Visitor {
-}
-exports.BaseTranslator = BaseTranslator;
-/**
- * Class that extends the BaseTranslator class and provides concrete
- * implementations for the abstract methods. Also declares three types:
- * VisitOperationOutput, VisitComparisonOutput, and
- * VisitStructuredQueryOutput, which are used as the return types for the
- * visitOperation, visitComparison, and visitStructuredQuery methods
- * respectively.
- */
-class BasicTranslator extends BaseTranslator {
-    constructor(opts) {
-        super();
-        Object.defineProperty(this, "allowedOperators", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "allowedComparators", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.allowedOperators = opts?.allowedOperators ?? [
-            ir_js_1.Operators.and,
-            ir_js_1.Operators.or,
-        ];
-        this.allowedComparators = opts?.allowedComparators ?? [
-            ir_js_1.Comparators.eq,
-            ir_js_1.Comparators.ne,
-            ir_js_1.Comparators.gt,
-            ir_js_1.Comparators.gte,
-            ir_js_1.Comparators.lt,
-            ir_js_1.Comparators.lte,
-        ];
-    }
-    formatFunction(func) {
-        if (func in ir_js_1.Comparators) {
-            if (this.allowedComparators.length > 0 &&
-                this.allowedComparators.indexOf(func) === -1) {
-                throw new Error(`Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join(", ")}`);
-            }
-        }
-        else if (func in ir_js_1.Operators) {
-            if (this.allowedOperators.length > 0 &&
-                this.allowedOperators.indexOf(func) === -1) {
-                throw new Error(`Operator ${func} not allowed. Allowed operators: ${this.allowedOperators.join(", ")}`);
-            }
-        }
-        else {
-            throw new Error("Unknown comparator or operator");
-        }
-        return `$${func}`;
-    }
-    /**
-     * Visits an operation and returns a result.
-     * @param operation The operation to visit.
-     * @returns The result of visiting the operation.
-     */
-    visitOperation(operation) {
-        const args = operation.args?.map((arg) => arg.accept(this));
-        return {
-            [this.formatFunction(operation.operator)]: args,
-        };
-    }
-    /**
-     * Visits a comparison and returns a result.
-     * @param comparison The comparison to visit.
-     * @returns The result of visiting the comparison.
-     */
-    visitComparison(comparison) {
-        return {
-            [comparison.attribute]: {
-                [this.formatFunction(comparison.comparator)]: (0, utils_js_1.castValue)(comparison.value),
-            },
-        };
-    }
-    /**
-     * Visits a structured query and returns a result.
-     * @param query The structured query to visit.
-     * @returns The result of visiting the structured query.
-     */
-    visitStructuredQuery(query) {
-        let nextArg = {};
-        if (query.filter) {
-            nextArg = {
-                filter: query.filter.accept(this),
-            };
-        }
-        return nextArg;
-    }
-    mergeFilters(defaultFilter, generatedFilter, mergeType = "and", forceDefaultFilter = false) {
-        if ((0, utils_js_1.isFilterEmpty)(defaultFilter) && (0, utils_js_1.isFilterEmpty)(generatedFilter)) {
-            return undefined;
-        }
-        if ((0, utils_js_1.isFilterEmpty)(defaultFilter) || mergeType === "replace") {
-            if ((0, utils_js_1.isFilterEmpty)(generatedFilter)) {
-                return undefined;
-            }
-            return generatedFilter;
-        }
-        if ((0, utils_js_1.isFilterEmpty)(generatedFilter)) {
-            if (forceDefaultFilter) {
-                return defaultFilter;
-            }
-            if (mergeType === "and") {
-                return undefined;
-            }
-            return defaultFilter;
-        }
-        if (mergeType === "and") {
-            return {
-                $and: [defaultFilter, generatedFilter],
-            };
-        }
-        else if (mergeType === "or") {
-            return {
-                $or: [defaultFilter, generatedFilter],
-            };
-        }
-        else {
-            throw new Error("Unknown merge type");
-        }
-    }
-}
-exports.BasicTranslator = BasicTranslator;
+var structured_query_1 = require("@langchain/core/structured_query");
+Object.defineProperty(exports, "BaseTranslator", { enumerable: true, get: function () { return structured_query_1.BaseTranslator; } });
+Object.defineProperty(exports, "BasicTranslator", { enumerable: true, get: function () { return structured_query_1.BasicTranslator; } });

package/dist/retrievers/self_query/base.d.ts

@@ -1,69 +1 @@
-import { VectorStore } from "@langchain/core/vectorstores";
-import { Comparator, Comparison, Operation, Operator, StructuredQuery, Visitor, VisitorComparisonResult, VisitorOperationResult, VisitorStructuredQueryResult } from "../../chains/query_constructor/ir.js";
-/**
- * Options object for the BasicTranslator class. Specifies the allowed
- * operators and comparators.
- */
-export type TranslatorOpts = {
-    allowedOperators: Operator[];
-    allowedComparators: Comparator[];
-};
-/**
- * Abstract class that provides a blueprint for creating specific
- * translator classes. Defines two abstract methods: formatFunction and
- * mergeFilters.
- */
-export declare abstract class BaseTranslator<T extends VectorStore = VectorStore> extends Visitor<T> {
-    /**
-     * Formats a given function (either an operator or a comparator) into a
-     * string.
-     * @param func The function to format.
-     * @returns Formatted string representation of the function.
-     */
-    abstract formatFunction(func: Operator | Comparator): string;
-    /**
-     * Merges two filters into one, using a specified merge type.
-     * @param defaultFilter The default filter.
-     * @param generatedFilter The generated filter.
-     * @param mergeType The type of merge to perform. Can be 'and', 'or', or 'replace'.
-     * @param forceDefaultFilter If true, the default filter will be used even if the generated filter is not empty.
-     * @returns The merged filter, or undefined if both filters are empty.
-     */
-    abstract mergeFilters(defaultFilter: this["VisitStructuredQueryOutput"]["filter"] | undefined, generatedFilter: this["VisitStructuredQueryOutput"]["filter"] | undefined, mergeType?: "and" | "or" | "replace", forceDefaultFilter?: boolean): this["VisitStructuredQueryOutput"]["filter"] | undefined;
-}
-/**
- * Class that extends the BaseTranslator class and provides concrete
- * implementations for the abstract methods. Also declares three types:
- * VisitOperationOutput, VisitComparisonOutput, and
- * VisitStructuredQueryOutput, which are used as the return types for the
- * visitOperation, visitComparison, and visitStructuredQuery methods
- * respectively.
- */
-export declare class BasicTranslator<T extends VectorStore = VectorStore> extends BaseTranslator<T> {
-    VisitOperationOutput: VisitorOperationResult;
-    VisitComparisonOutput: VisitorComparisonResult;
-    VisitStructuredQueryOutput: VisitorStructuredQueryResult;
-    allowedOperators: Operator[];
-    allowedComparators: Comparator[];
-    constructor(opts?: TranslatorOpts);
-    formatFunction(func: Operator | Comparator): string;
-    /**
-     * Visits an operation and returns a result.
-     * @param operation The operation to visit.
-     * @returns The result of visiting the operation.
-     */
-    visitOperation(operation: Operation): this["VisitOperationOutput"];
-    /**
-     * Visits a comparison and returns a result.
-     * @param comparison The comparison to visit.
-     * @returns The result of visiting the comparison.
-     */
-    visitComparison(comparison: Comparison): this["VisitComparisonOutput"];
-    /**
-     * Visits a structured query and returns a result.
-     * @param query The structured query to visit.
-     * @returns The result of visiting the structured query.
-     */
-    visitStructuredQuery(query: StructuredQuery): this["VisitStructuredQueryOutput"];
-    mergeFilters(defaultFilter: VisitorStructuredQueryResult["filter"] | undefined, generatedFilter: VisitorStructuredQueryResult["filter"] | undefined, mergeType?: string, forceDefaultFilter?: boolean): VisitorStructuredQueryResult["filter"] | undefined;
-}
+export { type TranslatorOpts, BaseTranslator, BasicTranslator, } from "@langchain/core/structured_query";