langchain 0.1.35 → 0.1.36

package/dist/document_loaders/web/firecrawl.d.ts

@@ -0,0 +1,48 @@
+import { type DocumentInterface } from "@langchain/core/documents";
+import { BaseDocumentLoader } from "../base.js";
+/**
+ * Interface representing the parameters for the Firecrawl loader. It
+ * includes properties such as the URL to scrape or crawl and the API key.
+ */
+interface FirecrawlLoaderParameters {
+    /**
+     * URL to scrape or crawl
+     */
+    url: string;
+    /**
+     * API key for Firecrawl. If not provided, the default value is the value of the FIRECRAWL_API_KEY environment variable.
+     */
+    apiKey?: string;
+    /**
+     * Mode of operation. Can be either "crawl" or "scrape". If not provided, the default value is "crawl".
+     */
+    mode?: "crawl" | "scrape";
+    params?: Record<string, unknown>;
+}
+/**
+ * Class representing a document loader for loading data from
+ * Firecrawl (firecrawl.dev). It extends the BaseDocumentLoader class.
+ * @example
+ * ```typescript
+ * const loader = new FireCrawlLoader({
+ *   url: "{url}",
+ *   apiKey: "{apiKey}",
+ *   mode: "crawl"
+ * });
+ * const docs = await loader.load();
+ * ```
+ */
+export declare class FireCrawlLoader extends BaseDocumentLoader {
+    private apiKey;
+    private url;
+    private mode;
+    private params?;
+    constructor(loaderParams: FirecrawlLoaderParameters);
+    /**
+     * Loads the data from the Firecrawl.
+     * @returns An array of Documents representing the retrieved data.
+     * @throws An error if the data could not be loaded.
+     */
+    load(): Promise<DocumentInterface[]>;
+}
+export {};

package/dist/document_loaders/web/firecrawl.js

@@ -0,0 +1,81 @@
+import FirecrawlApp from "@mendable/firecrawl-js";
+import { Document } from "@langchain/core/documents";
+import { getEnvironmentVariable } from "@langchain/core/utils/env";
+import { BaseDocumentLoader } from "../base.js";
+/**
+ * Class representing a document loader for loading data from
+ * Firecrawl (firecrawl.dev). It extends the BaseDocumentLoader class.
+ * @example
+ * ```typescript
+ * const loader = new FireCrawlLoader({
+ *   url: "{url}",
+ *   apiKey: "{apiKey}",
+ *   mode: "crawl"
+ * });
+ * const docs = await loader.load();
+ * ```
+ */
+export class FireCrawlLoader extends BaseDocumentLoader {
+    constructor(loaderParams) {
+        super();
+        Object.defineProperty(this, "apiKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "mode", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "params", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        const { apiKey = getEnvironmentVariable("FIRECRAWL_API_KEY"), url, mode = "crawl", params, } = loaderParams;
+        if (!apiKey) {
+            throw new Error("Firecrawl API key not set. You can set it as FIRECRAWL_API_KEY in your .env file, or pass it to Firecrawl.");
+        }
+        this.apiKey = apiKey;
+        this.url = url;
+        this.mode = mode;
+        this.params = params;
+    }
+    /**
+     * Loads the data from the Firecrawl.
+     * @returns An array of Documents representing the retrieved data.
+     * @throws An error if the data could not be loaded.
+     */
+    async load() {
+        const app = new FirecrawlApp({ apiKey: this.apiKey });
+        let firecrawlDocs;
+        if (this.mode === "scrape") {
+            const response = await app.scrapeUrl(this.url, this.params);
+            if (!response.success) {
+                throw new Error(`Firecrawl: Failed to scrape URL. Error: ${response.error}`);
+            }
+            firecrawlDocs = [response.data];
+        }
+        else if (this.mode === "crawl") {
+            const response = await app.crawlUrl(this.url, this.params, true);
+            firecrawlDocs = response;
+        }
+        else {
+            throw new Error(`Unrecognized mode '${this.mode}'. Expected one of 'crawl', 'scrape'.`);
+        }
+        return firecrawlDocs.map((doc) => new Document({
+            pageContent: doc.markdown || "",
+            metadata: doc.metadata || {},
+        }));
+    }
+}

package/dist/load/import_constants.cjs

@@ -93,6 +93,7 @@ exports.optionalImportEntrypoints = [
     "langchain/document_loaders/web/hn",
     "langchain/document_loaders/web/imsdb",
     "langchain/document_loaders/web/figma",
+    "langchain/document_loaders/web/firecrawl",
     "langchain/document_loaders/web/github",
     "langchain/document_loaders/web/notiondb",
     "langchain/document_loaders/web/notionapi",

package/dist/load/import_constants.js

@@ -90,6 +90,7 @@ export const optionalImportEntrypoints = [
     "langchain/document_loaders/web/hn",
     "langchain/document_loaders/web/imsdb",
     "langchain/document_loaders/web/figma",
+    "langchain/document_loaders/web/firecrawl",
     "langchain/document_loaders/web/github",
     "langchain/document_loaders/web/notiondb",
     "langchain/document_loaders/web/notionapi",

package/dist/output_parsers/expression.cjs

@@ -19,7 +19,7 @@ const output_parsers_1 = require("@langchain/core/output_parsers");
 const factory_js_1 = require("./expression_type_handlers/factory.cjs");
 const base_js_1 = require("./expression_type_handlers/base.cjs");
 /**
- * okay so we need to be able to handle the following cases:
+ * We need to be able to handle the following cases:
  * ExpressionStatement
  *  CallExpression
  *      Identifier | MemberExpression

package/dist/output_parsers/expression.d.ts

@@ -1,7 +1,7 @@
 import { BaseOutputParser } from "@langchain/core/output_parsers";
 import { ParsedType } from "./expression_type_handlers/types.js";
 /**
- * okay so we need to be able to handle the following cases:
+ * We need to be able to handle the following cases:
  * ExpressionStatement
  *  CallExpression
  *      Identifier | MemberExpression

package/dist/output_parsers/expression.js

@@ -2,7 +2,7 @@ import { BaseOutputParser } from "@langchain/core/output_parsers";
 import { MasterHandler } from "./expression_type_handlers/factory.js";
 import { ASTParser } from "./expression_type_handlers/base.js";
 /**
- * okay so we need to be able to handle the following cases:
+ * We need to be able to handle the following cases:
  * ExpressionStatement
  *  CallExpression
  *      Identifier | MemberExpression

package/dist/retrievers/self_query/base.cjs

@@ -1,139 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BasicTranslator = exports.BaseTranslator = void 0;
-const ir_js_1 = require("../../chains/query_constructor/ir.cjs");
-const utils_js_1 = require("./utils.cjs");
-/**
- * Abstract class that provides a blueprint for creating specific
- * translator classes. Defines two abstract methods: formatFunction and
- * mergeFilters.
- */
-class BaseTranslator extends ir_js_1.Visitor {
-}
-exports.BaseTranslator = BaseTranslator;
-/**
- * Class that extends the BaseTranslator class and provides concrete
- * implementations for the abstract methods. Also declares three types:
- * VisitOperationOutput, VisitComparisonOutput, and
- * VisitStructuredQueryOutput, which are used as the return types for the
- * visitOperation, visitComparison, and visitStructuredQuery methods
- * respectively.
- */
-class BasicTranslator extends BaseTranslator {
-    constructor(opts) {
-        super();
-        Object.defineProperty(this, "allowedOperators", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "allowedComparators", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.allowedOperators = opts?.allowedOperators ?? [
-            ir_js_1.Operators.and,
-            ir_js_1.Operators.or,
-        ];
-        this.allowedComparators = opts?.allowedComparators ?? [
-            ir_js_1.Comparators.eq,
-            ir_js_1.Comparators.ne,
-            ir_js_1.Comparators.gt,
-            ir_js_1.Comparators.gte,
-            ir_js_1.Comparators.lt,
-            ir_js_1.Comparators.lte,
-        ];
-    }
-    formatFunction(func) {
-        if (func in ir_js_1.Comparators) {
-            if (this.allowedComparators.length > 0 &&
-                this.allowedComparators.indexOf(func) === -1) {
-                throw new Error(`Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join(", ")}`);
-            }
-        }
-        else if (func in ir_js_1.Operators) {
-            if (this.allowedOperators.length > 0 &&
-                this.allowedOperators.indexOf(func) === -1) {
-                throw new Error(`Operator ${func} not allowed. Allowed operators: ${this.allowedOperators.join(", ")}`);
-            }
-        }
-        else {
-            throw new Error("Unknown comparator or operator");
-        }
-        return `$${func}`;
-    }
-    /**
-     * Visits an operation and returns a result.
-     * @param operation The operation to visit.
-     * @returns The result of visiting the operation.
-     */
-    visitOperation(operation) {
-        const args = operation.args?.map((arg) => arg.accept(this));
-        return {
-            [this.formatFunction(operation.operator)]: args,
-        };
-    }
-    /**
-     * Visits a comparison and returns a result.
-     * @param comparison The comparison to visit.
-     * @returns The result of visiting the comparison.
-     */
-    visitComparison(comparison) {
-        return {
-            [comparison.attribute]: {
-                [this.formatFunction(comparison.comparator)]: (0, utils_js_1.castValue)(comparison.value),
-            },
-        };
-    }
-    /**
-     * Visits a structured query and returns a result.
-     * @param query The structured query to visit.
-     * @returns The result of visiting the structured query.
-     */
-    visitStructuredQuery(query) {
-        let nextArg = {};
-        if (query.filter) {
-            nextArg = {
-                filter: query.filter.accept(this),
-            };
-        }
-        return nextArg;
-    }
-    mergeFilters(defaultFilter, generatedFilter, mergeType = "and", forceDefaultFilter = false) {
-        if ((0, utils_js_1.isFilterEmpty)(defaultFilter) && (0, utils_js_1.isFilterEmpty)(generatedFilter)) {
-            return undefined;
-        }
-        if ((0, utils_js_1.isFilterEmpty)(defaultFilter) || mergeType === "replace") {
-            if ((0, utils_js_1.isFilterEmpty)(generatedFilter)) {
-                return undefined;
-            }
-            return generatedFilter;
-        }
-        if ((0, utils_js_1.isFilterEmpty)(generatedFilter)) {
-            if (forceDefaultFilter) {
-                return defaultFilter;
-            }
-            if (mergeType === "and") {
-                return undefined;
-            }
-            return defaultFilter;
-        }
-        if (mergeType === "and") {
-            return {
-                $and: [defaultFilter, generatedFilter],
-            };
-        }
-        else if (mergeType === "or") {
-            return {
-                $or: [defaultFilter, generatedFilter],
-            };
-        }
-        else {
-            throw new Error("Unknown merge type");
-        }
-    }
-}
-exports.BasicTranslator = BasicTranslator;
+var structured_query_1 = require("@langchain/core/structured_query");
+Object.defineProperty(exports, "BaseTranslator", { enumerable: true, get: function () { return structured_query_1.BaseTranslator; } });
+Object.defineProperty(exports, "BasicTranslator", { enumerable: true, get: function () { return structured_query_1.BasicTranslator; } });

package/dist/retrievers/self_query/base.d.ts

@@ -1,69 +1 @@
-import { VectorStore } from "@langchain/core/vectorstores";
-import { Comparator, Comparison, Operation, Operator, StructuredQuery, Visitor, VisitorComparisonResult, VisitorOperationResult, VisitorStructuredQueryResult } from "../../chains/query_constructor/ir.js";
-/**
- * Options object for the BasicTranslator class. Specifies the allowed
- * operators and comparators.
- */
-export type TranslatorOpts = {
-    allowedOperators: Operator[];
-    allowedComparators: Comparator[];
-};
-/**
- * Abstract class that provides a blueprint for creating specific
- * translator classes. Defines two abstract methods: formatFunction and
- * mergeFilters.
- */
-export declare abstract class BaseTranslator<T extends VectorStore = VectorStore> extends Visitor<T> {
-    /**
-     * Formats a given function (either an operator or a comparator) into a
-     * string.
-     * @param func The function to format.
-     * @returns Formatted string representation of the function.
-     */
-    abstract formatFunction(func: Operator | Comparator): string;
-    /**
-     * Merges two filters into one, using a specified merge type.
-     * @param defaultFilter The default filter.
-     * @param generatedFilter The generated filter.
-     * @param mergeType The type of merge to perform. Can be 'and', 'or', or 'replace'.
-     * @param forceDefaultFilter If true, the default filter will be used even if the generated filter is not empty.
-     * @returns The merged filter, or undefined if both filters are empty.
-     */
-    abstract mergeFilters(defaultFilter: this["VisitStructuredQueryOutput"]["filter"] | undefined, generatedFilter: this["VisitStructuredQueryOutput"]["filter"] | undefined, mergeType?: "and" | "or" | "replace", forceDefaultFilter?: boolean): this["VisitStructuredQueryOutput"]["filter"] | undefined;
-}
-/**
- * Class that extends the BaseTranslator class and provides concrete
- * implementations for the abstract methods. Also declares three types:
- * VisitOperationOutput, VisitComparisonOutput, and
- * VisitStructuredQueryOutput, which are used as the return types for the
- * visitOperation, visitComparison, and visitStructuredQuery methods
- * respectively.
- */
-export declare class BasicTranslator<T extends VectorStore = VectorStore> extends BaseTranslator<T> {
-    VisitOperationOutput: VisitorOperationResult;
-    VisitComparisonOutput: VisitorComparisonResult;
-    VisitStructuredQueryOutput: VisitorStructuredQueryResult;
-    allowedOperators: Operator[];
-    allowedComparators: Comparator[];
-    constructor(opts?: TranslatorOpts);
-    formatFunction(func: Operator | Comparator): string;
-    /**
-     * Visits an operation and returns a result.
-     * @param operation The operation to visit.
-     * @returns The result of visiting the operation.
-     */
-    visitOperation(operation: Operation): this["VisitOperationOutput"];
-    /**
-     * Visits a comparison and returns a result.
-     * @param comparison The comparison to visit.
-     * @returns The result of visiting the comparison.
-     */
-    visitComparison(comparison: Comparison): this["VisitComparisonOutput"];
-    /**
-     * Visits a structured query and returns a result.
-     * @param query The structured query to visit.
-     * @returns The result of visiting the structured query.
-     */
-    visitStructuredQuery(query: StructuredQuery): this["VisitStructuredQueryOutput"];
-    mergeFilters(defaultFilter: VisitorStructuredQueryResult["filter"] | undefined, generatedFilter: VisitorStructuredQueryResult["filter"] | undefined, mergeType?: string, forceDefaultFilter?: boolean): VisitorStructuredQueryResult["filter"] | undefined;
-}
+export { type TranslatorOpts, BaseTranslator, BasicTranslator, } from "@langchain/core/structured_query";

package/dist/retrievers/self_query/base.js

@@ -1,134 +1 @@
-import { Comparators, Operators, Visitor, } from "../../chains/query_constructor/ir.js";
-import { isFilterEmpty, castValue } from "./utils.js";
-/**
- * Abstract class that provides a blueprint for creating specific
- * translator classes. Defines two abstract methods: formatFunction and
- * mergeFilters.
- */
-export class BaseTranslator extends Visitor {
-}
-/**
- * Class that extends the BaseTranslator class and provides concrete
- * implementations for the abstract methods. Also declares three types:
- * VisitOperationOutput, VisitComparisonOutput, and
- * VisitStructuredQueryOutput, which are used as the return types for the
- * visitOperation, visitComparison, and visitStructuredQuery methods
- * respectively.
- */
-export class BasicTranslator extends BaseTranslator {
-    constructor(opts) {
-        super();
-        Object.defineProperty(this, "allowedOperators", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "allowedComparators", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.allowedOperators = opts?.allowedOperators ?? [
-            Operators.and,
-            Operators.or,
-        ];
-        this.allowedComparators = opts?.allowedComparators ?? [
-            Comparators.eq,
-            Comparators.ne,
-            Comparators.gt,
-            Comparators.gte,
-            Comparators.lt,
-            Comparators.lte,
-        ];
-    }
-    formatFunction(func) {
-        if (func in Comparators) {
-            if (this.allowedComparators.length > 0 &&
-                this.allowedComparators.indexOf(func) === -1) {
-                throw new Error(`Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join(", ")}`);
-            }
-        }
-        else if (func in Operators) {
-            if (this.allowedOperators.length > 0 &&
-                this.allowedOperators.indexOf(func) === -1) {
-                throw new Error(`Operator ${func} not allowed. Allowed operators: ${this.allowedOperators.join(", ")}`);
-            }
-        }
-        else {
-            throw new Error("Unknown comparator or operator");
-        }
-        return `$${func}`;
-    }
-    /**
-     * Visits an operation and returns a result.
-     * @param operation The operation to visit.
-     * @returns The result of visiting the operation.
-     */
-    visitOperation(operation) {
-        const args = operation.args?.map((arg) => arg.accept(this));
-        return {
-            [this.formatFunction(operation.operator)]: args,
-        };
-    }
-    /**
-     * Visits a comparison and returns a result.
-     * @param comparison The comparison to visit.
-     * @returns The result of visiting the comparison.
-     */
-    visitComparison(comparison) {
-        return {
-            [comparison.attribute]: {
-                [this.formatFunction(comparison.comparator)]: castValue(comparison.value),
-            },
-        };
-    }
-    /**
-     * Visits a structured query and returns a result.
-     * @param query The structured query to visit.
-     * @returns The result of visiting the structured query.
-     */
-    visitStructuredQuery(query) {
-        let nextArg = {};
-        if (query.filter) {
-            nextArg = {
-                filter: query.filter.accept(this),
-            };
-        }
-        return nextArg;
-    }
-    mergeFilters(defaultFilter, generatedFilter, mergeType = "and", forceDefaultFilter = false) {
-        if (isFilterEmpty(defaultFilter) && isFilterEmpty(generatedFilter)) {
-            return undefined;
-        }
-        if (isFilterEmpty(defaultFilter) || mergeType === "replace") {
-            if (isFilterEmpty(generatedFilter)) {
-                return undefined;
-            }
-            return generatedFilter;
-        }
-        if (isFilterEmpty(generatedFilter)) {
-            if (forceDefaultFilter) {
-                return defaultFilter;
-            }
-            if (mergeType === "and") {
-                return undefined;
-            }
-            return defaultFilter;
-        }
-        if (mergeType === "and") {
-            return {
-                $and: [defaultFilter, generatedFilter],
-            };
-        }
-        else if (mergeType === "or") {
-            return {
-                $or: [defaultFilter, generatedFilter],
-            };
-        }
-        else {
-            throw new Error("Unknown merge type");
-        }
-    }
-}
+export { BaseTranslator, BasicTranslator, } from "@langchain/core/structured_query";

package/dist/retrievers/self_query/chroma.cjs

@@ -1,8 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ChromaTranslator = void 0;
-const ir_js_1 = require("../../chains/query_constructor/ir.cjs");
-const base_js_1 = require("./base.cjs");
+const structured_query_1 = require("@langchain/core/structured_query");
 /**
  * Specialized translator for the Chroma vector database. It extends the
  * BasicTranslator class and translates internal query language elements
@@ -24,17 +23,17 @@ const base_js_1 = require("./base.cjs");
  * );
  * ```
  */
-class ChromaTranslator extends base_js_1.BasicTranslator {
+class ChromaTranslator extends structured_query_1.BasicTranslator {
     constructor() {
         super({
-            allowedOperators: [ir_js_1.Operators.and, ir_js_1.Operators.or],
+            allowedOperators: [structured_query_1.Operators.and, structured_query_1.Operators.or],
             allowedComparators: [
-                ir_js_1.Comparators.eq,
-                ir_js_1.Comparators.ne,
-                ir_js_1.Comparators.gt,
-                ir_js_1.Comparators.gte,
-                ir_js_1.Comparators.lt,
-                ir_js_1.Comparators.lte,
+                structured_query_1.Comparators.eq,
+                structured_query_1.Comparators.ne,
+                structured_query_1.Comparators.gt,
+                structured_query_1.Comparators.gte,
+                structured_query_1.Comparators.lt,
+                structured_query_1.Comparators.lte,
             ],
         });
     }

package/dist/retrievers/self_query/chroma.d.ts

@@ -1,5 +1,5 @@
 import { Chroma } from "@langchain/community/vectorstores/chroma";
-import { BasicTranslator } from "./base.js";
+import { BasicTranslator } from "@langchain/core/structured_query";
 /**
  * Specialized translator for the Chroma vector database. It extends the
  * BasicTranslator class and translates internal query language elements

package/dist/retrievers/self_query/chroma.js

@@ -1,5 +1,4 @@
-import { Comparators, Operators } from "../../chains/query_constructor/ir.js";
-import { BasicTranslator } from "./base.js";
+import { BasicTranslator, Comparators, Operators, } from "@langchain/core/structured_query";
 /**
  * Specialized translator for the Chroma vector database. It extends the
  * BasicTranslator class and translates internal query language elements