langchain 0.2.6 → 0.2.7

package/dist/document_loaders/fs/multi_file.cjs

@@ -0,0 +1,98 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MultiFileLoader = void 0;
+const node_path_1 = require("node:path");
+const promises_1 = require("node:fs/promises");
+const base_js_1 = require("../base.cjs");
+const directory_js_1 = require("./directory.cjs");
+/**
+ * A document loader that loads documents from multiple files. It extends the
+ * `BaseDocumentLoader` class and implements the `load()` method.
+ * @example
+ * ```typescript
+ *
+ * const multiFileLoader = new MultiFileLoader(
+ *   ["path/to/file1.pdf", "path/to/file2.txt"],
+ *   {
+ *     ".pdf": (path: string) => new PDFLoader(path),
+ *   },
+ * );
+ *
+ * const docs = await multiFileLoader.load();
+ * console.log({ docs });
+ *
+ * ```
+ */
+class MultiFileLoader extends base_js_1.BaseDocumentLoader {
+    constructor(filePaths, loaders, unknown = directory_js_1.UnknownHandling.Warn) {
+        super();
+        Object.defineProperty(this, "filePaths", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: filePaths
+        });
+        Object.defineProperty(this, "loaders", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: loaders
+        });
+        Object.defineProperty(this, "unknown", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: unknown
+        });
+        if (Object.keys(loaders).length === 0) {
+            throw new Error("Must provide at least one loader");
+        }
+        for (const extension in loaders) {
+            if (Object.hasOwn(loaders, extension)) {
+                if (extension[0] !== ".") {
+                    throw new Error(`Extension must start with a dot: ${extension}`);
+                }
+            }
+        }
+    }
+    /**
+     * Loads the documents from the provided file paths. It checks if the file
+     * is a directory and ignores it. If a file is a file, it checks if there
+     * is a corresponding loader function for the file extension in the `loaders`
+     * mapping. If there is, it loads the documents. If there is no
+     * corresponding loader function and `unknown` is set to `Warn`, it logs a
+     * warning message. If `unknown` is set to `Error`, it throws an error.
+     * @returns A promise that resolves to an array of loaded documents.
+     */
+    async load() {
+        const documents = [];
+        for (const filePath of this.filePaths) {
+            const fullPath = (0, node_path_1.resolve)(filePath);
+            const fileStat = await (0, promises_1.stat)(fullPath);
+            if (fileStat.isDirectory()) {
+                console.warn(`Ignoring directory: ${fullPath}`);
+                continue;
+            }
+            const loaderFactory = this.loaders[(0, node_path_1.extname)(fullPath)];
+            if (loaderFactory) {
+                const loader = loaderFactory(fullPath);
+                documents.push(...(await loader.load()));
+            }
+            else {
+                switch (this.unknown) {
+                    case directory_js_1.UnknownHandling.Ignore:
+                        break;
+                    case directory_js_1.UnknownHandling.Warn:
+                        console.warn(`Unknown file type: ${fullPath}`);
+                        break;
+                    case directory_js_1.UnknownHandling.Error:
+                        throw new Error(`Unknown file type: ${fullPath}`);
+                    default:
+                        throw new Error(`Unknown unknown handling: ${this.unknown}`);
+                }
+            }
+        }
+        return documents;
+    }
+}
+exports.MultiFileLoader = MultiFileLoader;

package/dist/document_loaders/fs/multi_file.d.ts

@@ -0,0 +1,37 @@
+import { Document } from "@langchain/core/documents";
+import { BaseDocumentLoader } from "../base.js";
+import { type LoadersMapping, UnknownHandling } from "./directory.js";
+/**
+ * A document loader that loads documents from multiple files. It extends the
+ * `BaseDocumentLoader` class and implements the `load()` method.
+ * @example
+ * ```typescript
+ *
+ * const multiFileLoader = new MultiFileLoader(
+ *   ["path/to/file1.pdf", "path/to/file2.txt"],
+ *   {
+ *     ".pdf": (path: string) => new PDFLoader(path),
+ *   },
+ * );
+ *
+ * const docs = await multiFileLoader.load();
+ * console.log({ docs });
+ *
+ * ```
+ */
+export declare class MultiFileLoader extends BaseDocumentLoader {
+    filePaths: string[];
+    loaders: LoadersMapping;
+    unknown: UnknownHandling;
+    constructor(filePaths: string[], loaders: LoadersMapping, unknown?: UnknownHandling);
+    /**
+     * Loads the documents from the provided file paths. It checks if the file
+     * is a directory and ignores it. If a file is a file, it checks if there
+     * is a corresponding loader function for the file extension in the `loaders`
+     * mapping. If there is, it loads the documents. If there is no
+     * corresponding loader function and `unknown` is set to `Warn`, it logs a
+     * warning message. If `unknown` is set to `Error`, it throws an error.
+     * @returns A promise that resolves to an array of loaded documents.
+     */
+    load(): Promise<Document[]>;
+}

package/dist/document_loaders/fs/multi_file.js

@@ -0,0 +1,94 @@
+import { extname, resolve } from "node:path";
+import { stat } from "node:fs/promises";
+import { BaseDocumentLoader } from "../base.js";
+import { UnknownHandling } from "./directory.js";
+/**
+ * A document loader that loads documents from multiple files. It extends the
+ * `BaseDocumentLoader` class and implements the `load()` method.
+ * @example
+ * ```typescript
+ *
+ * const multiFileLoader = new MultiFileLoader(
+ *   ["path/to/file1.pdf", "path/to/file2.txt"],
+ *   {
+ *     ".pdf": (path: string) => new PDFLoader(path),
+ *   },
+ * );
+ *
+ * const docs = await multiFileLoader.load();
+ * console.log({ docs });
+ *
+ * ```
+ */
+export class MultiFileLoader extends BaseDocumentLoader {
+    constructor(filePaths, loaders, unknown = UnknownHandling.Warn) {
+        super();
+        Object.defineProperty(this, "filePaths", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: filePaths
+        });
+        Object.defineProperty(this, "loaders", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: loaders
+        });
+        Object.defineProperty(this, "unknown", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: unknown
+        });
+        if (Object.keys(loaders).length === 0) {
+            throw new Error("Must provide at least one loader");
+        }
+        for (const extension in loaders) {
+            if (Object.hasOwn(loaders, extension)) {
+                if (extension[0] !== ".") {
+                    throw new Error(`Extension must start with a dot: ${extension}`);
+                }
+            }
+        }
+    }
+    /**
+     * Loads the documents from the provided file paths. It checks if the file
+     * is a directory and ignores it. If a file is a file, it checks if there
+     * is a corresponding loader function for the file extension in the `loaders`
+     * mapping. If there is, it loads the documents. If there is no
+     * corresponding loader function and `unknown` is set to `Warn`, it logs a
+     * warning message. If `unknown` is set to `Error`, it throws an error.
+     * @returns A promise that resolves to an array of loaded documents.
+     */
+    async load() {
+        const documents = [];
+        for (const filePath of this.filePaths) {
+            const fullPath = resolve(filePath);
+            const fileStat = await stat(fullPath);
+            if (fileStat.isDirectory()) {
+                console.warn(`Ignoring directory: ${fullPath}`);
+                continue;
+            }
+            const loaderFactory = this.loaders[extname(fullPath)];
+            if (loaderFactory) {
+                const loader = loaderFactory(fullPath);
+                documents.push(...(await loader.load()));
+            }
+            else {
+                switch (this.unknown) {
+                    case UnknownHandling.Ignore:
+                        break;
+                    case UnknownHandling.Warn:
+                        console.warn(`Unknown file type: ${fullPath}`);
+                        break;
+                    case UnknownHandling.Error:
+                        throw new Error(`Unknown file type: ${fullPath}`);
+                    default:
+                        throw new Error(`Unknown unknown handling: ${this.unknown}`);
+                }
+            }
+        }
+        return documents;
+    }
+}

package/dist/document_loaders/tests/multi_file.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/document_loaders/tests/multi_file.test.js

@@ -0,0 +1,49 @@
+import * as url from "node:url";
+import * as path from "node:path";
+import { test, expect } from "@jest/globals";
+import { MultiFileLoader } from "../fs/multi_file.js";
+import { CSVLoader } from "../fs/csv.js";
+import { PDFLoader } from "../fs/pdf.js";
+import { TextLoader } from "../fs/text.js";
+import { JSONLoader } from "../fs/json.js";
+import { UnknownHandling } from "../fs/directory.js";
+test("Test MultiFileLoader", async () => {
+    const baseDirectory = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data");
+    const filePaths = [
+        path.resolve(baseDirectory, "1706.03762.pdf"),
+        path.resolve(baseDirectory, "Jacob_Lee_Resume_2023.pdf"),
+        path.resolve(baseDirectory, "Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.csv"),
+        path.resolve(baseDirectory, "Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.json"),
+        path.resolve(baseDirectory, "complex.json"),
+        path.resolve(baseDirectory, "example.txt"),
+        path.resolve(baseDirectory, "example_separator.csv"),
+    ];
+    const loader = new MultiFileLoader(filePaths, {
+        ".csv": (p) => {
+            if (p.includes("separator.csv")) {
+                return new CSVLoader(p, { column: "html", separator: "｜" });
+            }
+            return new CSVLoader(p, "html");
+        },
+        ".pdf": (p) => new PDFLoader(p),
+        ".txt": (p) => new TextLoader(p),
+        ".json": (p) => new JSONLoader(p),
+    }, UnknownHandling.Ignore);
+    const docs = await loader.load();
+    expect(docs.length).toBe(123);
+    const expectedSources = [
+        // PDF
+        ...Array.from({ length: 15 }, (_) => path.resolve(baseDirectory, "1706.03762.pdf")),
+        path.resolve(baseDirectory, "Jacob_Lee_Resume_2023.pdf"),
+        // CSV
+        ...Array.from({ length: 32 }, (_) => path.resolve(baseDirectory, "Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.csv")),
+        // JSON
+        ...Array.from({ length: 32 }, (_) => path.resolve(baseDirectory, "Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.json")),
+        ...Array.from({ length: 10 }, (_) => path.resolve(baseDirectory, "complex.json")),
+        // TXT
+        path.resolve(baseDirectory, "example.txt"),
+        // CSV
+        ...Array.from({ length: 32 }, (_) => path.resolve(baseDirectory, "example_separator.csv")),
+    ];
+    expect(docs.map((d) => d.metadata.source).sort()).toEqual(expectedSources);
+});

package/dist/load/import_constants.cjs

@@ -38,6 +38,7 @@ exports.optionalImportEntrypoints = [
     "langchain/document_loaders/web/couchbase",
     "langchain/document_loaders/web/youtube",
     "langchain/document_loaders/fs/directory",
+    "langchain/document_loaders/fs/multi_file",
     "langchain/document_loaders/fs/buffer",
     "langchain/document_loaders/fs/chatgpt",
     "langchain/document_loaders/fs/text",

package/dist/load/import_constants.js

@@ -35,6 +35,7 @@ export const optionalImportEntrypoints = [
     "langchain/document_loaders/web/couchbase",
     "langchain/document_loaders/web/youtube",
     "langchain/document_loaders/fs/directory",
+    "langchain/document_loaders/fs/multi_file",
     "langchain/document_loaders/fs/buffer",
     "langchain/document_loaders/fs/chatgpt",
     "langchain/document_loaders/fs/text",

package/dist/retrievers/multi_query.cjs

@@ -162,7 +162,7 @@ class MultiQueryRetriever extends retrievers_1.BaseRetriever {
         const uniqueDocuments = this._uniqueUnion(documents);
         let outputDocs = uniqueDocuments;
         if (this.documentCompressor && uniqueDocuments.length) {
-            outputDocs = await this.documentCompressor.compressDocuments(uniqueDocuments, question);
+            outputDocs = await this.documentCompressor.compressDocuments(uniqueDocuments, question, runManager?.getChild());
             if (this.documentCompressorFilteringFn) {
                 outputDocs = this.documentCompressorFilteringFn(outputDocs);
             }

package/dist/retrievers/multi_query.js

@@ -159,7 +159,7 @@ export class MultiQueryRetriever extends BaseRetriever {
         const uniqueDocuments = this._uniqueUnion(documents);
         let outputDocs = uniqueDocuments;
         if (this.documentCompressor && uniqueDocuments.length) {
-            outputDocs = await this.documentCompressor.compressDocuments(uniqueDocuments, question);
+            outputDocs = await this.documentCompressor.compressDocuments(uniqueDocuments, question, runManager?.getChild());
             if (this.documentCompressorFilteringFn) {
                 outputDocs = this.documentCompressorFilteringFn(outputDocs);
             }

package/document_loaders/fs/multi_file.cjs

@@ -0,0 +1 @@
+module.exports = require('../../dist/document_loaders/fs/multi_file.cjs');

package/document_loaders/fs/multi_file.d.cts

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/fs/multi_file.js'

package/document_loaders/fs/multi_file.d.ts

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/fs/multi_file.js'

package/document_loaders/fs/multi_file.js

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/fs/multi_file.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -286,6 +286,10 @@
     "document_loaders/fs/directory.js",
     "document_loaders/fs/directory.d.ts",
     "document_loaders/fs/directory.d.cts",
+    "document_loaders/fs/multi_file.cjs",
+    "document_loaders/fs/multi_file.js",
+    "document_loaders/fs/multi_file.d.ts",
+    "document_loaders/fs/multi_file.d.cts",
     "document_loaders/fs/buffer.cjs",
     "document_loaders/fs/buffer.js",
     "document_loaders/fs/buffer.d.ts",
@@ -1540,6 +1544,15 @@
       "import": "./document_loaders/fs/directory.js",
       "require": "./document_loaders/fs/directory.cjs"
     },
+    "./document_loaders/fs/multi_file": {
+      "types": {
+        "import": "./document_loaders/fs/multi_file.d.ts",
+        "require": "./document_loaders/fs/multi_file.d.cts",
+        "default": "./document_loaders/fs/multi_file.d.ts"
+      },
+      "import": "./document_loaders/fs/multi_file.js",
+      "require": "./document_loaders/fs/multi_file.cjs"
+    },
     "./document_loaders/fs/buffer": {
       "types": {
         "import": "./document_loaders/fs/buffer.d.ts",