langchain 0.2.17 → 0.3.0-rc.0

package/dist/document_loaders/fs/srt.js

@@ -1,50 +0,0 @@
-import srtParser2 from "srt-parser-2";
-import { TextLoader } from "./text.js";
-import { logVersion020MigrationWarning } from "../../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion020MigrationWarning({
-    oldEntrypointName: "document_loaders/fs/srt",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/fs/srt" instead. This entrypoint will be removed in 0.3.0.
- *
- * A class that extends the `TextLoader` class. It represents a document
- * loader that loads documents from SRT (SubRip) subtitle files. It has a
- * constructor that takes a `filePathOrBlob` parameter representing the
- * path to the SRT file or a `Blob` object. The `parse()` method is
- * implemented to parse the SRT file and extract the text content of each
- * subtitle.
- * @example
- * ```typescript
- * const loader = new SRTLoader("path/to/file.srt");
- * const docs = await loader.load();
- * console.log({ docs });
- * ```
- */
-export class SRTLoader extends TextLoader {
-    constructor(filePathOrBlob) {
-        super(filePathOrBlob);
-    }
-    /**
-     * A protected method that takes a `raw` string as a parameter and returns
-     * a promise that resolves to an array of strings. It parses the raw SRT
-     * string using the `SRTParser2` class from the `srt-parser-2` module. It
-     * retrieves the subtitle objects from the parsed SRT data and extracts
-     * the text content from each subtitle object. It filters out any empty
-     * text content and joins the non-empty text content with a space
-     * separator.
-     * @param raw The raw SRT string to be parsed.
-     * @returns A promise that resolves to an array of strings representing the text content of each subtitle.
-     */
-    async parse(raw) {
-        // eslint-disable-next-line new-cap
-        const parser = new srtParser2();
-        const srts = parser.fromSrt(raw);
-        return [
-            srts
-                .map((srt) => srt.text)
-                .filter(Boolean)
-                .join(" "),
-        ];
-    }
-}

package/dist/document_loaders/fs/unstructured.cjs

@@ -1,338 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.UnknownHandling = exports.UnstructuredDirectoryLoader = exports.UnstructuredLoader = void 0;
-const documents_1 = require("@langchain/core/documents");
-const env_1 = require("@langchain/core/utils/env");
-const directory_js_1 = require("./directory.cjs");
-Object.defineProperty(exports, "UnknownHandling", { enumerable: true, get: function () { return directory_js_1.UnknownHandling; } });
-const base_js_1 = require("../base.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/fs/unstructured",
-    newPackageName: "@langchain/community",
-});
-const UNSTRUCTURED_API_FILETYPES = [
-    ".txt",
-    ".text",
-    ".pdf",
-    ".docx",
-    ".doc",
-    ".jpg",
-    ".jpeg",
-    ".eml",
-    ".html",
-    ".htm",
-    ".md",
-    ".pptx",
-    ".ppt",
-    ".msg",
-    ".rtf",
-    ".xlsx",
-    ".xls",
-    ".odt",
-    ".epub",
-];
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/fs/unstructured" instead. This entrypoint will be removed in 0.3.0.
- *
- * A document loader that uses the Unstructured API to load unstructured
- * documents. It supports both the new syntax with options object and the
- * legacy syntax for backward compatibility. The load() method sends a
- * partitioning request to the Unstructured API and retrieves the
- * partitioned elements. It creates a Document instance for each element
- * and returns an array of Document instances.
- */
-class UnstructuredLoader extends base_js_1.BaseDocumentLoader {
-    constructor(filePathOrLegacyApiUrlOrMemoryBuffer, optionsOrLegacyFilePath = {}) {
-        super();
-        Object.defineProperty(this, "filePath", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "buffer", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "fileName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "apiUrl", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: "https://api.unstructured.io/general/v0/general"
-        });
-        Object.defineProperty(this, "apiKey", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "strategy", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: "hi_res"
-        });
-        Object.defineProperty(this, "encoding", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "ocrLanguages", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: []
-        });
-        Object.defineProperty(this, "coordinates", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "pdfInferTableStructure", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "xmlKeepTags", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "skipInferTableTypes", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "hiResModelName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "includePageBreaks", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "chunkingStrategy", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "multiPageSections", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "combineUnderNChars", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "newAfterNChars", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "maxCharacters", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        // Temporary shim to avoid breaking existing users
-        // Remove when API keys are enforced by Unstructured and existing code will break anyway
-        const isLegacySyntax = typeof optionsOrLegacyFilePath === "string";
-        const isMemorySyntax = typeof filePathOrLegacyApiUrlOrMemoryBuffer === "object";
-        if (isMemorySyntax) {
-            this.buffer = filePathOrLegacyApiUrlOrMemoryBuffer.buffer;
-            this.fileName = filePathOrLegacyApiUrlOrMemoryBuffer.fileName;
-        }
-        else if (isLegacySyntax) {
-            this.filePath = optionsOrLegacyFilePath;
-            this.apiUrl = filePathOrLegacyApiUrlOrMemoryBuffer;
-        }
-        else {
-            this.filePath = filePathOrLegacyApiUrlOrMemoryBuffer;
-        }
-        if (!isLegacySyntax) {
-            const options = optionsOrLegacyFilePath;
-            this.apiKey = options.apiKey;
-            this.apiUrl = options.apiUrl ?? this.apiUrl;
-            this.strategy = options.strategy ?? this.strategy;
-            this.encoding = options.encoding;
-            this.ocrLanguages = options.ocrLanguages ?? this.ocrLanguages;
-            this.coordinates = options.coordinates;
-            this.pdfInferTableStructure = options.pdfInferTableStructure;
-            this.xmlKeepTags = options.xmlKeepTags;
-            this.skipInferTableTypes = options.skipInferTableTypes;
-            this.hiResModelName = options.hiResModelName;
-            this.includePageBreaks = options.includePageBreaks;
-            this.chunkingStrategy = options.chunkingStrategy;
-            this.multiPageSections = options.multiPageSections;
-            this.combineUnderNChars = options.combineUnderNChars;
-            this.newAfterNChars = options.newAfterNChars;
-            this.maxCharacters = options.maxCharacters;
-        }
-    }
-    async _partition() {
-        let { buffer } = this;
-        let { fileName } = this;
-        if (!buffer) {
-            const { readFile, basename } = await this.imports();
-            buffer = await readFile(this.filePath);
-            fileName = basename(this.filePath);
-            // I'm aware this reads the file into memory first, but we have lots of work
-            // to do on then consuming Documents in a streaming fashion anyway, so not
-            // worried about this for now.
-        }
-        const formData = new FormData();
-        formData.append("files", new Blob([buffer]), fileName);
-        formData.append("strategy", this.strategy);
-        this.ocrLanguages.forEach((language) => {
-            formData.append("ocr_languages", language);
-        });
-        if (this.encoding) {
-            formData.append("encoding", this.encoding);
-        }
-        if (this.coordinates === true) {
-            formData.append("coordinates", "true");
-        }
-        if (this.pdfInferTableStructure === true) {
-            formData.append("pdf_infer_table_structure", "true");
-        }
-        if (this.xmlKeepTags === true) {
-            formData.append("xml_keep_tags", "true");
-        }
-        if (this.skipInferTableTypes) {
-            formData.append("skip_infer_table_types", JSON.stringify(this.skipInferTableTypes));
-        }
-        if (this.hiResModelName) {
-            formData.append("hi_res_model_name", this.hiResModelName);
-        }
-        if (this.includePageBreaks) {
-            formData.append("include_page_breaks", "true");
-        }
-        if (this.chunkingStrategy) {
-            formData.append("chunking_strategy", this.chunkingStrategy);
-        }
-        if (this.multiPageSections !== undefined) {
-            formData.append("multipage_sections", this.multiPageSections ? "true" : "false");
-        }
-        if (this.combineUnderNChars !== undefined) {
-            formData.append("combine_under_n_chars", String(this.combineUnderNChars));
-        }
-        if (this.newAfterNChars !== undefined) {
-            formData.append("new_after_n_chars", String(this.newAfterNChars));
-        }
-        if (this.maxCharacters !== undefined) {
-            formData.append("max_characters", String(this.maxCharacters));
-        }
-        const headers = {
-            "UNSTRUCTURED-API-KEY": this.apiKey ?? "",
-        };
-        const response = await fetch(this.apiUrl, {
-            method: "POST",
-            body: formData,
-            headers,
-        });
-        if (!response.ok) {
-            throw new Error(`Failed to partition file ${this.filePath} with error ${response.status} and message ${await response.text()}`);
-        }
-        const elements = await response.json();
-        if (!Array.isArray(elements)) {
-            throw new Error(`Expected partitioning request to return an array, but got ${elements}`);
-        }
-        return elements.filter((el) => typeof el.text === "string");
-    }
-    async load() {
-        const elements = await this._partition();
-        const documents = [];
-        for (const element of elements) {
-            const { metadata, text } = element;
-            if (typeof text === "string" && text !== "") {
-                documents.push(new documents_1.Document({
-                    pageContent: text,
-                    metadata: {
-                        ...metadata,
-                        category: element.type,
-                    },
-                }));
-            }
-        }
-        return documents;
-    }
-    async imports() {
-        try {
-            const { readFile } = await import("node:fs/promises");
-            const { basename } = await import("node:path");
-            return { readFile, basename };
-        }
-        catch (e) {
-            console.error(e);
-            throw new Error(`Failed to load fs/promises. TextLoader available only on environment 'node'. It appears you are running environment '${(0, env_1.getEnv)()}'. See https://<link to docs> for alternatives.`);
-        }
-    }
-}
-exports.UnstructuredLoader = UnstructuredLoader;
-/**
- * A document loader that loads unstructured documents from a directory
- * using the UnstructuredLoader. It creates a UnstructuredLoader instance
- * for each supported file type and passes it to the DirectoryLoader
- * constructor.
- * @example
- * ```typescript
- * const loader = new UnstructuredDirectoryLoader("path/to/directory", {
- *   apiKey: "MY_API_KEY",
- * });
- * const docs = await loader.load();
- * ```
- */
-class UnstructuredDirectoryLoader extends directory_js_1.DirectoryLoader {
-    constructor(directoryPathOrLegacyApiUrl, optionsOrLegacyDirectoryPath, legacyOptionRecursive = true, legacyOptionUnknown = directory_js_1.UnknownHandling.Warn) {
-        let directoryPath;
-        let options;
-        // Temporary shim to avoid breaking existing users
-        // Remove when API keys are enforced by Unstructured and existing code will break anyway
-        const isLegacySyntax = typeof optionsOrLegacyDirectoryPath === "string";
-        if (isLegacySyntax) {
-            directoryPath = optionsOrLegacyDirectoryPath;
-            options = {
-                apiUrl: directoryPathOrLegacyApiUrl,
-                recursive: legacyOptionRecursive,
-                unknown: legacyOptionUnknown,
-            };
-        }
-        else {
-            directoryPath = directoryPathOrLegacyApiUrl;
-            options = optionsOrLegacyDirectoryPath;
-        }
-        const loader = (p) => new UnstructuredLoader(p, options);
-        const loaders = UNSTRUCTURED_API_FILETYPES.reduce((loadersObject, filetype) => {
-            // eslint-disable-next-line no-param-reassign
-            loadersObject[filetype] = loader;
-            return loadersObject;
-        }, {});
-        super(directoryPath, loaders, options.recursive, options.unknown);
-    }
-}
-exports.UnstructuredDirectoryLoader = UnstructuredDirectoryLoader;

package/dist/document_loaders/fs/unstructured.d.ts

@@ -1,125 +0,0 @@
-/// <reference types="node" resolution-mode="require"/>
-/// <reference types="node" resolution-mode="require"/>
-/// <reference types="node" resolution-mode="require"/>
-import type { basename as BasenameT } from "node:path";
-import type { readFile as ReadFileT } from "node:fs/promises";
-import { Document } from "@langchain/core/documents";
-import { StringWithAutocomplete } from "@langchain/core/utils/types";
-import { DirectoryLoader, UnknownHandling } from "./directory.js";
-import { BaseDocumentLoader } from "../base.js";
-/**
- * Represents an element returned by the Unstructured API. It has
- * properties for the element type, text content, and metadata.
- */
-type Element = {
-    type: string;
-    text: string;
-    metadata: {
-        [key: string]: unknown;
-    };
-};
-/**
- * Represents the available strategies for the UnstructuredLoader. It can
- * be one of "hi_res", "fast", "ocr_only", or "auto".
- */
-export type UnstructuredLoaderStrategy = "hi_res" | "fast" | "ocr_only" | "auto";
-/**
- * Represents the available hi-res models for the UnstructuredLoader. It can
- * be one of "chipper".
- */
-export type HiResModelName = "chipper";
-/**
- * To enable or disable table extraction for file types other than PDF, set
- * the skipInferTableTypes property in the UnstructuredLoaderOptions object.
- * The skipInferTableTypes property is an array of file types for which table
- * extraction is disabled. For example, to disable table extraction for .docx
- * and .doc files, set the skipInferTableTypes property to ["docx", "doc"].
- * You can also disable table extraction for all file types other than PDF by
- * setting the skipInferTableTypes property to [].
- */
-export type SkipInferTableTypes = "txt" | "text" | "pdf" | "docx" | "doc" | "jpg" | "jpeg" | "eml" | "html" | "htm" | "md" | "pptx" | "ppt" | "msg" | "rtf" | "xlsx" | "xls" | "odt" | "epub";
-/**
- * Set the chunking_strategy to chunk text into larger or smaller elements. Defaults to None with optional arg of by_title
- */
-type ChunkingStrategy = "None" | "by_title";
-export type UnstructuredLoaderOptions = {
-    apiKey?: string;
-    apiUrl?: string;
-    strategy?: StringWithAutocomplete<UnstructuredLoaderStrategy>;
-    encoding?: string;
-    ocrLanguages?: Array<string>;
-    coordinates?: boolean;
-    pdfInferTableStructure?: boolean;
-    xmlKeepTags?: boolean;
-    skipInferTableTypes?: Array<StringWithAutocomplete<SkipInferTableTypes>>;
-    hiResModelName?: StringWithAutocomplete<HiResModelName>;
-    includePageBreaks?: boolean;
-    chunkingStrategy?: StringWithAutocomplete<ChunkingStrategy>;
-    multiPageSections?: boolean;
-    combineUnderNChars?: number;
-    newAfterNChars?: number;
-    maxCharacters?: number;
-};
-type UnstructuredDirectoryLoaderOptions = UnstructuredLoaderOptions & {
-    recursive?: boolean;
-    unknown?: UnknownHandling;
-};
-type UnstructuredMemoryLoaderOptions = {
-    buffer: Buffer;
-    fileName: string;
-};
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/fs/unstructured" instead. This entrypoint will be removed in 0.3.0.
- *
- * A document loader that uses the Unstructured API to load unstructured
- * documents. It supports both the new syntax with options object and the
- * legacy syntax for backward compatibility. The load() method sends a
- * partitioning request to the Unstructured API and retrieves the
- * partitioned elements. It creates a Document instance for each element
- * and returns an array of Document instances.
- */
-export declare class UnstructuredLoader extends BaseDocumentLoader {
-    filePath: string;
-    private buffer?;
-    private fileName?;
-    private apiUrl;
-    private apiKey?;
-    private strategy;
-    private encoding?;
-    private ocrLanguages;
-    private coordinates?;
-    private pdfInferTableStructure?;
-    private xmlKeepTags?;
-    private skipInferTableTypes?;
-    private hiResModelName?;
-    private includePageBreaks?;
-    private chunkingStrategy?;
-    private multiPageSections?;
-    private combineUnderNChars?;
-    private newAfterNChars?;
-    private maxCharacters?;
-    constructor(filePathOrLegacyApiUrlOrMemoryBuffer: string | UnstructuredMemoryLoaderOptions, optionsOrLegacyFilePath?: UnstructuredLoaderOptions | string);
-    _partition(): Promise<Element[]>;
-    load(): Promise<Document[]>;
-    imports(): Promise<{
-        readFile: typeof ReadFileT;
-        basename: typeof BasenameT;
-    }>;
-}
-/**
- * A document loader that loads unstructured documents from a directory
- * using the UnstructuredLoader. It creates a UnstructuredLoader instance
- * for each supported file type and passes it to the DirectoryLoader
- * constructor.
- * @example
- * ```typescript
- * const loader = new UnstructuredDirectoryLoader("path/to/directory", {
- *   apiKey: "MY_API_KEY",
- * });
- * const docs = await loader.load();
- * ```
- */
-export declare class UnstructuredDirectoryLoader extends DirectoryLoader {
-    constructor(directoryPathOrLegacyApiUrl: string, optionsOrLegacyDirectoryPath: UnstructuredDirectoryLoaderOptions | string, legacyOptionRecursive?: boolean, legacyOptionUnknown?: UnknownHandling);
-}
-export { UnknownHandling };