langchain 0.0.195 → 0.0.197-rc.0

package/dist/document_loaders/web/github.cjs

@@ -216,6 +216,22 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
         }
         return documents;
     }
+    /**
+     * Asynchronously streams documents from the entire GitHub repository.
+     * It is suitable for situations where processing large repositories in a memory-efficient manner is required.
+     * @yields Yields a Promise that resolves to a Document object for each file or submodule content found in the repository.
+     */
+    async *loadAsStream() {
+        this.log(`Loading documents from ${this.baseUrl}/${this.owner}/${this.repo}/${this.initialPath}...`);
+        yield* await this.processRepoAsStream(this.initialPath);
+        if (!this.processSubmodules) {
+            return;
+        }
+        await this.getSubmoduleInfo();
+        for (const submoduleInfo of this.submoduleInfos) {
+            yield* await this.loadSubmoduleAsStream(submoduleInfo);
+        }
+    }
     /**
      * Loads the information about Git submodules from the repository, if available.
      */
@@ -323,6 +339,37 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             }).load();
         }
     }
+    /**
+     * Asynchronously processes and streams the contents of a specified submodule in the GitHub repository.
+     * @param submoduleInfo the info about the submodule to be loaded
+     * @yields Yields a Promise that resolves to a Document object for each file found in the submodule.
+     */
+    async *loadSubmoduleAsStream(submoduleInfo) {
+        if (!submoduleInfo.url.startsWith(this.baseUrl)) {
+            this.log(`Ignoring external submodule ${submoduleInfo.url}.`);
+            yield* [];
+        }
+        if (!submoduleInfo.path.startsWith(this.initialPath)) {
+            this.log(`Ignoring submodule ${submoduleInfo.url}, as it is not on initial path.`);
+            yield* [];
+        }
+        this.log(`Accessing submodule ${submoduleInfo.name} (${submoduleInfo.url})...`);
+        const submoduleLoader = new GithubRepoLoader(submoduleInfo.url, {
+            accessToken: this.accessToken,
+            baseUrl: this.baseUrl,
+            apiUrl: this.apiUrl,
+            branch: submoduleInfo.ref,
+            recursive: this.recursive,
+            processSubmodules: this.processSubmodules,
+            unknown: this.unknown,
+            ignoreFiles: this.ignoreFiles,
+            ignorePaths: this.ignorePaths,
+            verbose: this.verbose,
+            maxConcurrency: this.maxConcurrency,
+            maxRetries: this.maxRetries,
+        });
+        yield* await submoduleLoader.processRepoAsStream(submoduleInfo.path);
+    }
     /**
      * Determines whether a file or directory should be ignored based on its
      * path and type.
@@ -414,6 +461,35 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             return Promise.reject(error);
         }
     }
+    /**
+     * Asynchronously processes the contents of the entire GitHub repository,
+     * streaming each file as a Document object.
+     * @param path The path of the directory to process.
+     * @yields Yields a Promise that resolves to a Document object for each file found in the repository.
+     */
+    async *processRepoAsStream(path) {
+        const files = await this.fetchRepoFiles(path);
+        for (const file of files) {
+            if (this.shouldIgnore(file.path, file.type)) {
+                continue;
+            }
+            if (file.type === "file") {
+                try {
+                    const fileResponse = await this.fetchFileContentWrapper(file);
+                    yield new document_js_1.Document({
+                        pageContent: fileResponse.contents,
+                        metadata: fileResponse.metadata,
+                    });
+                }
+                catch (error) {
+                    this.handleError(`Failed to fetch file content: ${file.path}, ${error}`);
+                }
+            }
+            else if (this.recursive) {
+                yield* await this.processDirectoryAsStream(file.path);
+            }
+        }
+    }
     /**
      * Fetches the contents of a directory and maps the file / directory paths
      * to promises that will fetch the file / directory contents.
@@ -430,6 +506,35 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
             return Promise.reject(error);
         }
     }
+    /**
+     * Asynchronously processes the contents of a given directory in the GitHub repository,
+     * streaming each file as a Document object.
+     * @param path The path of the directory to process.
+     * @yields Yields a Promise that resolves to a Document object for each file in the directory.
+     */
+    async *processDirectoryAsStream(path) {
+        const files = await this.fetchRepoFiles(path);
+        for (const file of files) {
+            if (this.shouldIgnore(file.path, file.type)) {
+                continue;
+            }
+            if (file.type === "file") {
+                try {
+                    const fileResponse = await this.fetchFileContentWrapper(file);
+                    yield new document_js_1.Document({
+                        pageContent: fileResponse.contents,
+                        metadata: fileResponse.metadata,
+                    });
+                }
+                catch {
+                    this.handleError(`Failed to fetch file content: ${file.path}`);
+                }
+            }
+            else if (this.recursive) {
+                yield* await this.processDirectoryAsStream(file.path);
+            }
+        }
+    }
     /**
      * Fetches the files from a GitHub repository.
      * If the path denotes a single file, the resulting array contains only one element.

package/dist/document_loaders/web/github.d.ts

@@ -99,6 +99,12 @@ export declare class GithubRepoLoader extends BaseDocumentLoader implements Gith
      * @returns A promise that resolves to an array of Document instances.
      */
     load(): Promise<Document[]>;
+    /**
+     * Asynchronously streams documents from the entire GitHub repository.
+     * It is suitable for situations where processing large repositories in a memory-efficient manner is required.
+     * @yields Yields a Promise that resolves to a Document object for each file or submodule content found in the repository.
+     */
+    loadAsStream(): AsyncGenerator<Document, void, undefined>;
     /**
      * Loads the information about Git submodules from the repository, if available.
      */
@@ -115,6 +121,12 @@ export declare class GithubRepoLoader extends BaseDocumentLoader implements Gith
      * @param submoduleInfo the info about the submodule to be loaded
      */
     private loadSubmodule;
+    /**
+     * Asynchronously processes and streams the contents of a specified submodule in the GitHub repository.
+     * @param submoduleInfo the info about the submodule to be loaded
+     * @yields Yields a Promise that resolves to a Document object for each file found in the submodule.
+     */
+    private loadSubmoduleAsStream;
     /**
      * Determines whether a file or directory should be ignored based on its
      * path and type.
@@ -137,6 +149,13 @@ export declare class GithubRepoLoader extends BaseDocumentLoader implements Gith
      * Begins the process of fetching the contents of the repository
      */
     private processRepo;
+    /**
+     * Asynchronously processes the contents of the entire GitHub repository,
+     * streaming each file as a Document object.
+     * @param path The path of the directory to process.
+     * @yields Yields a Promise that resolves to a Document object for each file found in the repository.
+     */
+    private processRepoAsStream;
     /**
      * Fetches the contents of a directory and maps the file / directory paths
      * to promises that will fetch the file / directory contents.
@@ -144,6 +163,13 @@ export declare class GithubRepoLoader extends BaseDocumentLoader implements Gith
      * @returns A promise that resolves to an array of promises that will fetch the file / directory contents.
      */
     private processDirectory;
+    /**
+     * Asynchronously processes the contents of a given directory in the GitHub repository,
+     * streaming each file as a Document object.
+     * @param path The path of the directory to process.
+     * @yields Yields a Promise that resolves to a Document object for each file in the directory.
+     */
+    private processDirectoryAsStream;
     /**
      * Fetches the files from a GitHub repository.
      * If the path denotes a single file, the resulting array contains only one element.

package/dist/document_loaders/web/github.js

@@ -210,6 +210,22 @@ export class GithubRepoLoader extends BaseDocumentLoader {
         }
         return documents;
     }
+    /**
+     * Asynchronously streams documents from the entire GitHub repository.
+     * It is suitable for situations where processing large repositories in a memory-efficient manner is required.
+     * @yields Yields a Promise that resolves to a Document object for each file or submodule content found in the repository.
+     */
+    async *loadAsStream() {
+        this.log(`Loading documents from ${this.baseUrl}/${this.owner}/${this.repo}/${this.initialPath}...`);
+        yield* await this.processRepoAsStream(this.initialPath);
+        if (!this.processSubmodules) {
+            return;
+        }
+        await this.getSubmoduleInfo();
+        for (const submoduleInfo of this.submoduleInfos) {
+            yield* await this.loadSubmoduleAsStream(submoduleInfo);
+        }
+    }
     /**
      * Loads the information about Git submodules from the repository, if available.
      */
@@ -317,6 +333,37 @@ export class GithubRepoLoader extends BaseDocumentLoader {
             }).load();
         }
     }
+    /**
+     * Asynchronously processes and streams the contents of a specified submodule in the GitHub repository.
+     * @param submoduleInfo the info about the submodule to be loaded
+     * @yields Yields a Promise that resolves to a Document object for each file found in the submodule.
+     */
+    async *loadSubmoduleAsStream(submoduleInfo) {
+        if (!submoduleInfo.url.startsWith(this.baseUrl)) {
+            this.log(`Ignoring external submodule ${submoduleInfo.url}.`);
+            yield* [];
+        }
+        if (!submoduleInfo.path.startsWith(this.initialPath)) {
+            this.log(`Ignoring submodule ${submoduleInfo.url}, as it is not on initial path.`);
+            yield* [];
+        }
+        this.log(`Accessing submodule ${submoduleInfo.name} (${submoduleInfo.url})...`);
+        const submoduleLoader = new GithubRepoLoader(submoduleInfo.url, {
+            accessToken: this.accessToken,
+            baseUrl: this.baseUrl,
+            apiUrl: this.apiUrl,
+            branch: submoduleInfo.ref,
+            recursive: this.recursive,
+            processSubmodules: this.processSubmodules,
+            unknown: this.unknown,
+            ignoreFiles: this.ignoreFiles,
+            ignorePaths: this.ignorePaths,
+            verbose: this.verbose,
+            maxConcurrency: this.maxConcurrency,
+            maxRetries: this.maxRetries,
+        });
+        yield* await submoduleLoader.processRepoAsStream(submoduleInfo.path);
+    }
     /**
      * Determines whether a file or directory should be ignored based on its
      * path and type.
@@ -408,6 +455,35 @@ export class GithubRepoLoader extends BaseDocumentLoader {
             return Promise.reject(error);
         }
     }
+    /**
+     * Asynchronously processes the contents of the entire GitHub repository,
+     * streaming each file as a Document object.
+     * @param path The path of the directory to process.
+     * @yields Yields a Promise that resolves to a Document object for each file found in the repository.
+     */
+    async *processRepoAsStream(path) {
+        const files = await this.fetchRepoFiles(path);
+        for (const file of files) {
+            if (this.shouldIgnore(file.path, file.type)) {
+                continue;
+            }
+            if (file.type === "file") {
+                try {
+                    const fileResponse = await this.fetchFileContentWrapper(file);
+                    yield new Document({
+                        pageContent: fileResponse.contents,
+                        metadata: fileResponse.metadata,
+                    });
+                }
+                catch (error) {
+                    this.handleError(`Failed to fetch file content: ${file.path}, ${error}`);
+                }
+            }
+            else if (this.recursive) {
+                yield* await this.processDirectoryAsStream(file.path);
+            }
+        }
+    }
     /**
      * Fetches the contents of a directory and maps the file / directory paths
      * to promises that will fetch the file / directory contents.
@@ -424,6 +500,35 @@ export class GithubRepoLoader extends BaseDocumentLoader {
             return Promise.reject(error);
         }
     }
+    /**
+     * Asynchronously processes the contents of a given directory in the GitHub repository,
+     * streaming each file as a Document object.
+     * @param path The path of the directory to process.
+     * @yields Yields a Promise that resolves to a Document object for each file in the directory.
+     */
+    async *processDirectoryAsStream(path) {
+        const files = await this.fetchRepoFiles(path);
+        for (const file of files) {
+            if (this.shouldIgnore(file.path, file.type)) {
+                continue;
+            }
+            if (file.type === "file") {
+                try {
+                    const fileResponse = await this.fetchFileContentWrapper(file);
+                    yield new Document({
+                        pageContent: fileResponse.contents,
+                        metadata: fileResponse.metadata,
+                    });
+                }
+                catch {
+                    this.handleError(`Failed to fetch file content: ${file.path}`);
+                }
+            }
+            else if (this.recursive) {
+                yield* await this.processDirectoryAsStream(file.path);
+            }
+        }
+    }
     /**
      * Fetches the files from a GitHub repository.
      * If the path denotes a single file, the resulting array contains only one element.

package/dist/document_loaders/web/s3.d.ts

@@ -70,5 +70,5 @@ export declare class S3Loader extends BaseDocumentLoader {
      * and then uses the UnstructuredLoader to load the file as a document.
      * @returns An array of Document objects representing the loaded documents.
      */
-    load(): Promise<import("../../document.js").Document<Record<string, any>>[]>;
+    load(): Promise<import("langchain-core/documents").Document<Record<string, any>>[]>;
 }

package/dist/embeddings/base.cjs

@@ -1,24 +1,17 @@
 "use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Embeddings = void 0;
-const async_caller_js_1 = require("../util/async_caller.cjs");
-/**
- * An abstract class that provides methods for embedding documents and
- * queries using LangChain.
- */
-class Embeddings {
-    constructor(params) {
-        /**
-         * The async caller should be used by subclasses to make any async calls,
-         * which will thus benefit from the concurrency and retry logic.
-         */
-        Object.defineProperty(this, "caller", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.caller = new async_caller_js_1.AsyncCaller(params ?? {});
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
     }
-}
-exports.Embeddings = Embeddings;
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("langchain-core/embeddings"), exports);

package/dist/embeddings/base.d.ts

@@ -1,33 +1 @@
-import { AsyncCaller, AsyncCallerParams } from "../util/async_caller.js";
-/**
- * The parameters required to initialize an instance of the Embeddings
- * class.
- */
-export type EmbeddingsParams = AsyncCallerParams;
-/**
- * An abstract class that provides methods for embedding documents and
- * queries using LangChain.
- */
-export declare abstract class Embeddings {
-    /**
-     * The async caller should be used by subclasses to make any async calls,
-     * which will thus benefit from the concurrency and retry logic.
-     */
-    caller: AsyncCaller;
-    constructor(params: EmbeddingsParams);
-    /**
-     * An abstract method that takes an array of documents as input and
-     * returns a promise that resolves to an array of vectors for each
-     * document.
-     * @param documents An array of documents to be embedded.
-     * @returns A promise that resolves to an array of vectors for each document.
-     */
-    abstract embedDocuments(documents: string[]): Promise<number[][]>;
-    /**
-     * An abstract method that takes a single document as input and returns a
-     * promise that resolves to a vector for the query document.
-     * @param document A single document to be embedded.
-     * @returns A promise that resolves to a vector for the query document.
-     */
-    abstract embedQuery(document: string): Promise<number[]>;
-}
+export * from "langchain-core/embeddings";

package/dist/embeddings/base.js

@@ -1,20 +1 @@
-import { AsyncCaller } from "../util/async_caller.js";
-/**
- * An abstract class that provides methods for embedding documents and
- * queries using LangChain.
- */
-export class Embeddings {
-    constructor(params) {
-        /**
-         * The async caller should be used by subclasses to make any async calls,
-         * which will thus benefit from the concurrency and retry logic.
-         */
-        Object.defineProperty(this, "caller", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.caller = new AsyncCaller(params ?? {});
-    }
-}
+export * from "langchain-core/embeddings";

package/dist/embeddings/cache_backed.cjs

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.CacheBackedEmbeddings = void 0;
-const hash_js_1 = require("../util/js-sha1/hash.cjs");
+const hash_1 = require("langchain-core/utils/hash");
 const encoder_backed_js_1 = require("../storage/encoder_backed.cjs");
 const base_js_1 = require("./base.cjs");
 /**
@@ -130,7 +130,7 @@ class CacheBackedEmbeddings extends base_js_1.Embeddings {
         const decoder = new TextDecoder();
         const encoderBackedStore = new encoder_backed_js_1.EncoderBackedStore({
             store: documentEmbeddingStore,
-            keyEncoder: (key) => (options?.namespace ?? "") + (0, hash_js_1.insecureHash)(key),
+            keyEncoder: (key) => (options?.namespace ?? "") + (0, hash_1.insecureHash)(key),
             valueSerializer: (value) => encoder.encode(JSON.stringify(value)),
             valueDeserializer: (serializedValue) => JSON.parse(decoder.decode(serializedValue)),
         });

package/dist/embeddings/cache_backed.js

@@ -1,4 +1,4 @@
-import { insecureHash } from "../util/js-sha1/hash.js";
+import { insecureHash } from "langchain-core/utils/hash";
 import { EncoderBackedStore } from "../storage/encoder_backed.js";
 import { Embeddings } from "./base.js";
 /**

package/dist/evaluation/agents/trajectory.d.ts

@@ -27,7 +27,7 @@ export declare class TrajectoryEvalChain extends AgentTrajectoryEvaluator {
     requiresInput: boolean;
     requiresReference: boolean;
     outputParser: TrajectoryOutputParser;
-    static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredTool[]): import("../../prompts/chat.js").ChatPromptTemplate<any, any> | BasePromptTemplate<any, import("../../schema/index.js").BasePromptValue, any>;
+    static resolveTrajectoryPrompt(prompt?: BasePromptTemplate | undefined, agentTools?: StructuredTool[]): BasePromptTemplate<any, import("../../schema/index.js").BasePromptValue, any> | import("../../prompts/index.js").ChatPromptTemplate<any, any>;
     /**
      * Get the description of the agent tools.
      *

package/dist/evaluation/criteria/prompt.d.ts

@@ -1,12 +1,12 @@
 import { PromptTemplate } from "../../prompts/index.js";
 export declare const CRITERIA_PROMPT: PromptTemplate<{
-    output: any;
     input: any;
+    output: any;
     criteria: any;
 }, any>;
 export declare const PROMPT_WITH_REFERENCES: PromptTemplate<{
-    output: any;
     input: any;
+    output: any;
     criteria: any;
     reference: any;
 }, any>;

package/dist/evaluation/qa/prompt.d.ts

@@ -1,11 +1,11 @@
 import { PromptTemplate } from "../../prompts/prompt.js";
 export declare const QA_PROMPT: PromptTemplate<{
-    query: any;
     answer: any;
+    query: any;
     result: any;
 }, any>;
 export declare const SQL_PROMPT: PromptTemplate<{
-    query: any;
     answer: any;
+    query: any;
     result: any;
 }, any>;

package/dist/experimental/hubs/makersuite/googlemakersuitehub.d.ts

@@ -118,7 +118,7 @@ export declare class MakerSuitePrompt {
      * will have the parameters (model name, temperature, etc) from those in
      * MakerSuite.
      */
-    toChain(): import("../../../schema/runnable/base.js").RunnableSequence<any, any>;
+    toChain(): import("../../../schema/runnable/index.js").RunnableSequence<any, any>;
 }
 interface DriveFileReadParams extends GoogleVertexAIConnectionParams<GoogleAuthOptions> {
     fileId: string;

package/dist/experimental/plan_and_execute/prompt.d.ts

@@ -10,4 +10,4 @@ export declare const DEFAULT_STEP_EXECUTOR_HUMAN_CHAT_MESSAGE_TEMPLATE = "Previo
  * @param tools the tools available to the `planner`
  * @returns
  */
-export declare const getPlannerChatPrompt: (tools: Tool[] | DynamicStructuredTool[]) => Promise<ChatPromptTemplate<import("../../schema/index.js").InputValues<string>, any>>;
+export declare const getPlannerChatPrompt: (tools: Tool[] | DynamicStructuredTool[]) => Promise<ChatPromptTemplate<import("langchain-core/utils/types").InputValues<string>, any>>;