langchain 0.1.33 → 0.1.35

package/dist/chains/history_aware_retriever.cjs

@@ -18,8 +18,7 @@ const output_parsers_1 = require("@langchain/core/output_parsers");
  *
  * import { ChatOpenAI } from "@langchain/openai";
  * import { pull } from "langchain/hub";
- * import { createRetrievalChain } from "langchain/chains/retrieval";
- * import { createStuffDocumentsChain } from "langchain/chains/combine_documents";
+ * import { createHistoryAwareRetriever } from "langchain/chains/history_aware_retriever";
  *
  * const rephrasePrompt = await pull("langchain-ai/chat-langchain-rephrase");
  * const llm = new ChatOpenAI({});

package/dist/chains/history_aware_retriever.d.ts

@@ -35,8 +35,7 @@ export type CreateHistoryAwareRetrieverParams = {
  *
  * import { ChatOpenAI } from "@langchain/openai";
  * import { pull } from "langchain/hub";
- * import { createRetrievalChain } from "langchain/chains/retrieval";
- * import { createStuffDocumentsChain } from "langchain/chains/combine_documents";
+ * import { createHistoryAwareRetriever } from "langchain/chains/history_aware_retriever";
  *
  * const rephrasePrompt = await pull("langchain-ai/chat-langchain-rephrase");
  * const llm = new ChatOpenAI({});

package/dist/chains/history_aware_retriever.js

@@ -15,8 +15,7 @@ import { StringOutputParser } from "@langchain/core/output_parsers";
  *
  * import { ChatOpenAI } from "@langchain/openai";
  * import { pull } from "langchain/hub";
- * import { createRetrievalChain } from "langchain/chains/retrieval";
- * import { createStuffDocumentsChain } from "langchain/chains/combine_documents";
+ * import { createHistoryAwareRetriever } from "langchain/chains/history_aware_retriever";
  *
  * const rephrasePrompt = await pull("langchain-ai/chat-langchain-rephrase");
  * const llm = new ChatOpenAI({});

package/dist/document_loaders/web/github.cjs

@@ -262,12 +262,17 @@ class GithubRepoLoader extends base_js_1.BaseDocumentLoader {
      * @param gitmodulesContent the content of a .gitmodules file
      */
     async parseGitmodules(gitmodulesContent) {
+        let validGitmodulesContent = gitmodulesContent;
+        // in case the .gitmodules file does not end with a newline, we add one to make the regex work
+        if (!validGitmodulesContent.endsWith("\n")) {
+            validGitmodulesContent += "\n";
+        }
         // catches the initial line of submodule entries
         const submodulePattern = /\[submodule "(.*?)"]\n((\s+.*?\s*=\s*.*?\n)*)/g;
         // catches the properties of a submodule
         const keyValuePattern = /\s+(.*?)\s*=\s*(.*?)\s/g;
         const submoduleInfos = [];
-        for (const [, name, propertyLines] of gitmodulesContent.matchAll(submodulePattern)) {
+        for (const [, name, propertyLines] of validGitmodulesContent.matchAll(submodulePattern)) {
             if (!name || !propertyLines) {
                 throw new Error("Could not parse submodule entry");
             }

package/dist/document_loaders/web/github.js

@@ -256,12 +256,17 @@ export class GithubRepoLoader extends BaseDocumentLoader {
      * @param gitmodulesContent the content of a .gitmodules file
      */
     async parseGitmodules(gitmodulesContent) {
+        let validGitmodulesContent = gitmodulesContent;
+        // in case the .gitmodules file does not end with a newline, we add one to make the regex work
+        if (!validGitmodulesContent.endsWith("\n")) {
+            validGitmodulesContent += "\n";
+        }
         // catches the initial line of submodule entries
         const submodulePattern = /\[submodule "(.*?)"]\n((\s+.*?\s*=\s*.*?\n)*)/g;
         // catches the properties of a submodule
         const keyValuePattern = /\s+(.*?)\s*=\s*(.*?)\s/g;
         const submoduleInfos = [];
-        for (const [, name, propertyLines] of gitmodulesContent.matchAll(submodulePattern)) {
+        for (const [, name, propertyLines] of validGitmodulesContent.matchAll(submodulePattern)) {
             if (!name || !propertyLines) {
                 throw new Error("Could not parse submodule entry");
             }

package/dist/experimental/openai_assistant/index.cjs

@@ -210,7 +210,7 @@ class OpenAIAssistantRunnable extends runnables_1.Runnable {
         const run = await this._waitForRun(runId, threadId);
         if (run.status === "completed") {
             const messages = await this.client.beta.threads.messages.list(threadId, {
-                order: "asc",
+                order: "desc",
             });
             const newMessages = messages.data.filter((msg) => msg.run_id === runId);
             if (!this.asAgent) {

package/dist/experimental/openai_assistant/index.js

@@ -207,7 +207,7 @@ export class OpenAIAssistantRunnable extends Runnable {
         const run = await this._waitForRun(runId, threadId);
         if (run.status === "completed") {
             const messages = await this.client.beta.threads.messages.list(threadId, {
-                order: "asc",
+                order: "desc",
             });
             const newMessages = messages.data.filter((msg) => msg.run_id === runId);
             if (!this.asAgent) {

package/dist/memory/vector_store.cjs

@@ -62,10 +62,17 @@ class VectorStoreRetrieverMemory extends memory_1.BaseMemory {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "metadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.vectorStoreRetriever = fields.vectorStoreRetriever;
         this.inputKey = fields.inputKey;
         this.memoryKey = fields.memoryKey ?? "memory";
         this.returnDocs = fields.returnDocs ?? false;
+        this.metadata = fields.metadata;
     }
     get memoryKeys() {
         return [this.memoryKey];
@@ -95,13 +102,16 @@ class VectorStoreRetrieverMemory extends memory_1.BaseMemory {
      * @returns A Promise that resolves to void.
      */
     async saveContext(inputValues, outputValues) {
+        const metadata = typeof this.metadata === "function"
+            ? this.metadata(inputValues, outputValues)
+            : this.metadata;
         const text = Object.entries(inputValues)
             .filter(([k]) => k !== this.memoryKey)
             .concat(Object.entries(outputValues))
             .map(([k, v]) => `${k}: ${v}`)
             .join("\n");
         await this.vectorStoreRetriever.addDocuments([
-            new documents_1.Document({ pageContent: text }),
+            new documents_1.Document({ pageContent: text, metadata }),
         ]);
     }
 }

package/dist/memory/vector_store.d.ts

@@ -1,5 +1,7 @@
 import type { VectorStoreRetrieverInterface } from "@langchain/core/vectorstores";
 import { BaseMemory, InputValues, MemoryVariables, OutputValues } from "@langchain/core/memory";
+type Metadata = Record<string, unknown>;
+type MetadataFunction = (inputValues?: InputValues, outputValues?: OutputValues) => Metadata;
 /**
  * Interface for the parameters required to initialize a
  * VectorStoreRetrieverMemory instance.
@@ -10,6 +12,10 @@ export interface VectorStoreRetrieverMemoryParams {
     outputKey?: string;
     memoryKey?: string;
     returnDocs?: boolean;
+    /**
+     * Metadata to be added to the document when saving context.
+     */
+    metadata?: Metadata | MetadataFunction;
 }
 /**
  * Class for managing long-term memory in Large Language Model (LLM)
@@ -47,6 +53,7 @@ export declare class VectorStoreRetrieverMemory extends BaseMemory implements Ve
     inputKey?: string;
     memoryKey: string;
     returnDocs: boolean;
+    metadata?: Metadata | MetadataFunction;
     constructor(fields: VectorStoreRetrieverMemoryParams);
     get memoryKeys(): string[];
     /**
@@ -67,3 +74,4 @@ export declare class VectorStoreRetrieverMemory extends BaseMemory implements Ve
      */
     saveContext(inputValues: InputValues, outputValues: OutputValues): Promise<void>;
 }
+export {};

package/dist/memory/vector_store.js

@@ -59,10 +59,17 @@ export class VectorStoreRetrieverMemory extends BaseMemory {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "metadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.vectorStoreRetriever = fields.vectorStoreRetriever;
         this.inputKey = fields.inputKey;
         this.memoryKey = fields.memoryKey ?? "memory";
         this.returnDocs = fields.returnDocs ?? false;
+        this.metadata = fields.metadata;
     }
     get memoryKeys() {
         return [this.memoryKey];
@@ -92,13 +99,16 @@ export class VectorStoreRetrieverMemory extends BaseMemory {
      * @returns A Promise that resolves to void.
      */
     async saveContext(inputValues, outputValues) {
+        const metadata = typeof this.metadata === "function"
+            ? this.metadata(inputValues, outputValues)
+            : this.metadata;
         const text = Object.entries(inputValues)
             .filter(([k]) => k !== this.memoryKey)
             .concat(Object.entries(outputValues))
             .map(([k, v]) => `${k}: ${v}`)
             .join("\n");
         await this.vectorStoreRetriever.addDocuments([
-            new Document({ pageContent: text }),
+            new Document({ pageContent: text, metadata }),
         ]);
     }
 }

package/dist/retrievers/multi_query.cjs

@@ -103,10 +103,24 @@ class MultiQueryRetriever extends retrievers_1.BaseRetriever {
             writable: true,
             value: "lines"
         });
+        Object.defineProperty(this, "documentCompressor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "documentCompressorFilteringFn", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.retriever = fields.retriever;
         this.llmChain = fields.llmChain;
         this.queryCount = fields.queryCount ?? this.queryCount;
         this.parserKey = fields.parserKey ?? this.parserKey;
+        this.documentCompressor = fields.documentCompressor;
+        this.documentCompressorFilteringFn = fields.documentCompressorFilteringFn;
     }
     static fromLLM(fields) {
         const { retriever, llm, prompt = DEFAULT_QUERY_PROMPT, queryCount, parserKey, ...rest } = fields;
@@ -126,10 +140,10 @@ class MultiQueryRetriever extends retrievers_1.BaseRetriever {
     // Retrieve documents using the original retriever
     async _retrieveDocuments(queries, runManager) {
         const documents = [];
-        for (const query of queries) {
+        await Promise.all(queries.map(async (query) => {
             const docs = await this.retriever.getRelevantDocuments(query, runManager?.getChild());
             documents.push(...docs);
-        }
+        }));
         return documents;
     }
     // Deduplicate the documents that were returned in multiple retrievals
@@ -146,7 +160,14 @@ class MultiQueryRetriever extends retrievers_1.BaseRetriever {
         const queries = await this._generateQueries(question, runManager);
         const documents = await this._retrieveDocuments(queries, runManager);
         const uniqueDocuments = this._uniqueUnion(documents);
-        return uniqueDocuments;
+        let outputDocs = uniqueDocuments;
+        if (this.documentCompressor && uniqueDocuments.length) {
+            outputDocs = await this.documentCompressor.compressDocuments(uniqueDocuments, question);
+            if (this.documentCompressorFilteringFn) {
+                outputDocs = this.documentCompressorFilteringFn(outputDocs);
+            }
+        }
+        return outputDocs;
     }
 }
 exports.MultiQueryRetriever = MultiQueryRetriever;

package/dist/retrievers/multi_query.d.ts

@@ -4,14 +4,18 @@ import { Document } from "@langchain/core/documents";
 import { BasePromptTemplate } from "@langchain/core/prompts";
 import { CallbackManagerForRetrieverRun } from "@langchain/core/callbacks/manager";
 import { LLMChain } from "../chains/llm_chain.js";
+import type { BaseDocumentCompressor } from "./document_compressors/index.js";
 interface LineList {
     lines: string[];
 }
+export type MultiDocs = Document<Record<string, any>>[];
 export interface MultiQueryRetrieverInput extends BaseRetrieverInput {
     retriever: BaseRetrieverInterface;
     llmChain: LLMChain<LineList>;
     queryCount?: number;
     parserKey?: string;
+    documentCompressor?: BaseDocumentCompressor | undefined;
+    documentCompressorFilteringFn?: (docs: MultiDocs) => MultiDocs;
 }
 /**
  * @example
@@ -33,6 +37,8 @@ export declare class MultiQueryRetriever extends BaseRetriever {
     private llmChain;
     private queryCount;
     private parserKey;
+    documentCompressor: BaseDocumentCompressor | undefined;
+    documentCompressorFilteringFn?: MultiQueryRetrieverInput["documentCompressorFilteringFn"];
     constructor(fields: MultiQueryRetrieverInput);
     static fromLLM(fields: Omit<MultiQueryRetrieverInput, "llmChain"> & {
         llm: BaseLanguageModelInterface;

package/dist/retrievers/multi_query.js

@@ -100,10 +100,24 @@ export class MultiQueryRetriever extends BaseRetriever {
             writable: true,
             value: "lines"
         });
+        Object.defineProperty(this, "documentCompressor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "documentCompressorFilteringFn", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.retriever = fields.retriever;
         this.llmChain = fields.llmChain;
         this.queryCount = fields.queryCount ?? this.queryCount;
         this.parserKey = fields.parserKey ?? this.parserKey;
+        this.documentCompressor = fields.documentCompressor;
+        this.documentCompressorFilteringFn = fields.documentCompressorFilteringFn;
     }
     static fromLLM(fields) {
         const { retriever, llm, prompt = DEFAULT_QUERY_PROMPT, queryCount, parserKey, ...rest } = fields;
@@ -123,10 +137,10 @@ export class MultiQueryRetriever extends BaseRetriever {
     // Retrieve documents using the original retriever
     async _retrieveDocuments(queries, runManager) {
         const documents = [];
-        for (const query of queries) {
+        await Promise.all(queries.map(async (query) => {
             const docs = await this.retriever.getRelevantDocuments(query, runManager?.getChild());
             documents.push(...docs);
-        }
+        }));
         return documents;
     }
     // Deduplicate the documents that were returned in multiple retrievals
@@ -143,6 +157,13 @@ export class MultiQueryRetriever extends BaseRetriever {
         const queries = await this._generateQueries(question, runManager);
         const documents = await this._retrieveDocuments(queries, runManager);
         const uniqueDocuments = this._uniqueUnion(documents);
-        return uniqueDocuments;
+        let outputDocs = uniqueDocuments;
+        if (this.documentCompressor && uniqueDocuments.length) {
+            outputDocs = await this.documentCompressor.compressDocuments(uniqueDocuments, question);
+            if (this.documentCompressorFilteringFn) {
+                outputDocs = this.documentCompressorFilteringFn(outputDocs);
+            }
+        }
+        return outputDocs;
     }
 }

package/dist/retrievers/parent_document.cjs

@@ -111,6 +111,18 @@ class ParentDocumentRetriever extends multi_vector_js_1.MultiVectorRetriever {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "documentCompressor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "documentCompressorFilteringFn", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.vectorstore = fields.vectorstore;
         this.childSplitter = fields.childSplitter;
         this.parentSplitter = fields.parentSplitter;
@@ -118,9 +130,10 @@ class ParentDocumentRetriever extends multi_vector_js_1.MultiVectorRetriever {
         this.childK = fields.childK;
         this.parentK = fields.parentK;
         this.childDocumentRetriever = fields.childDocumentRetriever;
+        this.documentCompressor = fields.documentCompressor;
+        this.documentCompressorFilteringFn = fields.documentCompressorFilteringFn;
     }
     async _getRelevantDocuments(query) {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         let subDocs = [];
         if (this.childDocumentRetriever) {
             subDocs = await this.childDocumentRetriever.getRelevantDocuments(query);
@@ -128,6 +141,12 @@ class ParentDocumentRetriever extends multi_vector_js_1.MultiVectorRetriever {
         else {
             subDocs = await this.vectorstore.similaritySearch(query, this.childK);
         }
+        if (this.documentCompressor && subDocs.length) {
+            subDocs = await this.documentCompressor.compressDocuments(subDocs, query);
+            if (this.documentCompressorFilteringFn) {
+                subDocs = this.documentCompressorFilteringFn(subDocs);
+            }
+        }
         // Maintain order
         const parentDocIds = [];
         for (const doc of subDocs) {

package/dist/retrievers/parent_document.d.ts

@@ -1,7 +1,9 @@
 import { type VectorStoreInterface, type VectorStoreRetrieverInterface } from "@langchain/core/vectorstores";
 import { Document } from "@langchain/core/documents";
+import type { BaseDocumentCompressor } from "./document_compressors/index.js";
 import { TextSplitter, TextSplitterChunkHeaderOptions } from "../text_splitter.js";
 import { MultiVectorRetriever, type MultiVectorRetrieverInput } from "./multi_vector.js";
+export type SubDocs = Document<Record<string, any>>[];
 /**
  * Interface for the fields required to initialize a
  * ParentDocumentRetriever instance.
@@ -14,6 +16,8 @@ export type ParentDocumentRetrieverFields = MultiVectorRetrieverInput & {
      * the `.similaritySearch` method of the vectorstore.
      */
     childDocumentRetriever?: VectorStoreRetrieverInterface<VectorStoreInterface>;
+    documentCompressor?: BaseDocumentCompressor | undefined;
+    documentCompressorFilteringFn?: (docs: SubDocs) => SubDocs;
 };
 /**
  * A type of document retriever that splits input documents into smaller chunks
@@ -55,6 +59,8 @@ export declare class ParentDocumentRetriever extends MultiVectorRetriever {
     protected childK?: number;
     protected parentK?: number;
     childDocumentRetriever: VectorStoreRetrieverInterface<VectorStoreInterface> | undefined;
+    documentCompressor: BaseDocumentCompressor | undefined;
+    documentCompressorFilteringFn?: ParentDocumentRetrieverFields["documentCompressorFilteringFn"];
     constructor(fields: ParentDocumentRetrieverFields);
     _getRelevantDocuments(query: string): Promise<Document[]>;
     /**

package/dist/retrievers/parent_document.js

@@ -85,6 +85,18 @@ export class ParentDocumentRetriever extends MultiVectorRetriever {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "documentCompressor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "documentCompressorFilteringFn", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.vectorstore = fields.vectorstore;
         this.childSplitter = fields.childSplitter;
         this.parentSplitter = fields.parentSplitter;
@@ -92,9 +104,10 @@ export class ParentDocumentRetriever extends MultiVectorRetriever {
         this.childK = fields.childK;
         this.parentK = fields.parentK;
         this.childDocumentRetriever = fields.childDocumentRetriever;
+        this.documentCompressor = fields.documentCompressor;
+        this.documentCompressorFilteringFn = fields.documentCompressorFilteringFn;
     }
     async _getRelevantDocuments(query) {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         let subDocs = [];
         if (this.childDocumentRetriever) {
             subDocs = await this.childDocumentRetriever.getRelevantDocuments(query);
@@ -102,6 +115,12 @@ export class ParentDocumentRetriever extends MultiVectorRetriever {
         else {
             subDocs = await this.vectorstore.similaritySearch(query, this.childK);
         }
+        if (this.documentCompressor && subDocs.length) {
+            subDocs = await this.documentCompressor.compressDocuments(subDocs, query);
+            if (this.documentCompressorFilteringFn) {
+                subDocs = this.documentCompressorFilteringFn(subDocs);
+            }
+        }
         // Maintain order
         const parentDocIds = [];
         for (const doc of subDocs) {

package/dist/retrievers/self_query/index.cjs

@@ -14,7 +14,7 @@ Object.defineProperty(exports, "BasicTranslator", { enumerable: true, get: funct
  * implements the SelfQueryRetrieverArgs interface.
  * @example
  * ```typescript
- * const selfQueryRetriever = await SelfQueryRetriever.fromLLM({
+ * const selfQueryRetriever = SelfQueryRetriever.fromLLM({
  *   llm: new ChatOpenAI(),
  *   vectorStore: await HNSWLib.fromDocuments(docs, new OpenAIEmbeddings()),
  *   documentContents: "Brief summary of a movie",

package/dist/retrievers/self_query/index.d.ts

@@ -30,7 +30,7 @@ export interface SelfQueryRetrieverArgs<T extends VectorStore> extends BaseRetri
  * implements the SelfQueryRetrieverArgs interface.
  * @example
  * ```typescript
- * const selfQueryRetriever = await SelfQueryRetriever.fromLLM({
+ * const selfQueryRetriever = SelfQueryRetriever.fromLLM({
  *   llm: new ChatOpenAI(),
  *   vectorStore: await HNSWLib.fromDocuments(docs, new OpenAIEmbeddings()),
  *   documentContents: "Brief summary of a movie",

package/dist/retrievers/self_query/index.js

@@ -9,7 +9,7 @@ export { BaseTranslator, BasicTranslator, FunctionalTranslator };
  * implements the SelfQueryRetrieverArgs interface.
  * @example
  * ```typescript
- * const selfQueryRetriever = await SelfQueryRetriever.fromLLM({
+ * const selfQueryRetriever = SelfQueryRetriever.fromLLM({
  *   llm: new ChatOpenAI(),
  *   vectorStore: await HNSWLib.fromDocuments(docs, new OpenAIEmbeddings()),
  *   documentContents: "Brief summary of a movie",

package/dist/retrievers/self_query/pinecone.cjs

@@ -11,7 +11,7 @@ const base_js_1 = require("./base.cjs");
  * queries and compare results.
  * @example
  * ```typescript
- * const selfQueryRetriever = await SelfQueryRetriever.fromLLM({
+ * const selfQueryRetriever = SelfQueryRetriever.fromLLM({
  *   llm: new ChatOpenAI(),
  *   vectorStore: new PineconeStore(),
  *   documentContents: "Brief summary of a movie",

package/dist/retrievers/self_query/pinecone.d.ts

@@ -8,7 +8,7 @@ import { BasicTranslator } from "./base.js";
  * queries and compare results.
  * @example
  * ```typescript
- * const selfQueryRetriever = await SelfQueryRetriever.fromLLM({
+ * const selfQueryRetriever = SelfQueryRetriever.fromLLM({
  *   llm: new ChatOpenAI(),
  *   vectorStore: new PineconeStore(),
  *   documentContents: "Brief summary of a movie",

package/dist/retrievers/self_query/pinecone.js

@@ -8,7 +8,7 @@ import { BasicTranslator } from "./base.js";
  * queries and compare results.
  * @example
  * ```typescript
- * const selfQueryRetriever = await SelfQueryRetriever.fromLLM({
+ * const selfQueryRetriever = SelfQueryRetriever.fromLLM({
  *   llm: new ChatOpenAI(),
  *   vectorStore: new PineconeStore(),
  *   documentContents: "Brief summary of a movie",

package/dist/smith/runner_utils.cjs

@@ -5,6 +5,7 @@ const messages_1 = require("@langchain/core/messages");
 const runnables_1 = require("@langchain/core/runnables");
 const tracer_langchain_1 = require("@langchain/core/tracers/tracer_langchain");
 const base_1 = require("@langchain/core/tracers/base");
+const async_caller_1 = require("@langchain/core/utils/async_caller");
 const langsmith_1 = require("langsmith");
 const loader_js_1 = require("../evaluation/loader.cjs");
 const config_js_1 = require("./config.cjs");
@@ -403,31 +404,37 @@ const loadExamples = async ({ datasetName, client, projectName, }) => {
         runExtractors,
     };
 };
-const applyEvaluators = async ({ evaluation, runs, examples, client, }) => {
+const applyEvaluators = async ({ evaluation, runs, examples, client, maxConcurrency, }) => {
     // TODO: Parallelize and/or put in callbacks to speed up evals.
     const { evaluators } = evaluation;
     const progress = new progress_js_1.ProgressBar({
         total: examples.length,
         format: "Running Evaluators: {bar} {percentage}% | {value}/{total}\n",
     });
-    const results = {};
-    for (let i = 0; i < runs.length; i += 1) {
-        const run = runs[i];
-        const example = examples[i];
+    const caller = new async_caller_1.AsyncCaller({
+        maxConcurrency,
+    });
+    const requests = runs.map(async (run, i) => caller.call(async () => {
         const evaluatorResults = await Promise.allSettled(evaluators.map((evaluator) => client.evaluateRun(run, evaluator, {
-            referenceExample: example,
+            referenceExample: examples[i],
             loadChildRuns: false,
         })));
         progress.increment();
-        results[example.id] = {
+        return {
             execution_time: run?.end_time && run.start_time
                 ? run.end_time - run.start_time
                 : undefined,
-            feedback: evaluatorResults.map((evalResult) => evalResult.status === "fulfilled" ? evalResult.value : evalResult.reason),
+            feedback: evaluatorResults.map((evalResult) => evalResult.status === "fulfilled"
+                ? evalResult.value
+                : evalResult.reason),
             run_id: run.id,
         };
-    }
-    return results;
+    }));
+    const results = await Promise.all(requests);
+    return results.reduce((acc, result, i) => ({
+        ...acc,
+        [examples[i].id]: result,
+    }), {});
 };
 const getExamplesInputs = (examples, chainOrFactory, dataType) => {
     if (dataType === "chat") {
@@ -553,6 +560,7 @@ async function runOnDataset(chainOrFactory, datasetName, options) {
             runs,
             examples,
             client: testClient,
+            maxConcurrency: testConcurrency,
         });
     }
     const results = {

package/dist/smith/runner_utils.js

@@ -2,6 +2,7 @@ import { mapStoredMessagesToChatMessages } from "@langchain/core/messages";
 import { Runnable, RunnableLambda, getCallbackManagerForConfig, } from "@langchain/core/runnables";
 import { LangChainTracer } from "@langchain/core/tracers/tracer_langchain";
 import { BaseTracer } from "@langchain/core/tracers/base";
+import { AsyncCaller } from "@langchain/core/utils/async_caller";
 import { Client, RunTree, } from "langsmith";
 import { loadEvaluator } from "../evaluation/loader.js";
 import { isOffTheShelfEvaluator, isCustomEvaluator, } from "./config.js";
@@ -400,31 +401,37 @@ const loadExamples = async ({ datasetName, client, projectName, }) => {
         runExtractors,
     };
 };
-const applyEvaluators = async ({ evaluation, runs, examples, client, }) => {
+const applyEvaluators = async ({ evaluation, runs, examples, client, maxConcurrency, }) => {
     // TODO: Parallelize and/or put in callbacks to speed up evals.
     const { evaluators } = evaluation;
     const progress = new ProgressBar({
         total: examples.length,
         format: "Running Evaluators: {bar} {percentage}% | {value}/{total}\n",
     });
-    const results = {};
-    for (let i = 0; i < runs.length; i += 1) {
-        const run = runs[i];
-        const example = examples[i];
+    const caller = new AsyncCaller({
+        maxConcurrency,
+    });
+    const requests = runs.map(async (run, i) => caller.call(async () => {
         const evaluatorResults = await Promise.allSettled(evaluators.map((evaluator) => client.evaluateRun(run, evaluator, {
-            referenceExample: example,
+            referenceExample: examples[i],
             loadChildRuns: false,
         })));
         progress.increment();
-        results[example.id] = {
+        return {
             execution_time: run?.end_time && run.start_time
                 ? run.end_time - run.start_time
                 : undefined,
-            feedback: evaluatorResults.map((evalResult) => evalResult.status === "fulfilled" ? evalResult.value : evalResult.reason),
+            feedback: evaluatorResults.map((evalResult) => evalResult.status === "fulfilled"
+                ? evalResult.value
+                : evalResult.reason),
             run_id: run.id,
         };
-    }
-    return results;
+    }));
+    const results = await Promise.all(requests);
+    return results.reduce((acc, result, i) => ({
+        ...acc,
+        [examples[i].id]: result,
+    }), {});
 };
 const getExamplesInputs = (examples, chainOrFactory, dataType) => {
     if (dataType === "chat") {
@@ -550,6 +557,7 @@ export async function runOnDataset(chainOrFactory, datasetName, options) {
             runs,
             examples,
             client: testClient,
+            maxConcurrency: testConcurrency,
         });
     }
     const results = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.1.33",
+  "version": "0.1.35",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {