langchain 0.1.19 → 0.1.20

package/dist/agents/executor.cjs

@@ -367,7 +367,7 @@ class AgentExecutor extends base_js_1.BaseChain {
         return this.maxIterations === undefined || iterations < this.maxIterations;
     }
     /** @ignore */
-    async _call(inputs, runManager) {
+    async _call(inputs, runManager, config) {
         const toolsByName = Object.fromEntries(this.tools.map((t) => [t.name.toLowerCase(), t]));
         const steps = [];
         let iterations = 0;
@@ -444,7 +444,7 @@ class AgentExecutor extends base_js_1.BaseChain {
                 let observation;
                 try {
                     observation = tool
-                        ? await tool.call(action.toolInput, runManager?.getChild())
+                        ? await tool.invoke(action.toolInput, (0, runnables_1.patchConfig)(config, { callbacks: runManager?.getChild() }))
                         : `${action.tool} is not a valid tool, try another one.`;
                 }
                 catch (e) {

package/dist/agents/executor.d.ts

@@ -140,7 +140,7 @@ export declare class AgentExecutor extends BaseChain<ChainValues, AgentExecutorO
      */
     private shouldContinue;
     /** @ignore */
-    _call(inputs: ChainValues, runManager?: CallbackManagerForChainRun): Promise<AgentExecutorOutput>;
+    _call(inputs: ChainValues, runManager?: CallbackManagerForChainRun, config?: RunnableConfig): Promise<AgentExecutorOutput>;
     _takeNextStep(nameToolMap: Record<string, ToolInterface>, inputs: ChainValues, intermediateSteps: AgentStep[], runManager?: CallbackManagerForChainRun): Promise<AgentFinish | AgentStep[]>;
     _return(output: AgentFinish, intermediateSteps: AgentStep[], runManager?: CallbackManagerForChainRun): Promise<AgentExecutorOutput>;
     _getToolReturn(nextStepOutput: AgentStep): Promise<AgentFinish | null>;

package/dist/agents/executor.js

@@ -1,5 +1,5 @@
 import { ToolInputParsingException, Tool, } from "@langchain/core/tools";
-import { Runnable } from "@langchain/core/runnables";
+import { Runnable, patchConfig, } from "@langchain/core/runnables";
 import { CallbackManager, } from "@langchain/core/callbacks/manager";
 import { OutputParserException } from "@langchain/core/output_parsers";
 import { Serializable } from "@langchain/core/load/serializable";
@@ -362,7 +362,7 @@ export class AgentExecutor extends BaseChain {
         return this.maxIterations === undefined || iterations < this.maxIterations;
     }
     /** @ignore */
-    async _call(inputs, runManager) {
+    async _call(inputs, runManager, config) {
         const toolsByName = Object.fromEntries(this.tools.map((t) => [t.name.toLowerCase(), t]));
         const steps = [];
         let iterations = 0;
@@ -439,7 +439,7 @@ export class AgentExecutor extends BaseChain {
                 let observation;
                 try {
                     observation = tool
-                        ? await tool.call(action.toolInput, runManager?.getChild())
+                        ? await tool.invoke(action.toolInput, patchConfig(config, { callbacks: runManager?.getChild() }))
                         : `${action.tool} is not a valid tool, try another one.`;
                 }
                 catch (e) {

package/dist/chains/base.cjs

@@ -55,14 +55,14 @@ class BaseChain extends base_1.BaseLangChain {
         try {
             outputValues = await (fullValues.signal
                 ? Promise.race([
-                    this._call(fullValues, runManager),
+                    this._call(fullValues, runManager, config),
                     new Promise((_, reject) => {
                         fullValues.signal?.addEventListener("abort", () => {
                             reject(new Error("AbortError"));
                         });
                     }),
                 ])
-                : this._call(fullValues, runManager));
+                : this._call(fullValues, runManager, config));
         }
         catch (e) {
             await runManager?.handleChainError(e);

package/dist/chains/base.d.ts

@@ -37,7 +37,7 @@ export declare abstract class BaseChain<RunInput extends ChainValues = ChainValu
     /**
      * Run the core logic of this chain and return the output
      */
-    abstract _call(values: RunInput, runManager?: CallbackManagerForChainRun): Promise<RunOutput>;
+    abstract _call(values: RunInput, runManager?: CallbackManagerForChainRun, config?: RunnableConfig): Promise<RunOutput>;
     /**
      * Return the string type key uniquely identifying this class of chain.
      */

package/dist/chains/base.js

@@ -52,14 +52,14 @@ export class BaseChain extends BaseLangChain {
         try {
             outputValues = await (fullValues.signal
                 ? Promise.race([
-                    this._call(fullValues, runManager),
+                    this._call(fullValues, runManager, config),
                     new Promise((_, reject) => {
                         fullValues.signal?.addEventListener("abort", () => {
                             reject(new Error("AbortError"));
                         });
                     }),
                 ])
-                : this._call(fullValues, runManager));
+                : this._call(fullValues, runManager, config));
         }
         catch (e) {
             await runManager?.handleChainError(e);

package/dist/document_loaders/web/azure_blob_storage_container.cjs

@@ -43,7 +43,11 @@ class AzureBlobStorageContainerLoader extends base_js_1.BaseDocumentLoader {
      * @returns An array of loaded documents.
      */
     async load() {
-        const blobServiceClient = storage_blob_1.BlobServiceClient.fromConnectionString(this.connectionString);
+        const blobServiceClient = storage_blob_1.BlobServiceClient.fromConnectionString(this.connectionString, {
+            userAgentOptions: {
+                userAgentPrefix: "langchainjs-blob-storage-container",
+            },
+        });
         const containerClient = blobServiceClient.getContainerClient(this.container);
         let docs = [];
         for await (const blob of containerClient.listBlobsFlat()) {

package/dist/document_loaders/web/azure_blob_storage_container.js

@@ -40,7 +40,11 @@ export class AzureBlobStorageContainerLoader extends BaseDocumentLoader {
      * @returns An array of loaded documents.
      */
     async load() {
-        const blobServiceClient = BlobServiceClient.fromConnectionString(this.connectionString);
+        const blobServiceClient = BlobServiceClient.fromConnectionString(this.connectionString, {
+            userAgentOptions: {
+                userAgentPrefix: "langchainjs-blob-storage-container",
+            },
+        });
         const containerClient = blobServiceClient.getContainerClient(this.container);
         let docs = [];
         for await (const blob of containerClient.listBlobsFlat()) {

package/dist/document_loaders/web/azure_blob_storage_file.cjs

@@ -89,7 +89,11 @@ class AzureBlobStorageFileLoader extends base_js_1.BaseDocumentLoader {
         const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "azureblobfileloader-"));
         const filePath = path.join(tempDir, this.blobName);
         try {
-            const blobServiceClient = storage_blob_1.BlobServiceClient.fromConnectionString(this.connectionString);
+            const blobServiceClient = storage_blob_1.BlobServiceClient.fromConnectionString(this.connectionString, {
+                userAgentOptions: {
+                    userAgentPrefix: "langchainjs-blob-storage-file",
+                },
+            });
             const containerClient = blobServiceClient.getContainerClient(this.container);
             const blobClient = containerClient.getBlobClient(this.blobName);
             fs.mkdirSync(path.dirname(filePath), { recursive: true });

package/dist/document_loaders/web/azure_blob_storage_file.js

@@ -63,7 +63,11 @@ export class AzureBlobStorageFileLoader extends BaseDocumentLoader {
         const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "azureblobfileloader-"));
         const filePath = path.join(tempDir, this.blobName);
         try {
-            const blobServiceClient = BlobServiceClient.fromConnectionString(this.connectionString);
+            const blobServiceClient = BlobServiceClient.fromConnectionString(this.connectionString, {
+                userAgentOptions: {
+                    userAgentPrefix: "langchainjs-blob-storage-file",
+                },
+            });
             const containerClient = blobServiceClient.getContainerClient(this.container);
             const blobClient = containerClient.getBlobClient(this.blobName);
             fs.mkdirSync(path.dirname(filePath), { recursive: true });

package/dist/smith/config.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.LabeledCriteria = exports.Criteria = exports.isCustomEvaluator = exports.isOffTheShelfEvaluator = void 0;
+exports.EmbeddingDistance = exports.LabeledCriteria = exports.Criteria = exports.isCustomEvaluator = exports.isOffTheShelfEvaluator = void 0;
 function isOffTheShelfEvaluator(evaluator) {
     return typeof evaluator === "string" || "evaluatorType" in evaluator;
 }
@@ -36,6 +36,7 @@ function Criteria(criteria, config) {
         evaluatorType: "criteria",
         criteria,
         feedbackKey: config?.feedbackKey ?? criteria,
+        llm: config?.llm,
         formatEvaluatorInputs,
     };
 }
@@ -51,7 +52,23 @@ function LabeledCriteria(criteria, config) {
         evaluatorType: "labeled_criteria",
         criteria,
         feedbackKey: config?.feedbackKey ?? criteria,
+        llm: config?.llm,
         formatEvaluatorInputs,
     };
 }
 exports.LabeledCriteria = LabeledCriteria;
+function EmbeddingDistance(distanceMetric, config) {
+    const formatEvaluatorInputs = config?.formatEvaluatorInputs ??
+        ((payload) => ({
+            prediction: getSingleStringifiedValue(payload.rawPrediction),
+            reference: getSingleStringifiedValue(payload.rawReferenceOutput),
+        }));
+    return {
+        evaluatorType: "embedding_distance",
+        embedding: config?.embedding,
+        distanceMetric,
+        feedbackKey: config?.feedbackKey ?? "embedding_distance",
+        formatEvaluatorInputs,
+    };
+}
+exports.EmbeddingDistance = EmbeddingDistance;

package/dist/smith/config.d.ts

@@ -2,7 +2,7 @@ import { BaseLanguageModel } from "@langchain/core/language_models/base";
 import { RunnableConfig } from "@langchain/core/runnables";
 import { Example, Run } from "langsmith";
 import { EvaluationResult, RunEvaluator } from "langsmith/evaluation";
-import { Criteria as CriteriaType } from "../evaluation/index.js";
+import { Criteria as CriteriaType, type EmbeddingDistanceEvalChainInput } from "../evaluation/index.js";
 import { LoadEvaluatorOptions } from "../evaluation/loader.js";
 import { EvaluatorType } from "../evaluation/types.js";
 export type EvaluatorInputs = {
@@ -35,6 +35,7 @@ export type RunEvaluatorLike = ((props: DynamicRunEvaluatorParams, options?: {
 }) => EvaluationResult);
 export declare function isOffTheShelfEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is T | EvalConfig;
 export declare function isCustomEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is U;
+export type RunEvalType<T extends keyof EvaluatorType = "criteria" | "labeled_criteria" | "embedding_distance", U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = T | EvalConfig | U;
 /**
  * Configuration class for running evaluations on datasets.
  *
@@ -44,22 +45,13 @@ export declare function isCustomEvaluator<T extends keyof EvaluatorType, U exten
  * @typeparam T - The type of evaluators.
  * @typeparam U - The type of custom evaluators.
  */
-export type RunEvalConfig<T extends keyof EvaluatorType = keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = {
+export type RunEvalConfig<T extends keyof EvaluatorType = "criteria" | "labeled_criteria" | "embedding_distance", U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike> = {
     /**
-     * Custom evaluators to apply to a dataset run.
-     * Each evaluator is provided with a run trace containing the model
-     * outputs, as well as an "example" object representing a record
-     * in the dataset.
-     *
-     * @deprecated Use `evaluators` instead.
-     */
-    customEvaluators?: U[];
-    /**
-     * LangChain evaluators to apply to a dataset run.
+     * Evaluators to apply to a dataset run.
      * You can optionally specify these by name, or by
      * configuring them with an EvalConfig object.
      */
-    evaluators?: (T | EvalConfig | U)[];
+    evaluators?: RunEvalType<T, U>[];
     /**
      * Convert the evaluation data into formats that can be used by the evaluator.
      * This should most commonly be a string.
@@ -85,9 +77,14 @@ export type RunEvalConfig<T extends keyof EvaluatorType = keyof EvaluatorType, U
      */
     formatEvaluatorInputs?: EvaluatorInputFormatter;
     /**
-     * The language model specification for evaluators that require one.
+     * Custom evaluators to apply to a dataset run.
+     * Each evaluator is provided with a run trace containing the model
+     * outputs, as well as an "example" object representing a record
+     * in the dataset.
+     *
+     * @deprecated Use `evaluators` instead.
      */
-    evalLlm?: string;
+    customEvaluators?: U[];
 };
 export interface EvalConfig extends LoadEvaluatorOptions {
     /**
@@ -151,7 +148,7 @@ export interface EvalConfig extends LoadEvaluatorOptions {
  *   }]
  * };
  */
-export type CriteriaEvalChainConfig = EvalConfig & {
+export type Criteria = EvalConfig & {
     evaluatorType: "criteria";
     /**
      * The "criteria" to insert into the prompt template
@@ -161,16 +158,12 @@ export type CriteriaEvalChainConfig = EvalConfig & {
      */
     criteria?: CriteriaType | Record<string, string>;
     /**
-     * The feedback (or metric) name to use for the logged
-     * evaluation results. If none provided, we default to
-     * the evaluationName.
-     */
-    feedbackKey?: string;
-    /**
-     * The language model to use as the evaluator.
+     * The language model to use as the evaluator, defaults to GPT-4
      */
     llm?: BaseLanguageModel;
 };
+export type CriteriaEvalChainConfig = Criteria;
+export declare function Criteria(criteria: CriteriaType, config?: Pick<Partial<LabeledCriteria>, "formatEvaluatorInputs" | "llm" | "feedbackKey">): EvalConfig;
 /**
  * Configuration to load a "LabeledCriteriaEvalChain" evaluator,
  * which prompts an LLM to determine whether the model's
@@ -208,39 +201,17 @@ export type LabeledCriteria = EvalConfig & {
      */
     criteria?: CriteriaType | Record<string, string>;
     /**
-     * The feedback (or metric) name to use for the logged
-     * evaluation results. If none provided, we default to
-     * the evaluationName.
-     */
-    feedbackKey?: string;
-    /**
-     * The language model to use as the evaluator.
+     * The language model to use as the evaluator, defaults to GPT-4
      */
     llm?: BaseLanguageModel;
 };
-export declare function Criteria(criteria: CriteriaType, config?: {
-    formatEvaluatorInputs?: EvaluatorInputFormatter;
-    feedbackKey?: string;
-}): {
-    evaluatorType: "criteria";
-    criteria: CriteriaType;
-    feedbackKey: string;
-    formatEvaluatorInputs: EvaluatorInputFormatter | ((payload: {
-        rawInput: any;
-        rawPrediction: any;
-        rawReferenceOutput?: any;
-        run: Run;
-    }) => {
-        prediction: string;
-        input: string;
-    });
-};
-export declare function LabeledCriteria(criteria: CriteriaType, config?: {
-    formatEvaluatorInputs?: EvaluatorInputFormatter;
-    feedbackKey?: string;
-}): {
-    evaluatorType: "labeled_criteria";
-    criteria: CriteriaType;
-    feedbackKey: string;
-    formatEvaluatorInputs: EvaluatorInputFormatter;
+export declare function LabeledCriteria(criteria: CriteriaType, config?: Pick<Partial<LabeledCriteria>, "formatEvaluatorInputs" | "llm" | "feedbackKey">): LabeledCriteria;
+/**
+ * Configuration to load a "EmbeddingDistanceEvalChain" evaluator,
+ * which embeds distances to score semantic difference between
+ * a prediction and reference.
+ */
+export type EmbeddingDistance = EvalConfig & EmbeddingDistanceEvalChainInput & {
+    evaluatorType: "embedding_distance";
 };
+export declare function EmbeddingDistance(distanceMetric: EmbeddingDistanceEvalChainInput["distanceMetric"], config?: Pick<Partial<LabeledCriteria>, "formatEvaluatorInputs" | "embedding" | "feedbackKey">): EmbeddingDistance;

package/dist/smith/config.js

@@ -31,6 +31,7 @@ export function Criteria(criteria, config) {
         evaluatorType: "criteria",
         criteria,
         feedbackKey: config?.feedbackKey ?? criteria,
+        llm: config?.llm,
         formatEvaluatorInputs,
     };
 }
@@ -45,6 +46,21 @@ export function LabeledCriteria(criteria, config) {
         evaluatorType: "labeled_criteria",
         criteria,
         feedbackKey: config?.feedbackKey ?? criteria,
+        llm: config?.llm,
+        formatEvaluatorInputs,
+    };
+}
+export function EmbeddingDistance(distanceMetric, config) {
+    const formatEvaluatorInputs = config?.formatEvaluatorInputs ??
+        ((payload) => ({
+            prediction: getSingleStringifiedValue(payload.rawPrediction),
+            reference: getSingleStringifiedValue(payload.rawReferenceOutput),
+        }));
+    return {
+        evaluatorType: "embedding_distance",
+        embedding: config?.embedding,
+        distanceMetric,
+        feedbackKey: config?.feedbackKey ?? "embedding_distance",
         formatEvaluatorInputs,
     };
 }

package/dist/smith/runner_utils.cjs

@@ -115,6 +115,61 @@ class DynamicRunEvaluator {
 function isLLMStringEvaluator(evaluator) {
     return evaluator && typeof evaluator.evaluateStrings === "function";
 }
+/**
+ * Internal implementation of RunTree, which uses the
+ * provided callback manager instead of the internal LangSmith client.
+ *
+ * The goal of this class is to ensure seamless interop when intergrated
+ * with other Runnables.
+ */
+class CallbackManagerRunTree extends langsmith_1.RunTree {
+    constructor(config, callbackManager) {
+        super(config);
+        Object.defineProperty(this, "callbackManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "activeCallbackManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        this.callbackManager = callbackManager;
+    }
+    async createChild(config) {
+        const child = new CallbackManagerRunTree({
+            ...config,
+            parent_run: this,
+            project_name: this.project_name,
+            client: this.client,
+        }, this.activeCallbackManager?.getChild() ?? this.callbackManager);
+        this.child_runs.push(child);
+        return child;
+    }
+    async postRun() {
+        // how it is translated in comparison to basic RunTree?
+        this.activeCallbackManager = await this.callbackManager.handleChainStart(typeof this.serialized !== "object" &&
+            this.serialized != null &&
+            "lc" in this.serialized
+            ? this.serialized
+            : {
+                id: ["langchain", "smith", "CallbackManagerRunTree"],
+                lc: 1,
+                type: "not_implemented",
+            }, this.inputs, this.id, this.run_type, undefined, undefined, this.name);
+    }
+    async patchRun() {
+        if (this.error) {
+            await this.activeCallbackManager?.handleChainError(this.error, this.id, this.parent_run?.id, undefined, undefined);
+        }
+        else {
+            await this.activeCallbackManager?.handleChainEnd(this.outputs ?? {}, this.id, this.parent_run?.id, undefined, undefined);
+        }
+    }
+}
 class RunnableTraceable extends runnables_1.Runnable {
     constructor(fields) {
         super(fields);
@@ -147,12 +202,14 @@ class RunnableTraceable extends runnables_1.Runnable {
         const partialConfig = "langsmith:traceable" in this.func
             ? this.func["langsmith:traceable"]
             : { name: "<lambda>" };
-        const runTree = new langsmith_1.RunTree({
+        if (!callbackManager)
+            throw new Error("CallbackManager not found");
+        const runTree = new CallbackManagerRunTree({
             ...partialConfig,
             parent_run: callbackManager?._parentRunId
                 ? new langsmith_1.RunTree({ name: "<parent>", id: callbackManager?._parentRunId })
                 : undefined,
-        });
+        }, callbackManager);
         if (typeof input === "object" &&
             input != null &&
             Object.keys(input).length === 1) {
@@ -217,14 +274,14 @@ class PreparedRunEvaluator {
         const evalConfig = typeof config === "string" ? {} : config;
         const evaluator = await (0, loader_js_1.loadEvaluator)(evaluatorType, evalConfig);
         const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;
-        if (!feedbackKey) {
-            throw new Error(`Evaluator of type ${evaluatorType} must have an evaluationName` +
-                ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`);
-        }
         if (!isLLMStringEvaluator(evaluator)) {
             throw new Error(`Evaluator of type ${evaluatorType} not yet supported. ` +
                 "Please use a string evaluator, or implement your " +
-                "evaluation logic as a customEvaluator.");
+                "evaluation logic as a custom evaluator.");
+        }
+        if (!feedbackKey) {
+            throw new Error(`Evaluator of type ${evaluatorType} must have an evaluationName` +
+                ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`);
         }
         return new PreparedRunEvaluator(evaluator, feedbackKey, evalConfig?.formatEvaluatorInputs);
     }
@@ -261,7 +318,7 @@ class PreparedRunEvaluator {
         }
         throw new Error("Evaluator not yet supported. " +
             "Please use a string evaluator, or implement your " +
-            "evaluation logic as a customEvaluator.");
+            "evaluation logic as a custom evaluator.");
     }
 }
 class LoadedEvalConfig {
@@ -302,6 +359,10 @@ const createWrappedModel = async (modelOrFactory) => {
         return () => modelOrFactory;
     }
     if (typeof modelOrFactory === "function") {
+        if (isLangsmithTraceableFunction(modelOrFactory)) {
+            const wrappedModel = new RunnableTraceable({ func: modelOrFactory });
+            return () => wrappedModel;
+        }
         try {
             // If it works with no arguments, assume it's a factory
             let res = modelOrFactory();
@@ -313,11 +374,7 @@ const createWrappedModel = async (modelOrFactory) => {
         }
         catch (err) {
             // Otherwise, it's a custom UDF, and we'll wrap
-            // in a lambda or a traceable function
-            if (isLangsmithTraceableFunction(modelOrFactory)) {
-                const wrappedModel = new RunnableTraceable({ func: modelOrFactory });
-                return () => wrappedModel;
-            }
+            // the function in a lambda
             const wrappedModel = new runnables_1.RunnableLambda({ func: modelOrFactory });
             return () => wrappedModel;
         }
@@ -389,10 +446,65 @@ const getExamplesInputs = (examples, chainOrFactory, dataType) => {
     }
     return examples.map(({ inputs }) => inputs);
 };
+/**
+ * Evaluates a given model or chain against a specified LangSmith dataset.
+ *
+ * This function fetches example records from the specified dataset,
+ * runs the model or chain against each example, and returns the evaluation
+ * results.
+ *
+ * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a
+ * Runnable instance, a factory function that returns a Runnable, or a user-defined
+ * function or factory.
+ *
+ * @param datasetName - The name of the dataset against which the evaluation will be
+ * performed. This dataset should already be defined and contain the relevant data
+ * for evaluation.
+ *
+ * @param options - (Optional) Additional parameters for the evaluation process:
+ *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.
+ *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.
+ *   - `projectName` (string): Name of the project for logging and tracking.
+ *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.
+ *   - `client` (Client): Client instance for LangSmith service interaction.
+ *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.
+ *
+ * @returns A promise that resolves to an `EvalResults` object. This object includes
+ * detailed results of the evaluation, such as execution time, run IDs, and feedback
+ * for each entry in the dataset.
+ *
+ * @example
+ * ```typescript
+ * // Example usage for evaluating a model on a dataset
+ * async function evaluateModel() {
+ *   const chain = /* ...create your model or chain...*\//
+ *   const datasetName = 'example-dataset';
+ *   const client = new Client(/* ...config... *\//);
+ *
+ *   const results = await runOnDataset(chain, datasetName, {
+ *     evaluators: [/* ...evaluators... *\//],
+ *     client,
+ *   });
+ *
+ *   console.log('Evaluation Results:', results);
+ * }
+ *
+ * evaluateModel();
+ * ```
+ * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against
+ * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams["evaluators"]`, which can
+ * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
+ * The function returns the evaluation results, which can be logged or further processed as needed.
+ */
 async function runOnDataset(chainOrFactory, datasetName, options) {
-    const { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, } = Array.isArray(options)
-        ? { evaluationConfig: { evaluators: options } }
-        : options ?? {};
+    const { projectName, projectMetadata, client, maxConcurrency, } = options ?? {};
+    const evaluationConfig = options?.evaluationConfig ??
+        (options?.evaluators != null
+            ? {
+                evaluators: options.evaluators,
+                formatEvaluatorInputs: options.formatEvaluatorInputs,
+            }
+            : undefined);
     const wrappedModel = await createWrappedModel(chainOrFactory);
     const testClient = client ?? new langsmith_1.Client();
     const testProjectName = projectName ?? (0, name_generation_js_1.randomName)();

package/dist/smith/runner_utils.d.ts

@@ -4,13 +4,28 @@ import type { TraceableFunction } from "langsmith/traceable";
 import { type RunEvalConfig } from "./config.js";
 export type ChainOrFactory = Runnable | (() => Runnable) | AnyTraceableFunction | ((obj: any) => any) | ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);
 type AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;
-export type RunOnDatasetParams = {
-    evaluationConfig?: RunEvalConfig;
-    projectMetadata?: Record<string, unknown>;
+export interface RunOnDatasetParams extends Omit<RunEvalConfig, "customEvaluators"> {
+    /**
+     * Name of the project for logging and tracking.
+     */
     projectName?: string;
+    /**
+     * Additional metadata for the project.
+     */
+    projectMetadata?: Record<string, unknown>;
+    /**
+     * Client instance for LangSmith service interaction.
+     */
     client?: Client;
+    /**
+     * Maximum concurrency level for dataset processing.
+     */
     maxConcurrency?: number;
-};
+    /**
+     * @deprecated Pass keys directly to the RunOnDatasetParams instead
+     */
+    evaluationConfig?: RunEvalConfig;
+}
 export type EvalResults = {
     projectName: string;
     results: {
@@ -37,11 +52,11 @@ export type EvalResults = {
  * for evaluation.
  *
  * @param options - (Optional) Additional parameters for the evaluation process:
- *   - `evaluationConfig` (RunEvalConfig): Configuration for the evaluation, including
- *     standard and custom evaluators.
+ *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.
+ *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.
  *   - `projectName` (string): Name of the project for logging and tracking.
  *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.
- *   - `client` (Client): Client instance for LangChain service interaction.
+ *   - `client` (Client): Client instance for LangSmith service interaction.
  *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.
  *
  * @returns A promise that resolves to an `EvalResults` object. This object includes
@@ -56,13 +71,8 @@ export type EvalResults = {
  *   const datasetName = 'example-dataset';
  *   const client = new Client(/* ...config... *\//);
  *
- *   const evaluationConfig = {
- *     evaluators: [/* ...evaluators... *\//],
- *     customEvaluators: [/* ...custom evaluators... *\//],
- *   };
- *
  *   const results = await runOnDataset(chain, datasetName, {
- *     evaluationConfig,
+ *     evaluators: [/* ...evaluators... *\//],
  *     client,
  *   });
  *
@@ -72,10 +82,9 @@ export type EvalResults = {
  * evaluateModel();
  * ```
  * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against
- * a dataset named 'example-dataset'. The evaluation process is configured using `RunEvalConfig`, which can
+ * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams["evaluators"]`, which can
  * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
  * The function returns the evaluation results, which can be logged or further processed as needed.
  */
-export declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, }: RunOnDatasetParams): Promise<EvalResults>;
-export declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, evaluators: RunEvalConfig["evaluators"]): Promise<EvalResults>;
+export declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, options?: RunOnDatasetParams): Promise<EvalResults>;
 export {};

package/dist/smith/runner_utils.js

@@ -112,6 +112,61 @@ class DynamicRunEvaluator {
 function isLLMStringEvaluator(evaluator) {
     return evaluator && typeof evaluator.evaluateStrings === "function";
 }
+/**
+ * Internal implementation of RunTree, which uses the
+ * provided callback manager instead of the internal LangSmith client.
+ *
+ * The goal of this class is to ensure seamless interop when intergrated
+ * with other Runnables.
+ */
+class CallbackManagerRunTree extends RunTree {
+    constructor(config, callbackManager) {
+        super(config);
+        Object.defineProperty(this, "callbackManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "activeCallbackManager", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        this.callbackManager = callbackManager;
+    }
+    async createChild(config) {
+        const child = new CallbackManagerRunTree({
+            ...config,
+            parent_run: this,
+            project_name: this.project_name,
+            client: this.client,
+        }, this.activeCallbackManager?.getChild() ?? this.callbackManager);
+        this.child_runs.push(child);
+        return child;
+    }
+    async postRun() {
+        // how it is translated in comparison to basic RunTree?
+        this.activeCallbackManager = await this.callbackManager.handleChainStart(typeof this.serialized !== "object" &&
+            this.serialized != null &&
+            "lc" in this.serialized
+            ? this.serialized
+            : {
+                id: ["langchain", "smith", "CallbackManagerRunTree"],
+                lc: 1,
+                type: "not_implemented",
+            }, this.inputs, this.id, this.run_type, undefined, undefined, this.name);
+    }
+    async patchRun() {
+        if (this.error) {
+            await this.activeCallbackManager?.handleChainError(this.error, this.id, this.parent_run?.id, undefined, undefined);
+        }
+        else {
+            await this.activeCallbackManager?.handleChainEnd(this.outputs ?? {}, this.id, this.parent_run?.id, undefined, undefined);
+        }
+    }
+}
 class RunnableTraceable extends Runnable {
     constructor(fields) {
         super(fields);
@@ -144,12 +199,14 @@ class RunnableTraceable extends Runnable {
         const partialConfig = "langsmith:traceable" in this.func
             ? this.func["langsmith:traceable"]
             : { name: "<lambda>" };
-        const runTree = new RunTree({
+        if (!callbackManager)
+            throw new Error("CallbackManager not found");
+        const runTree = new CallbackManagerRunTree({
             ...partialConfig,
             parent_run: callbackManager?._parentRunId
                 ? new RunTree({ name: "<parent>", id: callbackManager?._parentRunId })
                 : undefined,
-        });
+        }, callbackManager);
         if (typeof input === "object" &&
             input != null &&
             Object.keys(input).length === 1) {
@@ -214,14 +271,14 @@ class PreparedRunEvaluator {
         const evalConfig = typeof config === "string" ? {} : config;
         const evaluator = await loadEvaluator(evaluatorType, evalConfig);
         const feedbackKey = evalConfig?.feedbackKey ?? evaluator?.evaluationName;
-        if (!feedbackKey) {
-            throw new Error(`Evaluator of type ${evaluatorType} must have an evaluationName` +
-                ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`);
-        }
         if (!isLLMStringEvaluator(evaluator)) {
             throw new Error(`Evaluator of type ${evaluatorType} not yet supported. ` +
                 "Please use a string evaluator, or implement your " +
-                "evaluation logic as a customEvaluator.");
+                "evaluation logic as a custom evaluator.");
+        }
+        if (!feedbackKey) {
+            throw new Error(`Evaluator of type ${evaluatorType} must have an evaluationName` +
+                ` or feedbackKey. Please manually provide a feedbackKey in the EvalConfig.`);
         }
         return new PreparedRunEvaluator(evaluator, feedbackKey, evalConfig?.formatEvaluatorInputs);
     }
@@ -258,7 +315,7 @@ class PreparedRunEvaluator {
         }
         throw new Error("Evaluator not yet supported. " +
             "Please use a string evaluator, or implement your " +
-            "evaluation logic as a customEvaluator.");
+            "evaluation logic as a custom evaluator.");
     }
 }
 class LoadedEvalConfig {
@@ -299,6 +356,10 @@ const createWrappedModel = async (modelOrFactory) => {
         return () => modelOrFactory;
     }
     if (typeof modelOrFactory === "function") {
+        if (isLangsmithTraceableFunction(modelOrFactory)) {
+            const wrappedModel = new RunnableTraceable({ func: modelOrFactory });
+            return () => wrappedModel;
+        }
         try {
             // If it works with no arguments, assume it's a factory
             let res = modelOrFactory();
@@ -310,11 +371,7 @@ const createWrappedModel = async (modelOrFactory) => {
         }
         catch (err) {
             // Otherwise, it's a custom UDF, and we'll wrap
-            // in a lambda or a traceable function
-            if (isLangsmithTraceableFunction(modelOrFactory)) {
-                const wrappedModel = new RunnableTraceable({ func: modelOrFactory });
-                return () => wrappedModel;
-            }
+            // the function in a lambda
             const wrappedModel = new RunnableLambda({ func: modelOrFactory });
             return () => wrappedModel;
         }
@@ -386,10 +443,65 @@ const getExamplesInputs = (examples, chainOrFactory, dataType) => {
     }
     return examples.map(({ inputs }) => inputs);
 };
+/**
+ * Evaluates a given model or chain against a specified LangSmith dataset.
+ *
+ * This function fetches example records from the specified dataset,
+ * runs the model or chain against each example, and returns the evaluation
+ * results.
+ *
+ * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a
+ * Runnable instance, a factory function that returns a Runnable, or a user-defined
+ * function or factory.
+ *
+ * @param datasetName - The name of the dataset against which the evaluation will be
+ * performed. This dataset should already be defined and contain the relevant data
+ * for evaluation.
+ *
+ * @param options - (Optional) Additional parameters for the evaluation process:
+ *   - `evaluators` (RunEvalType[]): Evaluators to apply to a dataset run.
+ *   - `formatEvaluatorInputs` (EvaluatorInputFormatter): Convert the evaluation data into formats that can be used by the evaluator.
+ *   - `projectName` (string): Name of the project for logging and tracking.
+ *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.
+ *   - `client` (Client): Client instance for LangSmith service interaction.
+ *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.
+ *
+ * @returns A promise that resolves to an `EvalResults` object. This object includes
+ * detailed results of the evaluation, such as execution time, run IDs, and feedback
+ * for each entry in the dataset.
+ *
+ * @example
+ * ```typescript
+ * // Example usage for evaluating a model on a dataset
+ * async function evaluateModel() {
+ *   const chain = /* ...create your model or chain...*\//
+ *   const datasetName = 'example-dataset';
+ *   const client = new Client(/* ...config... *\//);
+ *
+ *   const results = await runOnDataset(chain, datasetName, {
+ *     evaluators: [/* ...evaluators... *\//],
+ *     client,
+ *   });
+ *
+ *   console.log('Evaluation Results:', results);
+ * }
+ *
+ * evaluateModel();
+ * ```
+ * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against
+ * a dataset named 'example-dataset'. The evaluation process is configured using `RunOnDatasetParams["evaluators"]`, which can
+ * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
+ * The function returns the evaluation results, which can be logged or further processed as needed.
+ */
 export async function runOnDataset(chainOrFactory, datasetName, options) {
-    const { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, } = Array.isArray(options)
-        ? { evaluationConfig: { evaluators: options } }
-        : options ?? {};
+    const { projectName, projectMetadata, client, maxConcurrency, } = options ?? {};
+    const evaluationConfig = options?.evaluationConfig ??
+        (options?.evaluators != null
+            ? {
+                evaluators: options.evaluators,
+                formatEvaluatorInputs: options.formatEvaluatorInputs,
+            }
+            : undefined);
     const wrappedModel = await createWrappedModel(chainOrFactory);
     const testClient = client ?? new Client();
     const testProjectName = projectName ?? randomName();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.1.19",
+  "version": "0.1.20",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -1223,7 +1223,7 @@
     "@gomomento/sdk": "^1.51.1",
     "@gomomento/sdk-core": "^1.51.1",
     "@google-ai/generativelanguage": "^0.2.1",
-    "@google-cloud/storage": "^6.10.1",
+    "@google-cloud/storage": "^7.7.0",
     "@jest/globals": "^29.5.0",
     "@langchain/scripts": "~0.0",
     "@notionhq/client": "^2.2.10",
@@ -1310,7 +1310,7 @@
     "@gomomento/sdk-core": "^1.51.1",
     "@gomomento/sdk-web": "^1.51.1",
     "@google-ai/generativelanguage": "^0.2.1",
-    "@google-cloud/storage": "^6.10.1",
+    "@google-cloud/storage": "^6.10.1 || ^7.7.0",
     "@notionhq/client": "^2.2.10",
     "@pinecone-database/pinecone": "*",
     "@supabase/supabase-js": "^2.10.0",