langchain 0.1.18 → 0.1.19-rc.1

package/dist/smith/config.cjs

@@ -1,2 +1,57 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.LabeledCriteria = exports.Criteria = exports.isCustomEvaluator = exports.isOffTheShelfEvaluator = void 0;
+function isOffTheShelfEvaluator(evaluator) {
+    return typeof evaluator === "string" || "evaluatorType" in evaluator;
+}
+exports.isOffTheShelfEvaluator = isOffTheShelfEvaluator;
+function isCustomEvaluator(evaluator) {
+    return !isOffTheShelfEvaluator(evaluator);
+}
+exports.isCustomEvaluator = isCustomEvaluator;
+const isStringifiableValue = (value) => typeof value === "string" ||
+    typeof value === "number" ||
+    typeof value === "boolean" ||
+    typeof value === "bigint";
+const getSingleStringifiedValue = (value) => {
+    if (isStringifiableValue(value)) {
+        return `${value}`;
+    }
+    if (typeof value === "object" && value != null && !Array.isArray(value)) {
+        const entries = Object.entries(value);
+        if (entries.length === 1 && isStringifiableValue(entries[0][1])) {
+            return `${entries[0][1]}`;
+        }
+    }
+    console.warn("Non-stringifiable value found when coercing", value);
+    return `${value}`;
+};
+function Criteria(criteria, config) {
+    const formatEvaluatorInputs = config?.formatEvaluatorInputs ??
+        ((payload) => ({
+            prediction: getSingleStringifiedValue(payload.rawPrediction),
+            input: getSingleStringifiedValue(payload.rawInput),
+        }));
+    return {
+        evaluatorType: "criteria",
+        criteria,
+        feedbackKey: config?.feedbackKey ?? criteria,
+        formatEvaluatorInputs,
+    };
+}
+exports.Criteria = Criteria;
+function LabeledCriteria(criteria, config) {
+    const formatEvaluatorInputs = config?.formatEvaluatorInputs ??
+        ((payload) => ({
+            prediction: getSingleStringifiedValue(payload.rawPrediction),
+            input: getSingleStringifiedValue(payload.rawInput),
+            reference: getSingleStringifiedValue(payload.rawReferenceOutput),
+        }));
+    return {
+        evaluatorType: "labeled_criteria",
+        criteria,
+        feedbackKey: config?.feedbackKey ?? criteria,
+        formatEvaluatorInputs,
+    };
+}
+exports.LabeledCriteria = LabeledCriteria;

package/dist/smith/config.d.ts

@@ -2,7 +2,7 @@ import { BaseLanguageModel } from "@langchain/core/language_models/base";
 import { RunnableConfig } from "@langchain/core/runnables";
 import { Example, Run } from "langsmith";
 import { EvaluationResult, RunEvaluator } from "langsmith/evaluation";
-import { Criteria } from "../evaluation/index.js";
+import { Criteria as CriteriaType } from "../evaluation/index.js";
 import { LoadEvaluatorOptions } from "../evaluation/loader.js";
 import { EvaluatorType } from "../evaluation/types.js";
 export type EvaluatorInputs = {
@@ -33,6 +33,8 @@ export type RunEvaluatorLike = ((props: DynamicRunEvaluatorParams, options?: {
 }) => Promise<EvaluationResult>) | ((props: DynamicRunEvaluatorParams, options?: {
     config?: RunnableConfig;
 }) => EvaluationResult);
+export declare function isOffTheShelfEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is T | EvalConfig;
+export declare function isCustomEvaluator<T extends keyof EvaluatorType, U extends RunEvaluator | RunEvaluatorLike = RunEvaluator | RunEvaluatorLike>(evaluator: T | EvalConfig | U): evaluator is U;
 /**
  * Configuration class for running evaluations on datasets.
  *
@@ -48,6 +50,8 @@ export type RunEvalConfig<T extends keyof EvaluatorType = keyof EvaluatorType, U
      * Each evaluator is provided with a run trace containing the model
      * outputs, as well as an "example" object representing a record
      * in the dataset.
+     *
+     * @deprecated Use `evaluators` instead.
      */
     customEvaluators?: U[];
     /**
@@ -55,7 +59,7 @@ export type RunEvalConfig<T extends keyof EvaluatorType = keyof EvaluatorType, U
      * You can optionally specify these by name, or by
      * configuring them with an EvalConfig object.
      */
-    evaluators?: (T | EvalConfig)[];
+    evaluators?: (T | EvalConfig | U)[];
     /**
      * Convert the evaluation data into formats that can be used by the evaluator.
      * This should most commonly be a string.
@@ -155,7 +159,7 @@ export type CriteriaEvalChainConfig = EvalConfig & {
      * https://smith.langchain.com/hub/langchain-ai/criteria-evaluator
      * for more information.
      */
-    criteria?: Criteria | Record<string, string>;
+    criteria?: CriteriaType | Record<string, string>;
     /**
      * The feedback (or metric) name to use for the logged
      * evaluation results. If none provided, we default to
@@ -202,7 +206,7 @@ export type LabeledCriteria = EvalConfig & {
      * https://smith.langchain.com/hub/langchain-ai/labeled-criteria
      * for more information.
      */
-    criteria?: Criteria | Record<string, string>;
+    criteria?: CriteriaType | Record<string, string>;
     /**
      * The feedback (or metric) name to use for the logged
      * evaluation results. If none provided, we default to
@@ -214,3 +218,29 @@ export type LabeledCriteria = EvalConfig & {
      */
     llm?: BaseLanguageModel;
 };
+export declare function Criteria(criteria: CriteriaType, config?: {
+    formatEvaluatorInputs?: EvaluatorInputFormatter;
+    feedbackKey?: string;
+}): {
+    evaluatorType: "criteria";
+    criteria: CriteriaType;
+    feedbackKey: string;
+    formatEvaluatorInputs: EvaluatorInputFormatter | ((payload: {
+        rawInput: any;
+        rawPrediction: any;
+        rawReferenceOutput?: any;
+        run: Run;
+    }) => {
+        prediction: string;
+        input: string;
+    });
+};
+export declare function LabeledCriteria(criteria: CriteriaType, config?: {
+    formatEvaluatorInputs?: EvaluatorInputFormatter;
+    feedbackKey?: string;
+}): {
+    evaluatorType: "labeled_criteria";
+    criteria: CriteriaType;
+    feedbackKey: string;
+    formatEvaluatorInputs: EvaluatorInputFormatter;
+};

package/dist/smith/config.js

@@ -1 +1,50 @@
-export {};
+export function isOffTheShelfEvaluator(evaluator) {
+    return typeof evaluator === "string" || "evaluatorType" in evaluator;
+}
+export function isCustomEvaluator(evaluator) {
+    return !isOffTheShelfEvaluator(evaluator);
+}
+const isStringifiableValue = (value) => typeof value === "string" ||
+    typeof value === "number" ||
+    typeof value === "boolean" ||
+    typeof value === "bigint";
+const getSingleStringifiedValue = (value) => {
+    if (isStringifiableValue(value)) {
+        return `${value}`;
+    }
+    if (typeof value === "object" && value != null && !Array.isArray(value)) {
+        const entries = Object.entries(value);
+        if (entries.length === 1 && isStringifiableValue(entries[0][1])) {
+            return `${entries[0][1]}`;
+        }
+    }
+    console.warn("Non-stringifiable value found when coercing", value);
+    return `${value}`;
+};
+export function Criteria(criteria, config) {
+    const formatEvaluatorInputs = config?.formatEvaluatorInputs ??
+        ((payload) => ({
+            prediction: getSingleStringifiedValue(payload.rawPrediction),
+            input: getSingleStringifiedValue(payload.rawInput),
+        }));
+    return {
+        evaluatorType: "criteria",
+        criteria,
+        feedbackKey: config?.feedbackKey ?? criteria,
+        formatEvaluatorInputs,
+    };
+}
+export function LabeledCriteria(criteria, config) {
+    const formatEvaluatorInputs = config?.formatEvaluatorInputs ??
+        ((payload) => ({
+            prediction: getSingleStringifiedValue(payload.rawPrediction),
+            input: getSingleStringifiedValue(payload.rawInput),
+            reference: getSingleStringifiedValue(payload.rawReferenceOutput),
+        }));
+    return {
+        evaluatorType: "labeled_criteria",
+        criteria,
+        feedbackKey: config?.feedbackKey ?? criteria,
+        formatEvaluatorInputs,
+    };
+}

package/dist/smith/runner_utils.cjs

@@ -7,6 +7,7 @@ const tracer_langchain_1 = require("@langchain/core/tracers/tracer_langchain");
 const base_1 = require("@langchain/core/tracers/base");
 const langsmith_1 = require("langsmith");
 const loader_js_1 = require("../evaluation/loader.cjs");
+const config_js_1 = require("./config.cjs");
 const name_generation_js_1 = require("./name_generation.cjs");
 const progress_js_1 = require("./progress.cjs");
 class SingleRunIdExtractor {
@@ -114,6 +115,67 @@ class DynamicRunEvaluator {
 function isLLMStringEvaluator(evaluator) {
     return evaluator && typeof evaluator.evaluateStrings === "function";
 }
+class RunnableTraceable extends runnables_1.Runnable {
+    constructor(fields) {
+        super(fields);
+        Object.defineProperty(this, "lc_serializable", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "lc_namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ["langchain_core", "runnables"]
+        });
+        Object.defineProperty(this, "func", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        if (!isLangsmithTraceableFunction(fields.func)) {
+            throw new Error("RunnableTraceable requires a function that is wrapped in traceable higher-order function");
+        }
+        this.func = fields.func;
+    }
+    async invoke(input, options) {
+        const [config] = this._getOptionsList(options ?? {}, 1);
+        const callbackManager = await (0, runnables_1.getCallbackManagerForConfig)(config);
+        const partialConfig = "langsmith:traceable" in this.func
+            ? this.func["langsmith:traceable"]
+            : { name: "<lambda>" };
+        const runTree = new langsmith_1.RunTree({
+            ...partialConfig,
+            parent_run: callbackManager?._parentRunId
+                ? new langsmith_1.RunTree({ name: "<parent>", id: callbackManager?._parentRunId })
+                : undefined,
+        });
+        if (typeof input === "object" &&
+            input != null &&
+            Object.keys(input).length === 1) {
+            if ("args" in input && Array.isArray(input)) {
+                return (await this.func(runTree, ...input));
+            }
+            if ("input" in input &&
+                !(typeof input === "object" &&
+                    input != null &&
+                    !Array.isArray(input) &&
+                    // eslint-disable-next-line no-instanceof/no-instanceof
+                    !(input instanceof Date))) {
+                try {
+                    return (await this.func(runTree, input.input));
+                }
+                catch (err) {
+                    return (await this.func(runTree, input));
+                }
+            }
+        }
+        return (await this.func(runTree, input));
+    }
+}
 /**
  * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])
  * and composes with a prepareData function so the user can prepare the trace and
@@ -213,7 +275,7 @@ class LoadedEvalConfig {
     }
     static async fromRunEvalConfig(config) {
         // Custom evaluators are applied "as-is"
-        const customEvaluators = config?.customEvaluators?.map((evaluator) => {
+        const customEvaluators = (config?.customEvaluators ?? config.evaluators?.filter(config_js_1.isCustomEvaluator))?.map((evaluator) => {
             if (typeof evaluator === "function") {
                 return new DynamicRunEvaluator(evaluator);
             }
@@ -221,7 +283,9 @@ class LoadedEvalConfig {
                 return evaluator;
             }
         });
-        const offTheShelfEvaluators = await Promise.all(config?.evaluators?.map(async (evaluator) => await PreparedRunEvaluator.fromEvalConfig(evaluator)) ?? []);
+        const offTheShelfEvaluators = await Promise.all(config?.evaluators
+            ?.filter(config_js_1.isOffTheShelfEvaluator)
+            ?.map(async (evaluator) => await PreparedRunEvaluator.fromEvalConfig(evaluator)) ?? []);
         return new LoadedEvalConfig((customEvaluators ?? []).concat(offTheShelfEvaluators ?? []));
     }
 }
@@ -249,7 +313,11 @@ const createWrappedModel = async (modelOrFactory) => {
         }
         catch (err) {
             // Otherwise, it's a custom UDF, and we'll wrap
-            // in a lambda
+            // in a lambda or a traceable function
+            if (isLangsmithTraceableFunction(modelOrFactory)) {
+                const wrappedModel = new RunnableTraceable({ func: modelOrFactory });
+                return () => wrappedModel;
+            }
             const wrappedModel = new runnables_1.RunnableLambda({ func: modelOrFactory });
             return () => wrappedModel;
         }
@@ -321,62 +389,10 @@ const getExamplesInputs = (examples, chainOrFactory, dataType) => {
     }
     return examples.map(({ inputs }) => inputs);
 };
-/**
- * Evaluates a given model or chain against a specified LangSmith dataset.
- *
- * This function fetches example records from the specified dataset,
- * runs the model or chain against each example, and returns the evaluation
- * results.
- *
- * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a
- * Runnable instance, a factory function that returns a Runnable, or a user-defined
- * function or factory.
- *
- * @param datasetName - The name of the dataset against which the evaluation will be
- * performed. This dataset should already be defined and contain the relevant data
- * for evaluation.
- *
- * @param options - (Optional) Additional parameters for the evaluation process:
- *   - `evaluationConfig` (RunEvalConfig): Configuration for the evaluation, including
- *     standard and custom evaluators.
- *   - `projectName` (string): Name of the project for logging and tracking.
- *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.
- *   - `client` (Client): Client instance for LangChain service interaction.
- *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.
- *
- * @returns A promise that resolves to an `EvalResults` object. This object includes
- * detailed results of the evaluation, such as execution time, run IDs, and feedback
- * for each entry in the dataset.
- *
- * @example
- * ```typescript
- * // Example usage for evaluating a model on a dataset
- * async function evaluateModel() {
- *   const chain = /* ...create your model or chain...*\//
- *   const datasetName = 'example-dataset';
- *   const client = new Client(/* ...config... *\//);
- *
- *   const evaluationConfig = {
- *     evaluators: [/* ...evaluators... *\//],
- *     customEvaluators: [/* ...custom evaluators... *\//],
- *   };
- *
- *   const results = await runOnDataset(chain, datasetName, {
- *     evaluationConfig,
- *     client,
- *   });
- *
- *   console.log('Evaluation Results:', results);
- * }
- *
- * evaluateModel();
- * ```
- * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against
- * a dataset named 'example-dataset'. The evaluation process is configured using `RunEvalConfig`, which can
- * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
- * The function returns the evaluation results, which can be logged or further processed as needed.
- */
-const runOnDataset = async (chainOrFactory, datasetName, { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, }) => {
+async function runOnDataset(chainOrFactory, datasetName, options) {
+    const { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, } = Array.isArray(options)
+        ? { evaluationConfig: { evaluators: options } }
+        : options ?? {};
     const wrappedModel = await createWrappedModel(chainOrFactory);
     const testClient = client ?? new langsmith_1.Client();
     const testProjectName = projectName ?? (0, name_generation_js_1.randomName)();
@@ -432,5 +448,8 @@ const runOnDataset = async (chainOrFactory, datasetName, { evaluationConfig, pro
         results: evalResults ?? {},
     };
     return results;
-};
+}
 exports.runOnDataset = runOnDataset;
+function isLangsmithTraceableFunction(x) {
+    return typeof x === "function" && "langsmith:traceable" in x;
+}

package/dist/smith/runner_utils.d.ts

@@ -1,7 +1,9 @@
 import { Runnable } from "@langchain/core/runnables";
 import { Client, Feedback } from "langsmith";
-import type { RunEvalConfig } from "./config.js";
-export type ChainOrFactory = Runnable | (() => Runnable) | ((obj: any) => any) | ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);
+import type { TraceableFunction } from "langsmith/traceable";
+import { type RunEvalConfig } from "./config.js";
+export type ChainOrFactory = Runnable | (() => Runnable) | AnyTraceableFunction | ((obj: any) => any) | ((obj: any) => Promise<any>) | (() => (obj: unknown) => unknown) | (() => (obj: unknown) => Promise<unknown>);
+type AnyTraceableFunction = TraceableFunction<(...any: any[]) => any>;
 export type RunOnDatasetParams = {
     evaluationConfig?: RunEvalConfig;
     projectMetadata?: Record<string, unknown>;
@@ -74,4 +76,6 @@ export type EvalResults = {
  * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
  * The function returns the evaluation results, which can be logged or further processed as needed.
  */
-export declare const runOnDataset: (chainOrFactory: ChainOrFactory, datasetName: string, { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, }: RunOnDatasetParams) => Promise<EvalResults>;
+export declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, }: RunOnDatasetParams): Promise<EvalResults>;
+export declare function runOnDataset(chainOrFactory: ChainOrFactory, datasetName: string, evaluators: RunEvalConfig["evaluators"]): Promise<EvalResults>;
+export {};

package/dist/smith/runner_utils.js

@@ -1,9 +1,10 @@
 import { mapStoredMessagesToChatMessages } from "@langchain/core/messages";
-import { Runnable, RunnableLambda, } from "@langchain/core/runnables";
+import { Runnable, RunnableLambda, getCallbackManagerForConfig, } from "@langchain/core/runnables";
 import { LangChainTracer } from "@langchain/core/tracers/tracer_langchain";
 import { BaseTracer } from "@langchain/core/tracers/base";
-import { Client } from "langsmith";
+import { Client, RunTree, } from "langsmith";
 import { loadEvaluator } from "../evaluation/loader.js";
+import { isOffTheShelfEvaluator, isCustomEvaluator, } from "./config.js";
 import { randomName } from "./name_generation.js";
 import { ProgressBar } from "./progress.js";
 class SingleRunIdExtractor {
@@ -111,6 +112,67 @@ class DynamicRunEvaluator {
 function isLLMStringEvaluator(evaluator) {
     return evaluator && typeof evaluator.evaluateStrings === "function";
 }
+class RunnableTraceable extends Runnable {
+    constructor(fields) {
+        super(fields);
+        Object.defineProperty(this, "lc_serializable", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "lc_namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ["langchain_core", "runnables"]
+        });
+        Object.defineProperty(this, "func", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        if (!isLangsmithTraceableFunction(fields.func)) {
+            throw new Error("RunnableTraceable requires a function that is wrapped in traceable higher-order function");
+        }
+        this.func = fields.func;
+    }
+    async invoke(input, options) {
+        const [config] = this._getOptionsList(options ?? {}, 1);
+        const callbackManager = await getCallbackManagerForConfig(config);
+        const partialConfig = "langsmith:traceable" in this.func
+            ? this.func["langsmith:traceable"]
+            : { name: "<lambda>" };
+        const runTree = new RunTree({
+            ...partialConfig,
+            parent_run: callbackManager?._parentRunId
+                ? new RunTree({ name: "<parent>", id: callbackManager?._parentRunId })
+                : undefined,
+        });
+        if (typeof input === "object" &&
+            input != null &&
+            Object.keys(input).length === 1) {
+            if ("args" in input && Array.isArray(input)) {
+                return (await this.func(runTree, ...input));
+            }
+            if ("input" in input &&
+                !(typeof input === "object" &&
+                    input != null &&
+                    !Array.isArray(input) &&
+                    // eslint-disable-next-line no-instanceof/no-instanceof
+                    !(input instanceof Date))) {
+                try {
+                    return (await this.func(runTree, input.input));
+                }
+                catch (err) {
+                    return (await this.func(runTree, input));
+                }
+            }
+        }
+        return (await this.func(runTree, input));
+    }
+}
 /**
  * Wraps an off-the-shelf evaluator (loaded using loadEvaluator; of EvaluatorType[T])
  * and composes with a prepareData function so the user can prepare the trace and
@@ -210,7 +272,7 @@ class LoadedEvalConfig {
     }
     static async fromRunEvalConfig(config) {
         // Custom evaluators are applied "as-is"
-        const customEvaluators = config?.customEvaluators?.map((evaluator) => {
+        const customEvaluators = (config?.customEvaluators ?? config.evaluators?.filter(isCustomEvaluator))?.map((evaluator) => {
             if (typeof evaluator === "function") {
                 return new DynamicRunEvaluator(evaluator);
             }
@@ -218,7 +280,9 @@ class LoadedEvalConfig {
                 return evaluator;
             }
         });
-        const offTheShelfEvaluators = await Promise.all(config?.evaluators?.map(async (evaluator) => await PreparedRunEvaluator.fromEvalConfig(evaluator)) ?? []);
+        const offTheShelfEvaluators = await Promise.all(config?.evaluators
+            ?.filter(isOffTheShelfEvaluator)
+            ?.map(async (evaluator) => await PreparedRunEvaluator.fromEvalConfig(evaluator)) ?? []);
         return new LoadedEvalConfig((customEvaluators ?? []).concat(offTheShelfEvaluators ?? []));
     }
 }
@@ -246,7 +310,11 @@ const createWrappedModel = async (modelOrFactory) => {
         }
         catch (err) {
             // Otherwise, it's a custom UDF, and we'll wrap
-            // in a lambda
+            // in a lambda or a traceable function
+            if (isLangsmithTraceableFunction(modelOrFactory)) {
+                const wrappedModel = new RunnableTraceable({ func: modelOrFactory });
+                return () => wrappedModel;
+            }
             const wrappedModel = new RunnableLambda({ func: modelOrFactory });
             return () => wrappedModel;
         }
@@ -318,62 +386,10 @@ const getExamplesInputs = (examples, chainOrFactory, dataType) => {
     }
     return examples.map(({ inputs }) => inputs);
 };
-/**
- * Evaluates a given model or chain against a specified LangSmith dataset.
- *
- * This function fetches example records from the specified dataset,
- * runs the model or chain against each example, and returns the evaluation
- * results.
- *
- * @param chainOrFactory - A model or factory/constructor function to be evaluated. It can be a
- * Runnable instance, a factory function that returns a Runnable, or a user-defined
- * function or factory.
- *
- * @param datasetName - The name of the dataset against which the evaluation will be
- * performed. This dataset should already be defined and contain the relevant data
- * for evaluation.
- *
- * @param options - (Optional) Additional parameters for the evaluation process:
- *   - `evaluationConfig` (RunEvalConfig): Configuration for the evaluation, including
- *     standard and custom evaluators.
- *   - `projectName` (string): Name of the project for logging and tracking.
- *   - `projectMetadata` (Record<string, unknown>): Additional metadata for the project.
- *   - `client` (Client): Client instance for LangChain service interaction.
- *   - `maxConcurrency` (number): Maximum concurrency level for dataset processing.
- *
- * @returns A promise that resolves to an `EvalResults` object. This object includes
- * detailed results of the evaluation, such as execution time, run IDs, and feedback
- * for each entry in the dataset.
- *
- * @example
- * ```typescript
- * // Example usage for evaluating a model on a dataset
- * async function evaluateModel() {
- *   const chain = /* ...create your model or chain...*\//
- *   const datasetName = 'example-dataset';
- *   const client = new Client(/* ...config... *\//);
- *
- *   const evaluationConfig = {
- *     evaluators: [/* ...evaluators... *\//],
- *     customEvaluators: [/* ...custom evaluators... *\//],
- *   };
- *
- *   const results = await runOnDataset(chain, datasetName, {
- *     evaluationConfig,
- *     client,
- *   });
- *
- *   console.log('Evaluation Results:', results);
- * }
- *
- * evaluateModel();
- * ```
- * In this example, `runOnDataset` is used to evaluate a language model (or a chain of models) against
- * a dataset named 'example-dataset'. The evaluation process is configured using `RunEvalConfig`, which can
- * include both standard and custom evaluators. The `Client` instance is used to interact with LangChain services.
- * The function returns the evaluation results, which can be logged or further processed as needed.
- */
-export const runOnDataset = async (chainOrFactory, datasetName, { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, }) => {
+export async function runOnDataset(chainOrFactory, datasetName, options) {
+    const { evaluationConfig, projectName, projectMetadata, client, maxConcurrency, } = Array.isArray(options)
+        ? { evaluationConfig: { evaluators: options } }
+        : options ?? {};
     const wrappedModel = await createWrappedModel(chainOrFactory);
     const testClient = client ?? new Client();
     const testProjectName = projectName ?? randomName();
@@ -429,4 +445,7 @@ export const runOnDataset = async (chainOrFactory, datasetName, { evaluationConf
         results: evalResults ?? {},
     };
     return results;
-};
+}
+function isLangsmithTraceableFunction(x) {
+    return typeof x === "function" && "langsmith:traceable" in x;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.1.18",
+  "version": "0.1.19-rc.1",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -1513,7 +1513,7 @@
     "js-yaml": "^4.1.0",
     "jsonpointer": "^5.0.1",
     "langchainhub": "~0.0.8",
-    "langsmith": "~0.0.59",
+    "langsmith": "~0.1.1",
     "ml-distance": "^4.0.0",
     "openapi-types": "^12.1.3",
     "p-retry": "4",