agentevals 0.0.2 → 0.0.4

package/dist/trajectory/match.d.ts

@@ -0,0 +1,61 @@
+import { BaseMessage } from "@langchain/core/messages";
+import { ChatCompletionMessage, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
+export type TrajectoryMatchMode = "strict" | "unordered" | "subset" | "superset";
+/**
+ * Creates an evaluator that compares trajectories between model outputs and reference outputs.
+ *
+ * @param options - The configuration options
+ * @param options.trajectoryMatchMode - The mode for matching trajectories:
+ *   - `"strict"`: Requires exact match in order and content
+ *   - `"unordered"`: Allows matching in any order
+ *   - `"subset"`: Accepts if output trajectory is a subset of reference
+ *   - `"superset"`: Accepts if output trajectory is a superset of reference
+ * @param options.toolArgsMatchMode - Mode for matching tool arguments ("exact" by default, can be "ignore")
+ * @param options.toolArgsMatchOverrides - Object containing custom overrides for tool argument matching.
+ *   Each key should be a tool name, and each value should be either a match mode or a matcher function.
+ *   Matchers should be a function that takes two sets of tool call args and returns whether they are equal.
+ *
+ * @returns An async function that evaluates trajectory matches between outputs and references.
+ *   The returned evaluator accepts:
+ *   - outputs: List of messages or dict representing the model output trajectory
+ *   - referenceOutputs: List of messages or dict representing the reference trajectory
+ *   - Additional arguments passed to the underlying evaluator
+ *
+ * @example
+ * ```typescript
+ * const matcher = (
+ *   outputToolCallArgs: Record<string, any>,
+ *   referenceToolCallArgs: Record<string, any>
+ * ): boolean => {
+ *   const outputArgs = (outputToolCallArgs.query ?? "").toLowerCase();
+ *   const referenceArgs = (referenceToolCallArgs.query ?? "").toLowerCase();
+ *   return outputArgs === referenceArgs;
+ * };
+ *
+ * const evaluator = createAsyncTrajectoryMatchEvaluator({
+ *   trajectoryMatchMode: "strict",
+ *   toolArgsMatchMode: "exact",
+ *   toolArgsMatchOverrides: {
+ *     myToolName: matcher,
+ *   },
+ * });
+ *
+ * const result = await evaluator({
+ *   outputs: [...],
+ *   referenceOutputs: [...],
+ * });
+ * ```
+ */
+export declare function createTrajectoryMatchEvaluator({ trajectoryMatchMode, toolArgsMatchMode, toolArgsMatchOverrides, }: {
+    trajectoryMatchMode?: TrajectoryMatchMode;
+    toolArgsMatchMode?: ToolArgsMatchMode;
+    toolArgsMatchOverrides?: ToolArgsMatchOverrides;
+}): ({ outputs, referenceOutputs, ...extra }: {
+    [key: string]: unknown;
+    outputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+}) => Promise<import("langsmith/vitest").SimpleEvaluationResult>;

package/dist/trajectory/match.js

@@ -0,0 +1,80 @@
+import { _normalizeToOpenAIMessagesList, _runEvaluator } from "../utils.js";
+import { _scorer as trajectoryStrictScorer } from "./strict.js";
+import { _scorer as trajectoryUnorderedScorer } from "./unordered.js";
+import { _scorer as trajectorySubsetScorer } from "./subset.js";
+import { _scorer as trajectorySuperstScorer } from "./superset.js";
+/**
+ * Creates an evaluator that compares trajectories between model outputs and reference outputs.
+ *
+ * @param options - The configuration options
+ * @param options.trajectoryMatchMode - The mode for matching trajectories:
+ *   - `"strict"`: Requires exact match in order and content
+ *   - `"unordered"`: Allows matching in any order
+ *   - `"subset"`: Accepts if output trajectory is a subset of reference
+ *   - `"superset"`: Accepts if output trajectory is a superset of reference
+ * @param options.toolArgsMatchMode - Mode for matching tool arguments ("exact" by default, can be "ignore")
+ * @param options.toolArgsMatchOverrides - Object containing custom overrides for tool argument matching.
+ *   Each key should be a tool name, and each value should be either a match mode or a matcher function.
+ *   Matchers should be a function that takes two sets of tool call args and returns whether they are equal.
+ *
+ * @returns An async function that evaluates trajectory matches between outputs and references.
+ *   The returned evaluator accepts:
+ *   - outputs: List of messages or dict representing the model output trajectory
+ *   - referenceOutputs: List of messages or dict representing the reference trajectory
+ *   - Additional arguments passed to the underlying evaluator
+ *
+ * @example
+ * ```typescript
+ * const matcher = (
+ *   outputToolCallArgs: Record<string, any>,
+ *   referenceToolCallArgs: Record<string, any>
+ * ): boolean => {
+ *   const outputArgs = (outputToolCallArgs.query ?? "").toLowerCase();
+ *   const referenceArgs = (referenceToolCallArgs.query ?? "").toLowerCase();
+ *   return outputArgs === referenceArgs;
+ * };
+ *
+ * const evaluator = createAsyncTrajectoryMatchEvaluator({
+ *   trajectoryMatchMode: "strict",
+ *   toolArgsMatchMode: "exact",
+ *   toolArgsMatchOverrides: {
+ *     myToolName: matcher,
+ *   },
+ * });
+ *
+ * const result = await evaluator({
+ *   outputs: [...],
+ *   referenceOutputs: [...],
+ * });
+ * ```
+ */
+export function createTrajectoryMatchEvaluator({ trajectoryMatchMode = "strict", toolArgsMatchMode = "exact", toolArgsMatchOverrides, }) {
+    let scorer;
+    switch (trajectoryMatchMode) {
+        case "strict":
+            scorer = trajectoryStrictScorer;
+            break;
+        case "unordered":
+            scorer = trajectoryUnorderedScorer;
+            break;
+        case "subset":
+            scorer = trajectorySubsetScorer;
+            break;
+        case "superset":
+            scorer = trajectorySuperstScorer;
+            break;
+        default:
+            throw new Error(`Invalid trajectory match type: ${trajectoryMatchMode}`);
+    }
+    return async function _wrappedEvaluator({ outputs, referenceOutputs, ...extra }) {
+        const normalizedOutputs = _normalizeToOpenAIMessagesList(outputs);
+        const normalizedReferenceOutputs = _normalizeToOpenAIMessagesList(referenceOutputs);
+        return _runEvaluator(`trajectory_${trajectoryMatchMode}_match`, scorer, `trajectory_${trajectoryMatchMode}_match`, {
+            outputs: normalizedOutputs,
+            referenceOutputs: normalizedReferenceOutputs,
+            toolArgsMatchMode,
+            toolArgsMatchOverrides,
+            ...extra,
+        });
+    };
+}

package/dist/trajectory/strict.cjs

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.trajectoryStrictMatch = void 0;
+exports.trajectoryStrictMatch = exports._scorer = void 0;
 const utils_js_1 = require("../utils.cjs");
-function _scorer(params) {
-    const { outputs, referenceOutputs, toolCallArgsExactMatch = true, messageContentExactMatch = false, } = params;
+const utils_js_2 = require("./utils.cjs");
+async function _scorer(params) {
+    const { outputs, referenceOutputs, toolArgsMatchMode, toolArgsMatchOverrides, } = params;
     const normalizedOutputs = (0, utils_js_1._normalizeToOpenAIMessagesList)(outputs);
     const normalizedReferenceOutputs = (0, utils_js_1._normalizeToOpenAIMessagesList)(referenceOutputs);
     if (!normalizedOutputs || !normalizedReferenceOutputs) {
@@ -12,63 +13,62 @@ function _scorer(params) {
     if (normalizedOutputs.length !== normalizedReferenceOutputs.length) {
         return false;
     }
-    let exactMatch = true;
     for (let i = 0; i < normalizedOutputs.length; i++) {
         const output = normalizedOutputs[i];
         const referenceOutput = normalizedReferenceOutputs[i];
         if (output.role !== referenceOutput.role) {
-            exactMatch = false;
-            break;
+            return false;
         }
         const outputHasToolCalls = output.tool_calls != null;
         const referenceHasToolCalls = referenceOutput.tool_calls != null;
         if (outputHasToolCalls !== referenceHasToolCalls) {
-            exactMatch = false;
-            break;
+            return false;
         }
         if (outputHasToolCalls) {
             if (output.tool_calls.length !== referenceOutput.tool_calls.length) {
-                exactMatch = false;
-                break;
+                return false;
             }
-            for (let j = 0; j < output.tool_calls.length; j++) {
-                if (output.tool_calls[j].function.name !==
-                    referenceOutput.tool_calls[j].function.name) {
-                    exactMatch = false;
-                    break;
+            const referenceCalls = referenceOutput.tool_calls ?? [];
+            const seen = new Array(referenceCalls.length).fill(false);
+            for (const outputCall of output.tool_calls ?? []) {
+                let foundMatch = false;
+                for (let i = 0; i < referenceCalls.length; i++) {
+                    const referenceCall = referenceCalls[i];
+                    if (!seen[i] &&
+                        outputCall.function?.name === referenceCall.function?.name) {
+                        const matcher = (0, utils_js_2._getMatcherForToolName)(outputCall.function?.name ?? "", toolArgsMatchMode, toolArgsMatchOverrides);
+                        if (await matcher(JSON.parse(outputCall.function?.arguments ?? "{}"), JSON.parse(referenceCall.function?.arguments ?? "{}"))) {
+                            foundMatch = true;
+                            seen[i] = true;
+                            break;
+                        }
+                    }
                 }
-                if (toolCallArgsExactMatch &&
-                    output.tool_calls[j].function.arguments !==
-                        referenceOutput.tool_calls[j].function.arguments) {
-                    exactMatch = false;
-                    break;
+                if (!foundMatch) {
+                    return false;
                 }
             }
         }
-        if (messageContentExactMatch &&
-            output.content !== referenceOutput.content) {
-            exactMatch = false;
-            break;
-        }
     }
-    return exactMatch;
+    return true;
 }
+exports._scorer = _scorer;
+/**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "strict"` instead.
+ * Evaluate whether an input agent trajectory and called tools strictly matches a reference trajectory.
+ * This means that at each step, the agent called the same tools in the same order as specified in the reference trajectory.
+ *
+ * @param outputs - Actual trajectory the agent followed. May be a list of OpenAI messages,
+ *                 a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
+ * @param referenceOutputs - Ideal reference trajectory the agent should have followed. May be a list of OpenAI messages,
+ *                          a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
+ * @param toolCallArgsExactMatch - Whether to require exact matches for tool call arguments
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
 async function trajectoryStrictMatch(params) {
-    /**
-     * Evaluate whether an input agent trajectory and called tools strictly matches a reference trajectory.
-     * This means that at each step, the agent called the same tools in the same order as specified in the reference trajectory.
-     *
-     * @param outputs - Actual trajectory the agent followed. May be a list of OpenAI messages,
-     *                 a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
-     * @param referenceOutputs - Ideal reference trajectory the agent should have followed. May be a list of OpenAI messages,
-     *                          a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
-     * @param toolCallArgsExactMatch - Whether to require exact matches for tool call arguments
-     * @param messageContentExactMatch - Whether to require exact matches for message content
-     * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
-     */
-    function _wrapper() {
-        return _scorer(params);
-    }
-    return (0, utils_js_1._runEvaluator)("trajectory_strict_match", _wrapper, "trajectory_strict_match", params);
+    return (0, utils_js_1._runEvaluator)("trajectory_strict_match", _scorer, "trajectory_strict_match", {
+        ...params,
+        toolArgsMatchMode: params.toolCallArgsExactMatch ? "exact" : "ignore",
+    });
 }
 exports.trajectoryStrictMatch = trajectoryStrictMatch;

package/dist/trajectory/strict.d.ts

@@ -1,5 +1,27 @@
 import { BaseMessage } from "@langchain/core/messages";
-import { ChatCompletionMessage, EvaluatorResult } from "../types.js";
+import { ChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
+export declare function _scorer(params: {
+    outputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+    toolArgsMatchMode: ToolArgsMatchMode;
+    toolArgsMatchOverrides?: ToolArgsMatchOverrides;
+}): Promise<boolean>;
+/**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "strict"` instead.
+ * Evaluate whether an input agent trajectory and called tools strictly matches a reference trajectory.
+ * This means that at each step, the agent called the same tools in the same order as specified in the reference trajectory.
+ *
+ * @param outputs - Actual trajectory the agent followed. May be a list of OpenAI messages,
+ *                 a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
+ * @param referenceOutputs - Ideal reference trajectory the agent should have followed. May be a list of OpenAI messages,
+ *                          a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
+ * @param toolCallArgsExactMatch - Whether to require exact matches for tool call arguments
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
 export declare function trajectoryStrictMatch(params: {
     outputs: ChatCompletionMessage[] | BaseMessage[] | {
         messages: (BaseMessage | ChatCompletionMessage)[];
@@ -8,5 +30,4 @@ export declare function trajectoryStrictMatch(params: {
         messages: (BaseMessage | ChatCompletionMessage)[];
     };
     toolCallArgsExactMatch: boolean;
-    messageContentExactMatch: boolean;
 }): Promise<EvaluatorResult>;

package/dist/trajectory/strict.js

@@ -1,6 +1,7 @@
 import { _normalizeToOpenAIMessagesList, _runEvaluator } from "../utils.js";
-function _scorer(params) {
-    const { outputs, referenceOutputs, toolCallArgsExactMatch = true, messageContentExactMatch = false, } = params;
+import { _getMatcherForToolName } from "./utils.js";
+export async function _scorer(params) {
+    const { outputs, referenceOutputs, toolArgsMatchMode, toolArgsMatchOverrides, } = params;
     const normalizedOutputs = _normalizeToOpenAIMessagesList(outputs);
     const normalizedReferenceOutputs = _normalizeToOpenAIMessagesList(referenceOutputs);
     if (!normalizedOutputs || !normalizedReferenceOutputs) {
@@ -9,62 +10,60 @@ function _scorer(params) {
     if (normalizedOutputs.length !== normalizedReferenceOutputs.length) {
         return false;
     }
-    let exactMatch = true;
     for (let i = 0; i < normalizedOutputs.length; i++) {
         const output = normalizedOutputs[i];
         const referenceOutput = normalizedReferenceOutputs[i];
         if (output.role !== referenceOutput.role) {
-            exactMatch = false;
-            break;
+            return false;
         }
         const outputHasToolCalls = output.tool_calls != null;
         const referenceHasToolCalls = referenceOutput.tool_calls != null;
         if (outputHasToolCalls !== referenceHasToolCalls) {
-            exactMatch = false;
-            break;
+            return false;
         }
         if (outputHasToolCalls) {
             if (output.tool_calls.length !== referenceOutput.tool_calls.length) {
-                exactMatch = false;
-                break;
+                return false;
             }
-            for (let j = 0; j < output.tool_calls.length; j++) {
-                if (output.tool_calls[j].function.name !==
-                    referenceOutput.tool_calls[j].function.name) {
-                    exactMatch = false;
-                    break;
+            const referenceCalls = referenceOutput.tool_calls ?? [];
+            const seen = new Array(referenceCalls.length).fill(false);
+            for (const outputCall of output.tool_calls ?? []) {
+                let foundMatch = false;
+                for (let i = 0; i < referenceCalls.length; i++) {
+                    const referenceCall = referenceCalls[i];
+                    if (!seen[i] &&
+                        outputCall.function?.name === referenceCall.function?.name) {
+                        const matcher = _getMatcherForToolName(outputCall.function?.name ?? "", toolArgsMatchMode, toolArgsMatchOverrides);
+                        if (await matcher(JSON.parse(outputCall.function?.arguments ?? "{}"), JSON.parse(referenceCall.function?.arguments ?? "{}"))) {
+                            foundMatch = true;
+                            seen[i] = true;
+                            break;
+                        }
+                    }
                 }
-                if (toolCallArgsExactMatch &&
-                    output.tool_calls[j].function.arguments !==
-                        referenceOutput.tool_calls[j].function.arguments) {
-                    exactMatch = false;
-                    break;
+                if (!foundMatch) {
+                    return false;
                 }
             }
         }
-        if (messageContentExactMatch &&
-            output.content !== referenceOutput.content) {
-            exactMatch = false;
-            break;
-        }
     }
-    return exactMatch;
+    return true;
 }
+/**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "strict"` instead.
+ * Evaluate whether an input agent trajectory and called tools strictly matches a reference trajectory.
+ * This means that at each step, the agent called the same tools in the same order as specified in the reference trajectory.
+ *
+ * @param outputs - Actual trajectory the agent followed. May be a list of OpenAI messages,
+ *                 a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
+ * @param referenceOutputs - Ideal reference trajectory the agent should have followed. May be a list of OpenAI messages,
+ *                          a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
+ * @param toolCallArgsExactMatch - Whether to require exact matches for tool call arguments
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
 export async function trajectoryStrictMatch(params) {
-    /**
-     * Evaluate whether an input agent trajectory and called tools strictly matches a reference trajectory.
-     * This means that at each step, the agent called the same tools in the same order as specified in the reference trajectory.
-     *
-     * @param outputs - Actual trajectory the agent followed. May be a list of OpenAI messages,
-     *                 a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
-     * @param referenceOutputs - Ideal reference trajectory the agent should have followed. May be a list of OpenAI messages,
-     *                          a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
-     * @param toolCallArgsExactMatch - Whether to require exact matches for tool call arguments
-     * @param messageContentExactMatch - Whether to require exact matches for message content
-     * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
-     */
-    function _wrapper() {
-        return _scorer(params);
-    }
-    return _runEvaluator("trajectory_strict_match", _wrapper, "trajectory_strict_match", params);
+    return _runEvaluator("trajectory_strict_match", _scorer, "trajectory_strict_match", {
+        ...params,
+        toolArgsMatchMode: params.toolCallArgsExactMatch ? "exact" : "ignore",
+    });
 }

package/dist/trajectory/subset.cjs

@@ -1,9 +1,15 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.trajectorySubset = void 0;
+exports.trajectorySubset = exports._scorer = void 0;
 const utils_js_1 = require("../utils.cjs");
 const utils_js_2 = require("./utils.cjs");
+const _scorer = async (params) => {
+    const isSubset = await (0, utils_js_2._isTrajectorySuperset)(params.referenceOutputs, params.outputs, params.toolArgsMatchMode, params.toolArgsMatchOverrides);
+    return isSubset;
+};
+exports._scorer = _scorer;
 /**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "subset"` instead.
  * Evaluate whether an agent trajectory and called tools is a subset of a reference trajectory and called tools.
  * This means the agent called a subset of the tools specified in the reference trajectory.
  *
@@ -20,13 +26,11 @@ async function trajectorySubset(params) {
     const { outputs, referenceOutputs } = params;
     const outputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(outputs);
     const referenceOutputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(referenceOutputs);
-    const getScore = async () => {
-        if (outputsList == null || referenceOutputsList == null) {
-            throw new Error("Trajectory subset match requires both outputs and reference_outputs");
-        }
-        const isSubset = (0, utils_js_2._isTrajectorySuperset)(referenceOutputsList, outputsList);
-        return isSubset;
-    };
-    return (0, utils_js_1._runEvaluator)("trajectory_subset", getScore, "trajectory_subset", params);
+    return (0, utils_js_1._runEvaluator)("trajectory_subset", exports._scorer, "trajectory_subset", {
+        ...params,
+        outputs: outputsList,
+        referenceOutputs: referenceOutputsList,
+        toolArgsMatchMode: "ignore",
+    });
 }
 exports.trajectorySubset = trajectorySubset;

package/dist/trajectory/subset.d.ts

@@ -1,6 +1,13 @@
 import { BaseMessage } from "@langchain/core/messages";
-import { ChatCompletionMessage, EvaluatorResult } from "../types.js";
+import { ChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
+export declare const _scorer: (params: {
+    outputs: ChatCompletionMessage[];
+    referenceOutputs: ChatCompletionMessage[];
+    toolArgsMatchMode: ToolArgsMatchMode;
+    toolArgsMatchOverrides?: ToolArgsMatchOverrides;
+}) => Promise<boolean>;
 /**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "subset"` instead.
  * Evaluate whether an agent trajectory and called tools is a subset of a reference trajectory and called tools.
  * This means the agent called a subset of the tools specified in the reference trajectory.
  *

package/dist/trajectory/subset.js

@@ -1,6 +1,11 @@
 import { _normalizeToOpenAIMessagesList, _runEvaluator } from "../utils.js";
 import { _isTrajectorySuperset } from "./utils.js";
+export const _scorer = async (params) => {
+    const isSubset = await _isTrajectorySuperset(params.referenceOutputs, params.outputs, params.toolArgsMatchMode, params.toolArgsMatchOverrides);
+    return isSubset;
+};
 /**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "subset"` instead.
  * Evaluate whether an agent trajectory and called tools is a subset of a reference trajectory and called tools.
  * This means the agent called a subset of the tools specified in the reference trajectory.
  *
@@ -17,12 +22,10 @@ export async function trajectorySubset(params) {
     const { outputs, referenceOutputs } = params;
     const outputsList = _normalizeToOpenAIMessagesList(outputs);
     const referenceOutputsList = _normalizeToOpenAIMessagesList(referenceOutputs);
-    const getScore = async () => {
-        if (outputsList == null || referenceOutputsList == null) {
-            throw new Error("Trajectory subset match requires both outputs and reference_outputs");
-        }
-        const isSubset = _isTrajectorySuperset(referenceOutputsList, outputsList);
-        return isSubset;
-    };
-    return _runEvaluator("trajectory_subset", getScore, "trajectory_subset", params);
+    return _runEvaluator("trajectory_subset", _scorer, "trajectory_subset", {
+        ...params,
+        outputs: outputsList,
+        referenceOutputs: referenceOutputsList,
+        toolArgsMatchMode: "ignore",
+    });
 }

package/dist/trajectory/superset.cjs

@@ -1,9 +1,15 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.trajectorySuperset = void 0;
+exports.trajectorySuperset = exports._scorer = void 0;
 const utils_js_1 = require("../utils.cjs");
 const utils_js_2 = require("./utils.cjs");
+const _scorer = async (params) => {
+    const isSuperset = await (0, utils_js_2._isTrajectorySuperset)(params.outputs, params.referenceOutputs, params.toolArgsMatchMode, params.toolArgsMatchOverrides);
+    return isSuperset;
+};
+exports._scorer = _scorer;
 /**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "superset"` instead.
  * Evaluate whether an agent trajectory and called tools is a superset of a reference trajectory and called tools.
  * This means the agent called a superset of the tools specified in the reference trajectory.
  *
@@ -20,13 +26,11 @@ async function trajectorySuperset(params) {
     const { outputs, referenceOutputs } = params;
     const outputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(outputs);
     const referenceOutputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(referenceOutputs);
-    const getScore = async () => {
-        if (outputsList == null || referenceOutputsList == null) {
-            throw new Error("Trajectory superset match requires both outputs and reference_outputs");
-        }
-        const isSuperset = (0, utils_js_2._isTrajectorySuperset)(outputsList, referenceOutputsList);
-        return isSuperset;
-    };
-    return (0, utils_js_1._runEvaluator)("trajectory_superset", getScore, "trajectory_superset", params);
+    return (0, utils_js_1._runEvaluator)("trajectory_superset", exports._scorer, "trajectory_superset", {
+        ...params,
+        outputs: outputsList,
+        referenceOutputs: referenceOutputsList,
+        toolArgsMatchMode: "ignore",
+    });
 }
 exports.trajectorySuperset = trajectorySuperset;

package/dist/trajectory/superset.d.ts

@@ -1,6 +1,13 @@
 import { BaseMessage } from "@langchain/core/messages";
-import { ChatCompletionMessage, EvaluatorResult } from "../types.js";
+import { ChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
+export declare const _scorer: (params: {
+    outputs: ChatCompletionMessage[];
+    referenceOutputs: ChatCompletionMessage[];
+    toolArgsMatchMode: ToolArgsMatchMode;
+    toolArgsMatchOverrides?: ToolArgsMatchOverrides;
+}) => Promise<boolean>;
 /**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "superset"` instead.
  * Evaluate whether an agent trajectory and called tools is a superset of a reference trajectory and called tools.
  * This means the agent called a superset of the tools specified in the reference trajectory.
  *

package/dist/trajectory/superset.js

@@ -1,6 +1,11 @@
 import { _normalizeToOpenAIMessagesList, _runEvaluator } from "../utils.js";
 import { _isTrajectorySuperset } from "./utils.js";
+export const _scorer = async (params) => {
+    const isSuperset = await _isTrajectorySuperset(params.outputs, params.referenceOutputs, params.toolArgsMatchMode, params.toolArgsMatchOverrides);
+    return isSuperset;
+};
 /**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "superset"` instead.
  * Evaluate whether an agent trajectory and called tools is a superset of a reference trajectory and called tools.
  * This means the agent called a superset of the tools specified in the reference trajectory.
  *
@@ -17,12 +22,10 @@ export async function trajectorySuperset(params) {
     const { outputs, referenceOutputs } = params;
     const outputsList = _normalizeToOpenAIMessagesList(outputs);
     const referenceOutputsList = _normalizeToOpenAIMessagesList(referenceOutputs);
-    const getScore = async () => {
-        if (outputsList == null || referenceOutputsList == null) {
-            throw new Error("Trajectory superset match requires both outputs and reference_outputs");
-        }
-        const isSuperset = _isTrajectorySuperset(outputsList, referenceOutputsList);
-        return isSuperset;
-    };
-    return _runEvaluator("trajectory_superset", getScore, "trajectory_superset", params);
+    return _runEvaluator("trajectory_superset", _scorer, "trajectory_superset", {
+        ...params,
+        outputs: outputsList,
+        referenceOutputs: referenceOutputsList,
+        toolArgsMatchMode: "ignore",
+    });
 }

package/dist/trajectory/unordered.cjs

@@ -1,9 +1,16 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.trajectoryUnorderedMatch = void 0;
+exports.trajectoryUnorderedMatch = exports._scorer = void 0;
 const utils_js_1 = require("../utils.cjs");
 const utils_js_2 = require("./utils.cjs");
+const _scorer = async (params) => {
+    const isUnorderedMatch = (await (0, utils_js_2._isTrajectorySuperset)(params.outputs, params.referenceOutputs, params.toolArgsMatchMode, params.toolArgsMatchOverrides)) &&
+        (await (0, utils_js_2._isTrajectorySuperset)(params.referenceOutputs, params.outputs, params.toolArgsMatchMode, params.toolArgsMatchOverrides));
+    return isUnorderedMatch;
+};
+exports._scorer = _scorer;
 /**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "unordered"` instead.
  * Evaluate whether an input agent trajectory and called tools contains all the tools used in a reference trajectory.
  * This accounts for some differences in an LLM's reasoning process in a case-by-case basis.
  *
@@ -20,14 +27,11 @@ async function trajectoryUnorderedMatch(params) {
     const { outputs, referenceOutputs } = params;
     const outputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(outputs);
     const referenceOutputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(referenceOutputs);
-    const getScore = async () => {
-        if (outputsList == null || referenceOutputsList == null) {
-            throw new Error("Trajectory unordered match requires both outputs and reference_outputs");
-        }
-        const unorderedMatch = (0, utils_js_2._isTrajectorySuperset)(outputsList, referenceOutputsList) &&
-            (0, utils_js_2._isTrajectorySuperset)(referenceOutputsList, outputsList);
-        return unorderedMatch;
-    };
-    return (0, utils_js_1._runEvaluator)("trajectory_unordered_match", getScore, "trajectory_unordered_match", params);
+    return (0, utils_js_1._runEvaluator)("trajectory_unordered_match", exports._scorer, "trajectory_unordered_match", {
+        ...params,
+        outputs: outputsList,
+        referenceOutputs: referenceOutputsList,
+        toolArgsMatchMode: "ignore",
+    });
 }
 exports.trajectoryUnorderedMatch = trajectoryUnorderedMatch;

package/dist/trajectory/unordered.d.ts

@@ -1,6 +1,13 @@
 import { BaseMessage } from "@langchain/core/messages";
-import { ChatCompletionMessage, EvaluatorResult } from "../types.js";
+import { ChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
+export declare const _scorer: (params: {
+    outputs: ChatCompletionMessage[];
+    referenceOutputs: ChatCompletionMessage[];
+    toolArgsMatchMode: ToolArgsMatchMode;
+    toolArgsMatchOverrides?: ToolArgsMatchOverrides;
+}) => Promise<boolean>;
 /**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "unordered"` instead.
  * Evaluate whether an input agent trajectory and called tools contains all the tools used in a reference trajectory.
  * This accounts for some differences in an LLM's reasoning process in a case-by-case basis.
  *