agentevals 0.0.2 → 0.0.4

package/dist/trajectory/unordered.js

@@ -1,6 +1,12 @@
 import { _normalizeToOpenAIMessagesList, _runEvaluator } from "../utils.js";
 import { _isTrajectorySuperset } from "./utils.js";
+export const _scorer = async (params) => {
+    const isUnorderedMatch = (await _isTrajectorySuperset(params.outputs, params.referenceOutputs, params.toolArgsMatchMode, params.toolArgsMatchOverrides)) &&
+        (await _isTrajectorySuperset(params.referenceOutputs, params.outputs, params.toolArgsMatchMode, params.toolArgsMatchOverrides));
+    return isUnorderedMatch;
+};
 /**
+ * @deprecated Use `createTrajectoryMatchEvaluator` with `trajectoryMatchMode: "unordered"` instead.
  * Evaluate whether an input agent trajectory and called tools contains all the tools used in a reference trajectory.
  * This accounts for some differences in an LLM's reasoning process in a case-by-case basis.
  *
@@ -17,13 +23,10 @@ export async function trajectoryUnorderedMatch(params) {
     const { outputs, referenceOutputs } = params;
     const outputsList = _normalizeToOpenAIMessagesList(outputs);
     const referenceOutputsList = _normalizeToOpenAIMessagesList(referenceOutputs);
-    const getScore = async () => {
-        if (outputsList == null || referenceOutputsList == null) {
-            throw new Error("Trajectory unordered match requires both outputs and reference_outputs");
-        }
-        const unorderedMatch = _isTrajectorySuperset(outputsList, referenceOutputsList) &&
-            _isTrajectorySuperset(referenceOutputsList, outputsList);
-        return unorderedMatch;
-    };
-    return _runEvaluator("trajectory_unordered_match", getScore, "trajectory_unordered_match", params);
+    return _runEvaluator("trajectory_unordered_match", _scorer, "trajectory_unordered_match", {
+        ...params,
+        outputs: outputsList,
+        referenceOutputs: referenceOutputsList,
+        toolArgsMatchMode: "ignore",
+    });
 }

package/dist/trajectory/utils.cjs

@@ -1,14 +1,15 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports._chatCompletionMessagesToString = exports._isTrajectorySuperset = void 0;
+exports._chatCompletionMessagesToString = exports._getMatcherForToolName = exports._isTrajectorySuperset = void 0;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function _normalizeToolCall(toolCall) {
     if ("function" in toolCall &&
         toolCall.function != null &&
-        typeof toolCall.function === "object") {
+        typeof toolCall.function === "object" &&
+        typeof toolCall.function.arguments === "string") {
         return {
             name: toolCall.function.name,
-            args: toolCall.function.arguments,
+            args: JSON.parse(toolCall.function.arguments),
         };
     }
     return toolCall;
@@ -22,29 +23,117 @@ function _extractToolCalls(messages) {
     }
     return toolCalls;
 }
-function _isTrajectorySuperset(outputs, referenceOutputs) {
+async function _isTrajectorySuperset(outputs, referenceOutputs, toolArgsMatchMode, toolArgsMatchOverrides) {
     const outputToolCalls = _extractToolCalls(outputs);
     const referenceToolCalls = _extractToolCalls(referenceOutputs);
-    const outputToolCounts = new Map();
-    const referenceToolCounts = new Map();
-    for (const call of outputToolCalls) {
-        outputToolCounts.set(call.name, (outputToolCounts.get(call.name) ?? 0) + 1);
-    }
-    for (const call of referenceToolCalls) {
-        referenceToolCounts.set(call.name, (referenceToolCounts.get(call.name) ?? 0) + 1);
-    }
-    const allTools = new Set([
-        ...outputToolCounts.keys(),
-        ...referenceToolCounts.keys(),
-    ]);
-    for (const name of allTools) {
-        if ((outputToolCounts.get(name) ?? 0) < (referenceToolCounts.get(name) ?? 0)) {
+    // Keep track of which reference tool calls have been matched
+    const matchedReferenceCalls = new Set();
+    // For each reference tool call, find a matching output tool call
+    for (const refCall of referenceToolCalls) {
+        const refName = refCall.name;
+        const refArgs = refCall.args;
+        let foundMatch = false;
+        for (let outIdx = 0; outIdx < outputToolCalls.length; outIdx++) {
+            const outCall = outputToolCalls[outIdx];
+            const outName = outCall.name;
+            // Names must match
+            if (refName !== outName) {
+                continue;
+            }
+            // If we're already using this output call for a different match, skip
+            if (matchedReferenceCalls.has(outIdx)) {
+                continue;
+            }
+            // Check tool args according to match mode
+            const matcher = _getMatcherForToolName(refName, toolArgsMatchMode, toolArgsMatchOverrides);
+            const outArgs = outCall.args;
+            if (await matcher(outArgs, refArgs)) {
+                matchedReferenceCalls.add(outIdx);
+                foundMatch = true;
+                break;
+            }
+        }
+        // If we didn't find a match for this reference call, we're not a superset
+        if (!foundMatch) {
             return false;
         }
     }
     return true;
 }
 exports._isTrajectorySuperset = _isTrajectorySuperset;
+// Deep equality check function
+function _deepEqual(a, b) {
+    if (a == null && b == null)
+        return true;
+    if (a === b)
+        return true;
+    if (typeof a !== "object" || typeof b !== "object" || !a || !b)
+        return false;
+    if (Array.isArray(a) && Array.isArray(b)) {
+        if (a.length !== b.length)
+            return false;
+        return a.every((val, index) => _deepEqual(val, b[index]));
+    }
+    const keysA = Object.keys(a);
+    const keysB = Object.keys(b);
+    if (keysA.length !== keysB.length)
+        return false;
+    return (keysA.every((key) => keysB.includes(key)) &&
+        keysB.every((key) => keysA.includes(key)) &&
+        keysA.every((key) => _deepEqual(a[key], b[key])));
+}
+function _exactMatch(toolCall, referenceToolCall) {
+    return _deepEqual(toolCall, referenceToolCall);
+}
+function _ignoreMatch(_toolCall, _referenceToolCall) {
+    return true;
+}
+function _getMatcherForComparisonMode(mode) {
+    if (mode === "exact") {
+        return _exactMatch;
+    }
+    else {
+        return _ignoreMatch;
+    }
+}
+function _getPartialMatcherOnKeys(keys) {
+    const getNestedValue = (d, keyPath) => {
+        let current = d;
+        for (const part of keyPath.split(".")) {
+            if (current && typeof current === "object" && part in current) {
+                current = current[part];
+            }
+            else {
+                return undefined;
+            }
+        }
+        return current;
+    };
+    return (outputCall, referenceCall) => {
+        return keys.every((key) => {
+            const nestedOutputValue = getNestedValue(outputCall, key);
+            const nestedReferenceValue = getNestedValue(referenceCall, key);
+            return _deepEqual(nestedOutputValue, nestedReferenceValue);
+        });
+    };
+}
+function _getMatcherForToolName(toolCallName, toolArgsMatchMode, toolArgsMatchOverrides) {
+    let matcher = _getMatcherForComparisonMode(toolArgsMatchMode);
+    if (toolArgsMatchOverrides && toolCallName in toolArgsMatchOverrides) {
+        const override = toolArgsMatchOverrides[toolCallName];
+        if (typeof override === "string") {
+            matcher = _getMatcherForComparisonMode(override);
+        }
+        else if (typeof override === "function") {
+            matcher = override;
+        }
+        else if (Array.isArray(override)) {
+            matcher = _getPartialMatcherOnKeys(override);
+        }
+    }
+    return matcher;
+}
+exports._getMatcherForToolName = _getMatcherForToolName;
 function _chatCompletionMessagesToString(messages) {
     function formatMessage(message) {
         let content = message.content ?? "";

package/dist/trajectory/utils.d.ts

@@ -1,3 +1,4 @@
-import { ChatCompletionMessage } from "../types.js";
-export declare function _isTrajectorySuperset(outputs: ChatCompletionMessage[], referenceOutputs: ChatCompletionMessage[]): boolean;
+import { ChatCompletionMessage, ToolArgsMatchMode, ToolArgsMatchOverrides, ToolArgsMatcher } from "../types.js";
+export declare function _isTrajectorySuperset(outputs: ChatCompletionMessage[], referenceOutputs: ChatCompletionMessage[], toolArgsMatchMode: ToolArgsMatchMode, toolArgsMatchOverrides?: ToolArgsMatchOverrides): Promise<boolean>;
+export declare function _getMatcherForToolName(toolCallName: string, toolArgsMatchMode: ToolArgsMatchMode, toolArgsMatchOverrides?: ToolArgsMatchOverrides): ToolArgsMatcher;
 export declare function _chatCompletionMessagesToString(messages: ChatCompletionMessage[]): string;

package/dist/trajectory/utils.js

@@ -2,10 +2,11 @@
 function _normalizeToolCall(toolCall) {
     if ("function" in toolCall &&
         toolCall.function != null &&
-        typeof toolCall.function === "object") {
+        typeof toolCall.function === "object" &&
+        typeof toolCall.function.arguments === "string") {
         return {
             name: toolCall.function.name,
-            args: toolCall.function.arguments,
+            args: JSON.parse(toolCall.function.arguments),
         };
     }
     return toolCall;
@@ -19,28 +20,115 @@ function _extractToolCalls(messages) {
     }
     return toolCalls;
 }
-export function _isTrajectorySuperset(outputs, referenceOutputs) {
+export async function _isTrajectorySuperset(outputs, referenceOutputs, toolArgsMatchMode, toolArgsMatchOverrides) {
     const outputToolCalls = _extractToolCalls(outputs);
     const referenceToolCalls = _extractToolCalls(referenceOutputs);
-    const outputToolCounts = new Map();
-    const referenceToolCounts = new Map();
-    for (const call of outputToolCalls) {
-        outputToolCounts.set(call.name, (outputToolCounts.get(call.name) ?? 0) + 1);
-    }
-    for (const call of referenceToolCalls) {
-        referenceToolCounts.set(call.name, (referenceToolCounts.get(call.name) ?? 0) + 1);
-    }
-    const allTools = new Set([
-        ...outputToolCounts.keys(),
-        ...referenceToolCounts.keys(),
-    ]);
-    for (const name of allTools) {
-        if ((outputToolCounts.get(name) ?? 0) < (referenceToolCounts.get(name) ?? 0)) {
+    // Keep track of which reference tool calls have been matched
+    const matchedReferenceCalls = new Set();
+    // For each reference tool call, find a matching output tool call
+    for (const refCall of referenceToolCalls) {
+        const refName = refCall.name;
+        const refArgs = refCall.args;
+        let foundMatch = false;
+        for (let outIdx = 0; outIdx < outputToolCalls.length; outIdx++) {
+            const outCall = outputToolCalls[outIdx];
+            const outName = outCall.name;
+            // Names must match
+            if (refName !== outName) {
+                continue;
+            }
+            // If we're already using this output call for a different match, skip
+            if (matchedReferenceCalls.has(outIdx)) {
+                continue;
+            }
+            // Check tool args according to match mode
+            const matcher = _getMatcherForToolName(refName, toolArgsMatchMode, toolArgsMatchOverrides);
+            const outArgs = outCall.args;
+            if (await matcher(outArgs, refArgs)) {
+                matchedReferenceCalls.add(outIdx);
+                foundMatch = true;
+                break;
+            }
+        }
+        // If we didn't find a match for this reference call, we're not a superset
+        if (!foundMatch) {
             return false;
         }
     }
     return true;
 }
+// Deep equality check function
+function _deepEqual(a, b) {
+    if (a == null && b == null)
+        return true;
+    if (a === b)
+        return true;
+    if (typeof a !== "object" || typeof b !== "object" || !a || !b)
+        return false;
+    if (Array.isArray(a) && Array.isArray(b)) {
+        if (a.length !== b.length)
+            return false;
+        return a.every((val, index) => _deepEqual(val, b[index]));
+    }
+    const keysA = Object.keys(a);
+    const keysB = Object.keys(b);
+    if (keysA.length !== keysB.length)
+        return false;
+    return (keysA.every((key) => keysB.includes(key)) &&
+        keysB.every((key) => keysA.includes(key)) &&
+        keysA.every((key) => _deepEqual(a[key], b[key])));
+}
+function _exactMatch(toolCall, referenceToolCall) {
+    return _deepEqual(toolCall, referenceToolCall);
+}
+function _ignoreMatch(_toolCall, _referenceToolCall) {
+    return true;
+}
+function _getMatcherForComparisonMode(mode) {
+    if (mode === "exact") {
+        return _exactMatch;
+    }
+    else {
+        return _ignoreMatch;
+    }
+}
+function _getPartialMatcherOnKeys(keys) {
+    const getNestedValue = (d, keyPath) => {
+        let current = d;
+        for (const part of keyPath.split(".")) {
+            if (current && typeof current === "object" && part in current) {
+                current = current[part];
+            }
+            else {
+                return undefined;
+            }
+        }
+        return current;
+    };
+    return (outputCall, referenceCall) => {
+        return keys.every((key) => {
+            const nestedOutputValue = getNestedValue(outputCall, key);
+            const nestedReferenceValue = getNestedValue(referenceCall, key);
+            return _deepEqual(nestedOutputValue, nestedReferenceValue);
+        });
+    };
+}
+export function _getMatcherForToolName(toolCallName, toolArgsMatchMode, toolArgsMatchOverrides) {
+    let matcher = _getMatcherForComparisonMode(toolArgsMatchMode);
+    if (toolArgsMatchOverrides && toolCallName in toolArgsMatchOverrides) {
+        const override = toolArgsMatchOverrides[toolCallName];
+        if (typeof override === "string") {
+            matcher = _getMatcherForComparisonMode(override);
+        }
+        else if (typeof override === "function") {
+            matcher = override;
+        }
+        else if (Array.isArray(override)) {
+            matcher = _getPartialMatcherOnKeys(override);
+        }
+    }
+    return matcher;
+}
 export function _chatCompletionMessagesToString(messages) {
     function formatMessage(message) {
         let content = message.content ?? "";

package/dist/types.d.ts

@@ -12,3 +12,6 @@ export type ExtractedLangGraphThreadTrajectory = {
 export type TrajectoryLLMAsJudgeParams = Omit<Parameters<typeof createLLMAsJudge>[0], "prompt"> & {
     prompt?: string;
 };
+export type ToolArgsMatchMode = "exact" | "ignore";
+export type ToolArgsMatcher = (toolCall: Record<string, unknown>, referenceToolCall: Record<string, unknown>) => boolean | Promise<boolean>;
+export type ToolArgsMatchOverrides = Record<string, ToolArgsMatchMode | string[] | ToolArgsMatcher>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentevals",
-  "version": "0.0.2",
+  "version": "0.0.4",
   "packageManager": "yarn@3.5.1",
   "type": "module",
   "scripts": {