agentevals 0.0.5 → 0.0.6

package/README.md

@@ -28,6 +28,7 @@ Once you've done this, you can run your first trajectory evaluator. We represent
 ```ts
 import {
   createTrajectoryLLMAsJudge,
+  type FlexibleChatCompletionMessage,
   TRAJECTORY_ACCURACY_PROMPT,
 } from "agentevals";
 
@@ -55,7 +56,7 @@ const outputs = [
     role: "assistant",
     content: "The weather in SF is 80 degrees and sunny.",
   },
-];
+] satisfies FlexibleChatCompletionMessage[];
 
 const evalResult = await trajectoryEvaluator({
   outputs,
@@ -130,7 +131,10 @@ The `"strict"` `trajectory_match_mode` compares two trajectories and ensures tha
 in the same order with the same tool calls. Note that it does allow for differences in message content:
 
 ```ts
-import { createTrajectoryMatchEvaluator } from "agentevals";
+import {
+  createTrajectoryMatchEvaluator,
+  type FlexibleChatCompletionMessage,
+} from "agentevals";
 
 const outputs = [
   { role: "user", content: "What is the weather in SF?" },
@@ -151,7 +155,7 @@ const outputs = [
   },
   { role: "tool", content: "It's 80 degrees and sunny in SF." },
   { role: "assistant", content: "The weather in SF is 80 degrees and sunny." },
-];
+] satisfies FlexibleChatCompletionMessage[];
 
 const referenceOutputs = [
   { role: "user", content: "What is the weather in San Francisco?" },
@@ -166,7 +170,7 @@ const referenceOutputs = [
     }]
   },
   { role: "tool", content: "It's 80 degrees and sunny in San Francisco." },
-];
+] satisfies FlexibleChatCompletionMessage[];
 
 const evaluator = createTrajectoryMatchEvaluator({
   trajectoryMatchMode: "strict",
@@ -196,7 +200,10 @@ console.log(result);
 The `"unordered"` `trajectory_match_mode` compares two trajectories and ensures that they contain the same tool calls in any order. This is useful if you want to allow flexibility in how an agent obtains the proper information, but still do care that all information was retrieved.
 
 ```ts
-import { createTrajectoryMatchEvaluator } from "agentevals";
+import {
+  createTrajectoryMatchEvaluator,
+  type FlexibleChatCompletionMessage,
+} from "agentevals";
 
 const outputs = [
   { role: "user", content: "What is the weather in SF and is there anything fun happening?" },
@@ -223,7 +230,7 @@ const outputs = [
   },
   { role: "tool", content: "Nothing fun is happening, you should stay indoors and read!" },
   { role: "assistant", content: "The weather in SF is 80 degrees and sunny, but there is nothing fun happening." },
-];
+] satisifes FlexibleChatCompletionMessage[];
 
 const referenceOutputs = [
   { role: "user", content: "What is the weather in SF and is there anything fun happening?" },
@@ -248,7 +255,7 @@ const referenceOutputs = [
   { role: "tool", content: "Nothing fun is happening, you should stay indoors and read!" },
   { role: "tool", content: "It's 80 degrees and sunny in SF." },
   { role: "assistant", content: "In SF, it's 80˚ and sunny, but there is nothing fun happening." },
-];
+] satisfies FlexibleChatCompletionMessage[];
 
 const evaluator = createTrajectoryMatchEvaluator({
   trajectoryMatchMode: "unordered",
@@ -278,7 +285,10 @@ console.log(result)
 The `"subset"` and `"superset"` modes match partial trajectories (ensuring that a trajectory contains a subset/superset of tool calls contained in a reference trajectory).
 
 ```ts
-import { createTrajectoryMatchEvaluator } from "agentevals";
+import {
+  createTrajectoryMatchEvaluator,
+  type FlexibleChatCompletionMessage
+} from "agentevals";
 
 const outputs = [
   { role: "user", content: "What is the weather in SF and London?" },
@@ -300,7 +310,7 @@ const outputs = [
   { role: "tool", content: "It's 80 degrees and sunny in SF, and 90 degrees and rainy in London." },
   { role: "tool", content: "Unknown." },
   { role: "assistant", content: "The weather in SF is 80 degrees and sunny. In London, it's 90 degrees and rainy."},
-];
+] satisfies FlexibleChatCompletionMessage[];
 
 const referenceOutputs = [
   { role: "user", content: "What is the weather in SF and London?" },
@@ -318,7 +328,7 @@ const referenceOutputs = [
   },
   { role: "tool", content: "It's 80 degrees and sunny in San Francisco, and 90 degrees and rainy in London." },
   { role: "assistant", content: "The weather in SF is 80˚ and sunny. In London, it's 90˚ and rainy." },
-];
+] satisfies FlexibleChatCompletionMessage[];
 
 const evaluator = createTrajectoryMatchEvaluator({
   trajectoryMatchMode: "superset", // or "subset"
@@ -364,7 +374,10 @@ ToolArgsMatchOverrides = dict[str, Union[ToolArgsMatchMode, list[str],  Callable
 Here's an example that allows case insensitivity for the arguments to a tool named `get_weather`:
 
 ```ts
-import { createTrajectoryMatchEvaluator } from "agentevals";
+import {
+  createTrajectoryMatchEvaluator,
+  type FlexibleChatCompletionMessage,
+} from "agentevals";
 
 const outputs = [
   { role: "user", content: "What is the weather in SF?" },
@@ -380,7 +393,7 @@ const outputs = [
   },
   { role: "tool", content: "It's 80 degrees and sunny in SF." },
   { role: "assistant", content: "The weather in SF is 80 degrees and sunny." },
-];
+] satisfies FlexibleChatCompletionMessage[];
 
 const referenceOutputs = [
   { role: "user", content: "What is the weather in San Francisco?" },
@@ -395,7 +408,7 @@ const referenceOutputs = [
     }]
   },
   { role: "tool", content: "It's 80 degrees and sunny in San Francisco." },
-];
+] satisfies FlexibleChatCompletionMessage[];
 
 const evaluator = createTrajectoryMatchEvaluator({
   trajectoryMatchMode: "strict",
@@ -434,6 +447,7 @@ The LLM-as-judge trajectory evaluator that uses an LLM to evaluate the trajector
 import {
   createTrajectoryLLMAsJudge,
   TRAJECTORY_ACCURACY_PROMPT,
+  type FlexibleChatCompletionMessage,
 } from "agentevals";
 
 const evaluator = createTrajectoryLLMAsJudge({
@@ -457,7 +471,7 @@ const outputs = [
   },
   {role: "tool", content: "It's 80 degrees and sunny in SF."},
   {role: "assistant", content: "The weather in SF is 80 degrees and sunny."},
-];
+] satisfies FlexibleChatCompletionMessage[];
 
 const result = await evaluator({ outputs });
 
@@ -477,7 +491,8 @@ If you have a reference trajectory, you can add an extra variable to your prompt
 ```ts
 import {
   createTrajectoryLLMAsJudge,
-  TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE
+  TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE,
+  type FlexibleChatCompletionMessage,
 } from "agentevals";
 
 const evaluator = createTrajectoryLLMAsJudge({
@@ -501,7 +516,8 @@ const outputs = [
   },
   {role: "tool", content: "It's 80 degrees and sunny in SF."},
   {role: "assistant", content: "The weather in SF is 80 degrees and sunny."},
-]
+] satisfies FlexibleChatCompletionMessage[];
+
 const referenceOutputs = [
   {role: "user", content: "What is the weather in SF?"},
   {
@@ -518,7 +534,7 @@ const referenceOutputs = [
   },
   {role: "tool", content: "It's 80 degrees and sunny in San Francisco."},
   {role: "assistant", content: "The weather in SF is 80˚ and sunny."},
-]
+] satisfies FlexibleChatCompletionMessage[];
 
 const result = await evaluator({
   outputs,
@@ -677,10 +693,10 @@ const graphTrajectoryEvaluator = createGraphTrajectoryLLMAsJudge({
     model: "openai:o3-mini",
 })
 
-const res = await graphTrajectoryEvaluator(
-    inputs=extractedTrajectory.inputs,
-    outputs=extractedTrajectory.outputs,
-)
+const res = await graphTrajectoryEvaluator({
+  inputs: extractedTrajectory.inputs,
+  outputs: extractedTrajectory.outputs,
+});
 
 console.log(res);
 ```
@@ -724,10 +740,10 @@ const graphTrajectoryEvaluator = createGraphTrajectoryLLMAsJudge({
   prompt: CUSTOM_PROMPT,
   model: "openai:o3-mini",
 })
-res = await graphTrajectoryEvaluator(
+const res = await graphTrajectoryEvaluator({
   inputs: extractedTrajectory.inputs,
   outputs: extractedTrajectory.outputs,
-)
+});
 ```
 
 In order to format them properly into the prompt, `reference_outputs` should be passed in as a `GraphTrajectory` object like `outputs`.

package/dist/graph_trajectory/llm.d.ts

@@ -27,4 +27,4 @@ export declare const createGraphTrajectoryLLMAsJudge: ({ prompt, model, feedback
     };
     outputs: GraphTrajectory;
     referenceOutputs?: GraphTrajectory | undefined;
-}) => Promise<import("langsmith/vitest").SimpleEvaluationResult>;
+}) => Promise<import("../types.js").EvaluatorResult>;

package/dist/graph_trajectory/strict.d.ts

@@ -11,4 +11,4 @@ import { GraphTrajectory } from "../types.js";
 export declare const graphTrajectoryStrictMatch: ({ outputs, referenceOutputs, }: {
     outputs: GraphTrajectory;
     referenceOutputs: GraphTrajectory;
-}) => Promise<import("langsmith/vitest").SimpleEvaluationResult>;
+}) => Promise<import("../types.js").EvaluatorResult>;

package/dist/graph_trajectory/utils.cjs

@@ -56,7 +56,14 @@ const extractLangGraphTrajectoryFromSnapshots = (snapshots) => {
         }
         if (isAccumulatingSteps) {
             if (snapshot.metadata != null && snapshot.metadata.source === "input") {
-                inputs.push(snapshot.metadata.writes);
+                if ("writes" in snapshot.metadata &&
+                    snapshot.metadata.writes != null &&
+                    typeof snapshot.metadata.writes === "object") {
+                    inputs.push(snapshot.metadata.writes);
+                }
+                else {
+                    inputs.push(...snapshot.tasks.map((task) => ({ [task.name]: task.result })));
+                }
             }
             else if (i + 1 < snapshots.length &&
                 snapshots[i + 1].tasks?.find((task) => task.interrupts?.length > 0)) {

package/dist/graph_trajectory/utils.d.ts

@@ -2,11 +2,11 @@ import type { StateSnapshot, Pregel } from "@langchain/langgraph/web";
 import type { RunnableConfig } from "@langchain/core/runnables";
 import type { GraphTrajectory } from "../types.js";
 export declare const extractLangGraphTrajectoryFromSnapshots: (snapshots: StateSnapshot[]) => {
-    inputs: (string | Record<string, unknown> | null)[];
+    inputs: (string | Record<string, unknown>)[];
     outputs: GraphTrajectory;
 };
 export declare const _getLangGraphStateHistoryRecursive: (graph: Pregel<any, any>, config: RunnableConfig) => Promise<StateSnapshot[]>;
 export declare const extractLangGraphTrajectoryFromThread: (graph: Pregel<any, any>, config: RunnableConfig) => Promise<{
-    inputs: (string | Record<string, unknown> | null)[];
+    inputs: (string | Record<string, unknown>)[];
     outputs: GraphTrajectory;
 }>;

package/dist/graph_trajectory/utils.js

@@ -53,7 +53,14 @@ export const extractLangGraphTrajectoryFromSnapshots = (snapshots) => {
         }
         if (isAccumulatingSteps) {
             if (snapshot.metadata != null && snapshot.metadata.source === "input") {
-                inputs.push(snapshot.metadata.writes);
+                if ("writes" in snapshot.metadata &&
+                    snapshot.metadata.writes != null &&
+                    typeof snapshot.metadata.writes === "object") {
+                    inputs.push(snapshot.metadata.writes);
+                }
+                else {
+                    inputs.push(...snapshot.tasks.map((task) => ({ [task.name]: task.result })));
+                }
             }
             else if (i + 1 < snapshots.length &&
                 snapshots[i + 1].tasks?.find((task) => task.interrupts?.length > 0)) {

package/dist/trajectory/llm.d.ts

@@ -1,5 +1,5 @@
 import { BaseMessage } from "@langchain/core/messages";
-import { ChatCompletionMessage, EvaluatorResult, TrajectoryLLMAsJudgeParams } from "../types.js";
+import { ChatCompletionMessage, FlexibleChatCompletionMessage, EvaluatorResult, TrajectoryLLMAsJudgeParams } from "../types.js";
 export declare const TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE = "You are an expert data labeler.\nYour task is to grade the accuracy of an AI agent's internal trajectory.\n\n<Rubric>\n  An accurate trajectory:\n  - Makes logical sense between steps\n  - Shows clear progression\n  - Is relatively efficient, though it does not need to be perfectly efficient\n  - Is semantically equivalent to the provided reference trajectory\n</Rubric>\n\nBased on the following reference trajectory:\n\n<reference_trajectory>\n{reference_outputs}\n</reference_trajectory>\n\nGrade this actual trajectory:\n\n<trajectory>\n{outputs}\n</trajectory>\n";
 export declare const TRAJECTORY_ACCURACY_PROMPT = "You are an expert data labeler.\nYour task is to grade the accuracy of an AI agent's internal trajectory.\n\n<Rubric>\n  An accurate trajectory:\n  - Makes logical sense between steps\n  - Shows clear progression\n  - Is relatively efficient, though it does not need to be perfectly efficient\n</Rubric>\n\nFirst, try to understand the goal of the trajectory by looking at the input\n(if the input is not present try to infer it from the content of the first message),\nas well as the output of the final message. Once you understand the goal, grade the trajectory\nas it relates to achieving that goal.\n\nGrade the following trajectory:\n\n<trajectory>\n{outputs}\n</trajectory>";
 /**
@@ -25,10 +25,10 @@ export declare const TRAJECTORY_ACCURACY_PROMPT = "You are an expert data labele
  */
 export declare const createTrajectoryLLMAsJudge: ({ prompt, feedbackKey, model, system, judge, continuous, choices, useReasoning, fewShotExamples, }: TrajectoryLLMAsJudgeParams) => ({ inputs, outputs, referenceOutputs, ...extra }: {
     [key: string]: unknown;
-    outputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    outputs: ChatCompletionMessage[] | FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     };
-    referenceOutputs?: BaseMessage[] | ChatCompletionMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    referenceOutputs?: ChatCompletionMessage[] | BaseMessage[] | FlexibleChatCompletionMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     } | undefined;
 }) => Promise<EvaluatorResult>;

package/dist/trajectory/match.d.ts

@@ -1,5 +1,5 @@
 import { BaseMessage } from "@langchain/core/messages";
-import { ChatCompletionMessage, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
+import { ChatCompletionMessage, FlexibleChatCompletionMessage, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
 export type TrajectoryMatchMode = "strict" | "unordered" | "subset" | "superset";
 /**
  * Creates an evaluator that compares trajectories between model outputs and reference outputs.
@@ -52,10 +52,10 @@ export declare function createTrajectoryMatchEvaluator({ trajectoryMatchMode, to
     toolArgsMatchOverrides?: ToolArgsMatchOverrides;
 }): ({ outputs, referenceOutputs, ...extra }: {
     [key: string]: unknown;
-    outputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    outputs: ChatCompletionMessage[] | FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     };
-    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    referenceOutputs: ChatCompletionMessage[] | FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     };
-}) => Promise<import("langsmith/vitest").SimpleEvaluationResult>;
+}) => Promise<import("../types.js").EvaluatorResult>;

package/dist/trajectory/strict.cjs

@@ -5,8 +5,8 @@ const utils_js_1 = require("../utils.cjs");
 const utils_js_2 = require("./utils.cjs");
 async function _scorer(params) {
     const { outputs, referenceOutputs, toolArgsMatchMode, toolArgsMatchOverrides, } = params;
-    const normalizedOutputs = (0, utils_js_1._normalizeToOpenAIMessagesList)(outputs);
-    const normalizedReferenceOutputs = (0, utils_js_1._normalizeToOpenAIMessagesList)(referenceOutputs);
+    const normalizedOutputs = outputs;
+    const normalizedReferenceOutputs = referenceOutputs;
     if (!normalizedOutputs || !normalizedReferenceOutputs) {
         throw new Error("Strict trajectory match requires both outputs and reference_outputs");
     }
@@ -66,8 +66,11 @@ exports._scorer = _scorer;
  * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
  */
 async function trajectoryStrictMatch(params) {
+    const normalizedOutputs = (0, utils_js_1._normalizeToOpenAIMessagesList)(params.outputs);
+    const normalizedReferenceOutputs = (0, utils_js_1._normalizeToOpenAIMessagesList)(params.referenceOutputs);
     return (0, utils_js_1._runEvaluator)("trajectory_strict_match", _scorer, "trajectory_strict_match", {
-        ...params,
+        outputs: normalizedOutputs,
+        referenceOutputs: normalizedReferenceOutputs,
         toolArgsMatchMode: params.toolCallArgsExactMatch ? "exact" : "ignore",
     });
 }

package/dist/trajectory/strict.d.ts

@@ -1,12 +1,8 @@
 import { BaseMessage } from "@langchain/core/messages";
-import { ChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
+import { ChatCompletionMessage, FlexibleChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
 export declare function _scorer(params: {
-    outputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
-    };
-    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
-    };
+    outputs: ChatCompletionMessage[];
+    referenceOutputs: ChatCompletionMessage[];
     toolArgsMatchMode: ToolArgsMatchMode;
     toolArgsMatchOverrides?: ToolArgsMatchOverrides;
 }): Promise<boolean>;
@@ -23,11 +19,11 @@ export declare function _scorer(params: {
  * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
  */
 export declare function trajectoryStrictMatch(params: {
-    outputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    outputs: ChatCompletionMessage[] | FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     };
-    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    referenceOutputs: ChatCompletionMessage[] | FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     };
     toolCallArgsExactMatch: boolean;
 }): Promise<EvaluatorResult>;

package/dist/trajectory/strict.js

@@ -2,8 +2,8 @@ import { _normalizeToOpenAIMessagesList, _runEvaluator } from "../utils.js";
 import { _getMatcherForToolName } from "./utils.js";
 export async function _scorer(params) {
     const { outputs, referenceOutputs, toolArgsMatchMode, toolArgsMatchOverrides, } = params;
-    const normalizedOutputs = _normalizeToOpenAIMessagesList(outputs);
-    const normalizedReferenceOutputs = _normalizeToOpenAIMessagesList(referenceOutputs);
+    const normalizedOutputs = outputs;
+    const normalizedReferenceOutputs = referenceOutputs;
     if (!normalizedOutputs || !normalizedReferenceOutputs) {
         throw new Error("Strict trajectory match requires both outputs and reference_outputs");
     }
@@ -62,8 +62,11 @@ export async function _scorer(params) {
  * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
  */
 export async function trajectoryStrictMatch(params) {
+    const normalizedOutputs = _normalizeToOpenAIMessagesList(params.outputs);
+    const normalizedReferenceOutputs = _normalizeToOpenAIMessagesList(params.referenceOutputs);
     return _runEvaluator("trajectory_strict_match", _scorer, "trajectory_strict_match", {
-        ...params,
+        outputs: normalizedOutputs,
+        referenceOutputs: normalizedReferenceOutputs,
         toolArgsMatchMode: params.toolCallArgsExactMatch ? "exact" : "ignore",
     });
 }

package/dist/trajectory/subset.d.ts

@@ -1,5 +1,5 @@
 import { BaseMessage } from "@langchain/core/messages";
-import { ChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
+import { ChatCompletionMessage, FlexibleChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
 export declare const _scorer: (params: {
     outputs: ChatCompletionMessage[];
     referenceOutputs: ChatCompletionMessage[];
@@ -21,10 +21,10 @@ export declare const _scorer: (params: {
  * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
  */
 export declare function trajectorySubset(params: {
-    outputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    outputs: FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     };
-    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    referenceOutputs: FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     };
 }): Promise<EvaluatorResult>;

package/dist/trajectory/superset.d.ts

@@ -1,5 +1,5 @@
 import { BaseMessage } from "@langchain/core/messages";
-import { ChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
+import { ChatCompletionMessage, FlexibleChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
 export declare const _scorer: (params: {
     outputs: ChatCompletionMessage[];
     referenceOutputs: ChatCompletionMessage[];
@@ -21,10 +21,10 @@ export declare const _scorer: (params: {
  * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
  */
 export declare function trajectorySuperset(params: {
-    outputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    outputs: FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     };
-    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    referenceOutputs: FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     };
 }): Promise<EvaluatorResult>;

package/dist/trajectory/unordered.d.ts

@@ -1,5 +1,5 @@
 import { BaseMessage } from "@langchain/core/messages";
-import { ChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
+import { ChatCompletionMessage, FlexibleChatCompletionMessage, EvaluatorResult, ToolArgsMatchMode, ToolArgsMatchOverrides } from "../types.js";
 export declare const _scorer: (params: {
     outputs: ChatCompletionMessage[];
     referenceOutputs: ChatCompletionMessage[];
@@ -21,10 +21,10 @@ export declare const _scorer: (params: {
  * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
  */
 export declare function trajectoryUnorderedMatch(params: {
-    outputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    outputs: FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     };
-    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage)[];
+    referenceOutputs: FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
     };
 }): Promise<EvaluatorResult>;

package/dist/types.d.ts

@@ -1,5 +1,20 @@
 import { createLLMAsJudge } from "openevals/llm";
 export * from "openevals/types";
+export type FlexibleChatCompletionMessage = Record<string, any> & ({
+    content: any;
+    role: "user" | "system" | "developer";
+    id?: string;
+} | {
+    role: "assistant";
+    content: any;
+    tool_calls?: any[];
+    id?: string;
+} | {
+    role: "tool";
+    content: any;
+    tool_call_id?: string;
+    id?: string;
+});
 export type GraphTrajectory = {
     inputs?: (Record<string, unknown> | null)[];
     results: Record<string, unknown>[];
@@ -9,8 +24,8 @@ export type ExtractedLangGraphThreadTrajectory = {
     inputs: (Record<string, unknown> | null)[][];
     outputs: GraphTrajectory;
 };
-export type TrajectoryLLMAsJudgeParams = Omit<Parameters<typeof createLLMAsJudge>[0], "prompt"> & {
-    prompt?: string;
+export type TrajectoryLLMAsJudgeParams = Partial<Omit<Parameters<typeof createLLMAsJudge>[0], "prompt">> & {
+    prompt?: Parameters<typeof createLLMAsJudge>[0]["prompt"];
 };
 export type ToolArgsMatchMode = "exact" | "ignore" | "subset" | "superset";
 export type ToolArgsMatcher = (toolCall: Record<string, unknown>, referenceToolCall: Record<string, unknown>) => boolean | Promise<boolean>;

package/dist/utils.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports._runEvaluator = exports.processScore = exports._normalizeToOpenAIMessagesList = exports._convertToOpenAIMessage = void 0;
+exports._runEvaluator = exports.processScore = exports._normalizeToOpenAIMessagesList = exports._convertToChatCompletionMessage = exports._convertToOpenAIMessage = void 0;
 const messages_1 = require("@langchain/core/messages");
 const openai_1 = require("@langchain/openai");
 const utils_1 = require("openevals/utils");
@@ -14,6 +14,25 @@ const _convertToOpenAIMessage = (message) => {
     }
 };
 exports._convertToOpenAIMessage = _convertToOpenAIMessage;
+const _convertToChatCompletionMessage = (message) => {
+    let converted;
+    if ((0, messages_1.isBaseMessage)(message)) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        converted = (0, openai_1._convertMessagesToOpenAIParams)([message])[0];
+    }
+    else {
+        converted = message;
+    }
+    // For tool messages without tool_call_id, generate one for compatibility
+    if (converted.role === "tool" && !converted.tool_call_id) {
+        converted = {
+            ...converted,
+            tool_call_id: `generated-${Math.random().toString(36).substring(2)}`,
+        };
+    }
+    return converted;
+};
+exports._convertToChatCompletionMessage = _convertToChatCompletionMessage;
 const _normalizeToOpenAIMessagesList = (messages) => {
     if (!messages) {
         return [];
@@ -30,7 +49,7 @@ const _normalizeToOpenAIMessagesList = (messages) => {
     else {
         messagesList = messages;
     }
-    return messagesList.map(exports._convertToOpenAIMessage);
+    return messagesList.map(exports._convertToChatCompletionMessage);
 };
 exports._normalizeToOpenAIMessagesList = _normalizeToOpenAIMessagesList;
 const processScore = (_, value) => {

package/dist/utils.d.ts

@@ -1,9 +1,10 @@
 import { BaseMessage } from "@langchain/core/messages";
 import { EvaluationResultType } from "openevals/utils";
-import { ChatCompletionMessage, MultiResultScorerReturnType, SingleResultScorerReturnType } from "./types.js";
+import { ChatCompletionMessage, FlexibleChatCompletionMessage, MultiResultScorerReturnType, SingleResultScorerReturnType } from "./types.js";
 export declare const _convertToOpenAIMessage: (message: BaseMessage | ChatCompletionMessage) => ChatCompletionMessage;
-export declare const _normalizeToOpenAIMessagesList: (messages?: (BaseMessage | ChatCompletionMessage)[] | {
-    messages: (BaseMessage | ChatCompletionMessage)[];
+export declare const _convertToChatCompletionMessage: (message: BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage) => ChatCompletionMessage;
+export declare const _normalizeToOpenAIMessagesList: (messages?: (FlexibleChatCompletionMessage | ChatCompletionMessage | BaseMessage)[] | {
+    messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
 } | undefined) => ChatCompletionMessage[];
 export declare const processScore: (_: string, value: boolean | number | {
     score: boolean | number;

package/dist/utils.js

@@ -10,6 +10,24 @@ export const _convertToOpenAIMessage = (message) => {
         return message;
     }
 };
+export const _convertToChatCompletionMessage = (message) => {
+    let converted;
+    if (isBaseMessage(message)) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        converted = _convertMessagesToOpenAIParams([message])[0];
+    }
+    else {
+        converted = message;
+    }
+    // For tool messages without tool_call_id, generate one for compatibility
+    if (converted.role === "tool" && !converted.tool_call_id) {
+        converted = {
+            ...converted,
+            tool_call_id: `generated-${Math.random().toString(36).substring(2)}`,
+        };
+    }
+    return converted;
+};
 export const _normalizeToOpenAIMessagesList = (messages) => {
     if (!messages) {
         return [];
@@ -26,7 +44,7 @@ export const _normalizeToOpenAIMessagesList = (messages) => {
     else {
         messagesList = messages;
     }
-    return messagesList.map(_convertToOpenAIMessage);
+    return messagesList.map(_convertToChatCompletionMessage);
 };
 export const processScore = (_, value) => {
     if (typeof value === "object") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentevals",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "packageManager": "yarn@3.5.1",
   "type": "module",
   "scripts": {
@@ -14,18 +14,18 @@
     "test": "vitest run"
   },
   "dependencies": {
-    "@langchain/openai": "^0.4.4",
-    "langchain": "^0.3.18",
-    "langsmith": "^0.3.11",
-    "openevals": "^0.0.3"
+    "@langchain/openai": ">=0.4.4",
+    "langchain": ">=0.3.18",
+    "langsmith": ">=0.3.11",
+    "openevals": "^0.1.0"
   },
   "peerDependencies": {
-    "@langchain/core": "^0.3.40",
-    "@langchain/langgraph": "^0.2.46"
+    "@langchain/core": ">=0.3.73",
+    "@langchain/langgraph": ">=0.2.46"
   },
   "devDependencies": {
-    "@langchain/core": "^0.3.40",
-    "@langchain/langgraph": "^0.2.46",
+    "@langchain/core": "^0.3.73",
+    "@langchain/langgraph": "^0.4.9",
     "@langchain/scripts": "0.1.3",
     "@tsconfig/recommended": "^1.0.8",
     "@typescript-eslint/eslint-plugin": "^8.24.1",
@@ -43,7 +43,7 @@
     "prettier": "^3.5.1",
     "typescript": "~5.1.6",
     "vitest": "^3.0.5",
-    "zod": "^3.24.2"
+    "zod": "^4.1.5"
   },
   "files": [
     "dist/",