agentevals 0.0.6 → 0.0.7

package/dist/graph_trajectory/utils.cjs

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.extractLangGraphTrajectoryFromThread = exports._getLangGraphStateHistoryRecursive = exports.extractLangGraphTrajectoryFromSnapshots = void 0;
 const messages_1 = require("@langchain/core/messages");
-const openai_1 = require("@langchain/openai");
+const utils_js_1 = require("../utils.cjs");
 const extractLangGraphTrajectoryFromSnapshots = (snapshots) => {
     const inputs = [];
     const trajectory = {
@@ -29,7 +29,7 @@ const extractLangGraphTrajectoryFromSnapshots = (snapshots) => {
                 if ((0, messages_1.isBaseMessage)(lastMessage)) {
                     // Just append the last message in the output to the results to reduce context size
                     trajectory.results.push({
-                        messages: (0, openai_1._convertMessagesToOpenAIParams)([lastMessage]),
+                        messages: [(0, utils_js_1._convertToOpenAIMessage)(lastMessage)],
                     });
                 }
                 else {
@@ -49,9 +49,9 @@ const extractLangGraphTrajectoryFromSnapshots = (snapshots) => {
             }
             for (const task of snapshot.tasks) {
                 if (task.interrupts?.length) {
-                    trajectory.steps.at(-1)?.push("__interrupt__");
+                    trajectory.steps[trajectory.steps.length - 1]?.push("__interrupt__");
                 }
-                trajectory.steps.at(-1)?.push(`${subgraphPath}${task.name}`);
+                trajectory.steps[trajectory.steps.length - 1]?.push(`${subgraphPath}${task.name}`);
             }
         }
         if (isAccumulatingSteps) {

package/dist/graph_trajectory/utils.js

@@ -1,5 +1,5 @@
 import { isBaseMessage } from "@langchain/core/messages";
-import { _convertMessagesToOpenAIParams } from "@langchain/openai";
+import { _convertToOpenAIMessage } from "../utils.js";
 export const extractLangGraphTrajectoryFromSnapshots = (snapshots) => {
     const inputs = [];
     const trajectory = {
@@ -26,7 +26,7 @@ export const extractLangGraphTrajectoryFromSnapshots = (snapshots) => {
                 if (isBaseMessage(lastMessage)) {
                     // Just append the last message in the output to the results to reduce context size
                     trajectory.results.push({
-                        messages: _convertMessagesToOpenAIParams([lastMessage]),
+                        messages: [_convertToOpenAIMessage(lastMessage)],
                     });
                 }
                 else {
@@ -46,9 +46,9 @@ export const extractLangGraphTrajectoryFromSnapshots = (snapshots) => {
             }
             for (const task of snapshot.tasks) {
                 if (task.interrupts?.length) {
-                    trajectory.steps.at(-1)?.push("__interrupt__");
+                    trajectory.steps[trajectory.steps.length - 1]?.push("__interrupt__");
                 }
-                trajectory.steps.at(-1)?.push(`${subgraphPath}${task.name}`);
+                trajectory.steps[trajectory.steps.length - 1]?.push(`${subgraphPath}${task.name}`);
             }
         }
         if (isAccumulatingSteps) {

package/dist/trajectory/llm.d.ts

@@ -1,5 +1,14 @@
-import { BaseMessage } from "@langchain/core/messages";
+import type { BaseMessage } from "@langchain/core/messages";
 import { ChatCompletionMessage, FlexibleChatCompletionMessage, EvaluatorResult, TrajectoryLLMAsJudgeParams } from "../types.js";
+type TrajectoryEvaluatorFunction = (params: {
+    outputs: ChatCompletionMessage[] | FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
+    };
+    referenceOutputs?: ChatCompletionMessage[] | FlexibleChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
+    };
+    [key: string]: unknown;
+}) => Promise<EvaluatorResult>;
 export declare const TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE = "You are an expert data labeler.\nYour task is to grade the accuracy of an AI agent's internal trajectory.\n\n<Rubric>\n  An accurate trajectory:\n  - Makes logical sense between steps\n  - Shows clear progression\n  - Is relatively efficient, though it does not need to be perfectly efficient\n  - Is semantically equivalent to the provided reference trajectory\n</Rubric>\n\nBased on the following reference trajectory:\n\n<reference_trajectory>\n{reference_outputs}\n</reference_trajectory>\n\nGrade this actual trajectory:\n\n<trajectory>\n{outputs}\n</trajectory>\n";
 export declare const TRAJECTORY_ACCURACY_PROMPT = "You are an expert data labeler.\nYour task is to grade the accuracy of an AI agent's internal trajectory.\n\n<Rubric>\n  An accurate trajectory:\n  - Makes logical sense between steps\n  - Shows clear progression\n  - Is relatively efficient, though it does not need to be perfectly efficient\n</Rubric>\n\nFirst, try to understand the goal of the trajectory by looking at the input\n(if the input is not present try to infer it from the content of the first message),\nas well as the output of the final message. Once you understand the goal, grade the trajectory\nas it relates to achieving that goal.\n\nGrade the following trajectory:\n\n<trajectory>\n{outputs}\n</trajectory>";
 /**
@@ -23,12 +32,5 @@ export declare const TRAJECTORY_ACCURACY_PROMPT = "You are an expert data labele
  * @param options.fewShotExamples - Optional list of example evaluations to append to the prompt.
  * @returns A function that evaluates agent trajectories using the configured LLM judge.
  */
-export declare const createTrajectoryLLMAsJudge: ({ prompt, feedbackKey, model, system, judge, continuous, choices, useReasoning, fewShotExamples, }: TrajectoryLLMAsJudgeParams) => ({ inputs, outputs, referenceOutputs, ...extra }: {
-    [key: string]: unknown;
-    outputs: ChatCompletionMessage[] | FlexibleChatCompletionMessage[] | BaseMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
-    };
-    referenceOutputs?: ChatCompletionMessage[] | BaseMessage[] | FlexibleChatCompletionMessage[] | {
-        messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
-    } | undefined;
-}) => Promise<EvaluatorResult>;
+export declare const createTrajectoryLLMAsJudge: ({ prompt, feedbackKey, model, system, judge, continuous, choices, useReasoning, fewShotExamples, }: TrajectoryLLMAsJudgeParams) => TrajectoryEvaluatorFunction;
+export {};

package/dist/utils.cjs

@@ -1,13 +1,52 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports._runEvaluator = exports.processScore = exports._normalizeToOpenAIMessagesList = exports._convertToChatCompletionMessage = exports._convertToOpenAIMessage = void 0;
 const messages_1 = require("@langchain/core/messages");
-const openai_1 = require("@langchain/openai");
+const openAIImports = __importStar(require("@langchain/openai"));
 const utils_1 = require("openevals/utils");
+const { 
+// @ts-expect-error Shim for older versions of @langchain/openai
+_convertMessagesToOpenAIParams, convertMessagesToCompletionsMessageParams, } = openAIImports;
+function _convertMessagesShim(message) {
+    if (typeof _convertMessagesToOpenAIParams === "function") {
+        return _convertMessagesToOpenAIParams([
+            message,
+        ])[0];
+    }
+    return convertMessagesToCompletionsMessageParams({
+        messages: [message],
+    })[0];
+}
 const _convertToOpenAIMessage = (message) => {
     if ((0, messages_1.isBaseMessage)(message)) {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        return (0, openai_1._convertMessagesToOpenAIParams)([message])[0];
+        const converted = _convertMessagesShim(message);
+        if (message.id && !converted.id) {
+            converted.id = message.id;
+        }
+        return converted;
     }
     else {
         return message;
@@ -17,8 +56,7 @@ exports._convertToOpenAIMessage = _convertToOpenAIMessage;
 const _convertToChatCompletionMessage = (message) => {
     let converted;
     if ((0, messages_1.isBaseMessage)(message)) {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        converted = (0, openai_1._convertMessagesToOpenAIParams)([message])[0];
+        converted = _convertMessagesShim(message);
     }
     else {
         converted = message;

package/dist/utils.d.ts

@@ -1,13 +1,15 @@
-import { BaseMessage } from "@langchain/core/messages";
+import type { BaseMessage } from "@langchain/core/messages";
 import { EvaluationResultType } from "openevals/utils";
 import { ChatCompletionMessage, FlexibleChatCompletionMessage, MultiResultScorerReturnType, SingleResultScorerReturnType } from "./types.js";
+type NormalizeToOpenAIMessagesListFunction = (messages?: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[] | {
+    messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
+}) => ChatCompletionMessage[];
 export declare const _convertToOpenAIMessage: (message: BaseMessage | ChatCompletionMessage) => ChatCompletionMessage;
 export declare const _convertToChatCompletionMessage: (message: BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage) => ChatCompletionMessage;
-export declare const _normalizeToOpenAIMessagesList: (messages?: (FlexibleChatCompletionMessage | ChatCompletionMessage | BaseMessage)[] | {
-    messages: (BaseMessage | ChatCompletionMessage | FlexibleChatCompletionMessage)[];
-} | undefined) => ChatCompletionMessage[];
+export declare const _normalizeToOpenAIMessagesList: NormalizeToOpenAIMessagesListFunction;
 export declare const processScore: (_: string, value: boolean | number | {
     score: boolean | number;
     reasoning?: string;
 }) => readonly [number | boolean, string | undefined] | readonly [number | boolean];
 export declare const _runEvaluator: <T extends Record<string, unknown>, O extends SingleResultScorerReturnType | MultiResultScorerReturnType | Promise<SingleResultScorerReturnType | MultiResultScorerReturnType>>(runName: string, scorer: (params: T) => O, feedbackKey: string, extra?: T | undefined) => Promise<EvaluationResultType<O>>;
+export {};

package/dist/utils.js

@@ -1,10 +1,26 @@
 import { isBaseMessage } from "@langchain/core/messages";
-import { _convertMessagesToOpenAIParams } from "@langchain/openai";
+import * as openAIImports from "@langchain/openai";
 import { _runEvaluator as baseRunEvaluator, } from "openevals/utils";
+const { 
+// @ts-expect-error Shim for older versions of @langchain/openai
+_convertMessagesToOpenAIParams, convertMessagesToCompletionsMessageParams, } = openAIImports;
+function _convertMessagesShim(message) {
+    if (typeof _convertMessagesToOpenAIParams === "function") {
+        return _convertMessagesToOpenAIParams([
+            message,
+        ])[0];
+    }
+    return convertMessagesToCompletionsMessageParams({
+        messages: [message],
+    })[0];
+}
 export const _convertToOpenAIMessage = (message) => {
     if (isBaseMessage(message)) {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        return _convertMessagesToOpenAIParams([message])[0];
+        const converted = _convertMessagesShim(message);
+        if (message.id && !converted.id) {
+            converted.id = message.id;
+        }
+        return converted;
     }
     else {
         return message;
@@ -13,8 +29,7 @@ export const _convertToOpenAIMessage = (message) => {
 export const _convertToChatCompletionMessage = (message) => {
     let converted;
     if (isBaseMessage(message)) {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        converted = _convertMessagesToOpenAIParams([message])[0];
+        converted = _convertMessagesShim(message);
     }
     else {
         converted = message;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentevals",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "packageManager": "yarn@3.5.1",
   "type": "module",
   "scripts": {
@@ -15,17 +15,18 @@
   },
   "dependencies": {
     "@langchain/openai": ">=0.4.4",
-    "langchain": ">=0.3.18",
-    "langsmith": ">=0.3.11",
-    "openevals": "^0.1.0"
+    "langchain": ">=1.2.28",
+    "langsmith": ">=0.4.6",
+    "openevals": "^0.1.4"
   },
   "peerDependencies": {
-    "@langchain/core": ">=0.3.73",
+    "@langchain/core": ">=0.3.80",
     "@langchain/langgraph": ">=0.2.46"
   },
   "devDependencies": {
-    "@langchain/core": "^0.3.73",
-    "@langchain/langgraph": "^0.4.9",
+    "@langchain/core": "^1.1.29",
+    "@langchain/langgraph": "^1.2.0",
+    "@langchain/openai": "^1.2.11",
     "@langchain/scripts": "0.1.3",
     "@tsconfig/recommended": "^1.0.8",
     "@typescript-eslint/eslint-plugin": "^8.24.1",
@@ -39,12 +40,26 @@
     "eslint-plugin-jest": "^27.6.0",
     "eslint-plugin-no-instanceof": "^1.0.1",
     "eslint-plugin-prettier": "^4.2.1",
-    "openai": "^4.85.1",
+    "openai": "^6.25.0",
     "prettier": "^3.5.1",
     "typescript": "~5.1.6",
     "vitest": "^3.0.5",
     "zod": "^4.1.5"
   },
+  "resolutions": {
+    "form-data": "^4.0.4",
+    "tar": "^7.5.8",
+    "axios": "^1.8.2",
+    "lodash": "^4.17.23",
+    "js-yaml": "^4.1.1",
+    "vite": "^6.4.1",
+    "esbuild": "^0.25.0",
+    "@langchain/scripts/glob": "^10.5.0",
+    "cacache/glob": "^10.5.0",
+    "dpdm/glob": "^10.5.0",
+    "node-gyp/glob": "^10.5.0",
+    "rimraf@5.0.10/glob": "^10.5.0"
+  },
   "files": [
     "dist/",
     "index.cjs",