agentevals 0.0.1 → 0.0.3

package/dist/graph_trajectory/llm.cjs

@@ -0,0 +1,113 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createGraphTrajectoryLLMAsJudge = exports.GRAPH_TRAJECTORY_ACCURACY_PROMPT = void 0;
+const llm_1 = require("openevals/llm");
+const utils_js_1 = require("../utils.cjs");
+exports.GRAPH_TRAJECTORY_ACCURACY_PROMPT = `You are an expert data labeler.
+Your task is to grade the accuracy of an AI agent's internal steps in resolving a user queries.
+
+<Rubric>
+  An accurate trajectory:
+  - Makes logical sense between steps
+  - Shows clear progression
+  - Is relatively efficient, though it does not need to be perfectly efficient
+  - Is semantically equivalent to the provided reference trajectory, if present
+</Rubric>
+
+<Instructions>
+  Grade the following thread, evaluating whether the agent's overall steps are logical and relatively efficient.
+  For the trajectory, "__start__" denotes an initial entrypoint to the agent, and "__interrupt__" corresponds to the agent
+  interrupting to await additional data from another source ("human-in-the-loop").
+  
+  Steps containing a colon represent steps within subagents (e.g. "graph:step_name").
+</Instructions>
+
+<thread>
+{thread}
+</thread>
+
+{reference_outputs}
+`;
+function _formatThread(inputs, outputs) {
+    let formattedThread = "";
+    const zippedData = inputs.map((input, i) => ({
+        input: JSON.stringify(input ?? ""),
+        result: JSON.stringify(outputs.results[i]),
+        step: JSON.stringify(outputs.steps[i]),
+    }));
+    for (const { input, result, step } of zippedData) {
+        formattedThread += input ? `\n<input>\n${input}\n</input>\n` : "";
+        formattedThread += `\n<trajectory>\n${step}\n</trajectory>\n`;
+        formattedThread += `\n<result>\n${result}\n</result>\n`;
+    }
+    return formattedThread;
+}
+function _formatInputs(inputs, outputs, referenceOutputs) {
+    let processedInputs;
+    if (Array.isArray(inputs)) {
+        processedInputs = inputs;
+    }
+    else {
+        if (!("inputs" in inputs)) {
+            throw new Error("inputs must be an array or an object with an 'inputs' key");
+        }
+        processedInputs = inputs.inputs;
+    }
+    if (processedInputs.length !== outputs.results.length) {
+        throw new Error("Provided `inputs` and `results` within provided `outputs` must have the same length");
+    }
+    if (processedInputs.length !== outputs.steps.length) {
+        throw new Error("Provided `inputs` and `steps` within provided `outputs` must have the same length");
+    }
+    const formattedThread = _formatThread(processedInputs, outputs);
+    const formattedReferenceOutputs = referenceOutputs
+        ? `\nUse the following trajectory as an example reference when grading:\n<reference_thread>\n${_formatThread(referenceOutputs.inputs ?? [], referenceOutputs)}\n</reference_thread>\n`
+        : "";
+    return {
+        formattedThread,
+        formattedReferenceOutputs,
+    };
+}
+/**
+ * Creates an evaluator that uses an LLM to judge agent trajectories.
+ * @param options Configuration options
+ * @param [options.prompt] - The evaluation prompt. Can be a string template,
+ *        LangChain prompt template, or callable that returns a list of chat messages. Note that the default prompt allows a rubric
+ *        in addition to the typical "inputs", "outputs", and "reference_outputs" parameters.
+ * @param [options.feedbackKey="graph_trajectory_accuracy"] - Key used to store the evaluation result
+ * @param [options.judge] - The LLM used for evaluation. Can be an OpenAI client
+ *        or a LangChain chat model. If an OpenAI client, must specify "model" as well.
+ *        If omitted, "model" will be used to instantiate a LangChain model instance by model string.
+ * @param [options.model] - Model identifier to use. If "judge" is an OpenAI client,
+ *        this argument should be a model name directly. If "judge" is omitted, must be a valid
+ *        LangChain model identifier. See `init_chat_model` docs for more details:
+ *        https://python.langchain.com/docs/how_to/chat_models_universal_init/
+ * @param [options.continuous=false] - If true, score will be a float between 0 and 1. If false, score will be boolean.
+ * @param [options.choices] - Optional list of specific float values the score must be chosen from
+ * @param [options.useReasoning=true] - If true, includes explanation for the score in the output
+ * @param [options.fewShotExamples] - Optional list of example evaluations to append to the prompt
+ * @returns A function that evaluates agent trajectories using the configured LLM judge
+ */
+const createGraphTrajectoryLLMAsJudge = ({ prompt = exports.GRAPH_TRAJECTORY_ACCURACY_PROMPT, model, feedbackKey = "graph_trajectory_accuracy", judge, continuous = false, choices, useReasoning = true, fewShotExamples, }) => {
+    const scorer = (0, llm_1._createLLMAsJudgeScorer)({
+        prompt,
+        judge,
+        model,
+        continuous,
+        choices,
+        useReasoning,
+        fewShotExamples,
+    });
+    const _wrappedEvaluator = async ({ inputs, outputs, referenceOutputs, ...extra }) => {
+        const { formattedThread, formattedReferenceOutputs } = _formatInputs(inputs, outputs, referenceOutputs);
+        return (0, utils_js_1._runEvaluator)(`llm_as_${feedbackKey}_judge`, scorer, feedbackKey, {
+            outputs,
+            inputs,
+            thread: formattedThread,
+            referenceOutputs: formattedReferenceOutputs,
+            ...extra,
+        });
+    };
+    return _wrappedEvaluator;
+};
+exports.createGraphTrajectoryLLMAsJudge = createGraphTrajectoryLLMAsJudge;

package/dist/graph_trajectory/llm.d.ts

@@ -0,0 +1,30 @@
+import type { GraphTrajectory, TrajectoryLLMAsJudgeParams } from "../types.js";
+export declare const GRAPH_TRAJECTORY_ACCURACY_PROMPT = "You are an expert data labeler.\nYour task is to grade the accuracy of an AI agent's internal steps in resolving a user queries.\n\n<Rubric>\n  An accurate trajectory:\n  - Makes logical sense between steps\n  - Shows clear progression\n  - Is relatively efficient, though it does not need to be perfectly efficient\n  - Is semantically equivalent to the provided reference trajectory, if present\n</Rubric>\n\n<Instructions>\n  Grade the following thread, evaluating whether the agent's overall steps are logical and relatively efficient.\n  For the trajectory, \"__start__\" denotes an initial entrypoint to the agent, and \"__interrupt__\" corresponds to the agent\n  interrupting to await additional data from another source (\"human-in-the-loop\").\n  \n  Steps containing a colon represent steps within subagents (e.g. \"graph:step_name\").\n</Instructions>\n\n<thread>\n{thread}\n</thread>\n\n{reference_outputs}\n";
+/**
+ * Creates an evaluator that uses an LLM to judge agent trajectories.
+ * @param options Configuration options
+ * @param [options.prompt] - The evaluation prompt. Can be a string template,
+ *        LangChain prompt template, or callable that returns a list of chat messages. Note that the default prompt allows a rubric
+ *        in addition to the typical "inputs", "outputs", and "reference_outputs" parameters.
+ * @param [options.feedbackKey="graph_trajectory_accuracy"] - Key used to store the evaluation result
+ * @param [options.judge] - The LLM used for evaluation. Can be an OpenAI client
+ *        or a LangChain chat model. If an OpenAI client, must specify "model" as well.
+ *        If omitted, "model" will be used to instantiate a LangChain model instance by model string.
+ * @param [options.model] - Model identifier to use. If "judge" is an OpenAI client,
+ *        this argument should be a model name directly. If "judge" is omitted, must be a valid
+ *        LangChain model identifier. See `init_chat_model` docs for more details:
+ *        https://python.langchain.com/docs/how_to/chat_models_universal_init/
+ * @param [options.continuous=false] - If true, score will be a float between 0 and 1. If false, score will be boolean.
+ * @param [options.choices] - Optional list of specific float values the score must be chosen from
+ * @param [options.useReasoning=true] - If true, includes explanation for the score in the output
+ * @param [options.fewShotExamples] - Optional list of example evaluations to append to the prompt
+ * @returns A function that evaluates agent trajectories using the configured LLM judge
+ */
+export declare const createGraphTrajectoryLLMAsJudge: ({ prompt, model, feedbackKey, judge, continuous, choices, useReasoning, fewShotExamples, }: TrajectoryLLMAsJudgeParams) => ({ inputs, outputs, referenceOutputs, ...extra }: {
+    [key: string]: unknown;
+    inputs: (string | Record<string, unknown> | null)[] | {
+        inputs: (string | Record<string, unknown> | null)[];
+    };
+    outputs: GraphTrajectory;
+    referenceOutputs?: GraphTrajectory | undefined;
+}) => Promise<import("langsmith/vitest").SimpleEvaluationResult>;

package/dist/graph_trajectory/llm.js

@@ -0,0 +1,109 @@
+import { _createLLMAsJudgeScorer } from "openevals/llm";
+import { _runEvaluator } from "../utils.js";
+export const GRAPH_TRAJECTORY_ACCURACY_PROMPT = `You are an expert data labeler.
+Your task is to grade the accuracy of an AI agent's internal steps in resolving a user queries.
+
+<Rubric>
+  An accurate trajectory:
+  - Makes logical sense between steps
+  - Shows clear progression
+  - Is relatively efficient, though it does not need to be perfectly efficient
+  - Is semantically equivalent to the provided reference trajectory, if present
+</Rubric>
+
+<Instructions>
+  Grade the following thread, evaluating whether the agent's overall steps are logical and relatively efficient.
+  For the trajectory, "__start__" denotes an initial entrypoint to the agent, and "__interrupt__" corresponds to the agent
+  interrupting to await additional data from another source ("human-in-the-loop").
+  
+  Steps containing a colon represent steps within subagents (e.g. "graph:step_name").
+</Instructions>
+
+<thread>
+{thread}
+</thread>
+
+{reference_outputs}
+`;
+function _formatThread(inputs, outputs) {
+    let formattedThread = "";
+    const zippedData = inputs.map((input, i) => ({
+        input: JSON.stringify(input ?? ""),
+        result: JSON.stringify(outputs.results[i]),
+        step: JSON.stringify(outputs.steps[i]),
+    }));
+    for (const { input, result, step } of zippedData) {
+        formattedThread += input ? `\n<input>\n${input}\n</input>\n` : "";
+        formattedThread += `\n<trajectory>\n${step}\n</trajectory>\n`;
+        formattedThread += `\n<result>\n${result}\n</result>\n`;
+    }
+    return formattedThread;
+}
+function _formatInputs(inputs, outputs, referenceOutputs) {
+    let processedInputs;
+    if (Array.isArray(inputs)) {
+        processedInputs = inputs;
+    }
+    else {
+        if (!("inputs" in inputs)) {
+            throw new Error("inputs must be an array or an object with an 'inputs' key");
+        }
+        processedInputs = inputs.inputs;
+    }
+    if (processedInputs.length !== outputs.results.length) {
+        throw new Error("Provided `inputs` and `results` within provided `outputs` must have the same length");
+    }
+    if (processedInputs.length !== outputs.steps.length) {
+        throw new Error("Provided `inputs` and `steps` within provided `outputs` must have the same length");
+    }
+    const formattedThread = _formatThread(processedInputs, outputs);
+    const formattedReferenceOutputs = referenceOutputs
+        ? `\nUse the following trajectory as an example reference when grading:\n<reference_thread>\n${_formatThread(referenceOutputs.inputs ?? [], referenceOutputs)}\n</reference_thread>\n`
+        : "";
+    return {
+        formattedThread,
+        formattedReferenceOutputs,
+    };
+}
+/**
+ * Creates an evaluator that uses an LLM to judge agent trajectories.
+ * @param options Configuration options
+ * @param [options.prompt] - The evaluation prompt. Can be a string template,
+ *        LangChain prompt template, or callable that returns a list of chat messages. Note that the default prompt allows a rubric
+ *        in addition to the typical "inputs", "outputs", and "reference_outputs" parameters.
+ * @param [options.feedbackKey="graph_trajectory_accuracy"] - Key used to store the evaluation result
+ * @param [options.judge] - The LLM used for evaluation. Can be an OpenAI client
+ *        or a LangChain chat model. If an OpenAI client, must specify "model" as well.
+ *        If omitted, "model" will be used to instantiate a LangChain model instance by model string.
+ * @param [options.model] - Model identifier to use. If "judge" is an OpenAI client,
+ *        this argument should be a model name directly. If "judge" is omitted, must be a valid
+ *        LangChain model identifier. See `init_chat_model` docs for more details:
+ *        https://python.langchain.com/docs/how_to/chat_models_universal_init/
+ * @param [options.continuous=false] - If true, score will be a float between 0 and 1. If false, score will be boolean.
+ * @param [options.choices] - Optional list of specific float values the score must be chosen from
+ * @param [options.useReasoning=true] - If true, includes explanation for the score in the output
+ * @param [options.fewShotExamples] - Optional list of example evaluations to append to the prompt
+ * @returns A function that evaluates agent trajectories using the configured LLM judge
+ */
+export const createGraphTrajectoryLLMAsJudge = ({ prompt = GRAPH_TRAJECTORY_ACCURACY_PROMPT, model, feedbackKey = "graph_trajectory_accuracy", judge, continuous = false, choices, useReasoning = true, fewShotExamples, }) => {
+    const scorer = _createLLMAsJudgeScorer({
+        prompt,
+        judge,
+        model,
+        continuous,
+        choices,
+        useReasoning,
+        fewShotExamples,
+    });
+    const _wrappedEvaluator = async ({ inputs, outputs, referenceOutputs, ...extra }) => {
+        const { formattedThread, formattedReferenceOutputs } = _formatInputs(inputs, outputs, referenceOutputs);
+        return _runEvaluator(`llm_as_${feedbackKey}_judge`, scorer, feedbackKey, {
+            outputs,
+            inputs,
+            thread: formattedThread,
+            referenceOutputs: formattedReferenceOutputs,
+            ...extra,
+        });
+    };
+    return _wrappedEvaluator;
+};

package/dist/graph_trajectory/strict.cjs

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.graphTrajectoryStrictMatch = void 0;
+const utils_js_1 = require("../utils.cjs");
+const _scorer = (params) => {
+    const { outputs, referenceOutputs } = params;
+    if (!outputs || !referenceOutputs) {
+        throw new Error("Strict trajectory match requires both outputs and referenceOutputs");
+    }
+    if (outputs.steps.length !== referenceOutputs.steps.length) {
+        return false;
+    }
+    for (let i = 0; i < outputs.steps.length; i++) {
+        if (outputs.steps[i].length !== referenceOutputs.steps[i].length) {
+            return false;
+        }
+        for (let j = 0; j < outputs.steps[i].length; j++) {
+            if (outputs.steps[i][j] !== referenceOutputs.steps[i][j]) {
+                return false;
+            }
+        }
+    }
+    return true;
+};
+/**
+ * Evaluate whether an input graph trajectory strictly matches a reference graph trajectory.
+ * This means that at each step, the agent took the same steps in the same order as specified in the reference trajectory.
+ *
+ * @param params - The parameters object
+ * @param params.outputs - Actual trajectory the agent followed
+ * @param params.referenceOutputs - Ideal reference trajectory the agent should have followed
+ * @returns Contains a score of true if trajectory (including called tools) matches, false otherwise
+ */
+const graphTrajectoryStrictMatch = ({ outputs, referenceOutputs, }) => {
+    return (0, utils_js_1._runEvaluator)("graph_trajectory_strict_match", _scorer, "graph_trajectory_strict_match", {
+        outputs,
+        referenceOutputs,
+    });
+};
+exports.graphTrajectoryStrictMatch = graphTrajectoryStrictMatch;

package/dist/graph_trajectory/strict.d.ts

@@ -0,0 +1,14 @@
+import { GraphTrajectory } from "../types.js";
+/**
+ * Evaluate whether an input graph trajectory strictly matches a reference graph trajectory.
+ * This means that at each step, the agent took the same steps in the same order as specified in the reference trajectory.
+ *
+ * @param params - The parameters object
+ * @param params.outputs - Actual trajectory the agent followed
+ * @param params.referenceOutputs - Ideal reference trajectory the agent should have followed
+ * @returns Contains a score of true if trajectory (including called tools) matches, false otherwise
+ */
+export declare const graphTrajectoryStrictMatch: ({ outputs, referenceOutputs, }: {
+    outputs: GraphTrajectory;
+    referenceOutputs: GraphTrajectory;
+}) => Promise<import("langsmith/vitest").SimpleEvaluationResult>;

package/dist/graph_trajectory/strict.js

@@ -0,0 +1,36 @@
+import { _runEvaluator } from "../utils.js";
+const _scorer = (params) => {
+    const { outputs, referenceOutputs } = params;
+    if (!outputs || !referenceOutputs) {
+        throw new Error("Strict trajectory match requires both outputs and referenceOutputs");
+    }
+    if (outputs.steps.length !== referenceOutputs.steps.length) {
+        return false;
+    }
+    for (let i = 0; i < outputs.steps.length; i++) {
+        if (outputs.steps[i].length !== referenceOutputs.steps[i].length) {
+            return false;
+        }
+        for (let j = 0; j < outputs.steps[i].length; j++) {
+            if (outputs.steps[i][j] !== referenceOutputs.steps[i][j]) {
+                return false;
+            }
+        }
+    }
+    return true;
+};
+/**
+ * Evaluate whether an input graph trajectory strictly matches a reference graph trajectory.
+ * This means that at each step, the agent took the same steps in the same order as specified in the reference trajectory.
+ *
+ * @param params - The parameters object
+ * @param params.outputs - Actual trajectory the agent followed
+ * @param params.referenceOutputs - Ideal reference trajectory the agent should have followed
+ * @returns Contains a score of true if trajectory (including called tools) matches, false otherwise
+ */
+export const graphTrajectoryStrictMatch = ({ outputs, referenceOutputs, }) => {
+    return _runEvaluator("graph_trajectory_strict_match", _scorer, "graph_trajectory_strict_match", {
+        outputs,
+        referenceOutputs,
+    });
+};

package/dist/graph_trajectory/utils.cjs

@@ -0,0 +1,105 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractLangGraphTrajectoryFromThread = exports._getLangGraphStateHistoryRecursive = exports.extractLangGraphTrajectoryFromSnapshots = void 0;
+const messages_1 = require("@langchain/core/messages");
+const openai_1 = require("@langchain/openai");
+const extractLangGraphTrajectoryFromSnapshots = (snapshots) => {
+    const inputs = [];
+    const trajectory = {
+        results: [],
+        steps: [],
+    };
+    let isAccumulatingSteps = false;
+    for (let i = 0; i < snapshots.length; i += 1) {
+        const snapshot = snapshots[i];
+        const hasInterrupts = snapshot.tasks?.find((task) => {
+            return task.interrupts?.length;
+        });
+        if (!snapshot.next?.length || hasInterrupts) {
+            isAccumulatingSteps = true;
+            if (hasInterrupts) {
+                trajectory.results.push({});
+            }
+            else if (snapshot.values != null &&
+                typeof snapshot.values === "object" &&
+                !Array.isArray(snapshot.values) &&
+                "messages" in snapshot.values &&
+                Array.isArray(snapshot.values.messages)) {
+                const lastMessage = snapshot.values.messages.at(-1);
+                if ((0, messages_1.isBaseMessage)(lastMessage)) {
+                    // Just append the last message in the output to the results to reduce context size
+                    trajectory.results.push({
+                        messages: (0, openai_1._convertMessagesToOpenAIParams)([lastMessage]),
+                    });
+                }
+                else {
+                    trajectory.results.push({ messages: [lastMessage] });
+                }
+            }
+            else {
+                trajectory.results.push(snapshot.values);
+            }
+            trajectory.steps.push([]);
+        }
+        if (isAccumulatingSteps && snapshot.tasks?.length) {
+            const checkpointNs = snapshot.config?.configurable?.checkpoint_ns ?? "";
+            let subgraphPath = "";
+            if (checkpointNs.split(":").length > 1) {
+                subgraphPath = `${checkpointNs.split(":")[0]}:`;
+            }
+            for (const task of snapshot.tasks) {
+                if (task.interrupts?.length) {
+                    trajectory.steps.at(-1)?.push("__interrupt__");
+                }
+                trajectory.steps.at(-1)?.push(`${subgraphPath}${task.name}`);
+            }
+        }
+        if (isAccumulatingSteps) {
+            if (snapshot.metadata != null && snapshot.metadata.source === "input") {
+                inputs.push(snapshot.metadata.writes);
+            }
+            else if (i + 1 < snapshots.length &&
+                snapshots[i + 1].tasks?.find((task) => task.interrupts?.length > 0)) {
+                inputs.push("__resuming__");
+            }
+        }
+    }
+    inputs.reverse();
+    trajectory.results.reverse();
+    trajectory.steps.reverse();
+    for (const stepList of trajectory.steps) {
+        stepList.reverse();
+    }
+    if (inputs.length !== trajectory.results.length) {
+        console.warn("Trajectory parsing may be incomplete: inputs and results have different lengths");
+    }
+    else if (inputs.length !== trajectory.steps.length) {
+        console.warn("Trajectory parsing may be incomplete: inputs and steps have different lengths");
+    }
+    return { inputs, outputs: trajectory };
+};
+exports.extractLangGraphTrajectoryFromSnapshots = extractLangGraphTrajectoryFromSnapshots;
+const _getLangGraphStateHistoryRecursive = async (
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+graph, config) => {
+    const stateHistory = [];
+    for await (const history of graph.getStateHistory(config)) {
+        if (history.tasks?.length) {
+            for (const task of history.tasks) {
+                if (task.state?.configurable?.checkpoint_ns) {
+                    stateHistory.push(...(await (0, exports._getLangGraphStateHistoryRecursive)(graph, task.state)));
+                }
+            }
+        }
+        stateHistory.push(history);
+    }
+    return stateHistory;
+};
+exports._getLangGraphStateHistoryRecursive = _getLangGraphStateHistoryRecursive;
+const extractLangGraphTrajectoryFromThread = async (
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+graph, config) => {
+    const history = await (0, exports._getLangGraphStateHistoryRecursive)(graph, config);
+    return (0, exports.extractLangGraphTrajectoryFromSnapshots)(history);
+};
+exports.extractLangGraphTrajectoryFromThread = extractLangGraphTrajectoryFromThread;

package/dist/graph_trajectory/utils.d.ts

@@ -0,0 +1,12 @@
+import type { StateSnapshot, Pregel } from "@langchain/langgraph/web";
+import type { RunnableConfig } from "@langchain/core/runnables";
+import type { GraphTrajectory } from "../types.js";
+export declare const extractLangGraphTrajectoryFromSnapshots: (snapshots: StateSnapshot[]) => {
+    inputs: (string | Record<string, unknown> | null)[];
+    outputs: GraphTrajectory;
+};
+export declare const _getLangGraphStateHistoryRecursive: (graph: Pregel<any, any>, config: RunnableConfig) => Promise<StateSnapshot[]>;
+export declare const extractLangGraphTrajectoryFromThread: (graph: Pregel<any, any>, config: RunnableConfig) => Promise<{
+    inputs: (string | Record<string, unknown> | null)[];
+    outputs: GraphTrajectory;
+}>;

package/dist/graph_trajectory/utils.js

@@ -0,0 +1,99 @@
+import { isBaseMessage } from "@langchain/core/messages";
+import { _convertMessagesToOpenAIParams } from "@langchain/openai";
+export const extractLangGraphTrajectoryFromSnapshots = (snapshots) => {
+    const inputs = [];
+    const trajectory = {
+        results: [],
+        steps: [],
+    };
+    let isAccumulatingSteps = false;
+    for (let i = 0; i < snapshots.length; i += 1) {
+        const snapshot = snapshots[i];
+        const hasInterrupts = snapshot.tasks?.find((task) => {
+            return task.interrupts?.length;
+        });
+        if (!snapshot.next?.length || hasInterrupts) {
+            isAccumulatingSteps = true;
+            if (hasInterrupts) {
+                trajectory.results.push({});
+            }
+            else if (snapshot.values != null &&
+                typeof snapshot.values === "object" &&
+                !Array.isArray(snapshot.values) &&
+                "messages" in snapshot.values &&
+                Array.isArray(snapshot.values.messages)) {
+                const lastMessage = snapshot.values.messages.at(-1);
+                if (isBaseMessage(lastMessage)) {
+                    // Just append the last message in the output to the results to reduce context size
+                    trajectory.results.push({
+                        messages: _convertMessagesToOpenAIParams([lastMessage]),
+                    });
+                }
+                else {
+                    trajectory.results.push({ messages: [lastMessage] });
+                }
+            }
+            else {
+                trajectory.results.push(snapshot.values);
+            }
+            trajectory.steps.push([]);
+        }
+        if (isAccumulatingSteps && snapshot.tasks?.length) {
+            const checkpointNs = snapshot.config?.configurable?.checkpoint_ns ?? "";
+            let subgraphPath = "";
+            if (checkpointNs.split(":").length > 1) {
+                subgraphPath = `${checkpointNs.split(":")[0]}:`;
+            }
+            for (const task of snapshot.tasks) {
+                if (task.interrupts?.length) {
+                    trajectory.steps.at(-1)?.push("__interrupt__");
+                }
+                trajectory.steps.at(-1)?.push(`${subgraphPath}${task.name}`);
+            }
+        }
+        if (isAccumulatingSteps) {
+            if (snapshot.metadata != null && snapshot.metadata.source === "input") {
+                inputs.push(snapshot.metadata.writes);
+            }
+            else if (i + 1 < snapshots.length &&
+                snapshots[i + 1].tasks?.find((task) => task.interrupts?.length > 0)) {
+                inputs.push("__resuming__");
+            }
+        }
+    }
+    inputs.reverse();
+    trajectory.results.reverse();
+    trajectory.steps.reverse();
+    for (const stepList of trajectory.steps) {
+        stepList.reverse();
+    }
+    if (inputs.length !== trajectory.results.length) {
+        console.warn("Trajectory parsing may be incomplete: inputs and results have different lengths");
+    }
+    else if (inputs.length !== trajectory.steps.length) {
+        console.warn("Trajectory parsing may be incomplete: inputs and steps have different lengths");
+    }
+    return { inputs, outputs: trajectory };
+};
+export const _getLangGraphStateHistoryRecursive = async (
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+graph, config) => {
+    const stateHistory = [];
+    for await (const history of graph.getStateHistory(config)) {
+        if (history.tasks?.length) {
+            for (const task of history.tasks) {
+                if (task.state?.configurable?.checkpoint_ns) {
+                    stateHistory.push(...(await _getLangGraphStateHistoryRecursive(graph, task.state)));
+                }
+            }
+        }
+        stateHistory.push(history);
+    }
+    return stateHistory;
+};
+export const extractLangGraphTrajectoryFromThread = async (
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+graph, config) => {
+    const history = await _getLangGraphStateHistoryRecursive(graph, config);
+    return extractLangGraphTrajectoryFromSnapshots(history);
+};

package/dist/index.cjs

@@ -0,0 +1,35 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GRAPH_TRAJECTORY_ACCURACY_PROMPT = exports.createGraphTrajectoryLLMAsJudge = exports.TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE = exports.TRAJECTORY_ACCURACY_PROMPT = exports.createTrajectoryLLMAsJudge = exports.trajectoryUnorderedMatch = exports.trajectorySuperset = exports.trajectorySubset = exports.trajectoryStrictMatch = void 0;
+var strict_js_1 = require("./trajectory/strict.cjs");
+Object.defineProperty(exports, "trajectoryStrictMatch", { enumerable: true, get: function () { return strict_js_1.trajectoryStrictMatch; } });
+var subset_js_1 = require("./trajectory/subset.cjs");
+Object.defineProperty(exports, "trajectorySubset", { enumerable: true, get: function () { return subset_js_1.trajectorySubset; } });
+var superset_js_1 = require("./trajectory/superset.cjs");
+Object.defineProperty(exports, "trajectorySuperset", { enumerable: true, get: function () { return superset_js_1.trajectorySuperset; } });
+var unordered_js_1 = require("./trajectory/unordered.cjs");
+Object.defineProperty(exports, "trajectoryUnorderedMatch", { enumerable: true, get: function () { return unordered_js_1.trajectoryUnorderedMatch; } });
+var llm_js_1 = require("./trajectory/llm.cjs");
+Object.defineProperty(exports, "createTrajectoryLLMAsJudge", { enumerable: true, get: function () { return llm_js_1.createTrajectoryLLMAsJudge; } });
+Object.defineProperty(exports, "TRAJECTORY_ACCURACY_PROMPT", { enumerable: true, get: function () { return llm_js_1.TRAJECTORY_ACCURACY_PROMPT; } });
+Object.defineProperty(exports, "TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE", { enumerable: true, get: function () { return llm_js_1.TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE; } });
+var llm_js_2 = require("./graph_trajectory/llm.cjs");
+Object.defineProperty(exports, "createGraphTrajectoryLLMAsJudge", { enumerable: true, get: function () { return llm_js_2.createGraphTrajectoryLLMAsJudge; } });
+Object.defineProperty(exports, "GRAPH_TRAJECTORY_ACCURACY_PROMPT", { enumerable: true, get: function () { return llm_js_2.GRAPH_TRAJECTORY_ACCURACY_PROMPT; } });
+__exportStar(require("./types.cjs"), exports);
+__exportStar(require("./utils.cjs"), exports);
+__exportStar(require("./graph_trajectory/utils.cjs"), exports);

package/dist/index.d.ts

@@ -2,5 +2,8 @@ export { trajectoryStrictMatch } from "./trajectory/strict.js";
 export { trajectorySubset } from "./trajectory/subset.js";
 export { trajectorySuperset } from "./trajectory/superset.js";
 export { trajectoryUnorderedMatch } from "./trajectory/unordered.js";
-export { createTrajectoryLLMAsJudge, DEFAULT_PROMPT, } from "./trajectory/llm.js";
+export { createTrajectoryLLMAsJudge, TRAJECTORY_ACCURACY_PROMPT, TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE, } from "./trajectory/llm.js";
+export { createGraphTrajectoryLLMAsJudge, GRAPH_TRAJECTORY_ACCURACY_PROMPT, } from "./graph_trajectory/llm.js";
 export * from "./types.js";
+export * from "./utils.js";
+export * from "./graph_trajectory/utils.js";

package/dist/index.js

@@ -2,5 +2,8 @@ export { trajectoryStrictMatch } from "./trajectory/strict.js";
 export { trajectorySubset } from "./trajectory/subset.js";
 export { trajectorySuperset } from "./trajectory/superset.js";
 export { trajectoryUnorderedMatch } from "./trajectory/unordered.js";
-export { createTrajectoryLLMAsJudge, DEFAULT_PROMPT, } from "./trajectory/llm.js";
+export { createTrajectoryLLMAsJudge, TRAJECTORY_ACCURACY_PROMPT, TRAJECTORY_ACCURACY_PROMPT_WITH_REFERENCE, } from "./trajectory/llm.js";
+export { createGraphTrajectoryLLMAsJudge, GRAPH_TRAJECTORY_ACCURACY_PROMPT, } from "./graph_trajectory/llm.js";
 export * from "./types.js";
+export * from "./utils.js";
+export * from "./graph_trajectory/utils.js";