agentevals 0.0.0

package/dist/evaluators/string/levenshtein.cjs

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.levenshteinDistance = void 0;
+const utils_js_1 = require("../utils.cjs");
+function scorer(outputs, referenceOutputs) {
+    if (outputs === null || referenceOutputs === null) {
+        throw new Error("Levenshtein distance requires both outputs and reference_outputs");
+    }
+    // Convert to strings if needed
+    const outputStr = typeof outputs === "string" ? outputs : JSON.stringify(outputs);
+    const referenceStr = typeof referenceOutputs === "string"
+        ? referenceOutputs
+        : JSON.stringify(referenceOutputs);
+    // Create a matrix of size (m+1)x(n+1) where m and n are the string lengths
+    const m = outputStr.length;
+    const n = referenceStr.length;
+    const dp = Array(m + 1)
+        .fill(null)
+        .map(() => Array(n + 1).fill(0));
+    // Initialize first row and column
+    for (let i = 0; i <= m; i++) {
+        dp[i][0] = i;
+    }
+    for (let j = 0; j <= n; j++) {
+        dp[0][j] = j;
+    }
+    // Fill the matrix
+    for (let i = 1; i <= m; i++) {
+        for (let j = 1; j <= n; j++) {
+            if (outputStr[i - 1] === referenceStr[j - 1]) {
+                dp[i][j] = dp[i - 1][j - 1];
+            }
+            else {
+                dp[i][j] = Math.min(dp[i - 1][j] + 1, // deletion
+                dp[i][j - 1] + 1, // insertion
+                dp[i - 1][j - 1] + 1 // substitution
+                );
+            }
+        }
+    }
+    // Calculate the distance and normalize it to a score between 0 and 1
+    const distance = dp[m][n];
+    const maxLength = Math.max(m, n);
+    return maxLength > 0 ? 1.0 - distance / maxLength : 1.0;
+}
+/**
+ * Evaluates the actual output and reference output for similarity by Levenshtein distance.
+ * @param options - The options object containing outputs and reference outputs
+ * @returns EvaluatorResult containing match result with score between 0.0 and 1.0,
+ * where 1.0 indicates an exact match and lower values indicate greater differences
+ */
+async function levenshteinDistance(params) {
+    const { outputs, referenceOutputs } = params;
+    const getScore = () => scorer(outputs, referenceOutputs);
+    return (0, utils_js_1._runEvaluator)("levenshtein_distance", getScore, "levenshtein_distance");
+}
+exports.levenshteinDistance = levenshteinDistance;

package/dist/evaluators/string/levenshtein.d.ts

@@ -0,0 +1,11 @@
+import { EvaluatorResult } from "../types.js";
+/**
+ * Evaluates the actual output and reference output for similarity by Levenshtein distance.
+ * @param options - The options object containing outputs and reference outputs
+ * @returns EvaluatorResult containing match result with score between 0.0 and 1.0,
+ * where 1.0 indicates an exact match and lower values indicate greater differences
+ */
+export declare function levenshteinDistance(params: {
+    outputs: unknown;
+    referenceOutputs: unknown;
+}): Promise<EvaluatorResult>;

package/dist/evaluators/string/levenshtein.js

@@ -0,0 +1,53 @@
+import { _runEvaluator } from "../utils.js";
+function scorer(outputs, referenceOutputs) {
+    if (outputs === null || referenceOutputs === null) {
+        throw new Error("Levenshtein distance requires both outputs and reference_outputs");
+    }
+    // Convert to strings if needed
+    const outputStr = typeof outputs === "string" ? outputs : JSON.stringify(outputs);
+    const referenceStr = typeof referenceOutputs === "string"
+        ? referenceOutputs
+        : JSON.stringify(referenceOutputs);
+    // Create a matrix of size (m+1)x(n+1) where m and n are the string lengths
+    const m = outputStr.length;
+    const n = referenceStr.length;
+    const dp = Array(m + 1)
+        .fill(null)
+        .map(() => Array(n + 1).fill(0));
+    // Initialize first row and column
+    for (let i = 0; i <= m; i++) {
+        dp[i][0] = i;
+    }
+    for (let j = 0; j <= n; j++) {
+        dp[0][j] = j;
+    }
+    // Fill the matrix
+    for (let i = 1; i <= m; i++) {
+        for (let j = 1; j <= n; j++) {
+            if (outputStr[i - 1] === referenceStr[j - 1]) {
+                dp[i][j] = dp[i - 1][j - 1];
+            }
+            else {
+                dp[i][j] = Math.min(dp[i - 1][j] + 1, // deletion
+                dp[i][j - 1] + 1, // insertion
+                dp[i - 1][j - 1] + 1 // substitution
+                );
+            }
+        }
+    }
+    // Calculate the distance and normalize it to a score between 0 and 1
+    const distance = dp[m][n];
+    const maxLength = Math.max(m, n);
+    return maxLength > 0 ? 1.0 - distance / maxLength : 1.0;
+}
+/**
+ * Evaluates the actual output and reference output for similarity by Levenshtein distance.
+ * @param options - The options object containing outputs and reference outputs
+ * @returns EvaluatorResult containing match result with score between 0.0 and 1.0,
+ * where 1.0 indicates an exact match and lower values indicate greater differences
+ */
+export async function levenshteinDistance(params) {
+    const { outputs, referenceOutputs } = params;
+    const getScore = () => scorer(outputs, referenceOutputs);
+    return _runEvaluator("levenshtein_distance", getScore, "levenshtein_distance");
+}

package/dist/evaluators/trajectory/llm.cjs

@@ -0,0 +1,86 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTrajectoryLLMAsJudge = exports.DEFAULT_PROMPT = void 0;
+const llm_js_1 = require("../llm.cjs");
+const utils_js_1 = require("../utils.cjs");
+const utils_js_2 = require("../utils.cjs");
+const utils_js_3 = require("./utils.cjs");
+exports.DEFAULT_PROMPT = `Grade the following agent trajectory:
+
+<trajectory>
+{outputs}
+</trajectory>
+{inputs}
+{reference_outputs}
+{rubric}
+`;
+function _formatInputs(params) {
+    const { inputs, outputs, referenceOutputs, rubric } = params;
+    const normalizedOutputs = (0, utils_js_2._normalizeToOpenAIMessagesList)(outputs);
+    const normalizedReferenceOutputs = (0, utils_js_2._normalizeToOpenAIMessagesList)(referenceOutputs ?? []);
+    const formattedReferenceOutputs = normalizedReferenceOutputs
+        ? `\nUse the following trajectory as an example reference when grading:\n<reference_trajectory>\n${(0, utils_js_3._chatCompletionMessagesToString)(normalizedReferenceOutputs)}\n</reference_trajectory>\n`
+        : "";
+    const formattedInputs = inputs
+        ? `\nThe agent generated the trajectory from the following input:\n<input>\n${JSON.stringify(inputs)}\n</input>\n`
+        : "";
+    const formattedOutputs = typeof outputs === "object" && !Array.isArray(outputs)
+        ? outputs
+        : (0, utils_js_3._chatCompletionMessagesToString)(normalizedOutputs);
+    const formattedRubric = rubric
+        ? `\nGrade the agent trajectory along the following rubric:\n<rubric>\n${rubric}\n</rubric>\n`
+        : "";
+    return [
+        formattedOutputs,
+        formattedReferenceOutputs,
+        formattedInputs,
+        formattedRubric,
+    ];
+}
+/**
+ * Creates an evaluator that uses an LLM to judge agent trajectories.
+ *
+ * @param options - Configuration options
+ * @param options.prompt - The evaluation prompt. Can be a string template, LangChain prompt template,
+ *                        or callable that returns a list of chat messages. Note that the default prompt
+ *                        allows a rubric in addition to the typical "inputs", "outputs", and
+ *                        "reference_outputs" parameters.
+ * @param options.feedbackKey - Key used to store the evaluation result. Defaults to "trajectory_accuracy".
+ * @param options.model - Model identifier to use. Defaults to "openai:o3-mini". If judge is an OpenAI client,
+ *                       this should be a model name directly. If judge is omitted, must be a valid
+ *                       LangChain model identifier.
+ * @param options.system - Optional system message to prepend to the prompt.
+ * @param options.judge - The LLM used for evaluation. Can be an OpenAI client or a LangChainLikeModel.
+ *                       If an OpenAI client, must specify "model" as well. If omitted, "model" will be
+ *                       used to instantiate a LangChain model instance by model string.
+ * @param options.continuous - If true, score will be a float between 0 and 1. If false, score will be boolean.
+ *                           Defaults to false.
+ * @param options.choices - Optional list of specific float values the score must be chosen from.
+ * @param options.useReasoning - If true, includes explanation for the score in the output. Defaults to true.
+ * @param options.fewShotExamples - Optional list of example evaluations to append to the prompt.
+ * @returns A function that evaluates agent trajectories using the configured LLM judge.
+ */
+const createTrajectoryLLMAsJudge = ({ prompt = exports.DEFAULT_PROMPT, feedbackKey = "trajectory_accuracy", model = "openai:o3-mini", system, judge, continuous = false, choices, useReasoning = true, fewShotExamples, }) => {
+    const scorer = (0, llm_js_1._createLLMAsJudgeScorer)({
+        prompt,
+        judge,
+        model,
+        system,
+        continuous,
+        choices,
+        useReasoning,
+        fewShotExamples,
+    });
+    const wrappedEvaluator = async ({ inputs, outputs, referenceOutputs, rubric, ...extra }) => {
+        const [formattedOutputs, formattedReferenceOutputs, formattedInputs, formattedRubric,] = _formatInputs({ inputs, outputs, referenceOutputs, rubric });
+        return (0, utils_js_1._runEvaluator)(`llm_as_${feedbackKey}_judge`, scorer, feedbackKey, {
+            outputs: formattedOutputs,
+            referenceOutputs: formattedReferenceOutputs,
+            inputs: formattedInputs,
+            rubric: formattedRubric,
+            ...extra,
+        });
+    };
+    return wrappedEvaluator;
+};
+exports.createTrajectoryLLMAsJudge = createTrajectoryLLMAsJudge;

package/dist/evaluators/trajectory/llm.d.ts

@@ -0,0 +1,49 @@
+import { ChatCompletionMessage, EvaluatorResult, FewShotExample, ModelClient } from "../types.js";
+import { BaseMessage } from "@langchain/core/messages";
+import { BaseChatModel } from "@langchain/core/language_models/chat_models";
+import { RunnableInterface } from "@langchain/core/runnables";
+export declare const DEFAULT_PROMPT = "Grade the following agent trajectory:\n\n<trajectory>\n{outputs}\n</trajectory>\n{inputs}\n{reference_outputs}\n{rubric}\n";
+/**
+ * Creates an evaluator that uses an LLM to judge agent trajectories.
+ *
+ * @param options - Configuration options
+ * @param options.prompt - The evaluation prompt. Can be a string template, LangChain prompt template,
+ *                        or callable that returns a list of chat messages. Note that the default prompt
+ *                        allows a rubric in addition to the typical "inputs", "outputs", and
+ *                        "reference_outputs" parameters.
+ * @param options.feedbackKey - Key used to store the evaluation result. Defaults to "trajectory_accuracy".
+ * @param options.model - Model identifier to use. Defaults to "openai:o3-mini". If judge is an OpenAI client,
+ *                       this should be a model name directly. If judge is omitted, must be a valid
+ *                       LangChain model identifier.
+ * @param options.system - Optional system message to prepend to the prompt.
+ * @param options.judge - The LLM used for evaluation. Can be an OpenAI client or a LangChainLikeModel.
+ *                       If an OpenAI client, must specify "model" as well. If omitted, "model" will be
+ *                       used to instantiate a LangChain model instance by model string.
+ * @param options.continuous - If true, score will be a float between 0 and 1. If false, score will be boolean.
+ *                           Defaults to false.
+ * @param options.choices - Optional list of specific float values the score must be chosen from.
+ * @param options.useReasoning - If true, includes explanation for the score in the output. Defaults to true.
+ * @param options.fewShotExamples - Optional list of example evaluations to append to the prompt.
+ * @returns A function that evaluates agent trajectories using the configured LLM judge.
+ */
+export declare const createTrajectoryLLMAsJudge: ({ prompt, feedbackKey, model, system, judge, continuous, choices, useReasoning, fewShotExamples, }: {
+    prompt?: string | RunnableInterface<any, any, import("@langchain/core/runnables").RunnableConfig<Record<string, any>>> | ((...args: unknown[]) => ChatCompletionMessage[] | Promise<ChatCompletionMessage[]>) | undefined;
+    feedbackKey?: string | undefined;
+    model?: string | undefined;
+    system?: string | undefined;
+    judge?: ModelClient | BaseChatModel<import("@langchain/core/language_models/chat_models").BaseChatModelCallOptions, import("@langchain/core/messages").AIMessageChunk> | undefined;
+    continuous?: boolean | undefined;
+    choices?: number[] | undefined;
+    useReasoning?: boolean | undefined;
+    fewShotExamples?: FewShotExample[] | undefined;
+}) => ({ inputs, outputs, referenceOutputs, rubric, ...extra }: {
+    [key: string]: unknown;
+    inputs?: Record<string, any> | undefined;
+    outputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+    referenceOutputs?: BaseMessage[] | ChatCompletionMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    } | undefined;
+    rubric?: string | undefined;
+}) => Promise<EvaluatorResult>;

package/dist/evaluators/trajectory/llm.js

@@ -0,0 +1,82 @@
+import { _createLLMAsJudgeScorer } from "../llm.js";
+import { _runEvaluator } from "../utils.js";
+import { _normalizeToOpenAIMessagesList } from "../utils.js";
+import { _chatCompletionMessagesToString } from "./utils.js";
+export const DEFAULT_PROMPT = `Grade the following agent trajectory:
+
+<trajectory>
+{outputs}
+</trajectory>
+{inputs}
+{reference_outputs}
+{rubric}
+`;
+function _formatInputs(params) {
+    const { inputs, outputs, referenceOutputs, rubric } = params;
+    const normalizedOutputs = _normalizeToOpenAIMessagesList(outputs);
+    const normalizedReferenceOutputs = _normalizeToOpenAIMessagesList(referenceOutputs ?? []);
+    const formattedReferenceOutputs = normalizedReferenceOutputs
+        ? `\nUse the following trajectory as an example reference when grading:\n<reference_trajectory>\n${_chatCompletionMessagesToString(normalizedReferenceOutputs)}\n</reference_trajectory>\n`
+        : "";
+    const formattedInputs = inputs
+        ? `\nThe agent generated the trajectory from the following input:\n<input>\n${JSON.stringify(inputs)}\n</input>\n`
+        : "";
+    const formattedOutputs = typeof outputs === "object" && !Array.isArray(outputs)
+        ? outputs
+        : _chatCompletionMessagesToString(normalizedOutputs);
+    const formattedRubric = rubric
+        ? `\nGrade the agent trajectory along the following rubric:\n<rubric>\n${rubric}\n</rubric>\n`
+        : "";
+    return [
+        formattedOutputs,
+        formattedReferenceOutputs,
+        formattedInputs,
+        formattedRubric,
+    ];
+}
+/**
+ * Creates an evaluator that uses an LLM to judge agent trajectories.
+ *
+ * @param options - Configuration options
+ * @param options.prompt - The evaluation prompt. Can be a string template, LangChain prompt template,
+ *                        or callable that returns a list of chat messages. Note that the default prompt
+ *                        allows a rubric in addition to the typical "inputs", "outputs", and
+ *                        "reference_outputs" parameters.
+ * @param options.feedbackKey - Key used to store the evaluation result. Defaults to "trajectory_accuracy".
+ * @param options.model - Model identifier to use. Defaults to "openai:o3-mini". If judge is an OpenAI client,
+ *                       this should be a model name directly. If judge is omitted, must be a valid
+ *                       LangChain model identifier.
+ * @param options.system - Optional system message to prepend to the prompt.
+ * @param options.judge - The LLM used for evaluation. Can be an OpenAI client or a LangChainLikeModel.
+ *                       If an OpenAI client, must specify "model" as well. If omitted, "model" will be
+ *                       used to instantiate a LangChain model instance by model string.
+ * @param options.continuous - If true, score will be a float between 0 and 1. If false, score will be boolean.
+ *                           Defaults to false.
+ * @param options.choices - Optional list of specific float values the score must be chosen from.
+ * @param options.useReasoning - If true, includes explanation for the score in the output. Defaults to true.
+ * @param options.fewShotExamples - Optional list of example evaluations to append to the prompt.
+ * @returns A function that evaluates agent trajectories using the configured LLM judge.
+ */
+export const createTrajectoryLLMAsJudge = ({ prompt = DEFAULT_PROMPT, feedbackKey = "trajectory_accuracy", model = "openai:o3-mini", system, judge, continuous = false, choices, useReasoning = true, fewShotExamples, }) => {
+    const scorer = _createLLMAsJudgeScorer({
+        prompt,
+        judge,
+        model,
+        system,
+        continuous,
+        choices,
+        useReasoning,
+        fewShotExamples,
+    });
+    const wrappedEvaluator = async ({ inputs, outputs, referenceOutputs, rubric, ...extra }) => {
+        const [formattedOutputs, formattedReferenceOutputs, formattedInputs, formattedRubric,] = _formatInputs({ inputs, outputs, referenceOutputs, rubric });
+        return _runEvaluator(`llm_as_${feedbackKey}_judge`, scorer, feedbackKey, {
+            outputs: formattedOutputs,
+            referenceOutputs: formattedReferenceOutputs,
+            inputs: formattedInputs,
+            rubric: formattedRubric,
+            ...extra,
+        });
+    };
+    return wrappedEvaluator;
+};

package/dist/evaluators/trajectory/strict.cjs

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.trajectoryStrictMatch = void 0;
+const utils_js_1 = require("../utils.cjs");
+function _scorer(params) {
+    const { outputs, referenceOutputs } = params;
+    const normalizedOutputs = (0, utils_js_1._normalizeToOpenAIMessagesList)(outputs);
+    const normalizedReferenceOutputs = (0, utils_js_1._normalizeToOpenAIMessagesList)(referenceOutputs);
+    if (!normalizedOutputs || !normalizedReferenceOutputs) {
+        throw new Error("Strict trajectory match requires both outputs and reference_outputs");
+    }
+    if (normalizedOutputs.length !== normalizedReferenceOutputs.length) {
+        return false;
+    }
+    let exactMatch = true;
+    for (let i = 0; i < normalizedOutputs.length; i++) {
+        const output = normalizedOutputs[i];
+        const referenceOutput = normalizedReferenceOutputs[i];
+        if (output.role !== referenceOutput.role) {
+            exactMatch = false;
+            break;
+        }
+        const outputHasToolCalls = output.tool_calls != null;
+        const referenceHasToolCalls = referenceOutput.tool_calls != null;
+        if (outputHasToolCalls !== referenceHasToolCalls) {
+            exactMatch = false;
+            break;
+        }
+        if (outputHasToolCalls) {
+            if (output.tool_calls.length !== referenceOutput.tool_calls.length) {
+                exactMatch = false;
+                break;
+            }
+            for (let j = 0; j < output.tool_calls.length; j++) {
+                if (output.tool_calls[j].function.name !==
+                    referenceOutput.tool_calls[j].function.name) {
+                    exactMatch = false;
+                    break;
+                }
+            }
+        }
+    }
+    return exactMatch;
+}
+async function trajectoryStrictMatch(params) {
+    /**
+     * Evaluate whether an input agent trajectory and called tools strictly matches a reference trajectory.
+     * This means that at each step, the agent called the same tools in the same order as specified in the reference trajectory.
+     *
+     * @param outputs - Actual trajectory the agent followed. May be a list of OpenAI messages,
+     *                 a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
+     * @param referenceOutputs - Ideal reference trajectory the agent should have followed. May be a list of OpenAI messages,
+     *                          a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
+     * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+     */
+    return (0, utils_js_1._runEvaluator)("trajectory_strict_match", _scorer, "trajectory_strict_match", params);
+}
+exports.trajectoryStrictMatch = trajectoryStrictMatch;

package/dist/evaluators/trajectory/strict.d.ts

@@ -0,0 +1,10 @@
+import { BaseMessage } from "@langchain/core/messages";
+import { ChatCompletionMessage, EvaluatorResult } from "../types.js";
+export declare function trajectoryStrictMatch(params: {
+    outputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+}): Promise<EvaluatorResult>;

package/dist/evaluators/trajectory/strict.js

@@ -0,0 +1,54 @@
+import { _normalizeToOpenAIMessagesList, _runEvaluator } from "../utils.js";
+function _scorer(params) {
+    const { outputs, referenceOutputs } = params;
+    const normalizedOutputs = _normalizeToOpenAIMessagesList(outputs);
+    const normalizedReferenceOutputs = _normalizeToOpenAIMessagesList(referenceOutputs);
+    if (!normalizedOutputs || !normalizedReferenceOutputs) {
+        throw new Error("Strict trajectory match requires both outputs and reference_outputs");
+    }
+    if (normalizedOutputs.length !== normalizedReferenceOutputs.length) {
+        return false;
+    }
+    let exactMatch = true;
+    for (let i = 0; i < normalizedOutputs.length; i++) {
+        const output = normalizedOutputs[i];
+        const referenceOutput = normalizedReferenceOutputs[i];
+        if (output.role !== referenceOutput.role) {
+            exactMatch = false;
+            break;
+        }
+        const outputHasToolCalls = output.tool_calls != null;
+        const referenceHasToolCalls = referenceOutput.tool_calls != null;
+        if (outputHasToolCalls !== referenceHasToolCalls) {
+            exactMatch = false;
+            break;
+        }
+        if (outputHasToolCalls) {
+            if (output.tool_calls.length !== referenceOutput.tool_calls.length) {
+                exactMatch = false;
+                break;
+            }
+            for (let j = 0; j < output.tool_calls.length; j++) {
+                if (output.tool_calls[j].function.name !==
+                    referenceOutput.tool_calls[j].function.name) {
+                    exactMatch = false;
+                    break;
+                }
+            }
+        }
+    }
+    return exactMatch;
+}
+export async function trajectoryStrictMatch(params) {
+    /**
+     * Evaluate whether an input agent trajectory and called tools strictly matches a reference trajectory.
+     * This means that at each step, the agent called the same tools in the same order as specified in the reference trajectory.
+     *
+     * @param outputs - Actual trajectory the agent followed. May be a list of OpenAI messages,
+     *                 a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
+     * @param referenceOutputs - Ideal reference trajectory the agent should have followed. May be a list of OpenAI messages,
+     *                          a list of LangChain messages, or a dictionary containing a "messages" key with one of the above.
+     * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+     */
+    return _runEvaluator("trajectory_strict_match", _scorer, "trajectory_strict_match", params);
+}

package/dist/evaluators/trajectory/subset.cjs

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.trajectorySubset = void 0;
+const utils_js_1 = require("../utils.cjs");
+const utils_js_2 = require("./utils.cjs");
+/**
+ * Evaluate whether an agent trajectory and called tools is a subset of a reference trajectory and called tools.
+ * This means the agent called a subset of the tools specified in the reference trajectory.
+ *
+ * @param params - The parameters for trajectory subset evaluation
+ * @param params.outputs - Actual trajectory the agent followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @param params.reference_outputs - Ideal reference trajectory the agent should have followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
+async function trajectorySubset(params) {
+    const { outputs, referenceOutputs } = params;
+    const outputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(outputs);
+    const referenceOutputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(referenceOutputs);
+    const getScore = async () => {
+        if (outputsList == null || referenceOutputsList == null) {
+            throw new Error("Trajectory subset match requires both outputs and reference_outputs");
+        }
+        const isSubset = (0, utils_js_2._isTrajectorySuperset)(referenceOutputsList, outputsList);
+        return isSubset;
+    };
+    return (0, utils_js_1._runEvaluator)("trajectory_subset", getScore, "trajectory_subset", params);
+}
+exports.trajectorySubset = trajectorySubset;

package/dist/evaluators/trajectory/subset.d.ts

@@ -0,0 +1,23 @@
+import { BaseMessage } from "@langchain/core/messages";
+import { ChatCompletionMessage, EvaluatorResult } from "../types.js";
+/**
+ * Evaluate whether an agent trajectory and called tools is a subset of a reference trajectory and called tools.
+ * This means the agent called a subset of the tools specified in the reference trajectory.
+ *
+ * @param params - The parameters for trajectory subset evaluation
+ * @param params.outputs - Actual trajectory the agent followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @param params.reference_outputs - Ideal reference trajectory the agent should have followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
+export declare function trajectorySubset(params: {
+    outputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+}): Promise<EvaluatorResult>;

package/dist/evaluators/trajectory/subset.js

@@ -0,0 +1,28 @@
+import { _normalizeToOpenAIMessagesList, _runEvaluator } from "../utils.js";
+import { _isTrajectorySuperset } from "./utils.js";
+/**
+ * Evaluate whether an agent trajectory and called tools is a subset of a reference trajectory and called tools.
+ * This means the agent called a subset of the tools specified in the reference trajectory.
+ *
+ * @param params - The parameters for trajectory subset evaluation
+ * @param params.outputs - Actual trajectory the agent followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @param params.reference_outputs - Ideal reference trajectory the agent should have followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
+export async function trajectorySubset(params) {
+    const { outputs, referenceOutputs } = params;
+    const outputsList = _normalizeToOpenAIMessagesList(outputs);
+    const referenceOutputsList = _normalizeToOpenAIMessagesList(referenceOutputs);
+    const getScore = async () => {
+        if (outputsList == null || referenceOutputsList == null) {
+            throw new Error("Trajectory subset match requires both outputs and reference_outputs");
+        }
+        const isSubset = _isTrajectorySuperset(referenceOutputsList, outputsList);
+        return isSubset;
+    };
+    return _runEvaluator("trajectory_subset", getScore, "trajectory_subset", params);
+}

package/dist/evaluators/trajectory/superset.cjs

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.trajectorySuperset = void 0;
+const utils_js_1 = require("../utils.cjs");
+const utils_js_2 = require("./utils.cjs");
+/**
+ * Evaluate whether an agent trajectory and called tools is a superset of a reference trajectory and called tools.
+ * This means the agent called a superset of the tools specified in the reference trajectory.
+ *
+ * @param params - The parameters for trajectory superset evaluation
+ * @param params.outputs - Actual trajectory the agent followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @param params.reference_outputs - Ideal reference trajectory the agent should have followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
+async function trajectorySuperset(params) {
+    const { outputs, referenceOutputs } = params;
+    const outputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(outputs);
+    const referenceOutputsList = (0, utils_js_1._normalizeToOpenAIMessagesList)(referenceOutputs);
+    const getScore = async () => {
+        if (outputsList == null || referenceOutputsList == null) {
+            throw new Error("Trajectory superset match requires both outputs and reference_outputs");
+        }
+        const isSuperset = (0, utils_js_2._isTrajectorySuperset)(outputsList, referenceOutputsList);
+        return isSuperset;
+    };
+    return (0, utils_js_1._runEvaluator)("trajectory_superset", getScore, "trajectory_superset", params);
+}
+exports.trajectorySuperset = trajectorySuperset;

package/dist/evaluators/trajectory/superset.d.ts

@@ -0,0 +1,23 @@
+import { BaseMessage } from "@langchain/core/messages";
+import { ChatCompletionMessage, EvaluatorResult } from "../types.js";
+/**
+ * Evaluate whether an agent trajectory and called tools is a superset of a reference trajectory and called tools.
+ * This means the agent called a superset of the tools specified in the reference trajectory.
+ *
+ * @param params - The parameters for trajectory superset evaluation
+ * @param params.outputs - Actual trajectory the agent followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @param params.reference_outputs - Ideal reference trajectory the agent should have followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
+export declare function trajectorySuperset(params: {
+    outputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+    referenceOutputs: ChatCompletionMessage[] | BaseMessage[] | {
+        messages: (BaseMessage | ChatCompletionMessage)[];
+    };
+}): Promise<EvaluatorResult>;

package/dist/evaluators/trajectory/superset.js

@@ -0,0 +1,28 @@
+import { _normalizeToOpenAIMessagesList, _runEvaluator } from "../utils.js";
+import { _isTrajectorySuperset } from "./utils.js";
+/**
+ * Evaluate whether an agent trajectory and called tools is a superset of a reference trajectory and called tools.
+ * This means the agent called a superset of the tools specified in the reference trajectory.
+ *
+ * @param params - The parameters for trajectory superset evaluation
+ * @param params.outputs - Actual trajectory the agent followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @param params.reference_outputs - Ideal reference trajectory the agent should have followed.
+ *    May be a list of OpenAI messages, a list of LangChain messages, or a dictionary containing
+ *    a "messages" key with one of the above.
+ * @returns EvaluatorResult containing a score of true if trajectory (including called tools) matches, false otherwise
+ */
+export async function trajectorySuperset(params) {
+    const { outputs, referenceOutputs } = params;
+    const outputsList = _normalizeToOpenAIMessagesList(outputs);
+    const referenceOutputsList = _normalizeToOpenAIMessagesList(referenceOutputs);
+    const getScore = async () => {
+        if (outputsList == null || referenceOutputsList == null) {
+            throw new Error("Trajectory superset match requires both outputs and reference_outputs");
+        }
+        const isSuperset = _isTrajectorySuperset(outputsList, referenceOutputsList);
+        return isSuperset;
+    };
+    return _runEvaluator("trajectory_superset", getScore, "trajectory_superset", params);
+}