@mastra/evals 1.6.0 → 1.7.0

package/dist/scorers/utils.js

@@ -1,3 +1,940 @@
-export { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, createAgentTestRun, createTestMessage, createTestRun, createToolInvocation, createTrajectoryTestRun, extractAgentResponseMessages, extractInputMessages, extractToolCalls, extractToolResults, extractTrajectory, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getReasoningFromRunOutput, getSystemMessagesFromRunInput, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isCloserTo, isScorerRunInputForAgent, isScorerRunOutputForAgent, roundToTwoDecimals } from '../chunk-UJ4WCQ3F.js';
-//# sourceMappingURL=utils.js.map
+import { extractTrajectory } from "@mastra/core/evals";
+import { RequestContext } from "@mastra/core/request-context";
+//#region src/scorers/utils.ts
+/**
+* Extracts text content from a MastraDBMessage or ModelMessage-like object.
+*
+* @param message - The message to extract text from
+* @returns The extracted text content, or an empty string if no text is found
+*
+* @example
+* ```ts
+* const message: MastraDBMessage = {
+*   id: 'msg-1',
+*   role: 'assistant',
+*   content: { format: 2, parts: [{ type: 'text', text: 'Hello!' }] },
+*   createdAt: new Date(),
+* };
+* const text = getTextContentFromMastraDBMessage(message); // 'Hello!'
+* ```
+*/
+function getTextContentFromMastraDBMessage(message) {
+	const content = message.content;
+	if (typeof content === "string") return content;
+	if (Array.isArray(content)) {
+		const textParts = content.filter((p) => p.type === "text");
+		return textParts.length > 0 ? textParts[textParts.length - 1]?.text || "" : "";
+	}
+	if (typeof content?.content === "string" && content.content !== "") return content.content;
+	if (typeof content?.text === "string" && content.text !== "") return content.text;
+	if (content?.parts && Array.isArray(content.parts)) {
+		const textParts = content.parts.filter((p) => p.type === "text");
+		return textParts.length > 0 ? textParts[textParts.length - 1]?.text || "" : "";
+	}
+	return "";
+}
+const isRecord = (value) => {
+	return typeof value === "object" && value !== null;
+};
+const getTextFromValue = (value) => {
+	if (typeof value === "string") return value === "" ? void 0 : value;
+	if (Array.isArray(value)) {
+		const textParts = value.filter((part) => isRecord(part) && part.type === "text" && typeof part.text === "string").map((part) => part.text);
+		return textParts.length > 0 ? textParts[textParts.length - 1] : void 0;
+	}
+	if (!isRecord(value)) return void 0;
+	const fromParts = Array.isArray(value.parts) ? getTextFromValue(value.parts) : void 0;
+	return getTextFromValue(value.content) ?? (typeof value.text === "string" && value.text !== "" ? value.text : void 0) ?? (typeof value.body === "string" && value.body !== "" ? value.body : void 0) ?? fromParts;
+};
+const isScorerRunInputForAgent = (input) => {
+	return isRecord(input) && Array.isArray(input.inputMessages) && Array.isArray(input.rememberedMessages) && Array.isArray(input.systemMessages) && isRecord(input.taggedSystemMessages);
+};
+const isMastraDBMessageLike = (message) => {
+	return isRecord(message) && typeof message.id === "string" && typeof message.role === "string" && "content" in message && "createdAt" in message;
+};
+const isScorerRunOutputForAgent = (output) => {
+	return Array.isArray(output) && output.every(isMastraDBMessageLike);
+};
+/**
+* Resolves the effective role of a message, accounting for agent signal messages.
+*
+* Messages delivered through the agent subscription / signal API are persisted with
+* `role: 'signal'` and carry their semantic role (e.g. `user`) on `type` and on
+* `content.metadata.signal.{type,tagName}`. Treat those as their underlying role so
+* helpers like `getUserMessageFromRunInput` can find them.
+*/
+const getEffectiveMessageRole = (message) => {
+	if (message.role !== "signal") return typeof message.role === "string" ? message.role : void 0;
+	const signalMeta = isRecord(message.content) && isRecord(message.content.metadata) ? message.content.metadata.signal : void 0;
+	const tagName = isRecord(signalMeta) && typeof signalMeta.tagName === "string" ? signalMeta.tagName : void 0;
+	const signalType = isRecord(signalMeta) && typeof signalMeta.type === "string" ? signalMeta.type : void 0;
+	const topLevelType = typeof message.type === "string" ? message.type : void 0;
+	return tagName ?? signalType ?? topLevelType;
+};
+const getTextFromMessages = (messages, role) => {
+	if (!Array.isArray(messages)) return void 0;
+	const message = messages.find((message) => isRecord(message) && getEffectiveMessageRole(message) === role);
+	return message ? getTextFromValue(message) : void 0;
+};
+/**
+* Rounds a number to two decimal places.
+*
+* Uses `Number.EPSILON` to handle floating-point precision issues.
+*
+* @param num - The number to round
+* @returns The number rounded to two decimal places
+*
+* @example
+* ```ts
+* roundToTwoDecimals(0.1 + 0.2); // 0.3
+* roundToTwoDecimals(1.005); // 1.01
+* ```
+*/
+const roundToTwoDecimals = (num) => {
+	return Math.round((num + Number.EPSILON) * 100) / 100;
+};
+/**
+* Determines if a value is closer to the first target than the second.
+*
+* @param value - The value to compare
+* @param target1 - The first target value
+* @param target2 - The second target value
+* @returns `true` if `value` is closer to `target1` than `target2`
+*
+* @example
+* ```ts
+* isCloserTo(0.6, 1, 0); // true (0.6 is closer to 1)
+* isCloserTo(0.3, 1, 0); // false (0.3 is closer to 0)
+* ```
+*/
+function isCloserTo(value, target1, target2) {
+	return Math.abs(value - target1) < Math.abs(value - target2);
+}
+/**
+* Creates a scoring input object for testing purposes.
+*
+* @param input - The user input text
+* @param output - The assistant output text
+* @param additionalContext - Optional additional context data
+* @param requestContext - Optional request context data
+* @returns A ScoringInput object ready for use in scorer tests
+*
+* @example
+* ```ts
+* const run = createTestRun(
+*   'What is 2+2?',
+*   'The answer is 4.',
+*   { topic: 'math' }
+* );
+* ```
+*/
+const createTestRun = (input, output, additionalContext, requestContext) => {
+	return {
+		input: [{
+			role: "user",
+			content: input
+		}],
+		output: {
+			role: "assistant",
+			text: output
+		},
+		additionalContext: additionalContext ?? {},
+		requestContext: requestContext ?? {}
+	};
+};
+/**
+* Extracts the user message text from a scorer run input.
+*
+* Accepts the agent shape (`{ inputMessages }`), `ModelMessage[]`
+* (`{ messages }`), workflow input (`{ prompt }`), and a bare string.
+*
+* @param input - The scorer run input
+* @returns The user message text, or `undefined` if none can be extracted
+*
+* @example
+* ```ts
+* const scorer = createScorer({ ... })
+*   .preprocess(({ run }) => {
+*     const userText = getUserMessageFromRunInput(run.input);
+*     return { userText };
+*   });
+* ```
+*/
+const getUserMessageFromRunInput = (input) => {
+	if (typeof input === "string") return input;
+	if (!isRecord(input)) return void 0;
+	return getTextFromMessages(input.inputMessages, "user") ?? getTextFromMessages(input.messages, "user") ?? (typeof input.prompt === "string" ? input.prompt : void 0) ?? (typeof input.text === "string" ? input.text : void 0) ?? getTextFromValue(input.content) ?? getTextFromValue(input.input) ?? getTextFromValue(input.user);
+};
+/**
+* Extracts all system messages from a scorer run input.
+*
+* Collects text from both standard system messages and tagged system messages
+* (specialized system prompts like memory instructions).
+*
+* @param input - The scorer run input containing system messages
+* @returns An array of system message strings
+*
+* @example
+* ```ts
+* const scorer = createScorer({ ... })
+*   .preprocess(({ run }) => {
+*     const systemMessages = getSystemMessagesFromRunInput(run.input);
+*     return { systemPrompt: systemMessages.join('\n') };
+*   });
+* ```
+*/
+const getSystemMessagesFromRunInput = (input) => {
+	const systemMessages = [];
+	if (!isRecord(input)) return systemMessages;
+	if (Array.isArray(input.systemMessages)) systemMessages.push(...input.systemMessages.map((msg) => {
+		if (typeof msg.content === "string") return msg.content;
+		else if (Array.isArray(msg.content)) return msg.content.filter((part) => part.type === "text").map((part) => part.text || "").join(" ");
+		return "";
+	}).filter((content) => content));
+	const addSystemMessages = (messages) => {
+		if (!Array.isArray(messages)) return;
+		systemMessages.push(...messages.filter((message) => isRecord(message) && message.role === "system").map((message) => getTextFromValue(message)).filter((content) => Boolean(content)));
+	};
+	addSystemMessages(input.inputMessages);
+	addSystemMessages(input.messages);
+	if (isRecord(input.taggedSystemMessages)) Object.values(input.taggedSystemMessages).forEach((messages) => {
+		if (!Array.isArray(messages)) return;
+		messages.forEach((msg) => {
+			const content = getTextFromValue(msg);
+			if (content) systemMessages.push(content);
+		});
+	});
+	return systemMessages;
+};
+/**
+* Combines all system messages into a single prompt string.
+*
+* Joins all system messages (standard and tagged) with double newlines.
+*
+* @param input - The scorer run input containing system messages
+* @returns A combined system prompt string
+*
+* @example
+* ```ts
+* const scorer = createScorer({ ... })
+*   .preprocess(({ run }) => {
+*     const systemPrompt = getCombinedSystemPrompt(run.input);
+*     return { systemPrompt };
+*   });
+* ```
+*/
+const getCombinedSystemPrompt = (input) => {
+	return getSystemMessagesFromRunInput(input).join("\n\n");
+};
+/**
+* Extracts the assistant message text from a scorer run output.
+*
+* Accepts the agent shape (`MastraDBMessage[]` / `ModelMessage[]`), workflow
+* output (`{ text }`), task output (`{ content }`), a single assistant message
+* object, and a bare string.
+*
+* @param output - The scorer run output
+* @returns The assistant message text, or `undefined` if none can be extracted
+*
+* @example
+* ```ts
+* const scorer = createScorer({ ... })
+*   .preprocess(({ run }) => {
+*     const response = getAssistantMessageFromRunOutput(run.output);
+*     return { response };
+*   });
+* ```
+*/
+const getAssistantMessageFromRunOutput = (output) => {
+	if (typeof output === "string") return output;
+	if (Array.isArray(output)) return getTextFromMessages(output, "assistant");
+	if (!isRecord(output)) return void 0;
+	const isAssistantOutput = output.role === void 0 || output.role === "assistant";
+	if (isAssistantOutput && typeof output.text === "string") return output.text;
+	if (isAssistantOutput && typeof output.content === "string") return output.content;
+	if (isAssistantOutput && (isRecord(output.content) || Array.isArray(output.content))) return getTextContentFromMastraDBMessage(output) || getTextContentFromMastraDBMessage(output.content) || void 0;
+	if (output.role === "assistant") return getTextContentFromMastraDBMessage(output) || void 0;
+};
+/**
+* Extracts reasoning text from a scorer run output.
+*
+* This function extracts reasoning content from assistant messages, which is
+* produced by reasoning models like `deepseek-reasoner`. The reasoning can be
+* stored in two places:
+* 1. `content.reasoning` - a string field on the message content
+* 2. `content.parts` - as parts with `type: 'reasoning'` containing `details`
+*
+* @param output - The scorer run output (array of MastraDBMessage)
+* @returns The reasoning text, or `undefined` if no reasoning is present
+*
+* @example
+* ```ts
+* const reasoningScorer = createScorer({
+*   id: 'reasoning-scorer',
+*   name: 'Reasoning Quality',
+*   description: 'Evaluates the quality of model reasoning',
+*   type: 'agent',
+* })
+*   .preprocess(({ run }) => {
+*     const reasoning = getReasoningFromRunOutput(run.output);
+*     const response = getAssistantMessageFromRunOutput(run.output);
+*     return { reasoning, response };
+*   })
+*   .generateScore(({ results }) => {
+*     // Score based on reasoning quality
+*     return results.preprocessStepResult?.reasoning ? 1 : 0;
+*   });
+* ```
+*/
+const getReasoningFromRunOutput = (output) => {
+	if (!output) return void 0;
+	const message = output.find(({ role }) => role === "assistant");
+	if (!message) return void 0;
+	if (message.content.reasoning) return message.content.reasoning;
+	const reasoningParts = message.content.parts?.filter((p) => p.type === "reasoning");
+	if (reasoningParts && reasoningParts.length > 0) {
+		const reasoningTexts = reasoningParts.map((p) => {
+			if (p.details && Array.isArray(p.details)) return p.details.filter((d) => d.type === "text").map((d) => d.text).join("");
+			return p.reasoning || "";
+		}).filter(Boolean);
+		return reasoningTexts.length > 0 ? reasoningTexts.join("\n") : void 0;
+	}
+};
+/**
+* Creates a tool invocation object for testing purposes.
+*
+* @param options - The tool invocation configuration
+* @param options.toolCallId - Unique identifier for the tool call
+* @param options.toolName - Name of the tool being called
+* @param options.args - Arguments passed to the tool
+* @param options.result - Result returned by the tool
+* @param options.state - State of the invocation (default: 'result')
+* @returns A tool invocation object
+*
+* @example
+* ```ts
+* const invocation = createToolInvocation({
+*   toolCallId: 'call-123',
+*   toolName: 'weatherTool',
+*   args: { location: 'London' },
+*   result: { temperature: 20, condition: 'sunny' },
+* });
+* ```
+*/
+const createToolInvocation = ({ toolCallId, toolName, args, result, state = "result" }) => {
+	return {
+		toolCallId,
+		toolName,
+		args,
+		result,
+		state
+	};
+};
+/**
+* Creates a MastraDBMessage object for testing purposes.
+*
+* Supports optional tool invocations for testing tool call scenarios.
+*
+* @param options - The message configuration
+* @param options.content - The text content of the message
+* @param options.role - The role of the message sender ('user', 'assistant', or 'system')
+* @param options.id - Optional message ID (default: 'test-message')
+* @param options.toolInvocations - Optional array of tool invocations
+* @returns A MastraDBMessage object
+*
+* @example
+* ```ts
+* const message = createTestMessage({
+*   content: 'Hello, how can I help?',
+*   role: 'assistant',
+* });
+*
+* // With tool invocations
+* const messageWithTools = createTestMessage({
+*   content: 'Let me check the weather.',
+*   role: 'assistant',
+*   toolInvocations: [{
+*     toolCallId: 'call-1',
+*     toolName: 'weatherTool',
+*     args: { location: 'Paris' },
+*     result: { temp: 22 },
+*     state: 'result',
+*   }],
+* });
+* ```
+*/
+function createTestMessage({ content, role, id = "test-message", toolInvocations = [] }) {
+	return {
+		id,
+		role,
+		content: {
+			format: 2,
+			parts: [{
+				type: "text",
+				text: content
+			}],
+			content,
+			...toolInvocations.length > 0 && { toolInvocations: toolInvocations.map((ti) => ({
+				toolCallId: ti.toolCallId,
+				toolName: ti.toolName,
+				args: ti.args,
+				result: ti.result,
+				state: ti.state
+			})) }
+		},
+		createdAt: /* @__PURE__ */ new Date()
+	};
+}
+/**
+* Creates a complete agent test run object for testing scorers.
+*
+* Provides a convenient way to construct the full run object that scorers receive,
+* including input messages, output, system messages, and request context.
+*
+* @param options - The test run configuration
+* @param options.inputMessages - Array of input messages (default: [])
+* @param options.output - The output messages (required)
+* @param options.rememberedMessages - Array of remembered messages from memory (default: [])
+* @param options.systemMessages - Array of system messages (default: [])
+* @param options.taggedSystemMessages - Tagged system messages map (default: {})
+* @param options.requestContext - Request context (default: new RequestContext())
+* @param options.runId - Unique run ID (default: random UUID)
+* @returns A complete test run object
+*
+* @example
+* ```ts
+* const testRun = createAgentTestRun({
+*   inputMessages: [createTestMessage({ content: 'Hello', role: 'user' })],
+*   output: [createTestMessage({ content: 'Hi there!', role: 'assistant' })],
+* });
+*
+* const result = await scorer.run({
+*   input: testRun.input,
+*   output: testRun.output,
+* });
+* ```
+*/
+const createAgentTestRun = ({ inputMessages = [], output, rememberedMessages = [], systemMessages = [], taggedSystemMessages = {}, requestContext = new RequestContext(), runId = crypto.randomUUID() }) => {
+	return {
+		input: {
+			inputMessages,
+			rememberedMessages,
+			systemMessages,
+			taggedSystemMessages
+		},
+		output,
+		requestContext,
+		runId
+	};
+};
+/**
+* Creates a test run for trajectory scorers where `output` is a `Trajectory`
+* (pre-extracted by the `runEvals` pipeline).
+*
+* @example
+* ```ts
+* const testRun = createTrajectoryTestRun({
+*   inputMessages: [createTestMessage({ content: 'Do X', role: 'user', id: 'u1' })],
+*   trajectory: {
+*     steps: [
+*       { stepType: 'tool_call', name: 'search', toolArgs: { q: 'test' } },
+*     ],
+*   },
+* });
+* ```
+*/
+const createTrajectoryTestRun = ({ inputMessages = [], trajectory, rememberedMessages = [], systemMessages = [], taggedSystemMessages = {}, requestContext = new RequestContext(), runId = crypto.randomUUID(), expectedTrajectory }) => {
+	return {
+		input: {
+			inputMessages,
+			rememberedMessages,
+			systemMessages,
+			taggedSystemMessages
+		},
+		output: trajectory,
+		expectedTrajectory,
+		requestContext,
+		runId
+	};
+};
+/**
+* Extracts all tool calls from a scorer run output.
+*
+* Iterates through all messages and their tool invocations to collect
+* information about tools that were called (with state 'result' or 'call').
+*
+* @param output - The scorer run output (array of MastraDBMessage)
+* @returns An object containing tool names and detailed tool call info
+*
+* @example
+* ```ts
+* const scorer = createScorer({ ... })
+*   .preprocess(({ run }) => {
+*     const { tools, toolCallInfos } = extractToolCalls(run.output);
+*     return {
+*       toolsUsed: tools,
+*       toolCount: tools.length,
+*     };
+*   });
+* ```
+*/
+function extractToolCalls(output) {
+	const toolCalls = [];
+	const toolCallInfos = [];
+	for (let messageIndex = 0; messageIndex < output.length; messageIndex++) {
+		const message = output[messageIndex];
+		const legacy = message?.content?.toolInvocations;
+		const fromParts = legacy ? void 0 : message?.content?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation);
+		const toolInvocations = legacy ?? fromParts;
+		if (!toolInvocations?.length) continue;
+		for (let invocationIndex = 0; invocationIndex < toolInvocations.length; invocationIndex++) {
+			const invocation = toolInvocations[invocationIndex];
+			if (invocation && invocation.toolName && (invocation.state === "result" || invocation.state === "call")) {
+				toolCalls.push(invocation.toolName);
+				toolCallInfos.push({
+					toolName: invocation.toolName,
+					toolCallId: invocation.toolCallId || `${messageIndex}-${invocationIndex}`,
+					messageIndex,
+					invocationIndex
+				});
+			}
+		}
+	}
+	return {
+		tools: toolCalls,
+		toolCallInfos
+	};
+}
+/**
+* Extracts text content from all input messages.
+*
+* @param runInput - The scorer run input
+* @returns An array of text strings from each input message
+*
+* @example
+* ```ts
+* const scorer = createScorer({ ... })
+*   .preprocess(({ run }) => {
+*     const messages = extractInputMessages(run.input);
+*     return { allUserMessages: messages.join('\n') };
+*   });
+* ```
+*/
+const extractInputMessages = (runInput) => {
+	return runInput?.inputMessages?.map((msg) => getTextContentFromMastraDBMessage(msg)) || [];
+};
+/**
+* Extracts text content from all assistant response messages.
+*
+* Filters for messages with role 'assistant' and extracts their text content.
+*
+* @param runOutput - The scorer run output (array of MastraDBMessage)
+* @returns An array of text strings from each assistant message
+*
+* @example
+* ```ts
+* const scorer = createScorer({ ... })
+*   .preprocess(({ run }) => {
+*     const responses = extractAgentResponseMessages(run.output);
+*     return { allResponses: responses.join('\n') };
+*   });
+* ```
+*/
+const extractAgentResponseMessages = (runOutput) => {
+	return runOutput.filter((msg) => msg.role === "assistant").map((msg) => getTextContentFromMastraDBMessage(msg));
+};
+/**
+* Extracts tool results from a scorer run output.
+*
+* Returns structured objects that can be used with the hallucination scorer's
+* `getContext` hook or for other scorer logic.
+*
+* @param output - The scorer run output (array of MastraDBMessage)
+* @returns An array of ToolResultInfo objects
+*
+* @example
+* ```ts
+* import { extractToolResults } from '@mastra/evals/scorers';
+* import { createHallucinationScorer } from '@mastra/evals/scorers/prebuilt';
+*
+* const scorer = createHallucinationScorer({
+*   model: openai('gpt-4o'),
+*   options: {
+*     getContext: (run) => {
+*       const toolResults = extractToolResults(run.output);
+*       return toolResults.map(t => JSON.stringify({ tool: t.toolName, result: t.result }));
+*     },
+*   },
+* });
+* ```
+*/
+function extractToolResults(output) {
+	const results = [];
+	for (const message of output) {
+		const legacy = message?.content?.toolInvocations;
+		const fromParts = legacy ? void 0 : message?.content?.parts?.filter((p) => p.type === "tool-invocation").map((p) => p.toolInvocation);
+		const toolInvocations = legacy ?? fromParts;
+		if (!toolInvocations?.length) continue;
+		for (const invocation of toolInvocations) if (invocation.state === "result" && invocation.result !== void 0) results.push({
+			toolName: invocation.toolName,
+			toolCallId: invocation.toolCallId || "",
+			args: invocation.args || {},
+			result: invocation.result
+		});
+	}
+	return results;
+}
+/**
+* Compares two trajectories and returns detailed comparison results.
+*
+* This is the core comparison logic used by trajectory scorers. It supports
+* strict and non-strict ordering, optional step data comparison, and loop detection.
+*
+* @param actual - The trajectory the agent actually took
+* @param expected - The expected trajectory to compare against
+* @param options - Comparison configuration options
+* @returns Detailed comparison results including match scores and diagnostics
+*
+* @example
+* ```ts
+* const result = compareTrajectories(
+*   { steps: [{ stepType: 'tool_call', name: 'search' }, { stepType: 'tool_call', name: 'summarize' }] },
+*   { steps: [{ stepType: 'tool_call', name: 'search' }, { stepType: 'tool_call', name: 'summarize' }] },
+*   { ordering: 'strict' }
+* );
+* // result.score = 1.0
+* ```
+*/
+function compareTrajectories(actual, expected, options = {}) {
+	const { allowRepeatedSteps = true, ordering = "relaxed" } = options;
+	const normalizedExpected = { steps: expected.steps };
+	if (normalizedExpected.steps.length === 0) return {
+		score: actual.steps.length === 0 ? 1 : 0,
+		matchedSteps: 0,
+		totalExpectedSteps: 0,
+		totalActualSteps: actual.steps.length,
+		missingSteps: [],
+		extraSteps: actual.steps.map((s) => s.name),
+		outOfOrderSteps: [],
+		repeatedSteps: []
+	};
+	const actualNames = actual.steps.map((s) => s.name);
+	const nameCounts = /* @__PURE__ */ new Map();
+	for (const name of actualNames) nameCounts.set(name, (nameCounts.get(name) || 0) + 1);
+	const repeatedSteps = [...nameCounts.entries()].filter(([_, count]) => count > 1).map(([name]) => name);
+	if (ordering === "strict") return compareStrictOrder(actual, normalizedExpected, {
+		allowRepeatedSteps,
+		repeatedSteps
+	});
+	if (ordering === "unordered") return compareUnorderedPresence(actual, normalizedExpected, {
+		allowRepeatedSteps,
+		repeatedSteps
+	});
+	return compareRelaxedOrder(actual, normalizedExpected, {
+		allowRepeatedSteps,
+		repeatedSteps
+	});
+}
+function compareStrictOrder(actual, expected, opts) {
+	const actualNames = actual.steps.map((s) => s.name);
+	const expectedNames = expected.steps.map((s) => s.name);
+	let matchedSteps = 0;
+	const outOfOrderSteps = [];
+	const matchedExpectedIndices = /* @__PURE__ */ new Set();
+	const maxLen = Math.max(actualNames.length, expectedNames.length);
+	for (let i = 0; i < maxLen; i++) {
+		const actualName = actualNames[i];
+		if (actualName === expectedNames[i]) if (actual.steps[i] && expected.steps[i]) {
+			if (expectedStepMatches(actual.steps[i], expected.steps[i])) {
+				matchedSteps++;
+				matchedExpectedIndices.add(i);
+			}
+		} else {
+			matchedSteps++;
+			matchedExpectedIndices.add(i);
+		}
+		else if (actualName && expectedNames.includes(actualName)) outOfOrderSteps.push(actualName);
+	}
+	const missingSteps = expectedNames.filter((_, i) => !matchedExpectedIndices.has(i));
+	const extraSteps = actualNames.filter((name) => !expectedNames.includes(name));
+	let score = matchedSteps / expected.steps.length;
+	if (actualNames.length > expectedNames.length) {
+		const extraPenalty = (actualNames.length - expectedNames.length) / expectedNames.length;
+		score = Math.max(0, score - extraPenalty * .5);
+	}
+	if (!opts.allowRepeatedSteps && opts.repeatedSteps.length > 0) score = Math.max(0, score - opts.repeatedSteps.length * .1);
+	return {
+		score: roundToTwoDecimals(Math.max(0, Math.min(1, score))),
+		matchedSteps,
+		totalExpectedSteps: expected.steps.length,
+		totalActualSteps: actual.steps.length,
+		missingSteps,
+		extraSteps,
+		outOfOrderSteps,
+		repeatedSteps: opts.repeatedSteps
+	};
+}
+function compareRelaxedOrder(actual, expected, opts) {
+	const actualNames = actual.steps.map((s) => s.name);
+	const expectedNames = expected.steps.map((s) => s.name);
+	let matchedSteps = 0;
+	let lastMatchedIndex = -1;
+	const outOfOrderSteps = [];
+	const matchedExpectedIndices = /* @__PURE__ */ new Set();
+	for (let i = 0; i < expectedNames.length; i++) {
+		const expectedName = expectedNames[i];
+		let found = false;
+		for (let j = lastMatchedIndex + 1; j < actualNames.length; j++) if (actualNames[j] === expectedName) if (actual.steps[j] && expected.steps[i]) {
+			if (expectedStepMatches(actual.steps[j], expected.steps[i])) {
+				matchedSteps++;
+				lastMatchedIndex = j;
+				matchedExpectedIndices.add(i);
+				found = true;
+				break;
+			}
+		} else {
+			matchedSteps++;
+			lastMatchedIndex = j;
+			matchedExpectedIndices.add(i);
+			found = true;
+			break;
+		}
+		if (!found) {
+			if (actualNames.includes(expectedName)) outOfOrderSteps.push(expectedName);
+		}
+	}
+	const missingSteps = expectedNames.filter((_, i) => !matchedExpectedIndices.has(i));
+	const expectedSet = new Set(expectedNames);
+	const extraSteps = actualNames.filter((name) => !expectedSet.has(name));
+	let score = matchedSteps / expected.steps.length;
+	if (!opts.allowRepeatedSteps && opts.repeatedSteps.length > 0) score = Math.max(0, score - opts.repeatedSteps.length * .1);
+	return {
+		score: roundToTwoDecimals(Math.max(0, Math.min(1, score))),
+		matchedSteps,
+		totalExpectedSteps: expected.steps.length,
+		totalActualSteps: actual.steps.length,
+		missingSteps,
+		extraSteps,
+		outOfOrderSteps,
+		repeatedSteps: opts.repeatedSteps
+	};
+}
+/**
+* Fields on each ExpectedStep variant that are comparable data (not structural).
+* Used by `expectedStepMatches` to know which fields to compare when `compareData` is true.
+*/
+const COMPARABLE_FIELDS_BY_TYPE = {
+	tool_call: [
+		"toolArgs",
+		"toolResult",
+		"success"
+	],
+	mcp_tool_call: [
+		"toolArgs",
+		"toolResult",
+		"mcpServer",
+		"success"
+	],
+	model_generation: [
+		"modelId",
+		"promptTokens",
+		"completionTokens",
+		"finishReason"
+	],
+	agent_run: ["agentId"],
+	workflow_step: [
+		"stepId",
+		"status",
+		"output"
+	],
+	workflow_run: ["workflowId", "status"],
+	workflow_conditional: ["conditionCount", "selectedSteps"],
+	workflow_parallel: ["branchCount", "parallelSteps"],
+	workflow_loop: ["loopType", "totalIterations"],
+	workflow_sleep: ["sleepDurationMs", "sleepType"],
+	workflow_wait_event: ["eventName", "eventReceived"],
+	processor_run: ["processorId"]
+};
+/**
+* Check if an actual TrajectoryStep matches an ExpectedStep.
+* Matches by name, optionally by stepType, and auto-compares any variant-specific
+* fields that are present on the expected step.
+*/
+function expectedStepMatches(actual, expected) {
+	if (actual.name !== expected.name) return false;
+	if (expected.stepType && actual.stepType !== expected.stepType) return false;
+	if (expected.stepType) {
+		const fields = COMPARABLE_FIELDS_BY_TYPE[expected.stepType] ?? [];
+		for (const field of fields) {
+			const expectedVal = expected[field];
+			if (expectedVal === void 0) continue;
+			const actualVal = actual[field];
+			if (actualVal === void 0) return false;
+			try {
+				if (JSON.stringify(actualVal) !== JSON.stringify(expectedVal)) return false;
+			} catch {
+				return false;
+			}
+		}
+	}
+	return true;
+}
+function compareUnorderedPresence(actual, expected, opts) {
+	const actualNames = actual.steps.map((s) => s.name);
+	const expectedNames = expected.steps.map((s) => s.name);
+	let matchedSteps = 0;
+	const matchedExpectedIndices = /* @__PURE__ */ new Set();
+	const usedIndices = /* @__PURE__ */ new Set();
+	for (let i = 0; i < expected.steps.length; i++) {
+		const expectedStep = expected.steps[i];
+		for (let j = 0; j < actual.steps.length; j++) if (!usedIndices.has(j) && expectedStepMatches(actual.steps[j], expectedStep)) {
+			matchedSteps++;
+			matchedExpectedIndices.add(i);
+			usedIndices.add(j);
+			break;
+		}
+	}
+	const missingSteps = expectedNames.filter((_, i) => !matchedExpectedIndices.has(i));
+	const expectedSet = new Set(expectedNames);
+	const extraSteps = actualNames.filter((name) => !expectedSet.has(name));
+	let score = matchedSteps / expected.steps.length;
+	if (!opts.allowRepeatedSteps && opts.repeatedSteps.length > 0) score = Math.max(0, score - opts.repeatedSteps.length * .1);
+	return {
+		score: roundToTwoDecimals(Math.max(0, Math.min(1, score))),
+		matchedSteps,
+		totalExpectedSteps: expected.steps.length,
+		totalActualSteps: actual.steps.length,
+		missingSteps,
+		extraSteps,
+		outOfOrderSteps: [],
+		repeatedSteps: opts.repeatedSteps
+	};
+}
+/**
+* Evaluate trajectory efficiency against budgets and redundancy checks.
+*/
+function checkTrajectoryEfficiency(trajectory, options = {}) {
+	const { maxSteps, maxTotalTokens, maxTotalDurationMs, noRedundantCalls = true } = options;
+	const totalSteps = trajectory.steps.length;
+	let totalTokens = 0;
+	for (const step of trajectory.steps) if (step.stepType === "model_generation") totalTokens += (step.promptTokens ?? 0) + (step.completionTokens ?? 0);
+	const totalDurationMs = trajectory.totalDurationMs ?? trajectory.steps.reduce((sum, s) => sum + (s.durationMs ?? 0), 0);
+	const redundantCalls = [];
+	if (noRedundantCalls) for (let i = 1; i < trajectory.steps.length; i++) {
+		const prev = trajectory.steps[i - 1];
+		const curr = trajectory.steps[i];
+		if (prev.name === curr.name && prev.stepType === curr.stepType && (prev.stepType === "tool_call" || prev.stepType === "mcp_tool_call")) {
+			const prevArgs = prev.toolArgs;
+			const currArgs = curr.toolArgs;
+			try {
+				if (JSON.stringify(prevArgs) === JSON.stringify(currArgs)) redundantCalls.push({
+					name: curr.name,
+					index: i
+				});
+			} catch {}
+		}
+	}
+	const overStepBudget = maxSteps !== void 0 && totalSteps > maxSteps;
+	const overTokenBudget = maxTotalTokens !== void 0 && totalTokens > maxTotalTokens;
+	const overDurationBudget = maxTotalDurationMs !== void 0 && totalDurationMs > maxTotalDurationMs;
+	const dimensions = [];
+	if (maxSteps !== void 0) dimensions.push(overStepBudget ? Math.max(0, 1 - (totalSteps - maxSteps) / maxSteps) : 1);
+	if (maxTotalTokens !== void 0) dimensions.push(overTokenBudget ? Math.max(0, 1 - (totalTokens - maxTotalTokens) / maxTotalTokens) : 1);
+	if (maxTotalDurationMs !== void 0) dimensions.push(overDurationBudget ? Math.max(0, 1 - (totalDurationMs - maxTotalDurationMs) / maxTotalDurationMs) : 1);
+	if (noRedundantCalls) dimensions.push(redundantCalls.length === 0 ? 1 : Math.max(0, 1 - redundantCalls.length * .2));
+	const score = dimensions.length > 0 ? dimensions.reduce((a, b) => a + b, 0) / dimensions.length : 1;
+	return {
+		score: roundToTwoDecimals(Math.max(0, Math.min(1, score))),
+		totalSteps,
+		overStepBudget,
+		totalTokens,
+		overTokenBudget,
+		totalDurationMs,
+		overDurationBudget,
+		redundantCalls
+	};
+}
+/**
+* Check if a trajectory violates any blacklist rules.
+* Returns score 0.0 if any violation is found (hard fail).
+*/
+function checkTrajectoryBlacklist(trajectory, options = {}) {
+	const { blacklistedTools = [], blacklistedSequences = [] } = options;
+	const violatedTools = [];
+	const violatedSequences = [];
+	const stepNames = trajectory.steps.map((s) => s.name);
+	for (const forbidden of blacklistedTools) if (stepNames.includes(forbidden)) violatedTools.push(forbidden);
+	for (const sequence of blacklistedSequences) {
+		if (sequence.length === 0) continue;
+		for (let i = 0; i <= stepNames.length - sequence.length; i++) {
+			let match = true;
+			for (let j = 0; j < sequence.length; j++) if (stepNames[i + j] !== sequence[j]) {
+				match = false;
+				break;
+			}
+			if (match) {
+				violatedSequences.push(sequence);
+				break;
+			}
+		}
+	}
+	return {
+		score: violatedTools.length > 0 || violatedSequences.length > 0 ? 0 : 1,
+		violatedTools,
+		violatedSequences
+	};
+}
+/**
+* Analyze tool failure and retry patterns in a trajectory.
+*/
+function analyzeToolFailures(trajectory, options = {}) {
+	const { maxRetriesPerTool = 2 } = options;
+	const patterns = [];
+	let totalRetries = 0;
+	const toolCallSteps = trajectory.steps.filter((s) => s.stepType === "tool_call" || s.stepType === "mcp_tool_call");
+	if (toolCallSteps.length === 0) return {
+		score: 1,
+		patterns: [],
+		totalRetries: 0,
+		excessiveRetryTools: []
+	};
+	let i = 0;
+	while (i < toolCallSteps.length) {
+		const currentTool = toolCallSteps[i];
+		let retryCount = 0;
+		let j = i + 1;
+		while (j < toolCallSteps.length && toolCallSteps[j].name === currentTool.name) {
+			if (toolCallSteps[j - 1].success === false) retryCount++;
+			j++;
+		}
+		if (retryCount > 0) {
+			const nextDifferentTool = j < toolCallSteps.length ? toolCallSteps[j] : void 0;
+			const lastSuccess = toolCallSteps[j - 1].success !== false;
+			patterns.push({
+				toolName: currentTool.name,
+				retryCount,
+				fellBackToAlternative: nextDifferentTool !== void 0 && !lastSuccess,
+				alternativeTool: nextDifferentTool !== void 0 && !lastSuccess ? nextDifferentTool.name : void 0,
+				eventuallySucceeded: lastSuccess
+			});
+			totalRetries += retryCount;
+		}
+		i = j;
+	}
+	const excessiveRetryTools = patterns.filter((p) => p.retryCount > maxRetriesPerTool).map((p) => p.toolName);
+	let score = 1;
+	if (toolCallSteps.length > 0) {
+		const excessRetries = patterns.reduce((sum, p) => sum + Math.max(0, p.retryCount - maxRetriesPerTool), 0);
+		score = Math.max(0, 1 - excessRetries * .2);
+	}
+	return {
+		score: roundToTwoDecimals(Math.max(0, Math.min(1, score))),
+		patterns,
+		totalRetries,
+		excessiveRetryTools
+	};
+}
+//#endregion
+export { analyzeToolFailures, checkTrajectoryBlacklist, checkTrajectoryEfficiency, compareTrajectories, createAgentTestRun, createTestMessage, createTestRun, createToolInvocation, createTrajectoryTestRun, extractAgentResponseMessages, extractInputMessages, extractToolCalls, extractToolResults, extractTrajectory, getAssistantMessageFromRunOutput, getCombinedSystemPrompt, getReasoningFromRunOutput, getSystemMessagesFromRunInput, getTextContentFromMastraDBMessage, getUserMessageFromRunInput, isCloserTo, isScorerRunInputForAgent, isScorerRunOutputForAgent, roundToTwoDecimals };
+
 //# sourceMappingURL=utils.js.map