@mastra/evals 1.1.2 → 1.2.0-alpha.1

package/dist/scorers/prebuilt/index.js

@@ -1,4 +1,4 @@
-import { getAssistantMessageFromRunOutput, getUserMessageFromRunInput, roundToTwoDecimals, extractToolCalls, getCombinedSystemPrompt, getTextContentFromMastraDBMessage } from '../../chunk-OEOE7ZHN.js';
+import { getAssistantMessageFromRunOutput, getUserMessageFromRunInput, roundToTwoDecimals, extractToolCalls, getCombinedSystemPrompt, getTextContentFromMastraDBMessage, compareTrajectories, checkTrajectoryEfficiency, checkTrajectoryBlacklist, analyzeToolFailures } from '../../chunk-X4MKZ735.js';
 import { createScorer } from '@mastra/core/evals';
 import { z } from 'zod';
 import nlp from 'compromise';
@@ -2600,6 +2600,239 @@ function createPromptAlignmentScorerLLM({
     }
   });
 }
+
+// src/scorers/llm/trajectory/prompts.ts
+var TRAJECTORY_EVALUATION_INSTRUCTIONS = `
+You are an expert evaluator specializing in AI agent trajectory analysis. Your role is to assess whether an agent took an appropriate sequence of actions (tool calls, reasoning steps) to accomplish a user's request.
+
+CORE RESPONSIBILITIES:
+- Analyze the full sequence of actions the agent took
+- Evaluate whether each step was necessary and well-ordered
+- Identify unnecessary, redundant, or missing steps
+- Assess the overall quality of the agent's action path
+
+EVALUATION PHILOSOPHY:
+- Consider both the individual steps AND the overall flow
+- A good trajectory is efficient, logical, and complete
+- Redundant steps reduce quality even if the final result is correct
+- Missing critical steps are a significant issue
+- Order matters: logical dependencies should be respected
+
+OUTPUT REQUIREMENTS:
+- Provide clear reasoning for your trajectory assessment
+- Use provided JSON schema exactly as specified
+- Be consistent in your evaluation standards
+`;
+var createAnalyzePrompt6 = ({
+  userInput,
+  agentResponse,
+  actualTrajectory,
+  expectedTrajectory
+}) => {
+  let prompt = `
+You are evaluating whether an AI agent took an appropriate sequence of actions to fulfill a user request.
+
+USER REQUEST: "${userInput}"
+AGENT FINAL RESPONSE: "${agentResponse}"
+
+ACTUAL TRAJECTORY (sequence of actions the agent took):
+${actualTrajectory}
+`;
+  if (expectedTrajectory) {
+    prompt += `
+EXPECTED TRAJECTORY (the ideal sequence):
+${expectedTrajectory}
+
+EVALUATION CRITERIA:
+1. STEP PRESENCE: Did the agent perform all expected steps?
+2. STEP ORDER: Were the steps in a logical order? (Expected order is a guideline, not absolute)
+3. EXTRA STEPS: Did the agent take unnecessary steps not in the expected trajectory?
+4. MISSING STEPS: Are any expected steps missing from the actual trajectory?
+5. STEP QUALITY: For each step that matches, was it executed appropriately?
+
+For each actual step, evaluate:
+- Does it correspond to an expected step?
+- Was it necessary for the task?
+- Was it in the right position in the sequence?
+`;
+  } else {
+    prompt += `
+EVALUATION CRITERIA (no expected trajectory provided - evaluate based on the task):
+1. COMPLETENESS: Did the agent take all necessary steps to fulfill the request?
+2. EFFICIENCY: Were there any redundant or unnecessary steps?
+3. ORDERING: Were the steps in a logical order given their dependencies?
+4. APPROPRIATENESS: Was each step appropriate for the task?
+`;
+  }
+  prompt += `
+Evaluate each step and the overall trajectory quality.
+`;
+  return prompt;
+};
+var createReasonPrompt7 = ({
+  userInput,
+  score,
+  stepEvaluations,
+  missingSteps,
+  extraSteps
+}) => {
+  return `
+Explain this trajectory evaluation in ONE SENTENCE.
+
+User Request: "${userInput}"
+Score: ${score}/1
+Steps Evaluated: ${JSON.stringify(stepEvaluations)}
+Missing Steps: ${JSON.stringify(missingSteps)}
+Extra/Unnecessary Steps: ${JSON.stringify(extraSteps)}
+
+Provide a single, concise sentence explaining why this score was given.
+`;
+};
+
+// src/scorers/llm/trajectory/index.ts
+var analyzeOutputSchema6 = z.object({
+  stepEvaluations: z.array(
+    z.object({
+      stepName: z.string().describe("Name of the step (tool name or action)"),
+      wasNecessary: z.boolean().describe("Whether this step was necessary for the task"),
+      wasInOrder: z.boolean().describe("Whether this step was in a logical position in the sequence"),
+      reasoning: z.string().describe("Brief explanation of the evaluation")
+    })
+  ),
+  missingSteps: z.array(z.string()).optional().describe("Steps that should have been taken but were not"),
+  extraSteps: z.array(z.string()).optional().describe("Steps that were unnecessary or redundant"),
+  overallAssessment: z.string().describe("Brief overall assessment of the trajectory quality")
+});
+function formatStepDetails(step) {
+  switch (step.stepType) {
+    case "tool_call":
+    case "mcp_tool_call": {
+      const parts = [];
+      if (step.toolArgs !== void 0) parts.push(`args: ${JSON.stringify(step.toolArgs)}`);
+      if (step.toolResult !== void 0) parts.push(`result: ${JSON.stringify(step.toolResult)}`);
+      return parts.length > 0 ? ` (${parts.join(", ")})` : "";
+    }
+    case "model_generation":
+      return step.modelId ? ` (model: ${step.modelId})` : "";
+    case "workflow_step":
+      return step.output !== void 0 ? ` (output: ${JSON.stringify(step.output)})` : "";
+    default:
+      return "";
+  }
+}
+function formatTrajectory(trajectory, indent = 0) {
+  const prefix = "  ".repeat(indent);
+  return trajectory.steps.map((step, i) => {
+    let line = `${prefix}${i + 1}. [${step.stepType}] ${step.name}${formatStepDetails(step)}`;
+    if (step.children && step.children.length > 0) {
+      line += `
+${formatTrajectory({ steps: step.children }, indent + 1)}`;
+    }
+    return line;
+  }).join("\n");
+}
+function formatExpectedSteps(steps, indent = 0) {
+  const prefix = "  ".repeat(indent);
+  return steps.map((step, i) => {
+    const typeStr = step.stepType ? `[${step.stepType}] ` : "";
+    const { name: _, stepType: _t, children: _c, ...fields } = step;
+    const dataStr = Object.keys(fields).length > 0 ? ` (${JSON.stringify(fields)})` : "";
+    let line = `${prefix}${i + 1}. ${typeStr}${step.name}${dataStr}`;
+    if (step.children?.steps && step.children.steps.length > 0) {
+      line += `
+${formatExpectedSteps(step.children.steps, indent + 1)}`;
+    }
+    return line;
+  }).join("\n");
+}
+function createTrajectoryAccuracyScorerLLM({
+  model,
+  expectedTrajectory: staticExpectedTrajectory
+}) {
+  return createScorer({
+    id: "llm-trajectory-accuracy-scorer",
+    name: "Trajectory Accuracy (LLM)",
+    description: staticExpectedTrajectory ? "Evaluates the trajectory against an expected trajectory using LLM analysis" : "Evaluates the quality and appropriateness of the trajectory using LLM analysis",
+    judge: {
+      model,
+      instructions: TRAJECTORY_EVALUATION_INSTRUCTIONS
+    },
+    type: "trajectory"
+  }).preprocess(async ({ run }) => {
+    const actualTrajectory = run.output;
+    let expectedSteps;
+    if (staticExpectedTrajectory) {
+      if (Array.isArray(staticExpectedTrajectory)) {
+        expectedSteps = staticExpectedTrajectory;
+      } else {
+        const toExpectedStep = (s) => {
+          const { durationMs: _, metadata: _m, children, ...rest } = s;
+          const result = rest;
+          if (children && children.length > 0) {
+            result.children = { steps: children.map(toExpectedStep) };
+          }
+          return result;
+        };
+        expectedSteps = staticExpectedTrajectory.steps.map(toExpectedStep);
+      }
+    } else if (run.expectedTrajectory) {
+      const expectation = run.expectedTrajectory;
+      expectedSteps = expectation.steps && expectation.steps.length > 0 ? expectation.steps : void 0;
+    }
+    return {
+      actualTrajectory,
+      actualTrajectoryFormatted: formatTrajectory(actualTrajectory),
+      expectedTrajectoryFormatted: expectedSteps ? formatExpectedSteps(expectedSteps) : void 0,
+      hasSteps: actualTrajectory.steps.length > 0
+    };
+  }).analyze({
+    description: "Analyze the quality and appropriateness of the agent trajectory",
+    outputSchema: analyzeOutputSchema6,
+    createPrompt: ({ run, results }) => {
+      const userInput = getUserMessageFromRunInput(run.input) ?? "";
+      const agentResponse = getAssistantMessageFromRunOutput(run.output.rawOutput) ?? "";
+      return createAnalyzePrompt6({
+        userInput,
+        agentResponse,
+        actualTrajectory: results.preprocessStepResult?.actualTrajectoryFormatted ?? "No steps taken",
+        expectedTrajectory: results.preprocessStepResult?.expectedTrajectoryFormatted
+      });
+    }
+  }).generateScore(({ results }) => {
+    const stepEvaluations = results.analyzeStepResult?.stepEvaluations || [];
+    if (stepEvaluations.length === 0) {
+      const missingSteps2 = results.analyzeStepResult?.missingSteps || [];
+      const extraSteps = results.analyzeStepResult?.extraSteps || [];
+      if (missingSteps2.length > 0) return 0;
+      if (extraSteps.length > 0) return 0.5;
+      return 1;
+    }
+    const necessarySteps = stepEvaluations.filter((e) => e.wasNecessary).length;
+    const orderedSteps = stepEvaluations.filter((e) => e.wasInOrder).length;
+    const totalSteps = stepEvaluations.length;
+    const missingSteps = results.analyzeStepResult?.missingSteps || [];
+    const missingPenalty = missingSteps.length > 0 ? missingSteps.length / (totalSteps + missingSteps.length) : 0;
+    const necessityScore = necessarySteps / totalSteps;
+    const orderScore = orderedSteps / totalSteps;
+    const score = necessityScore * 0.6 + orderScore * 0.3 - missingPenalty * 0.1;
+    return roundToTwoDecimals(Math.max(0, Math.min(1, score)));
+  }).generateReason({
+    description: "Generate human-readable explanation of trajectory evaluation",
+    createPrompt: ({ run, results, score }) => {
+      const userInput = getUserMessageFromRunInput(run.input) ?? "";
+      const stepEvaluations = results.analyzeStepResult?.stepEvaluations || [];
+      const missingSteps = results.analyzeStepResult?.missingSteps || [];
+      const extraSteps = results.analyzeStepResult?.extraSteps || [];
+      return createReasonPrompt7({
+        userInput,
+        score,
+        stepEvaluations,
+        missingSteps,
+        extraSteps
+      });
+    }
+  });
+}
 function normalizeString(str) {
   return str.normalize("NFD").replace(/[\u0300-\u036f]/g, "").toLowerCase();
 }
@@ -2986,7 +3219,339 @@ function createToolCallAccuracyScorerCode(options) {
     });
   });
 }
+function trajectoryStepToExpectedStep(step) {
+  const { durationMs: _, metadata: _m, children, ...rest } = step;
+  const result = rest;
+  if (children && children.length > 0) {
+    result.children = {
+      steps: children.map(trajectoryStepToExpectedStep)
+    };
+  }
+  return result;
+}
+function expectationToExpectedSteps(expectation) {
+  if (!expectation.steps || expectation.steps.length === 0) return void 0;
+  return expectation.steps;
+}
+function createTrajectoryAccuracyScorerCode(options = {}) {
+  const { expectedTrajectory: staticExpectedTrajectory, comparisonOptions = {} } = options;
+  const { ordering = "relaxed", allowRepeatedSteps = true } = comparisonOptions;
+  const staticExpectedSteps = staticExpectedTrajectory ? Array.isArray(staticExpectedTrajectory) && staticExpectedTrajectory.length > 0 && !("steps" in staticExpectedTrajectory[0] || false) ? staticExpectedTrajectory : "steps" in staticExpectedTrajectory ? staticExpectedTrajectory.steps.map(trajectoryStepToExpectedStep) : void 0 : void 0;
+  const getDescription = () => {
+    if (staticExpectedSteps) {
+      const expectedStepNames = staticExpectedSteps.map((s) => s.name).join(" \u2192 ");
+      return `Evaluates whether the trajectory matches the expected path: [${expectedStepNames}] (${ordering} ordering)`;
+    }
+    return `Evaluates trajectory accuracy against expected trajectory from dataset items (${ordering} ordering)`;
+  };
+  return createScorer({
+    id: "code-trajectory-accuracy-scorer",
+    name: "Trajectory Accuracy Scorer",
+    description: getDescription(),
+    type: "trajectory"
+  }).preprocess(async ({ run }) => {
+    const actualTrajectory = run.output;
+    let resolvedExpectedSteps = staticExpectedSteps;
+    if (!resolvedExpectedSteps && run.expectedTrajectory) {
+      const expectation = run.expectedTrajectory;
+      resolvedExpectedSteps = expectationToExpectedSteps(expectation);
+    }
+    if (!resolvedExpectedSteps || resolvedExpectedSteps.length === 0) {
+      return {
+        actualTrajectory,
+        expectedTrajectory: void 0,
+        comparison: void 0,
+        actualStepNames: actualTrajectory.steps.map((s) => s.name),
+        expectedStepNames: [],
+        error: "No expected trajectory provided (pass via options or dataset item expectedTrajectory)"
+      };
+    }
+    const itemExpectation = run.expectedTrajectory;
+    const effectiveOrdering = itemExpectation?.ordering ?? ordering;
+    const effectiveAllowRepeated = itemExpectation?.allowRepeatedSteps ?? allowRepeatedSteps;
+    const comparison = compareTrajectories(
+      actualTrajectory,
+      { steps: resolvedExpectedSteps },
+      {
+        ordering: effectiveOrdering,
+        allowRepeatedSteps: effectiveAllowRepeated
+      }
+    );
+    return {
+      actualTrajectory,
+      expectedTrajectory: { steps: resolvedExpectedSteps },
+      comparison,
+      actualStepNames: actualTrajectory.steps.map((s) => s.name),
+      expectedStepNames: resolvedExpectedSteps.map((s) => s.name)
+    };
+  }).generateScore(({ results }) => {
+    const preprocessResult = results.preprocessStepResult;
+    if (!preprocessResult || !preprocessResult.comparison) {
+      return 0;
+    }
+    return preprocessResult.comparison.score;
+  });
+}
+function evaluateNestedExpectations(expectedSteps, actualSteps, weights = { accuracy: 0.4, efficiency: 0.3, toolFailures: 0.2, blacklist: 0.1 }) {
+  const results = [];
+  const matchedIndices = /* @__PURE__ */ new Set();
+  for (const expectedStep of expectedSteps) {
+    if (!expectedStep.children) continue;
+    const matchIndex = actualSteps.findIndex(
+      (s, i) => !matchedIndices.has(i) && s.name === expectedStep.name && (!expectedStep.stepType || s.stepType === expectedStep.stepType)
+    );
+    const actualStep = matchIndex >= 0 ? actualSteps[matchIndex] : void 0;
+    if (matchIndex >= 0) matchedIndices.add(matchIndex);
+    if (!actualStep?.children || actualStep.children.length === 0) {
+      const expectedStepCount = expectedStep.children.steps?.length ?? 0;
+      results.push({
+        stepName: expectedStep.name,
+        score: 0,
+        accuracy: expectedStepCount > 0 ? {
+          score: 0,
+          matchedSteps: 0,
+          totalExpectedSteps: expectedStepCount,
+          totalActualSteps: 0,
+          missingSteps: expectedStep.children.steps.map((s) => s.name),
+          extraSteps: [],
+          outOfOrderSteps: [],
+          repeatedSteps: []
+        } : void 0
+      });
+      continue;
+    }
+    const childTrajectory = {
+      steps: actualStep.children,
+      totalDurationMs: actualStep.durationMs
+    };
+    const childConfig = expectedStep.children;
+    let accuracy;
+    if (childConfig.steps && childConfig.steps.length > 0) {
+      accuracy = compareTrajectories(
+        childTrajectory,
+        { steps: childConfig.steps },
+        {
+          ordering: childConfig.ordering ?? "relaxed",
+          allowRepeatedSteps: childConfig.allowRepeatedSteps ?? true
+        }
+      );
+    }
+    const hasEfficiencyConfig = childConfig.maxSteps !== void 0 || childConfig.maxTotalTokens !== void 0 || childConfig.maxTotalDurationMs !== void 0 || childConfig.noRedundantCalls !== void 0;
+    const efficiency = hasEfficiencyConfig ? checkTrajectoryEfficiency(childTrajectory, {
+      maxSteps: childConfig.maxSteps,
+      maxTotalTokens: childConfig.maxTotalTokens,
+      maxTotalDurationMs: childConfig.maxTotalDurationMs,
+      noRedundantCalls: childConfig.noRedundantCalls ?? true
+    }) : void 0;
+    const hasBlacklistConfig = childConfig.blacklistedTools && childConfig.blacklistedTools.length > 0 || childConfig.blacklistedSequences && childConfig.blacklistedSequences.length > 0;
+    const blacklist = hasBlacklistConfig ? checkTrajectoryBlacklist(childTrajectory, {
+      blacklistedTools: childConfig.blacklistedTools,
+      blacklistedSequences: childConfig.blacklistedSequences
+    }) : void 0;
+    const toolFailures = analyzeToolFailures(childTrajectory, {
+      maxRetriesPerTool: childConfig.maxRetriesPerTool ?? 2
+    });
+    const nested = childConfig.steps ? evaluateNestedExpectations(childConfig.steps, actualStep.children, weights) : [];
+    const scores = [];
+    if (accuracy) scores.push({ weight: weights.accuracy, value: accuracy.score });
+    if (efficiency) scores.push({ weight: weights.efficiency, value: efficiency.score });
+    if (toolFailures && toolFailures.patterns.length > 0)
+      scores.push({ weight: weights.toolFailures, value: toolFailures.score });
+    if (blacklist) {
+      if (blacklist.score === 0) {
+        results.push({ stepName: expectedStep.name, score: 0, accuracy, efficiency, blacklist, toolFailures, nested });
+        continue;
+      }
+      scores.push({ weight: weights.blacklist, value: blacklist.score });
+    }
+    let levelScore = 1;
+    if (scores.length > 0) {
+      const totalWeight = scores.reduce((sum, s) => sum + s.weight, 0);
+      levelScore = totalWeight > 0 ? scores.reduce((sum, s) => sum + s.weight / totalWeight * s.value, 0) : 1;
+    }
+    let finalScore = levelScore;
+    if (nested.length > 0) {
+      const hasNestedBlacklistViolation = nested.some((r) => r.blacklist && r.blacklist.score === 0);
+      if (hasNestedBlacklistViolation) {
+        results.push({ stepName: expectedStep.name, score: 0, accuracy, efficiency, blacklist, toolFailures, nested });
+        continue;
+      }
+      const nestedAvg = nested.reduce((sum, r) => sum + r.score, 0) / nested.length;
+      finalScore = 0.7 * levelScore + 0.3 * nestedAvg;
+    }
+    results.push({
+      stepName: expectedStep.name,
+      score: Math.round(finalScore * 100) / 100,
+      accuracy,
+      efficiency,
+      blacklist,
+      toolFailures,
+      nested: nested.length > 0 ? nested : void 0
+    });
+  }
+  return results;
+}
+function createTrajectoryScorerCode(options = {}) {
+  const { defaults = {}, weights: userWeights = {} } = options;
+  const w = {
+    accuracy: Math.max(0, userWeights.accuracy ?? 0.4),
+    efficiency: Math.max(0, userWeights.efficiency ?? 0.3),
+    toolFailures: Math.max(0, userWeights.toolFailures ?? 0.2),
+    blacklist: Math.max(0, userWeights.blacklist ?? 0.1)
+  };
+  return createScorer({
+    id: "code-trajectory-scorer",
+    name: "Trajectory Scorer",
+    description: "Multi-dimensional trajectory evaluation: accuracy, efficiency, blacklist, and tool failures",
+    type: "trajectory"
+  }).preprocess(async ({ run }) => {
+    const actualTrajectory = run.output;
+    const itemExpectation = run.expectedTrajectory ?? {};
+    const config = { ...defaults, ...itemExpectation };
+    if (itemExpectation.steps !== void 0) {
+      config.steps = itemExpectation.steps;
+    }
+    let accuracy;
+    if (config.steps && config.steps.length > 0) {
+      accuracy = compareTrajectories(
+        actualTrajectory,
+        { steps: config.steps },
+        {
+          ordering: config.ordering ?? "relaxed",
+          allowRepeatedSteps: config.allowRepeatedSteps ?? true
+        }
+      );
+    }
+    const hasEfficiencyConfig = config.maxSteps !== void 0 || config.maxTotalTokens !== void 0 || config.maxTotalDurationMs !== void 0 || config.noRedundantCalls !== void 0;
+    const efficiency = hasEfficiencyConfig ? checkTrajectoryEfficiency(actualTrajectory, {
+      maxSteps: config.maxSteps,
+      maxTotalTokens: config.maxTotalTokens,
+      maxTotalDurationMs: config.maxTotalDurationMs,
+      noRedundantCalls: config.noRedundantCalls ?? true
+    }) : void 0;
+    const hasBlacklistConfig = config.blacklistedTools && config.blacklistedTools.length > 0 || config.blacklistedSequences && config.blacklistedSequences.length > 0;
+    const blacklist = hasBlacklistConfig ? checkTrajectoryBlacklist(actualTrajectory, {
+      blacklistedTools: config.blacklistedTools,
+      blacklistedSequences: config.blacklistedSequences
+    }) : void 0;
+    const toolFailures = analyzeToolFailures(actualTrajectory, {
+      maxRetriesPerTool: config.maxRetriesPerTool ?? 2
+    });
+    const nested = config.steps && config.steps.length > 0 ? evaluateNestedExpectations(config.steps, actualTrajectory.steps, w) : void 0;
+    return {
+      accuracy,
+      efficiency,
+      blacklist,
+      toolFailures,
+      nested: nested && nested.length > 0 ? nested : void 0,
+      config
+    };
+  }).generateScore(({ results }) => {
+    const { accuracy, efficiency, blacklist, toolFailures, nested } = results.preprocessStepResult ?? {};
+    if (blacklist && blacklist.score === 0) {
+      return 0;
+    }
+    const scores = [];
+    if (accuracy) {
+      scores.push({ weight: w.accuracy, value: accuracy.score });
+    }
+    if (efficiency) {
+      scores.push({ weight: w.efficiency, value: efficiency.score });
+    }
+    if (toolFailures && toolFailures.patterns.length > 0) {
+      scores.push({ weight: w.toolFailures, value: toolFailures.score });
+    }
+    if (blacklist) {
+      scores.push({ weight: w.blacklist, value: blacklist.score });
+    }
+    if (scores.length === 0 && !nested) {
+      return 1;
+    }
+    let levelScore = 1;
+    if (scores.length > 0) {
+      const totalWeight = scores.reduce((sum, s) => sum + s.weight, 0);
+      levelScore = totalWeight > 0 ? scores.reduce((sum, s) => sum + s.weight / totalWeight * s.value, 0) : 1;
+    }
+    if (nested && nested.length > 0) {
+      const hasNestedBlacklistViolation = nested.some((r) => r.blacklist && r.blacklist.score === 0);
+      if (hasNestedBlacklistViolation) {
+        return 0;
+      }
+      const nestedAvg = nested.reduce((sum, r) => sum + r.score, 0) / nested.length;
+      levelScore = 0.7 * levelScore + 0.3 * nestedAvg;
+    }
+    return Math.round(levelScore * 100) / 100;
+  }).generateReason(({ results, score }) => {
+    const { accuracy, efficiency, blacklist, toolFailures, nested } = results.preprocessStepResult ?? {};
+    const parts = [];
+    parts.push(`Score: ${score}`);
+    if (blacklist && blacklist.score === 0) {
+      const violations = [];
+      if (blacklist.violatedTools.length > 0) {
+        violations.push(`forbidden tools used: ${blacklist.violatedTools.join(", ")}`);
+      }
+      if (blacklist.violatedSequences.length > 0) {
+        violations.push(`forbidden sequences: ${blacklist.violatedSequences.map((s) => s.join(" \u2192 ")).join("; ")}`);
+      }
+      parts.push(`Blacklist violation: ${violations.join(". ")}.`);
+      return parts.join("\n");
+    }
+    if (nested && nested.some((r) => r.blacklist && r.blacklist.score === 0)) {
+      const violating = nested.filter((r) => r.blacklist && r.blacklist.score === 0).map((r) => r.stepName);
+      parts.push(`Nested blacklist violation in: ${violating.join(", ")}.`);
+      return parts.join("\n");
+    }
+    if (accuracy) {
+      const details = [`${accuracy.matchedSteps}/${accuracy.totalExpectedSteps} expected steps matched`];
+      if (accuracy.missingSteps.length > 0) {
+        details.push(`missing: ${accuracy.missingSteps.join(", ")}`);
+      }
+      if (accuracy.extraSteps.length > 0) {
+        details.push(`extra: ${accuracy.extraSteps.join(", ")}`);
+      }
+      if (accuracy.outOfOrderSteps.length > 0) {
+        details.push(`out of order: ${accuracy.outOfOrderSteps.join(", ")}`);
+      }
+      parts.push(`Accuracy (${accuracy.score}): ${details.join(". ")}.`);
+    }
+    if (efficiency) {
+      const details = [];
+      if (efficiency.overStepBudget) {
+        details.push(`over step budget (${efficiency.totalSteps} steps)`);
+      }
+      if (efficiency.overTokenBudget) {
+        details.push(`over token budget (${efficiency.totalTokens} tokens)`);
+      }
+      if (efficiency.overDurationBudget) {
+        details.push(`over duration budget (${efficiency.totalDurationMs}ms)`);
+      }
+      if (efficiency.redundantCalls.length > 0) {
+        details.push(`redundant calls: ${efficiency.redundantCalls.map((c) => c.name).join(", ")}`);
+      }
+      if (details.length > 0) {
+        parts.push(`Efficiency (${efficiency.score}): ${details.join(". ")}.`);
+      } else {
+        parts.push(`Efficiency (${efficiency.score}): all budgets met, no redundant calls.`);
+      }
+    }
+    if (toolFailures && toolFailures.patterns.length > 0) {
+      const details = [];
+      if (toolFailures.totalRetries > 0) {
+        details.push(`${toolFailures.totalRetries} total retries`);
+      }
+      if (toolFailures.excessiveRetryTools.length > 0) {
+        details.push(`excessive retries: ${toolFailures.excessiveRetryTools.join(", ")}`);
+      }
+      parts.push(`Tool failures (${toolFailures.score}): ${details.join(". ")}.`);
+    }
+    if (nested && nested.length > 0) {
+      const nestedSummary = nested.map((r) => `${r.stepName}: ${r.score}`).join(", ");
+      parts.push(`Nested scores: ${nestedSummary}.`);
+    }
+    return parts.join("\n");
+  });
+}
 
-export { ANSWER_RELEVANCY_AGENT_INSTRUCTIONS, ANSWER_SIMILARITY_DEFAULT_OPTIONS, ANSWER_SIMILARITY_INSTRUCTIONS, DEFAULT_OPTIONS, createAnswerRelevancyScorer, createAnswerSimilarityScorer, createBiasScorer, createCompletenessScorer, createContentSimilarityScorer, createContextPrecisionScorer, createContextRelevanceScorerLLM, createFaithfulnessScorer, createHallucinationScorer, createKeywordCoverageScorer, createNoiseSensitivityScorerLLM, createPromptAlignmentScorerLLM, createTextualDifferenceScorer, createToneScorer, createToolCallAccuracyScorerCode, createToolCallAccuracyScorerLLM, createToxicityScorer };
+export { ANSWER_RELEVANCY_AGENT_INSTRUCTIONS, ANSWER_SIMILARITY_DEFAULT_OPTIONS, ANSWER_SIMILARITY_INSTRUCTIONS, DEFAULT_OPTIONS, createAnswerRelevancyScorer, createAnswerSimilarityScorer, createBiasScorer, createCompletenessScorer, createContentSimilarityScorer, createContextPrecisionScorer, createContextRelevanceScorerLLM, createFaithfulnessScorer, createHallucinationScorer, createKeywordCoverageScorer, createNoiseSensitivityScorerLLM, createPromptAlignmentScorerLLM, createTextualDifferenceScorer, createToneScorer, createToolCallAccuracyScorerCode, createToolCallAccuracyScorerLLM, createToxicityScorer, createTrajectoryAccuracyScorerCode, createTrajectoryAccuracyScorerLLM, createTrajectoryScorerCode };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map