@mastra/code-sdk 0.0.0-fix-workflow-step-timing-and-parallel-status-20260715180716

package/dist/evals/scorers/efficiency.d.ts

@@ -0,0 +1,44 @@
+/**
+ * MastraCode Efficiency Scorer (Code-based, Agent type, 30% sampling)
+ *
+ * Measures how efficiently the agent completes work through quantitative ratios.
+ * Unlike the Outcome Scorer (which checks *what* happened), this checks
+ * *how much work* it took.
+ *
+ * This scorer produces a score from 0-1 but also exposes raw metrics in the
+ * reason text for trend analysis. The score represents "how cleanly did the
+ * agent work?" — not whether it got the right answer.
+ *
+ * Sampled at 30% because it's slightly more expensive (analyzes message sequences).
+ */
+export declare function createEfficiencyScorer(): import("@mastra/core/evals").MastraScorer<"mastracode-efficiency", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    empty: true;
+    totalCalls: number;
+    totalTurns: number;
+    redundancy?: undefined;
+    turnCount?: undefined;
+    retryEfficiency?: undefined;
+    readBeforeEdit?: undefined;
+} | {
+    empty: false;
+    redundancy: {
+        score: number;
+        ratio: number;
+        detail: string;
+    };
+    turnCount: {
+        score: number;
+        detail: string;
+    };
+    retryEfficiency: {
+        score: number;
+        detail: string;
+    };
+    readBeforeEdit: {
+        score: number;
+        detail: string;
+    };
+    totalCalls: number;
+    totalTurns: number;
+}> & Record<"generateScoreStepResult", number> & Record<"generateReasonStepResult", string>>;
+//# sourceMappingURL=efficiency.d.ts.map

package/dist/evals/scorers/efficiency.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"efficiency.d.ts","sourceRoot":"","sources":["../../../src/evals/scorers/efficiency.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAsOH,wBAAgB,sBAAsB;;;;;;;;;;;eA7JyB,MAAM;eAAS,MAAM;gBAAU,MAAM;;;eAqCnD,MAAM;gBAAU,MAAM;;;eAqBH,MAAM;gBAAU,MAAM;;;eAuDvB,MAAM;gBAAU,MAAM;;;;6FA8GxF"}

package/dist/evals/scorers/efficiency.js

@@ -0,0 +1,201 @@
+import { createScorer, filterRun } from "@mastra/core/evals";
+import { extractToolCalls } from "./extract-tools.js";
+const WEIGHTS = {
+  redundancy: 0.35,
+  turnCount: 0.3,
+  retryEfficiency: 0.2,
+  readBeforeEdit: 0.15
+};
+const THRESHOLDS = {
+  /** Redundancy ratio below this is considered acceptable (no penalty). */
+  redundancyAcceptableRate: 0.05,
+  /** Penalty multiplier applied to redundancy ratio above the acceptable rate. */
+  redundancyPenaltyMultiplier: 3,
+  /** Turns at or below this is considered normal (score 1.0). */
+  turnCountNormalMax: 8,
+  /** Penalty per turn beyond the normal maximum (for turns 9–15). */
+  turnCountPenaltyPerTurn: 0.04,
+  /** Penalty per turn beyond 15. */
+  turnCountExtendedPenaltyPerTurn: 0.03,
+  /** Number of consecutive failures before a retry chain is "excessive". */
+  retryExcessiveThreshold: 3,
+  /** Tools considered "edit" tools that should be preceded by a read. */
+  editTools: ["string_replace_lsp", "ast_smart_edit"],
+  /** Tools that satisfy read-before-edit (only `view` truly reads a specific file). */
+  readTools: ["view"],
+  /** Tools whose repeated identical calls are intentional (idempotent reads). */
+  redundancyWhitelist: ["view", "search_content", "find_files", "lsp_inspect", "web_search", "web_extract"],
+  /** Minimum tool calls required to produce a meaningful score. */
+  minToolCalls: 2
+};
+function countAssistantTurns(messages) {
+  return messages.filter((m) => m.role === "assistant").length;
+}
+function scoreRedundancy(calls) {
+  if (calls.length < 2) return { score: 1, ratio: 0, detail: "Too few calls" };
+  const mutationCalls = calls.filter((c) => !THRESHOLDS.redundancyWhitelist.includes(c.toolName));
+  if (mutationCalls.length < 2) return { score: 1, ratio: 0, detail: "No mutation tools to check" };
+  const fingerprints = /* @__PURE__ */ new Map();
+  for (const call of mutationCalls) {
+    const fp = `${call.toolName}:${JSON.stringify(call.args)}`;
+    const existing = fingerprints.get(fp) ?? { count: 0, successCount: 0 };
+    existing.count++;
+    if (!call.isError) existing.successCount++;
+    fingerprints.set(fp, existing);
+  }
+  let redundantCount = 0;
+  for (const { successCount } of fingerprints.values()) {
+    if (successCount > 1) redundantCount += successCount - 1;
+  }
+  const ratio = redundantCount / mutationCalls.length;
+  const acceptable = THRESHOLDS.redundancyAcceptableRate;
+  const score = ratio <= acceptable ? 1 : Math.max(0, 1 - (ratio - acceptable) * THRESHOLDS.redundancyPenaltyMultiplier);
+  return {
+    score,
+    ratio,
+    detail: `${redundantCount}/${mutationCalls.length} redundant mutations (${(ratio * 100).toFixed(0)}%)`
+  };
+}
+function scoreTurnCount(turns) {
+  const normalMax = THRESHOLDS.turnCountNormalMax;
+  const penaltyPerTurn = THRESHOLDS.turnCountPenaltyPerTurn;
+  const extendedPenalty = THRESHOLDS.turnCountExtendedPenaltyPerTurn;
+  if (turns <= 1) return { score: 1, detail: `${turns} turn` };
+  if (turns <= normalMax) return { score: 1, detail: `${turns} turns (within normal range)` };
+  if (turns <= 15) {
+    const score2 = 1 - (turns - normalMax) * penaltyPerTurn;
+    return { score: score2, detail: `${turns} turns (slightly extended)` };
+  }
+  const midPenalty = (15 - normalMax) * penaltyPerTurn;
+  const score = Math.max(0, 1 - midPenalty - (turns - 15) * extendedPenalty);
+  return { score, detail: `${turns} turns (extended session)` };
+}
+function scoreRetryEfficiency(calls) {
+  if (calls.length < 2) return { score: 1, detail: "Too few calls" };
+  const groupKey = (call) => {
+    const path = String(call.args.path ?? call.args.command ?? "");
+    return `${call.toolName}:${path}`;
+  };
+  const byGroup = /* @__PURE__ */ new Map();
+  for (const call of calls) {
+    const key = groupKey(call);
+    const arr = byGroup.get(key) ?? [];
+    arr.push(call);
+    byGroup.set(key, arr);
+  }
+  let totalRetryChains = 0;
+  let excessiveRetries = 0;
+  const threshold = THRESHOLDS.retryExcessiveThreshold;
+  for (const [, toolCalls] of byGroup) {
+    let consecutiveFailures = 0;
+    for (const call of toolCalls) {
+      if (call.isError) {
+        consecutiveFailures++;
+      } else {
+        if (consecutiveFailures > 0) {
+          totalRetryChains++;
+          if (consecutiveFailures >= threshold) excessiveRetries++;
+        }
+        consecutiveFailures = 0;
+      }
+    }
+    if (consecutiveFailures > 0) {
+      totalRetryChains++;
+      if (consecutiveFailures >= threshold) excessiveRetries++;
+    }
+  }
+  if (totalRetryChains === 0) return { score: 1, detail: "No retry chains" };
+  if (excessiveRetries === 0) return { score: 0.9, detail: `${totalRetryChains} retry chain(s), all resolved quickly` };
+  const score = Math.max(0, 0.8 - excessiveRetries * 0.2);
+  return {
+    score,
+    detail: `${excessiveRetries} excessive retry chain(s) (${threshold}+ failures before success)`
+  };
+}
+function scoreReadBeforeEdit(calls) {
+  const editTools = THRESHOLDS.editTools;
+  const readTools = THRESHOLDS.readTools;
+  const edits = calls.filter((c) => editTools.includes(c.toolName));
+  if (edits.length === 0) return { score: 1, detail: "No edits to check" };
+  const readPaths = /* @__PURE__ */ new Set();
+  let compliant = 0;
+  let total = 0;
+  for (const call of calls) {
+    const rawPath = String(call.args.path ?? "");
+    if (!rawPath) continue;
+    if (readTools.includes(call.toolName)) {
+      readPaths.add(rawPath);
+    } else if (editTools.includes(call.toolName)) {
+      total++;
+      if (hasReadPath(readPaths, rawPath)) {
+        compliant++;
+      }
+    }
+  }
+  if (total === 0) return { score: 1, detail: "No path-targeted edits" };
+  const ratio = compliant / total;
+  const violations = total - compliant;
+  if (violations === 0) return { score: 1, detail: `All ${total} edits had prior reads` };
+  return { score: ratio, detail: `${violations}/${total} edits without prior read` };
+}
+function hasReadPath(readPaths, editPath) {
+  if (readPaths.has(editPath)) return true;
+  for (const rp of readPaths) {
+    if (rp.endsWith(`/${editPath}`) || editPath.endsWith(`/${rp}`)) return true;
+  }
+  return false;
+}
+function createEfficiencyScorer() {
+  return createScorer({
+    id: "mastracode-efficiency",
+    name: "MastraCode Efficiency",
+    description: "Measures coding session efficiency: redundancy, turn count, retry patterns, read-before-edit. 30% sampled.",
+    type: "agent",
+    prepareRun: filterRun({ partTypes: ["tool-invocation", "text"] })
+  }).preprocess(async ({ run }) => {
+    const messages = run.output ?? [];
+    const calls = extractToolCalls(messages);
+    const turns = countAssistantTurns(messages);
+    if (calls.length < THRESHOLDS.minToolCalls) {
+      return { empty: true, totalCalls: calls.length, totalTurns: turns };
+    }
+    const redundancy = scoreRedundancy(calls);
+    const turnCount = scoreTurnCount(turns);
+    const retryEfficiency = scoreRetryEfficiency(calls);
+    const readBeforeEdit = scoreReadBeforeEdit(calls);
+    return {
+      empty: false,
+      redundancy,
+      turnCount,
+      retryEfficiency,
+      readBeforeEdit,
+      totalCalls: calls.length,
+      totalTurns: turns
+    };
+  }).generateScore(({ results }) => {
+    const p = results.preprocessStepResult;
+    if (p.empty) return 0;
+    const score = p.redundancy.score * WEIGHTS.redundancy + p.turnCount.score * WEIGHTS.turnCount + p.retryEfficiency.score * WEIGHTS.retryEfficiency + p.readBeforeEdit.score * WEIGHTS.readBeforeEdit;
+    return Math.round(score * 100) / 100;
+  }).generateReason(({ results, score }) => {
+    const p = results.preprocessStepResult;
+    if (p.empty) {
+      return `Score: 0 (${p.totalCalls} tool calls \u2014 below minimum threshold for scoring)`;
+    }
+    const pct = (w) => `${(w * 100).toFixed(0)}%`;
+    const parts = [`Score: ${score} (${p.totalCalls} calls, ${p.totalTurns} turns)`];
+    parts.push(`Redundancy (${pct(WEIGHTS.redundancy)}): ${p.redundancy.detail} [${p.redundancy.score}]`);
+    parts.push(`Turn count (${pct(WEIGHTS.turnCount)}): ${p.turnCount.detail} [${p.turnCount.score}]`);
+    parts.push(
+      `Retry efficiency (${pct(WEIGHTS.retryEfficiency)}): ${p.retryEfficiency.detail} [${p.retryEfficiency.score}]`
+    );
+    parts.push(
+      `Read-before-edit (${pct(WEIGHTS.readBeforeEdit)}): ${p.readBeforeEdit.detail} [${p.readBeforeEdit.score}]`
+    );
+    return parts.join("\n");
+  });
+}
+export {
+  createEfficiencyScorer
+};
+//# sourceMappingURL=efficiency.js.map

package/dist/evals/scorers/efficiency.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/evals/scorers/efficiency.ts"],"sourcesContent":["/**\n * MastraCode Efficiency Scorer (Code-based, Agent type, 30% sampling)\n *\n * Measures how efficiently the agent completes work through quantitative ratios.\n * Unlike the Outcome Scorer (which checks *what* happened), this checks\n * *how much work* it took.\n *\n * This scorer produces a score from 0-1 but also exposes raw metrics in the\n * reason text for trend analysis. The score represents \"how cleanly did the\n * agent work?\" — not whether it got the right answer.\n *\n * Sampled at 30% because it's slightly more expensive (analyzes message sequences).\n */\n\nimport type { MastraDBMessage } from '@mastra/core/agent';\nimport { createScorer, filterRun } from '@mastra/core/evals';\n\nimport type { ExtractedToolCall } from './extract-tools.js';\nimport { extractToolCalls } from './extract-tools.js';\n\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n// CONFIGURATION — Adjust weights and thresholds here\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n/**\n * Dimension weights (must sum to 1.0).\n *\n * - redundancy:     Duplicate mutation tool calls with identical args.\n * - turnCount:      Total assistant turns to complete the task.\n * - retryEfficiency: How quickly the agent recovers from tool failures.\n * - readBeforeEdit: Whether files are read before being edited.\n */\nconst WEIGHTS = {\n  redundancy: 0.35,\n  turnCount: 0.3,\n  retryEfficiency: 0.2,\n  readBeforeEdit: 0.15,\n} as const;\n\n/**\n * Scoring thresholds and multipliers.\n */\nconst THRESHOLDS = {\n  /** Redundancy ratio below this is considered acceptable (no penalty). */\n  redundancyAcceptableRate: 0.05,\n\n  /** Penalty multiplier applied to redundancy ratio above the acceptable rate. */\n  redundancyPenaltyMultiplier: 3,\n\n  /** Turns at or below this is considered normal (score 1.0). */\n  turnCountNormalMax: 8,\n\n  /** Penalty per turn beyond the normal maximum (for turns 9–15). */\n  turnCountPenaltyPerTurn: 0.04,\n\n  /** Penalty per turn beyond 15. */\n  turnCountExtendedPenaltyPerTurn: 0.03,\n\n  /** Number of consecutive failures before a retry chain is \"excessive\". */\n  retryExcessiveThreshold: 3,\n\n  /** Tools considered \"edit\" tools that should be preceded by a read. */\n  editTools: ['string_replace_lsp', 'ast_smart_edit'] as string[],\n\n  /** Tools that satisfy read-before-edit (only `view` truly reads a specific file). */\n  readTools: ['view'] as string[],\n\n  /** Tools whose repeated identical calls are intentional (idempotent reads). */\n  redundancyWhitelist: ['view', 'search_content', 'find_files', 'lsp_inspect', 'web_search', 'web_extract'] as string[],\n\n  /** Minimum tool calls required to produce a meaningful score. */\n  minToolCalls: 2,\n} as const;\n\n/**\n * Count assistant turns (messages with role === 'assistant').\n */\nfunction countAssistantTurns(messages: MastraDBMessage[]): number {\n  return messages.filter(m => m.role === 'assistant').length;\n}\n\n/**\n * Calculate redundant call ratio.\n * Only counts mutation/action tools as redundant — reads are whitelisted.\n */\nfunction scoreRedundancy(calls: ExtractedToolCall[]): { score: number; ratio: number; detail: string } {\n  if (calls.length < 2) return { score: 1, ratio: 0, detail: 'Too few calls' };\n\n  // Only consider non-whitelisted (mutation) tools for redundancy\n  const mutationCalls = calls.filter(c => !THRESHOLDS.redundancyWhitelist.includes(c.toolName));\n  if (mutationCalls.length < 2) return { score: 1, ratio: 0, detail: 'No mutation tools to check' };\n\n  const fingerprints = new Map<string, { count: number; successCount: number }>();\n  for (const call of mutationCalls) {\n    const fp = `${call.toolName}:${JSON.stringify(call.args)}`;\n    const existing = fingerprints.get(fp) ?? { count: 0, successCount: 0 };\n    existing.count++;\n    if (!call.isError) existing.successCount++;\n    fingerprints.set(fp, existing);\n  }\n\n  // Only count as redundant if the same successful call happened multiple times\n  let redundantCount = 0;\n  for (const { successCount } of fingerprints.values()) {\n    if (successCount > 1) redundantCount += successCount - 1;\n  }\n\n  const ratio = redundantCount / mutationCalls.length;\n  const acceptable = THRESHOLDS.redundancyAcceptableRate;\n  const score =\n    ratio <= acceptable ? 1 : Math.max(0, 1 - (ratio - acceptable) * THRESHOLDS.redundancyPenaltyMultiplier);\n  return {\n    score,\n    ratio,\n    detail: `${redundantCount}/${mutationCalls.length} redundant mutations (${(ratio * 100).toFixed(0)}%)`,\n  };\n}\n\n/**\n * Score turn count efficiency.\n * Fewer turns to accomplish the task = more efficient.\n */\nfunction scoreTurnCount(turns: number): { score: number; detail: string } {\n  const normalMax = THRESHOLDS.turnCountNormalMax;\n  const penaltyPerTurn = THRESHOLDS.turnCountPenaltyPerTurn;\n  const extendedPenalty = THRESHOLDS.turnCountExtendedPenaltyPerTurn;\n\n  if (turns <= 1) return { score: 1, detail: `${turns} turn` };\n  if (turns <= normalMax) return { score: 1, detail: `${turns} turns (within normal range)` };\n  if (turns <= 15) {\n    const score = 1 - (turns - normalMax) * penaltyPerTurn;\n    return { score, detail: `${turns} turns (slightly extended)` };\n  }\n\n  const midPenalty = (15 - normalMax) * penaltyPerTurn;\n  const score = Math.max(0, 1 - midPenalty - (turns - 15) * extendedPenalty);\n  return { score, detail: `${turns} turns (extended session)` };\n}\n\n/**\n * Score failed-then-succeeded ratio.\n * Groups retry chains by tool name + target file path for more accurate detection.\n */\nfunction scoreRetryEfficiency(calls: ExtractedToolCall[]): { score: number; detail: string } {\n  if (calls.length < 2) return { score: 1, detail: 'Too few calls' };\n\n  // Group calls by tool name + target file path (if available)\n  const groupKey = (call: ExtractedToolCall): string => {\n    const path = String(call.args.path ?? call.args.command ?? '');\n    return `${call.toolName}:${path}`;\n  };\n\n  const byGroup = new Map<string, ExtractedToolCall[]>();\n  for (const call of calls) {\n    const key = groupKey(call);\n    const arr = byGroup.get(key) ?? [];\n    arr.push(call);\n    byGroup.set(key, arr);\n  }\n\n  let totalRetryChains = 0;\n  let excessiveRetries = 0;\n  const threshold = THRESHOLDS.retryExcessiveThreshold;\n\n  for (const [, toolCalls] of byGroup) {\n    let consecutiveFailures = 0;\n    for (const call of toolCalls) {\n      if (call.isError) {\n        consecutiveFailures++;\n      } else {\n        if (consecutiveFailures > 0) {\n          totalRetryChains++;\n          if (consecutiveFailures >= threshold) excessiveRetries++;\n        }\n        consecutiveFailures = 0;\n      }\n    }\n    // Count unrecovered retry chains (failures at end of group with no success)\n    if (consecutiveFailures > 0) {\n      totalRetryChains++;\n      if (consecutiveFailures >= threshold) excessiveRetries++;\n    }\n  }\n\n  if (totalRetryChains === 0) return { score: 1, detail: 'No retry chains' };\n  if (excessiveRetries === 0) return { score: 0.9, detail: `${totalRetryChains} retry chain(s), all resolved quickly` };\n\n  const score = Math.max(0, 0.8 - excessiveRetries * 0.2);\n  return {\n    score,\n    detail: `${excessiveRetries} excessive retry chain(s) (${threshold}+ failures before success)`,\n  };\n}\n\n/**\n * Score read-before-edit compliance.\n * Only `view` satisfies a read — search/find don't provide specific file content.\n */\nfunction scoreReadBeforeEdit(calls: ExtractedToolCall[]): { score: number; detail: string } {\n  const editTools = THRESHOLDS.editTools;\n  const readTools = THRESHOLDS.readTools;\n\n  const edits = calls.filter(c => editTools.includes(c.toolName));\n  if (edits.length === 0) return { score: 1, detail: 'No edits to check' };\n\n  // Build set of files that were read before each edit.\n  // Use suffix-normalized paths so that \"/abs/path/src/foo.ts\" and \"src/foo.ts\" match.\n  const readPaths = new Set<string>();\n  let compliant = 0;\n  let total = 0;\n\n  for (const call of calls) {\n    const rawPath = String(call.args.path ?? '');\n    if (!rawPath) continue;\n\n    if (readTools.includes(call.toolName)) {\n      readPaths.add(rawPath);\n    } else if (editTools.includes(call.toolName)) {\n      total++;\n      if (hasReadPath(readPaths, rawPath)) {\n        compliant++;\n      }\n    }\n  }\n\n  if (total === 0) return { score: 1, detail: 'No path-targeted edits' };\n\n  const ratio = compliant / total;\n  const violations = total - compliant;\n  if (violations === 0) return { score: 1, detail: `All ${total} edits had prior reads` };\n  return { score: ratio, detail: `${violations}/${total} edits without prior read` };\n}\n\n/** Check if any read path matches the edit path, using suffix matching for mixed abs/relative paths. */\nfunction hasReadPath(readPaths: Set<string>, editPath: string): boolean {\n  if (readPaths.has(editPath)) return true;\n  for (const rp of readPaths) {\n    if (rp.endsWith(`/${editPath}`) || editPath.endsWith(`/${rp}`)) return true;\n  }\n  return false;\n}\n\nexport function createEfficiencyScorer() {\n  return createScorer({\n    id: 'mastracode-efficiency',\n    name: 'MastraCode Efficiency',\n    description:\n      'Measures coding session efficiency: redundancy, turn count, retry patterns, read-before-edit. 30% sampled.',\n    type: 'agent',\n    prepareRun: filterRun({ partTypes: ['tool-invocation', 'text'] }),\n  })\n    .preprocess(async ({ run }) => {\n      const messages = (run.output ?? []) as MastraDBMessage[];\n      const calls = extractToolCalls(messages);\n      const turns = countAssistantTurns(messages);\n\n      // H1: Empty sessions → score 0\n      if (calls.length < THRESHOLDS.minToolCalls) {\n        return { empty: true as const, totalCalls: calls.length, totalTurns: turns };\n      }\n\n      const redundancy = scoreRedundancy(calls);\n      const turnCount = scoreTurnCount(turns);\n      const retryEfficiency = scoreRetryEfficiency(calls);\n      const readBeforeEdit = scoreReadBeforeEdit(calls);\n\n      return {\n        empty: false as const,\n        redundancy,\n        turnCount,\n        retryEfficiency,\n        readBeforeEdit,\n        totalCalls: calls.length,\n        totalTurns: turns,\n      };\n    })\n    .generateScore(({ results }) => {\n      const p = results.preprocessStepResult;\n      if (p.empty) return 0;\n\n      const score =\n        p.redundancy.score * WEIGHTS.redundancy +\n        p.turnCount.score * WEIGHTS.turnCount +\n        p.retryEfficiency.score * WEIGHTS.retryEfficiency +\n        p.readBeforeEdit.score * WEIGHTS.readBeforeEdit;\n\n      return Math.round(score * 100) / 100;\n    })\n    .generateReason(({ results, score }) => {\n      const p = results.preprocessStepResult;\n      if (p.empty) {\n        return `Score: 0 (${p.totalCalls} tool calls — below minimum threshold for scoring)`;\n      }\n\n      const pct = (w: number) => `${(w * 100).toFixed(0)}%`;\n      const parts: string[] = [`Score: ${score} (${p.totalCalls} calls, ${p.totalTurns} turns)`];\n\n      parts.push(`Redundancy (${pct(WEIGHTS.redundancy)}): ${p.redundancy.detail} [${p.redundancy.score}]`);\n      parts.push(`Turn count (${pct(WEIGHTS.turnCount)}): ${p.turnCount.detail} [${p.turnCount.score}]`);\n      parts.push(\n        `Retry efficiency (${pct(WEIGHTS.retryEfficiency)}): ${p.retryEfficiency.detail} [${p.retryEfficiency.score}]`,\n      );\n      parts.push(\n        `Read-before-edit (${pct(WEIGHTS.readBeforeEdit)}): ${p.readBeforeEdit.detail} [${p.readBeforeEdit.score}]`,\n      );\n\n      return parts.join('\\n');\n    });\n}\n"],"mappings":"AAeA,SAAS,cAAc,iBAAiB;AAGxC,SAAS,wBAAwB;AAcjC,MAAM,UAAU;AAAA,EACd,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,gBAAgB;AAClB;AAKA,MAAM,aAAa;AAAA;AAAA,EAEjB,0BAA0B;AAAA;AAAA,EAG1B,6BAA6B;AAAA;AAAA,EAG7B,oBAAoB;AAAA;AAAA,EAGpB,yBAAyB;AAAA;AAAA,EAGzB,iCAAiC;AAAA;AAAA,EAGjC,yBAAyB;AAAA;AAAA,EAGzB,WAAW,CAAC,sBAAsB,gBAAgB;AAAA;AAAA,EAGlD,WAAW,CAAC,MAAM;AAAA;AAAA,EAGlB,qBAAqB,CAAC,QAAQ,kBAAkB,cAAc,eAAe,cAAc,aAAa;AAAA;AAAA,EAGxG,cAAc;AAChB;AAKA,SAAS,oBAAoB,UAAqC;AAChE,SAAO,SAAS,OAAO,OAAK,EAAE,SAAS,WAAW,EAAE;AACtD;AAMA,SAAS,gBAAgB,OAA8E;AACrG,MAAI,MAAM,SAAS,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,QAAQ,gBAAgB;AAG3E,QAAM,gBAAgB,MAAM,OAAO,OAAK,CAAC,WAAW,oBAAoB,SAAS,EAAE,QAAQ,CAAC;AAC5F,MAAI,cAAc,SAAS,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,QAAQ,6BAA6B;AAEhG,QAAM,eAAe,oBAAI,IAAqD;AAC9E,aAAW,QAAQ,eAAe;AAChC,UAAM,KAAK,GAAG,KAAK,QAAQ,IAAI,KAAK,UAAU,KAAK,IAAI,CAAC;AACxD,UAAM,WAAW,aAAa,IAAI,EAAE,KAAK,EAAE,OAAO,GAAG,cAAc,EAAE;AACrE,aAAS;AACT,QAAI,CAAC,KAAK,QAAS,UAAS;AAC5B,iBAAa,IAAI,IAAI,QAAQ;AAAA,EAC/B;AAGA,MAAI,iBAAiB;AACrB,aAAW,EAAE,aAAa,KAAK,aAAa,OAAO,GAAG;AACpD,QAAI,eAAe,EAAG,mBAAkB,eAAe;AAAA,EACzD;AAEA,QAAM,QAAQ,iBAAiB,cAAc;AAC7C,QAAM,aAAa,WAAW;AAC9B,QAAM,QACJ,SAAS,aAAa,IAAI,KAAK,IAAI,GAAG,KAAK,QAAQ,cAAc,WAAW,2BAA2B;AACzG,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,QAAQ,GAAG,cAAc,IAAI,cAAc,MAAM,0BAA0B,QAAQ,KAAK,QAAQ,CAAC,CAAC;AAAA,EACpG;AACF;AAMA,SAAS,eAAe,OAAkD;AACxE,QAAM,YAAY,WAAW;AAC7B,QAAM,iBAAiB,WAAW;AAClC,QAAM,kBAAkB,WAAW;AAEnC,MAAI,SAAS,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,GAAG,KAAK,QAAQ;AAC3D,MAAI,SAAS,UAAW,QAAO,EAAE,OAAO,GAAG,QAAQ,GAAG,KAAK,+BAA+B;AAC1F,MAAI,SAAS,IAAI;AACf,UAAMA,SAAQ,KAAK,QAAQ,aAAa;AACxC,WAAO,EAAE,OAAAA,QAAO,QAAQ,GAAG,KAAK,6BAA6B;AAAA,EAC/D;AAEA,QAAM,cAAc,KAAK,aAAa;AACtC,QAAM,QAAQ,KAAK,IAAI,GAAG,IAAI,cAAc,QAAQ,MAAM,eAAe;AACzE,SAAO,EAAE,OAAO,QAAQ,GAAG,KAAK,4BAA4B;AAC9D;AAMA,SAAS,qBAAqB,OAA+D;AAC3F,MAAI,MAAM,SAAS,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,gBAAgB;AAGjE,QAAM,WAAW,CAAC,SAAoC;AACpD,UAAM,OAAO,OAAO,KAAK,KAAK,QAAQ,KAAK,KAAK,WAAW,EAAE;AAC7D,WAAO,GAAG,KAAK,QAAQ,IAAI,IAAI;AAAA,EACjC;AAEA,QAAM,UAAU,oBAAI,IAAiC;AACrD,aAAW,QAAQ,OAAO;AACxB,UAAM,MAAM,SAAS,IAAI;AACzB,UAAM,MAAM,QAAQ,IAAI,GAAG,KAAK,CAAC;AACjC,QAAI,KAAK,IAAI;AACb,YAAQ,IAAI,KAAK,GAAG;AAAA,EACtB;AAEA,MAAI,mBAAmB;AACvB,MAAI,mBAAmB;AACvB,QAAM,YAAY,WAAW;AAE7B,aAAW,CAAC,EAAE,SAAS,KAAK,SAAS;AACnC,QAAI,sBAAsB;AAC1B,eAAW,QAAQ,WAAW;AAC5B,UAAI,KAAK,SAAS;AAChB;AAAA,MACF,OAAO;AACL,YAAI,sBAAsB,GAAG;AAC3B;AACA,cAAI,uBAAuB,UAAW;AAAA,QACxC;AACA,8BAAsB;AAAA,MACxB;AAAA,IACF;AAEA,QAAI,sBAAsB,GAAG;AAC3B;AACA,UAAI,uBAAuB,UAAW;AAAA,IACxC;AAAA,EACF;AAEA,MAAI,qBAAqB,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,kBAAkB;AACzE,MAAI,qBAAqB,EAAG,QAAO,EAAE,OAAO,KAAK,QAAQ,GAAG,gBAAgB,wCAAwC;AAEpH,QAAM,QAAQ,KAAK,IAAI,GAAG,MAAM,mBAAmB,GAAG;AACtD,SAAO;AAAA,IACL;AAAA,IACA,QAAQ,GAAG,gBAAgB,8BAA8B,SAAS;AAAA,EACpE;AACF;AAMA,SAAS,oBAAoB,OAA+D;AAC1F,QAAM,YAAY,WAAW;AAC7B,QAAM,YAAY,WAAW;AAE7B,QAAM,QAAQ,MAAM,OAAO,OAAK,UAAU,SAAS,EAAE,QAAQ,CAAC;AAC9D,MAAI,MAAM,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,oBAAoB;AAIvE,QAAM,YAAY,oBAAI,IAAY;AAClC,MAAI,YAAY;AAChB,MAAI,QAAQ;AAEZ,aAAW,QAAQ,OAAO;AACxB,UAAM,UAAU,OAAO,KAAK,KAAK,QAAQ,EAAE;AAC3C,QAAI,CAAC,QAAS;AAEd,QAAI,UAAU,SAAS,KAAK,QAAQ,GAAG;AACrC,gBAAU,IAAI,OAAO;AAAA,IACvB,WAAW,UAAU,SAAS,KAAK,QAAQ,GAAG;AAC5C;AACA,UAAI,YAAY,WAAW,OAAO,GAAG;AACnC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,UAAU,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,yBAAyB;AAErE,QAAM,QAAQ,YAAY;AAC1B,QAAM,aAAa,QAAQ;AAC3B,MAAI,eAAe,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,OAAO,KAAK,yBAAyB;AACtF,SAAO,EAAE,OAAO,OAAO,QAAQ,GAAG,UAAU,IAAI,KAAK,4BAA4B;AACnF;AAGA,SAAS,YAAY,WAAwB,UAA2B;AACtE,MAAI,UAAU,IAAI,QAAQ,EAAG,QAAO;AACpC,aAAW,MAAM,WAAW;AAC1B,QAAI,GAAG,SAAS,IAAI,QAAQ,EAAE,KAAK,SAAS,SAAS,IAAI,EAAE,EAAE,EAAG,QAAO;AAAA,EACzE;AACA,SAAO;AACT;AAEO,SAAS,yBAAyB;AACvC,SAAO,aAAa;AAAA,IAClB,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,aACE;AAAA,IACF,MAAM;AAAA,IACN,YAAY,UAAU,EAAE,WAAW,CAAC,mBAAmB,MAAM,EAAE,CAAC;AAAA,EAClE,CAAC,EACE,WAAW,OAAO,EAAE,IAAI,MAAM;AAC7B,UAAM,WAAY,IAAI,UAAU,CAAC;AACjC,UAAM,QAAQ,iBAAiB,QAAQ;AACvC,UAAM,QAAQ,oBAAoB,QAAQ;AAG1C,QAAI,MAAM,SAAS,WAAW,cAAc;AAC1C,aAAO,EAAE,OAAO,MAAe,YAAY,MAAM,QAAQ,YAAY,MAAM;AAAA,IAC7E;AAEA,UAAM,aAAa,gBAAgB,KAAK;AACxC,UAAM,YAAY,eAAe,KAAK;AACtC,UAAM,kBAAkB,qBAAqB,KAAK;AAClD,UAAM,iBAAiB,oBAAoB,KAAK;AAEhD,WAAO;AAAA,MACL,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,MAAM;AAAA,MAClB,YAAY;AAAA,IACd;AAAA,EACF,CAAC,EACA,cAAc,CAAC,EAAE,QAAQ,MAAM;AAC9B,UAAM,IAAI,QAAQ;AAClB,QAAI,EAAE,MAAO,QAAO;AAEpB,UAAM,QACJ,EAAE,WAAW,QAAQ,QAAQ,aAC7B,EAAE,UAAU,QAAQ,QAAQ,YAC5B,EAAE,gBAAgB,QAAQ,QAAQ,kBAClC,EAAE,eAAe,QAAQ,QAAQ;AAEnC,WAAO,KAAK,MAAM,QAAQ,GAAG,IAAI;AAAA,EACnC,CAAC,EACA,eAAe,CAAC,EAAE,SAAS,MAAM,MAAM;AACtC,UAAM,IAAI,QAAQ;AAClB,QAAI,EAAE,OAAO;AACX,aAAO,aAAa,EAAE,UAAU;AAAA,IAClC;AAEA,UAAM,MAAM,CAAC,MAAc,IAAI,IAAI,KAAK,QAAQ,CAAC,CAAC;AAClD,UAAM,QAAkB,CAAC,UAAU,KAAK,KAAK,EAAE,UAAU,WAAW,EAAE,UAAU,SAAS;AAEzF,UAAM,KAAK,eAAe,IAAI,QAAQ,UAAU,CAAC,MAAM,EAAE,WAAW,MAAM,KAAK,EAAE,WAAW,KAAK,GAAG;AACpG,UAAM,KAAK,eAAe,IAAI,QAAQ,SAAS,CAAC,MAAM,EAAE,UAAU,MAAM,KAAK,EAAE,UAAU,KAAK,GAAG;AACjG,UAAM;AAAA,MACJ,qBAAqB,IAAI,QAAQ,eAAe,CAAC,MAAM,EAAE,gBAAgB,MAAM,KAAK,EAAE,gBAAgB,KAAK;AAAA,IAC7G;AACA,UAAM;AAAA,MACJ,qBAAqB,IAAI,QAAQ,cAAc,CAAC,MAAM,EAAE,eAAe,MAAM,KAAK,EAAE,eAAe,KAAK;AAAA,IAC1G;AAEA,WAAO,MAAM,KAAK,IAAI;AAAA,EACxB,CAAC;AACL;","names":["score"]}

package/dist/evals/scorers/extract-tools.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Shared tool invocation extraction from MastraDBMessages.
+ *
+ * Used by both the Outcome and Efficiency scorers to extract completed
+ * tool calls from message content, handling both the `parts` array (primary)
+ * and the legacy `toolInvocations` array.
+ */
+import type { MastraDBMessage } from '@mastra/core/agent';
+export type ExtractedToolCall = {
+    toolCallId?: string;
+    toolName: string;
+    args: Record<string, unknown>;
+    result: unknown;
+    isError: boolean;
+    index: number;
+};
+/**
+ * Extract completed tool invocations from MastraDBMessages.
+ *
+ * Handles both:
+ * - `content.parts` (primary format, MastraToolInvocationPart)
+ * - `content.toolInvocations` (legacy format, AI SDK ToolInvocation[])
+ */
+export declare function extractToolCalls(messages: MastraDBMessage[]): ExtractedToolCall[];
+//# sourceMappingURL=extract-tools.d.ts.map

package/dist/evals/scorers/extract-tools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-tools.d.ts","sourceRoot":"","sources":["../../../src/evals/scorers/extract-tools.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAG1D,MAAM,MAAM,iBAAiB,GAAG;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAmBF;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,iBAAiB,EAAE,CAoDjF"}

package/dist/evals/scorers/extract-tools.js

@@ -0,0 +1,56 @@
+function isToolInvocationPart(part) {
+  return "type" in part && part.type === "tool-invocation" && "toolInvocation" in part;
+}
+function isCompleted(inv) {
+  return inv.state === "result" || inv.state === "error" || inv.state === "output-error" || inv.state === "output-denied";
+}
+function isErrorState(inv) {
+  return inv.state === "error" || inv.state === "output-error" || inv.state === "output-denied";
+}
+function extractToolCalls(messages) {
+  const results = [];
+  const seenToolCallIds = /* @__PURE__ */ new Set();
+  let index = 0;
+  for (const msg of messages) {
+    if (!msg.content) continue;
+    for (const part of msg.content.parts ?? []) {
+      if (isToolInvocationPart(part)) {
+        const inv = part.toolInvocation;
+        if (isCompleted(inv)) {
+          const toolCallId = "toolCallId" in inv ? inv.toolCallId : void 0;
+          if (toolCallId) seenToolCallIds.add(toolCallId);
+          results.push({
+            toolCallId,
+            toolName: inv.toolName ?? "",
+            args: inv.args ?? {},
+            result: inv.result,
+            isError: isErrorState(inv),
+            index: index++
+          });
+        }
+      }
+    }
+    if (msg.content.toolInvocations) {
+      for (const inv of msg.content.toolInvocations) {
+        if (inv.state === "result") {
+          const toolCallId = "toolCallId" in inv ? inv.toolCallId : void 0;
+          if (toolCallId && seenToolCallIds.has(toolCallId)) continue;
+          if (toolCallId) seenToolCallIds.add(toolCallId);
+          results.push({
+            toolCallId,
+            toolName: inv.toolName ?? "",
+            args: inv.args ?? {},
+            result: inv.result,
+            isError: false,
+            index: index++
+          });
+        }
+      }
+    }
+  }
+  return results;
+}
+export {
+  extractToolCalls
+};
+//# sourceMappingURL=extract-tools.js.map

package/dist/evals/scorers/extract-tools.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/evals/scorers/extract-tools.ts"],"sourcesContent":["/**\n * Shared tool invocation extraction from MastraDBMessages.\n *\n * Used by both the Outcome and Efficiency scorers to extract completed\n * tool calls from message content, handling both the `parts` array (primary)\n * and the legacy `toolInvocations` array.\n */\n\nimport type { MastraDBMessage } from '@mastra/core/agent';\nimport type { MastraMessagePart, MastraToolInvocation } from '@mastra/core/agent/message-list';\n\nexport type ExtractedToolCall = {\n  toolCallId?: string;\n  toolName: string;\n  args: Record<string, unknown>;\n  result: unknown;\n  isError: boolean;\n  index: number;\n};\n\nfunction isToolInvocationPart(part: MastraMessagePart): part is MastraMessagePart & {\n  type: 'tool-invocation';\n  toolInvocation: MastraToolInvocation;\n} {\n  return 'type' in part && part.type === 'tool-invocation' && 'toolInvocation' in part;\n}\n\nfunction isCompleted(inv: { state: string }): boolean {\n  return (\n    inv.state === 'result' || inv.state === 'error' || inv.state === 'output-error' || inv.state === 'output-denied'\n  );\n}\n\nfunction isErrorState(inv: { state: string }): boolean {\n  return inv.state === 'error' || inv.state === 'output-error' || inv.state === 'output-denied';\n}\n\n/**\n * Extract completed tool invocations from MastraDBMessages.\n *\n * Handles both:\n * - `content.parts` (primary format, MastraToolInvocationPart)\n * - `content.toolInvocations` (legacy format, AI SDK ToolInvocation[])\n */\nexport function extractToolCalls(messages: MastraDBMessage[]): ExtractedToolCall[] {\n  const results: ExtractedToolCall[] = [];\n  const seenToolCallIds = new Set<string>();\n  let index = 0;\n\n  for (const msg of messages) {\n    if (!msg.content) continue;\n\n    // Primary format: parts array with tool-invocation parts\n    for (const part of msg.content.parts ?? []) {\n      if (isToolInvocationPart(part)) {\n        const inv = part.toolInvocation;\n        if (isCompleted(inv)) {\n          const toolCallId = 'toolCallId' in inv ? (inv.toolCallId as string) : undefined;\n          if (toolCallId) seenToolCallIds.add(toolCallId);\n\n          results.push({\n            toolCallId,\n            toolName: inv.toolName ?? '',\n            args: (inv.args ?? {}) as Record<string, unknown>,\n            result: inv.result,\n            isError: isErrorState(inv),\n            index: index++,\n          });\n        }\n      }\n    }\n\n    // Legacy format: toolInvocations array (AI SDK ToolInvocation)\n    // Only states here are 'partial-call' | 'call' | 'result' — no error state.\n    if (msg.content.toolInvocations) {\n      for (const inv of msg.content.toolInvocations) {\n        if (inv.state === 'result') {\n          // Skip if we already extracted this from parts (deduplicate by toolCallId)\n          const toolCallId = 'toolCallId' in inv ? inv.toolCallId : undefined;\n          if (toolCallId && seenToolCallIds.has(toolCallId)) continue;\n          if (toolCallId) seenToolCallIds.add(toolCallId);\n\n          results.push({\n            toolCallId,\n            toolName: inv.toolName ?? '',\n            args: (inv.args ?? {}) as Record<string, unknown>,\n            result: inv.result,\n            isError: false,\n            index: index++,\n          });\n        }\n      }\n    }\n  }\n\n  return results;\n}\n"],"mappings":"AAoBA,SAAS,qBAAqB,MAG5B;AACA,SAAO,UAAU,QAAQ,KAAK,SAAS,qBAAqB,oBAAoB;AAClF;AAEA,SAAS,YAAY,KAAiC;AACpD,SACE,IAAI,UAAU,YAAY,IAAI,UAAU,WAAW,IAAI,UAAU,kBAAkB,IAAI,UAAU;AAErG;AAEA,SAAS,aAAa,KAAiC;AACrD,SAAO,IAAI,UAAU,WAAW,IAAI,UAAU,kBAAkB,IAAI,UAAU;AAChF;AASO,SAAS,iBAAiB,UAAkD;AACjF,QAAM,UAA+B,CAAC;AACtC,QAAM,kBAAkB,oBAAI,IAAY;AACxC,MAAI,QAAQ;AAEZ,aAAW,OAAO,UAAU;AAC1B,QAAI,CAAC,IAAI,QAAS;AAGlB,eAAW,QAAQ,IAAI,QAAQ,SAAS,CAAC,GAAG;AAC1C,UAAI,qBAAqB,IAAI,GAAG;AAC9B,cAAM,MAAM,KAAK;AACjB,YAAI,YAAY,GAAG,GAAG;AACpB,gBAAM,aAAa,gBAAgB,MAAO,IAAI,aAAwB;AACtE,cAAI,WAAY,iBAAgB,IAAI,UAAU;AAE9C,kBAAQ,KAAK;AAAA,YACX;AAAA,YACA,UAAU,IAAI,YAAY;AAAA,YAC1B,MAAO,IAAI,QAAQ,CAAC;AAAA,YACpB,QAAQ,IAAI;AAAA,YACZ,SAAS,aAAa,GAAG;AAAA,YACzB,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAIA,QAAI,IAAI,QAAQ,iBAAiB;AAC/B,iBAAW,OAAO,IAAI,QAAQ,iBAAiB;AAC7C,YAAI,IAAI,UAAU,UAAU;AAE1B,gBAAM,aAAa,gBAAgB,MAAM,IAAI,aAAa;AAC1D,cAAI,cAAc,gBAAgB,IAAI,UAAU,EAAG;AACnD,cAAI,WAAY,iBAAgB,IAAI,UAAU;AAE9C,kBAAQ,KAAK;AAAA,YACX;AAAA,YACA,UAAU,IAAI,YAAY;AAAA,YAC1B,MAAO,IAAI,QAAQ,CAAC;AAAA,YACpB,QAAQ,IAAI;AAAA,YACZ,SAAS;AAAA,YACT,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;","names":[]}

package/dist/evals/scorers/index.d.ts

@@ -0,0 +1,3 @@
+export { createOutcomeScorer } from './outcome.js';
+export { createEfficiencyScorer } from './efficiency.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/evals/scorers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/evals/scorers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/evals/scorers/index.js

@@ -0,0 +1,7 @@
+import { createOutcomeScorer } from "./outcome.js";
+import { createEfficiencyScorer } from "./efficiency.js";
+export {
+  createEfficiencyScorer,
+  createOutcomeScorer
+};
+//# sourceMappingURL=index.js.map

package/dist/evals/scorers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/evals/scorers/index.ts"],"sourcesContent":["export { createOutcomeScorer } from './outcome.js';\nexport { createEfficiencyScorer } from './efficiency.js';\n"],"mappings":"AAAA,SAAS,2BAA2B;AACpC,SAAS,8BAA8B;","names":[]}

package/dist/evals/scorers/outcome.d.ts

@@ -0,0 +1,49 @@
+/**
+ * MastraCode Outcome Scorer (Code-based, Agent type, always-on)
+ *
+ * Grades the objective outcome of a coding session by checking hard facts:
+ * - Did the build/typecheck pass?
+ * - Did tests pass?
+ * - What was the tool error rate?
+ * - Did the agent get stuck in a loop?
+ * - Was there a regression (build passed earlier, failed later)?
+ * - Did the agent use ask_user unnecessarily? (autonomy)
+ *
+ * This scorer is always-on (no sampling) because it's cheap — it only
+ * reads exit codes and error states from tool results already in the messages.
+ *
+ * Scoring philosophy (from Anthropic's eval guidance):
+ * "Grade outcomes, not paths" — we check what happened, not how.
+ * Partial credit: a session that passes build but fails tests scores
+ * higher than one that fails build entirely.
+ *
+ * Dimensions that don't apply to a session (e.g. build for a docs-only task)
+ * are excluded from the weighted average, so sessions aren't penalized for
+ * irrelevant criteria.
+ */
+type DimensionResult = {
+    score: number;
+    detail: string;
+    applicable: boolean;
+};
+export declare function createOutcomeScorer(): import("@mastra/core/evals").MastraScorer<"mastracode-outcome", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    empty: true;
+    build: DimensionResult;
+    tests: DimensionResult;
+    toolErrors: DimensionResult;
+    loops: DimensionResult;
+    regression: DimensionResult;
+    autonomy: DimensionResult;
+    totalCalls: number;
+} | {
+    empty: false;
+    build: DimensionResult;
+    tests: DimensionResult;
+    toolErrors: DimensionResult;
+    loops: DimensionResult;
+    regression: DimensionResult;
+    autonomy: DimensionResult;
+    totalCalls: number;
+}> & Record<"generateScoreStepResult", number> & Record<"generateReasonStepResult", string>>;
+export {};
+//# sourceMappingURL=outcome.d.ts.map

package/dist/evals/scorers/outcome.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outcome.d.ts","sourceRoot":"","sources":["../../../src/evals/scorers/outcome.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAsDH,KAAK,eAAe,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,OAAO,CAAA;CAAE,CAAC;AAuN9E,wBAAgB,mBAAmB;;;;;;;;;;;;;;;;;;6FAkFlC"}