@mastra/code-sdk 1.0.3-alpha.0 → 1.1.0-alpha.2

package/dist/evals/scorers/efficiency.js

@@ -1,201 +1,259 @@
-import { createScorer, filterRun } from "@mastra/core/evals";
 import { extractToolCalls } from "./extract-tools.js";
+import { createScorer, filterRun } from "@mastra/core/evals";
+//#region src/evals/scorers/efficiency.ts
+/**
+* Dimension weights (must sum to 1.0).
+*
+* - redundancy:     Duplicate mutation tool calls with identical args.
+* - turnCount:      Total assistant turns to complete the task.
+* - retryEfficiency: How quickly the agent recovers from tool failures.
+* - readBeforeEdit: Whether files are read before being edited.
+*/
 const WEIGHTS = {
-  redundancy: 0.35,
-  turnCount: 0.3,
-  retryEfficiency: 0.2,
-  readBeforeEdit: 0.15
+	redundancy: .35,
+	turnCount: .3,
+	retryEfficiency: .2,
+	readBeforeEdit: .15
 };
+/**
+* Scoring thresholds and multipliers.
+*/
 const THRESHOLDS = {
-  /** Redundancy ratio below this is considered acceptable (no penalty). */
-  redundancyAcceptableRate: 0.05,
-  /** Penalty multiplier applied to redundancy ratio above the acceptable rate. */
-  redundancyPenaltyMultiplier: 3,
-  /** Turns at or below this is considered normal (score 1.0). */
-  turnCountNormalMax: 8,
-  /** Penalty per turn beyond the normal maximum (for turns 9–15). */
-  turnCountPenaltyPerTurn: 0.04,
-  /** Penalty per turn beyond 15. */
-  turnCountExtendedPenaltyPerTurn: 0.03,
-  /** Number of consecutive failures before a retry chain is "excessive". */
-  retryExcessiveThreshold: 3,
-  /** Tools considered "edit" tools that should be preceded by a read. */
-  editTools: ["string_replace_lsp", "ast_smart_edit"],
-  /** Tools that satisfy read-before-edit (only `view` truly reads a specific file). */
-  readTools: ["view"],
-  /** Tools whose repeated identical calls are intentional (idempotent reads). */
-  redundancyWhitelist: ["view", "search_content", "find_files", "lsp_inspect", "web_search", "web_extract"],
-  /** Minimum tool calls required to produce a meaningful score. */
-  minToolCalls: 2
+	/** Redundancy ratio below this is considered acceptable (no penalty). */
+	redundancyAcceptableRate: .05,
+	/** Penalty multiplier applied to redundancy ratio above the acceptable rate. */
+	redundancyPenaltyMultiplier: 3,
+	/** Turns at or below this is considered normal (score 1.0). */
+	turnCountNormalMax: 8,
+	/** Penalty per turn beyond the normal maximum (for turns 9–15). */
+	turnCountPenaltyPerTurn: .04,
+	/** Penalty per turn beyond 15. */
+	turnCountExtendedPenaltyPerTurn: .03,
+	/** Number of consecutive failures before a retry chain is "excessive". */
+	retryExcessiveThreshold: 3,
+	/** Tools considered "edit" tools that should be preceded by a read. */
+	editTools: ["string_replace_lsp", "ast_smart_edit"],
+	/** Tools that satisfy read-before-edit (only `view` truly reads a specific file). */
+	readTools: ["view"],
+	/** Tools whose repeated identical calls are intentional (idempotent reads). */
+	redundancyWhitelist: [
+		"view",
+		"search_content",
+		"find_files",
+		"lsp_inspect",
+		"web_search",
+		"web_extract"
+	],
+	/** Minimum tool calls required to produce a meaningful score. */
+	minToolCalls: 2
 };
+/**
+* Count assistant turns (messages with role === 'assistant').
+*/
 function countAssistantTurns(messages) {
-  return messages.filter((m) => m.role === "assistant").length;
+	return messages.filter((m) => m.role === "assistant").length;
 }
+/**
+* Calculate redundant call ratio.
+* Only counts mutation/action tools as redundant — reads are whitelisted.
+*/
 function scoreRedundancy(calls) {
-  if (calls.length < 2) return { score: 1, ratio: 0, detail: "Too few calls" };
-  const mutationCalls = calls.filter((c) => !THRESHOLDS.redundancyWhitelist.includes(c.toolName));
-  if (mutationCalls.length < 2) return { score: 1, ratio: 0, detail: "No mutation tools to check" };
-  const fingerprints = /* @__PURE__ */ new Map();
-  for (const call of mutationCalls) {
-    const fp = `${call.toolName}:${JSON.stringify(call.args)}`;
-    const existing = fingerprints.get(fp) ?? { count: 0, successCount: 0 };
-    existing.count++;
-    if (!call.isError) existing.successCount++;
-    fingerprints.set(fp, existing);
-  }
-  let redundantCount = 0;
-  for (const { successCount } of fingerprints.values()) {
-    if (successCount > 1) redundantCount += successCount - 1;
-  }
-  const ratio = redundantCount / mutationCalls.length;
-  const acceptable = THRESHOLDS.redundancyAcceptableRate;
-  const score = ratio <= acceptable ? 1 : Math.max(0, 1 - (ratio - acceptable) * THRESHOLDS.redundancyPenaltyMultiplier);
-  return {
-    score,
-    ratio,
-    detail: `${redundantCount}/${mutationCalls.length} redundant mutations (${(ratio * 100).toFixed(0)}%)`
-  };
+	if (calls.length < 2) return {
+		score: 1,
+		ratio: 0,
+		detail: "Too few calls"
+	};
+	const mutationCalls = calls.filter((c) => !THRESHOLDS.redundancyWhitelist.includes(c.toolName));
+	if (mutationCalls.length < 2) return {
+		score: 1,
+		ratio: 0,
+		detail: "No mutation tools to check"
+	};
+	const fingerprints = /* @__PURE__ */ new Map();
+	for (const call of mutationCalls) {
+		const fp = `${call.toolName}:${JSON.stringify(call.args)}`;
+		const existing = fingerprints.get(fp) ?? {
+			count: 0,
+			successCount: 0
+		};
+		existing.count++;
+		if (!call.isError) existing.successCount++;
+		fingerprints.set(fp, existing);
+	}
+	let redundantCount = 0;
+	for (const { successCount } of fingerprints.values()) if (successCount > 1) redundantCount += successCount - 1;
+	const ratio = redundantCount / mutationCalls.length;
+	const acceptable = THRESHOLDS.redundancyAcceptableRate;
+	return {
+		score: ratio <= acceptable ? 1 : Math.max(0, 1 - (ratio - acceptable) * THRESHOLDS.redundancyPenaltyMultiplier),
+		ratio,
+		detail: `${redundantCount}/${mutationCalls.length} redundant mutations (${(ratio * 100).toFixed(0)}%)`
+	};
 }
+/**
+* Score turn count efficiency.
+* Fewer turns to accomplish the task = more efficient.
+*/
 function scoreTurnCount(turns) {
-  const normalMax = THRESHOLDS.turnCountNormalMax;
-  const penaltyPerTurn = THRESHOLDS.turnCountPenaltyPerTurn;
-  const extendedPenalty = THRESHOLDS.turnCountExtendedPenaltyPerTurn;
-  if (turns <= 1) return { score: 1, detail: `${turns} turn` };
-  if (turns <= normalMax) return { score: 1, detail: `${turns} turns (within normal range)` };
-  if (turns <= 15) {
-    const score2 = 1 - (turns - normalMax) * penaltyPerTurn;
-    return { score: score2, detail: `${turns} turns (slightly extended)` };
-  }
-  const midPenalty = (15 - normalMax) * penaltyPerTurn;
-  const score = Math.max(0, 1 - midPenalty - (turns - 15) * extendedPenalty);
-  return { score, detail: `${turns} turns (extended session)` };
+	const normalMax = THRESHOLDS.turnCountNormalMax;
+	const penaltyPerTurn = THRESHOLDS.turnCountPenaltyPerTurn;
+	const extendedPenalty = THRESHOLDS.turnCountExtendedPenaltyPerTurn;
+	if (turns <= 1) return {
+		score: 1,
+		detail: `${turns} turn`
+	};
+	if (turns <= normalMax) return {
+		score: 1,
+		detail: `${turns} turns (within normal range)`
+	};
+	if (turns <= 15) return {
+		score: 1 - (turns - normalMax) * penaltyPerTurn,
+		detail: `${turns} turns (slightly extended)`
+	};
+	const midPenalty = (15 - normalMax) * penaltyPerTurn;
+	return {
+		score: Math.max(0, 1 - midPenalty - (turns - 15) * extendedPenalty),
+		detail: `${turns} turns (extended session)`
+	};
 }
+/**
+* Score failed-then-succeeded ratio.
+* Groups retry chains by tool name + target file path for more accurate detection.
+*/
 function scoreRetryEfficiency(calls) {
-  if (calls.length < 2) return { score: 1, detail: "Too few calls" };
-  const groupKey = (call) => {
-    const path = String(call.args.path ?? call.args.command ?? "");
-    return `${call.toolName}:${path}`;
-  };
-  const byGroup = /* @__PURE__ */ new Map();
-  for (const call of calls) {
-    const key = groupKey(call);
-    const arr = byGroup.get(key) ?? [];
-    arr.push(call);
-    byGroup.set(key, arr);
-  }
-  let totalRetryChains = 0;
-  let excessiveRetries = 0;
-  const threshold = THRESHOLDS.retryExcessiveThreshold;
-  for (const [, toolCalls] of byGroup) {
-    let consecutiveFailures = 0;
-    for (const call of toolCalls) {
-      if (call.isError) {
-        consecutiveFailures++;
-      } else {
-        if (consecutiveFailures > 0) {
-          totalRetryChains++;
-          if (consecutiveFailures >= threshold) excessiveRetries++;
-        }
-        consecutiveFailures = 0;
-      }
-    }
-    if (consecutiveFailures > 0) {
-      totalRetryChains++;
-      if (consecutiveFailures >= threshold) excessiveRetries++;
-    }
-  }
-  if (totalRetryChains === 0) return { score: 1, detail: "No retry chains" };
-  if (excessiveRetries === 0) return { score: 0.9, detail: `${totalRetryChains} retry chain(s), all resolved quickly` };
-  const score = Math.max(0, 0.8 - excessiveRetries * 0.2);
-  return {
-    score,
-    detail: `${excessiveRetries} excessive retry chain(s) (${threshold}+ failures before success)`
-  };
+	if (calls.length < 2) return {
+		score: 1,
+		detail: "Too few calls"
+	};
+	const groupKey = (call) => {
+		const path = String(call.args.path ?? call.args.command ?? "");
+		return `${call.toolName}:${path}`;
+	};
+	const byGroup = /* @__PURE__ */ new Map();
+	for (const call of calls) {
+		const key = groupKey(call);
+		const arr = byGroup.get(key) ?? [];
+		arr.push(call);
+		byGroup.set(key, arr);
+	}
+	let totalRetryChains = 0;
+	let excessiveRetries = 0;
+	const threshold = THRESHOLDS.retryExcessiveThreshold;
+	for (const [, toolCalls] of byGroup) {
+		let consecutiveFailures = 0;
+		for (const call of toolCalls) if (call.isError) consecutiveFailures++;
+		else {
+			if (consecutiveFailures > 0) {
+				totalRetryChains++;
+				if (consecutiveFailures >= threshold) excessiveRetries++;
+			}
+			consecutiveFailures = 0;
+		}
+		if (consecutiveFailures > 0) {
+			totalRetryChains++;
+			if (consecutiveFailures >= threshold) excessiveRetries++;
+		}
+	}
+	if (totalRetryChains === 0) return {
+		score: 1,
+		detail: "No retry chains"
+	};
+	if (excessiveRetries === 0) return {
+		score: .9,
+		detail: `${totalRetryChains} retry chain(s), all resolved quickly`
+	};
+	return {
+		score: Math.max(0, .8 - excessiveRetries * .2),
+		detail: `${excessiveRetries} excessive retry chain(s) (${threshold}+ failures before success)`
+	};
 }
+/**
+* Score read-before-edit compliance.
+* Only `view` satisfies a read — search/find don't provide specific file content.
+*/
 function scoreReadBeforeEdit(calls) {
-  const editTools = THRESHOLDS.editTools;
-  const readTools = THRESHOLDS.readTools;
-  const edits = calls.filter((c) => editTools.includes(c.toolName));
-  if (edits.length === 0) return { score: 1, detail: "No edits to check" };
-  const readPaths = /* @__PURE__ */ new Set();
-  let compliant = 0;
-  let total = 0;
-  for (const call of calls) {
-    const rawPath = String(call.args.path ?? "");
-    if (!rawPath) continue;
-    if (readTools.includes(call.toolName)) {
-      readPaths.add(rawPath);
-    } else if (editTools.includes(call.toolName)) {
-      total++;
-      if (hasReadPath(readPaths, rawPath)) {
-        compliant++;
-      }
-    }
-  }
-  if (total === 0) return { score: 1, detail: "No path-targeted edits" };
-  const ratio = compliant / total;
-  const violations = total - compliant;
-  if (violations === 0) return { score: 1, detail: `All ${total} edits had prior reads` };
-  return { score: ratio, detail: `${violations}/${total} edits without prior read` };
+	const editTools = THRESHOLDS.editTools;
+	const readTools = THRESHOLDS.readTools;
+	if (calls.filter((c) => editTools.includes(c.toolName)).length === 0) return {
+		score: 1,
+		detail: "No edits to check"
+	};
+	const readPaths = /* @__PURE__ */ new Set();
+	let compliant = 0;
+	let total = 0;
+	for (const call of calls) {
+		const rawPath = String(call.args.path ?? "");
+		if (!rawPath) continue;
+		if (readTools.includes(call.toolName)) readPaths.add(rawPath);
+		else if (editTools.includes(call.toolName)) {
+			total++;
+			if (hasReadPath(readPaths, rawPath)) compliant++;
+		}
+	}
+	if (total === 0) return {
+		score: 1,
+		detail: "No path-targeted edits"
+	};
+	const ratio = compliant / total;
+	const violations = total - compliant;
+	if (violations === 0) return {
+		score: 1,
+		detail: `All ${total} edits had prior reads`
+	};
+	return {
+		score: ratio,
+		detail: `${violations}/${total} edits without prior read`
+	};
 }
+/** Check if any read path matches the edit path, using suffix matching for mixed abs/relative paths. */
 function hasReadPath(readPaths, editPath) {
-  if (readPaths.has(editPath)) return true;
-  for (const rp of readPaths) {
-    if (rp.endsWith(`/${editPath}`) || editPath.endsWith(`/${rp}`)) return true;
-  }
-  return false;
+	if (readPaths.has(editPath)) return true;
+	for (const rp of readPaths) if (rp.endsWith(`/${editPath}`) || editPath.endsWith(`/${rp}`)) return true;
+	return false;
 }
 function createEfficiencyScorer() {
-  return createScorer({
-    id: "mastracode-efficiency",
-    name: "MastraCode Efficiency",
-    description: "Measures coding session efficiency: redundancy, turn count, retry patterns, read-before-edit. 30% sampled.",
-    type: "agent",
-    prepareRun: filterRun({ partTypes: ["tool-invocation", "text"] })
-  }).preprocess(async ({ run }) => {
-    const messages = run.output ?? [];
-    const calls = extractToolCalls(messages);
-    const turns = countAssistantTurns(messages);
-    if (calls.length < THRESHOLDS.minToolCalls) {
-      return { empty: true, totalCalls: calls.length, totalTurns: turns };
-    }
-    const redundancy = scoreRedundancy(calls);
-    const turnCount = scoreTurnCount(turns);
-    const retryEfficiency = scoreRetryEfficiency(calls);
-    const readBeforeEdit = scoreReadBeforeEdit(calls);
-    return {
-      empty: false,
-      redundancy,
-      turnCount,
-      retryEfficiency,
-      readBeforeEdit,
-      totalCalls: calls.length,
-      totalTurns: turns
-    };
-  }).generateScore(({ results }) => {
-    const p = results.preprocessStepResult;
-    if (p.empty) return 0;
-    const score = p.redundancy.score * WEIGHTS.redundancy + p.turnCount.score * WEIGHTS.turnCount + p.retryEfficiency.score * WEIGHTS.retryEfficiency + p.readBeforeEdit.score * WEIGHTS.readBeforeEdit;
-    return Math.round(score * 100) / 100;
-  }).generateReason(({ results, score }) => {
-    const p = results.preprocessStepResult;
-    if (p.empty) {
-      return `Score: 0 (${p.totalCalls} tool calls \u2014 below minimum threshold for scoring)`;
-    }
-    const pct = (w) => `${(w * 100).toFixed(0)}%`;
-    const parts = [`Score: ${score} (${p.totalCalls} calls, ${p.totalTurns} turns)`];
-    parts.push(`Redundancy (${pct(WEIGHTS.redundancy)}): ${p.redundancy.detail} [${p.redundancy.score}]`);
-    parts.push(`Turn count (${pct(WEIGHTS.turnCount)}): ${p.turnCount.detail} [${p.turnCount.score}]`);
-    parts.push(
-      `Retry efficiency (${pct(WEIGHTS.retryEfficiency)}): ${p.retryEfficiency.detail} [${p.retryEfficiency.score}]`
-    );
-    parts.push(
-      `Read-before-edit (${pct(WEIGHTS.readBeforeEdit)}): ${p.readBeforeEdit.detail} [${p.readBeforeEdit.score}]`
-    );
-    return parts.join("\n");
-  });
+	return createScorer({
+		id: "mastracode-efficiency",
+		name: "MastraCode Efficiency",
+		description: "Measures coding session efficiency: redundancy, turn count, retry patterns, read-before-edit. 30% sampled.",
+		type: "agent",
+		prepareRun: filterRun({ partTypes: ["tool-invocation", "text"] })
+	}).preprocess(async ({ run }) => {
+		const messages = run.output ?? [];
+		const calls = extractToolCalls(messages);
+		const turns = countAssistantTurns(messages);
+		if (calls.length < THRESHOLDS.minToolCalls) return {
+			empty: true,
+			totalCalls: calls.length,
+			totalTurns: turns
+		};
+		return {
+			empty: false,
+			redundancy: scoreRedundancy(calls),
+			turnCount: scoreTurnCount(turns),
+			retryEfficiency: scoreRetryEfficiency(calls),
+			readBeforeEdit: scoreReadBeforeEdit(calls),
+			totalCalls: calls.length,
+			totalTurns: turns
+		};
+	}).generateScore(({ results }) => {
+		const p = results.preprocessStepResult;
+		if (p.empty) return 0;
+		const score = p.redundancy.score * WEIGHTS.redundancy + p.turnCount.score * WEIGHTS.turnCount + p.retryEfficiency.score * WEIGHTS.retryEfficiency + p.readBeforeEdit.score * WEIGHTS.readBeforeEdit;
+		return Math.round(score * 100) / 100;
+	}).generateReason(({ results, score }) => {
+		const p = results.preprocessStepResult;
+		if (p.empty) return `Score: 0 (${p.totalCalls} tool calls — below minimum threshold for scoring)`;
+		const pct = (w) => `${(w * 100).toFixed(0)}%`;
+		const parts = [`Score: ${score} (${p.totalCalls} calls, ${p.totalTurns} turns)`];
+		parts.push(`Redundancy (${pct(WEIGHTS.redundancy)}): ${p.redundancy.detail} [${p.redundancy.score}]`);
+		parts.push(`Turn count (${pct(WEIGHTS.turnCount)}): ${p.turnCount.detail} [${p.turnCount.score}]`);
+		parts.push(`Retry efficiency (${pct(WEIGHTS.retryEfficiency)}): ${p.retryEfficiency.detail} [${p.retryEfficiency.score}]`);
+		parts.push(`Read-before-edit (${pct(WEIGHTS.readBeforeEdit)}): ${p.readBeforeEdit.detail} [${p.readBeforeEdit.score}]`);
+		return parts.join("\n");
+	});
 }
-export {
-  createEfficiencyScorer
-};
+//#endregion
+export { createEfficiencyScorer };
+
 //# sourceMappingURL=efficiency.js.map

package/dist/evals/scorers/efficiency.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/evals/scorers/efficiency.ts"],"sourcesContent":["/**\n * MastraCode Efficiency Scorer (Code-based, Agent type, 30% sampling)\n *\n * Measures how efficiently the agent completes work through quantitative ratios.\n * Unlike the Outcome Scorer (which checks *what* happened), this checks\n * *how much work* it took.\n *\n * This scorer produces a score from 0-1 but also exposes raw metrics in the\n * reason text for trend analysis. The score represents \"how cleanly did the\n * agent work?\" — not whether it got the right answer.\n *\n * Sampled at 30% because it's slightly more expensive (analyzes message sequences).\n */\n\nimport type { MastraDBMessage } from '@mastra/core/agent';\nimport { createScorer, filterRun } from '@mastra/core/evals';\n\nimport type { ExtractedToolCall } from './extract-tools.js';\nimport { extractToolCalls } from './extract-tools.js';\n\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n// CONFIGURATION — Adjust weights and thresholds here\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n/**\n * Dimension weights (must sum to 1.0).\n *\n * - redundancy:     Duplicate mutation tool calls with identical args.\n * - turnCount:      Total assistant turns to complete the task.\n * - retryEfficiency: How quickly the agent recovers from tool failures.\n * - readBeforeEdit: Whether files are read before being edited.\n */\nconst WEIGHTS = {\n  redundancy: 0.35,\n  turnCount: 0.3,\n  retryEfficiency: 0.2,\n  readBeforeEdit: 0.15,\n} as const;\n\n/**\n * Scoring thresholds and multipliers.\n */\nconst THRESHOLDS = {\n  /** Redundancy ratio below this is considered acceptable (no penalty). */\n  redundancyAcceptableRate: 0.05,\n\n  /** Penalty multiplier applied to redundancy ratio above the acceptable rate. */\n  redundancyPenaltyMultiplier: 3,\n\n  /** Turns at or below this is considered normal (score 1.0). */\n  turnCountNormalMax: 8,\n\n  /** Penalty per turn beyond the normal maximum (for turns 9–15). */\n  turnCountPenaltyPerTurn: 0.04,\n\n  /** Penalty per turn beyond 15. */\n  turnCountExtendedPenaltyPerTurn: 0.03,\n\n  /** Number of consecutive failures before a retry chain is \"excessive\". */\n  retryExcessiveThreshold: 3,\n\n  /** Tools considered \"edit\" tools that should be preceded by a read. */\n  editTools: ['string_replace_lsp', 'ast_smart_edit'] as string[],\n\n  /** Tools that satisfy read-before-edit (only `view` truly reads a specific file). */\n  readTools: ['view'] as string[],\n\n  /** Tools whose repeated identical calls are intentional (idempotent reads). */\n  redundancyWhitelist: ['view', 'search_content', 'find_files', 'lsp_inspect', 'web_search', 'web_extract'] as string[],\n\n  /** Minimum tool calls required to produce a meaningful score. */\n  minToolCalls: 2,\n} as const;\n\n/**\n * Count assistant turns (messages with role === 'assistant').\n */\nfunction countAssistantTurns(messages: MastraDBMessage[]): number {\n  return messages.filter(m => m.role === 'assistant').length;\n}\n\n/**\n * Calculate redundant call ratio.\n * Only counts mutation/action tools as redundant — reads are whitelisted.\n */\nfunction scoreRedundancy(calls: ExtractedToolCall[]): { score: number; ratio: number; detail: string } {\n  if (calls.length < 2) return { score: 1, ratio: 0, detail: 'Too few calls' };\n\n  // Only consider non-whitelisted (mutation) tools for redundancy\n  const mutationCalls = calls.filter(c => !THRESHOLDS.redundancyWhitelist.includes(c.toolName));\n  if (mutationCalls.length < 2) return { score: 1, ratio: 0, detail: 'No mutation tools to check' };\n\n  const fingerprints = new Map<string, { count: number; successCount: number }>();\n  for (const call of mutationCalls) {\n    const fp = `${call.toolName}:${JSON.stringify(call.args)}`;\n    const existing = fingerprints.get(fp) ?? { count: 0, successCount: 0 };\n    existing.count++;\n    if (!call.isError) existing.successCount++;\n    fingerprints.set(fp, existing);\n  }\n\n  // Only count as redundant if the same successful call happened multiple times\n  let redundantCount = 0;\n  for (const { successCount } of fingerprints.values()) {\n    if (successCount > 1) redundantCount += successCount - 1;\n  }\n\n  const ratio = redundantCount / mutationCalls.length;\n  const acceptable = THRESHOLDS.redundancyAcceptableRate;\n  const score =\n    ratio <= acceptable ? 1 : Math.max(0, 1 - (ratio - acceptable) * THRESHOLDS.redundancyPenaltyMultiplier);\n  return {\n    score,\n    ratio,\n    detail: `${redundantCount}/${mutationCalls.length} redundant mutations (${(ratio * 100).toFixed(0)}%)`,\n  };\n}\n\n/**\n * Score turn count efficiency.\n * Fewer turns to accomplish the task = more efficient.\n */\nfunction scoreTurnCount(turns: number): { score: number; detail: string } {\n  const normalMax = THRESHOLDS.turnCountNormalMax;\n  const penaltyPerTurn = THRESHOLDS.turnCountPenaltyPerTurn;\n  const extendedPenalty = THRESHOLDS.turnCountExtendedPenaltyPerTurn;\n\n  if (turns <= 1) return { score: 1, detail: `${turns} turn` };\n  if (turns <= normalMax) return { score: 1, detail: `${turns} turns (within normal range)` };\n  if (turns <= 15) {\n    const score = 1 - (turns - normalMax) * penaltyPerTurn;\n    return { score, detail: `${turns} turns (slightly extended)` };\n  }\n\n  const midPenalty = (15 - normalMax) * penaltyPerTurn;\n  const score = Math.max(0, 1 - midPenalty - (turns - 15) * extendedPenalty);\n  return { score, detail: `${turns} turns (extended session)` };\n}\n\n/**\n * Score failed-then-succeeded ratio.\n * Groups retry chains by tool name + target file path for more accurate detection.\n */\nfunction scoreRetryEfficiency(calls: ExtractedToolCall[]): { score: number; detail: string } {\n  if (calls.length < 2) return { score: 1, detail: 'Too few calls' };\n\n  // Group calls by tool name + target file path (if available)\n  const groupKey = (call: ExtractedToolCall): string => {\n    const path = String(call.args.path ?? call.args.command ?? '');\n    return `${call.toolName}:${path}`;\n  };\n\n  const byGroup = new Map<string, ExtractedToolCall[]>();\n  for (const call of calls) {\n    const key = groupKey(call);\n    const arr = byGroup.get(key) ?? [];\n    arr.push(call);\n    byGroup.set(key, arr);\n  }\n\n  let totalRetryChains = 0;\n  let excessiveRetries = 0;\n  const threshold = THRESHOLDS.retryExcessiveThreshold;\n\n  for (const [, toolCalls] of byGroup) {\n    let consecutiveFailures = 0;\n    for (const call of toolCalls) {\n      if (call.isError) {\n        consecutiveFailures++;\n      } else {\n        if (consecutiveFailures > 0) {\n          totalRetryChains++;\n          if (consecutiveFailures >= threshold) excessiveRetries++;\n        }\n        consecutiveFailures = 0;\n      }\n    }\n    // Count unrecovered retry chains (failures at end of group with no success)\n    if (consecutiveFailures > 0) {\n      totalRetryChains++;\n      if (consecutiveFailures >= threshold) excessiveRetries++;\n    }\n  }\n\n  if (totalRetryChains === 0) return { score: 1, detail: 'No retry chains' };\n  if (excessiveRetries === 0) return { score: 0.9, detail: `${totalRetryChains} retry chain(s), all resolved quickly` };\n\n  const score = Math.max(0, 0.8 - excessiveRetries * 0.2);\n  return {\n    score,\n    detail: `${excessiveRetries} excessive retry chain(s) (${threshold}+ failures before success)`,\n  };\n}\n\n/**\n * Score read-before-edit compliance.\n * Only `view` satisfies a read — search/find don't provide specific file content.\n */\nfunction scoreReadBeforeEdit(calls: ExtractedToolCall[]): { score: number; detail: string } {\n  const editTools = THRESHOLDS.editTools;\n  const readTools = THRESHOLDS.readTools;\n\n  const edits = calls.filter(c => editTools.includes(c.toolName));\n  if (edits.length === 0) return { score: 1, detail: 'No edits to check' };\n\n  // Build set of files that were read before each edit.\n  // Use suffix-normalized paths so that \"/abs/path/src/foo.ts\" and \"src/foo.ts\" match.\n  const readPaths = new Set<string>();\n  let compliant = 0;\n  let total = 0;\n\n  for (const call of calls) {\n    const rawPath = String(call.args.path ?? '');\n    if (!rawPath) continue;\n\n    if (readTools.includes(call.toolName)) {\n      readPaths.add(rawPath);\n    } else if (editTools.includes(call.toolName)) {\n      total++;\n      if (hasReadPath(readPaths, rawPath)) {\n        compliant++;\n      }\n    }\n  }\n\n  if (total === 0) return { score: 1, detail: 'No path-targeted edits' };\n\n  const ratio = compliant / total;\n  const violations = total - compliant;\n  if (violations === 0) return { score: 1, detail: `All ${total} edits had prior reads` };\n  return { score: ratio, detail: `${violations}/${total} edits without prior read` };\n}\n\n/** Check if any read path matches the edit path, using suffix matching for mixed abs/relative paths. */\nfunction hasReadPath(readPaths: Set<string>, editPath: string): boolean {\n  if (readPaths.has(editPath)) return true;\n  for (const rp of readPaths) {\n    if (rp.endsWith(`/${editPath}`) || editPath.endsWith(`/${rp}`)) return true;\n  }\n  return false;\n}\n\nexport function createEfficiencyScorer() {\n  return createScorer({\n    id: 'mastracode-efficiency',\n    name: 'MastraCode Efficiency',\n    description:\n      'Measures coding session efficiency: redundancy, turn count, retry patterns, read-before-edit. 30% sampled.',\n    type: 'agent',\n    prepareRun: filterRun({ partTypes: ['tool-invocation', 'text'] }),\n  })\n    .preprocess(async ({ run }) => {\n      const messages = (run.output ?? []) as MastraDBMessage[];\n      const calls = extractToolCalls(messages);\n      const turns = countAssistantTurns(messages);\n\n      // H1: Empty sessions → score 0\n      if (calls.length < THRESHOLDS.minToolCalls) {\n        return { empty: true as const, totalCalls: calls.length, totalTurns: turns };\n      }\n\n      const redundancy = scoreRedundancy(calls);\n      const turnCount = scoreTurnCount(turns);\n      const retryEfficiency = scoreRetryEfficiency(calls);\n      const readBeforeEdit = scoreReadBeforeEdit(calls);\n\n      return {\n        empty: false as const,\n        redundancy,\n        turnCount,\n        retryEfficiency,\n        readBeforeEdit,\n        totalCalls: calls.length,\n        totalTurns: turns,\n      };\n    })\n    .generateScore(({ results }) => {\n      const p = results.preprocessStepResult;\n      if (p.empty) return 0;\n\n      const score =\n        p.redundancy.score * WEIGHTS.redundancy +\n        p.turnCount.score * WEIGHTS.turnCount +\n        p.retryEfficiency.score * WEIGHTS.retryEfficiency +\n        p.readBeforeEdit.score * WEIGHTS.readBeforeEdit;\n\n      return Math.round(score * 100) / 100;\n    })\n    .generateReason(({ results, score }) => {\n      const p = results.preprocessStepResult;\n      if (p.empty) {\n        return `Score: 0 (${p.totalCalls} tool calls — below minimum threshold for scoring)`;\n      }\n\n      const pct = (w: number) => `${(w * 100).toFixed(0)}%`;\n      const parts: string[] = [`Score: ${score} (${p.totalCalls} calls, ${p.totalTurns} turns)`];\n\n      parts.push(`Redundancy (${pct(WEIGHTS.redundancy)}): ${p.redundancy.detail} [${p.redundancy.score}]`);\n      parts.push(`Turn count (${pct(WEIGHTS.turnCount)}): ${p.turnCount.detail} [${p.turnCount.score}]`);\n      parts.push(\n        `Retry efficiency (${pct(WEIGHTS.retryEfficiency)}): ${p.retryEfficiency.detail} [${p.retryEfficiency.score}]`,\n      );\n      parts.push(\n        `Read-before-edit (${pct(WEIGHTS.readBeforeEdit)}): ${p.readBeforeEdit.detail} [${p.readBeforeEdit.score}]`,\n      );\n\n      return parts.join('\\n');\n    });\n}\n"],"mappings":"AAeA,SAAS,cAAc,iBAAiB;AAGxC,SAAS,wBAAwB;AAcjC,MAAM,UAAU;AAAA,EACd,YAAY;AAAA,EACZ,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,gBAAgB;AAClB;AAKA,MAAM,aAAa;AAAA;AAAA,EAEjB,0BAA0B;AAAA;AAAA,EAG1B,6BAA6B;AAAA;AAAA,EAG7B,oBAAoB;AAAA;AAAA,EAGpB,yBAAyB;AAAA;AAAA,EAGzB,iCAAiC;AAAA;AAAA,EAGjC,yBAAyB;AAAA;AAAA,EAGzB,WAAW,CAAC,sBAAsB,gBAAgB;AAAA;AAAA,EAGlD,WAAW,CAAC,MAAM;AAAA;AAAA,EAGlB,qBAAqB,CAAC,QAAQ,kBAAkB,cAAc,eAAe,cAAc,aAAa;AAAA;AAAA,EAGxG,cAAc;AAChB;AAKA,SAAS,oBAAoB,UAAqC;AAChE,SAAO,SAAS,OAAO,OAAK,EAAE,SAAS,WAAW,EAAE;AACtD;AAMA,SAAS,gBAAgB,OAA8E;AACrG,MAAI,MAAM,SAAS,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,QAAQ,gBAAgB;AAG3E,QAAM,gBAAgB,MAAM,OAAO,OAAK,CAAC,WAAW,oBAAoB,SAAS,EAAE,QAAQ,CAAC;AAC5F,MAAI,cAAc,SAAS,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,QAAQ,6BAA6B;AAEhG,QAAM,eAAe,oBAAI,IAAqD;AAC9E,aAAW,QAAQ,eAAe;AAChC,UAAM,KAAK,GAAG,KAAK,QAAQ,IAAI,KAAK,UAAU,KAAK,IAAI,CAAC;AACxD,UAAM,WAAW,aAAa,IAAI,EAAE,KAAK,EAAE,OAAO,GAAG,cAAc,EAAE;AACrE,aAAS;AACT,QAAI,CAAC,KAAK,QAAS,UAAS;AAC5B,iBAAa,IAAI,IAAI,QAAQ;AAAA,EAC/B;AAGA,MAAI,iBAAiB;AACrB,aAAW,EAAE,aAAa,KAAK,aAAa,OAAO,GAAG;AACpD,QAAI,eAAe,EAAG,mBAAkB,eAAe;AAAA,EACzD;AAEA,QAAM,QAAQ,iBAAiB,cAAc;AAC7C,QAAM,aAAa,WAAW;AAC9B,QAAM,QACJ,SAAS,aAAa,IAAI,KAAK,IAAI,GAAG,KAAK,QAAQ,cAAc,WAAW,2BAA2B;AACzG,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,QAAQ,GAAG,cAAc,IAAI,cAAc,MAAM,0BAA0B,QAAQ,KAAK,QAAQ,CAAC,CAAC;AAAA,EACpG;AACF;AAMA,SAAS,eAAe,OAAkD;AACxE,QAAM,YAAY,WAAW;AAC7B,QAAM,iBAAiB,WAAW;AAClC,QAAM,kBAAkB,WAAW;AAEnC,MAAI,SAAS,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,GAAG,KAAK,QAAQ;AAC3D,MAAI,SAAS,UAAW,QAAO,EAAE,OAAO,GAAG,QAAQ,GAAG,KAAK,+BAA+B;AAC1F,MAAI,SAAS,IAAI;AACf,UAAMA,SAAQ,KAAK,QAAQ,aAAa;AACxC,WAAO,EAAE,OAAAA,QAAO,QAAQ,GAAG,KAAK,6BAA6B;AAAA,EAC/D;AAEA,QAAM,cAAc,KAAK,aAAa;AACtC,QAAM,QAAQ,KAAK,IAAI,GAAG,IAAI,cAAc,QAAQ,MAAM,eAAe;AACzE,SAAO,EAAE,OAAO,QAAQ,GAAG,KAAK,4BAA4B;AAC9D;AAMA,SAAS,qBAAqB,OAA+D;AAC3F,MAAI,MAAM,SAAS,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,gBAAgB;AAGjE,QAAM,WAAW,CAAC,SAAoC;AACpD,UAAM,OAAO,OAAO,KAAK,KAAK,QAAQ,KAAK,KAAK,WAAW,EAAE;AAC7D,WAAO,GAAG,KAAK,QAAQ,IAAI,IAAI;AAAA,EACjC;AAEA,QAAM,UAAU,oBAAI,IAAiC;AACrD,aAAW,QAAQ,OAAO;AACxB,UAAM,MAAM,SAAS,IAAI;AACzB,UAAM,MAAM,QAAQ,IAAI,GAAG,KAAK,CAAC;AACjC,QAAI,KAAK,IAAI;AACb,YAAQ,IAAI,KAAK,GAAG;AAAA,EACtB;AAEA,MAAI,mBAAmB;AACvB,MAAI,mBAAmB;AACvB,QAAM,YAAY,WAAW;AAE7B,aAAW,CAAC,EAAE,SAAS,KAAK,SAAS;AACnC,QAAI,sBAAsB;AAC1B,eAAW,QAAQ,WAAW;AAC5B,UAAI,KAAK,SAAS;AAChB;AAAA,MACF,OAAO;AACL,YAAI,sBAAsB,GAAG;AAC3B;AACA,cAAI,uBAAuB,UAAW;AAAA,QACxC;AACA,8BAAsB;AAAA,MACxB;AAAA,IACF;AAEA,QAAI,sBAAsB,GAAG;AAC3B;AACA,UAAI,uBAAuB,UAAW;AAAA,IACxC;AAAA,EACF;AAEA,MAAI,qBAAqB,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,kBAAkB;AACzE,MAAI,qBAAqB,EAAG,QAAO,EAAE,OAAO,KAAK,QAAQ,GAAG,gBAAgB,wCAAwC;AAEpH,QAAM,QAAQ,KAAK,IAAI,GAAG,MAAM,mBAAmB,GAAG;AACtD,SAAO;AAAA,IACL;AAAA,IACA,QAAQ,GAAG,gBAAgB,8BAA8B,SAAS;AAAA,EACpE;AACF;AAMA,SAAS,oBAAoB,OAA+D;AAC1F,QAAM,YAAY,WAAW;AAC7B,QAAM,YAAY,WAAW;AAE7B,QAAM,QAAQ,MAAM,OAAO,OAAK,UAAU,SAAS,EAAE,QAAQ,CAAC;AAC9D,MAAI,MAAM,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,oBAAoB;AAIvE,QAAM,YAAY,oBAAI,IAAY;AAClC,MAAI,YAAY;AAChB,MAAI,QAAQ;AAEZ,aAAW,QAAQ,OAAO;AACxB,UAAM,UAAU,OAAO,KAAK,KAAK,QAAQ,EAAE;AAC3C,QAAI,CAAC,QAAS;AAEd,QAAI,UAAU,SAAS,KAAK,QAAQ,GAAG;AACrC,gBAAU,IAAI,OAAO;AAAA,IACvB,WAAW,UAAU,SAAS,KAAK,QAAQ,GAAG;AAC5C;AACA,UAAI,YAAY,WAAW,OAAO,GAAG;AACnC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,MAAI,UAAU,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,yBAAyB;AAErE,QAAM,QAAQ,YAAY;AAC1B,QAAM,aAAa,QAAQ;AAC3B,MAAI,eAAe,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,OAAO,KAAK,yBAAyB;AACtF,SAAO,EAAE,OAAO,OAAO,QAAQ,GAAG,UAAU,IAAI,KAAK,4BAA4B;AACnF;AAGA,SAAS,YAAY,WAAwB,UAA2B;AACtE,MAAI,UAAU,IAAI,QAAQ,EAAG,QAAO;AACpC,aAAW,MAAM,WAAW;AAC1B,QAAI,GAAG,SAAS,IAAI,QAAQ,EAAE,KAAK,SAAS,SAAS,IAAI,EAAE,EAAE,EAAG,QAAO;AAAA,EACzE;AACA,SAAO;AACT;AAEO,SAAS,yBAAyB;AACvC,SAAO,aAAa;AAAA,IAClB,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,aACE;AAAA,IACF,MAAM;AAAA,IACN,YAAY,UAAU,EAAE,WAAW,CAAC,mBAAmB,MAAM,EAAE,CAAC;AAAA,EAClE,CAAC,EACE,WAAW,OAAO,EAAE,IAAI,MAAM;AAC7B,UAAM,WAAY,IAAI,UAAU,CAAC;AACjC,UAAM,QAAQ,iBAAiB,QAAQ;AACvC,UAAM,QAAQ,oBAAoB,QAAQ;AAG1C,QAAI,MAAM,SAAS,WAAW,cAAc;AAC1C,aAAO,EAAE,OAAO,MAAe,YAAY,MAAM,QAAQ,YAAY,MAAM;AAAA,IAC7E;AAEA,UAAM,aAAa,gBAAgB,KAAK;AACxC,UAAM,YAAY,eAAe,KAAK;AACtC,UAAM,kBAAkB,qBAAqB,KAAK;AAClD,UAAM,iBAAiB,oBAAoB,KAAK;AAEhD,WAAO;AAAA,MACL,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY,MAAM;AAAA,MAClB,YAAY;AAAA,IACd;AAAA,EACF,CAAC,EACA,cAAc,CAAC,EAAE,QAAQ,MAAM;AAC9B,UAAM,IAAI,QAAQ;AAClB,QAAI,EAAE,MAAO,QAAO;AAEpB,UAAM,QACJ,EAAE,WAAW,QAAQ,QAAQ,aAC7B,EAAE,UAAU,QAAQ,QAAQ,YAC5B,EAAE,gBAAgB,QAAQ,QAAQ,kBAClC,EAAE,eAAe,QAAQ,QAAQ;AAEnC,WAAO,KAAK,MAAM,QAAQ,GAAG,IAAI;AAAA,EACnC,CAAC,EACA,eAAe,CAAC,EAAE,SAAS,MAAM,MAAM;AACtC,UAAM,IAAI,QAAQ;AAClB,QAAI,EAAE,OAAO;AACX,aAAO,aAAa,EAAE,UAAU;AAAA,IAClC;AAEA,UAAM,MAAM,CAAC,MAAc,IAAI,IAAI,KAAK,QAAQ,CAAC,CAAC;AAClD,UAAM,QAAkB,CAAC,UAAU,KAAK,KAAK,EAAE,UAAU,WAAW,EAAE,UAAU,SAAS;AAEzF,UAAM,KAAK,eAAe,IAAI,QAAQ,UAAU,CAAC,MAAM,EAAE,WAAW,MAAM,KAAK,EAAE,WAAW,KAAK,GAAG;AACpG,UAAM,KAAK,eAAe,IAAI,QAAQ,SAAS,CAAC,MAAM,EAAE,UAAU,MAAM,KAAK,EAAE,UAAU,KAAK,GAAG;AACjG,UAAM;AAAA,MACJ,qBAAqB,IAAI,QAAQ,eAAe,CAAC,MAAM,EAAE,gBAAgB,MAAM,KAAK,EAAE,gBAAgB,KAAK;AAAA,IAC7G;AACA,UAAM;AAAA,MACJ,qBAAqB,IAAI,QAAQ,cAAc,CAAC,MAAM,EAAE,eAAe,MAAM,KAAK,EAAE,eAAe,KAAK;AAAA,IAC1G;AAEA,WAAO,MAAM,KAAK,IAAI;AAAA,EACxB,CAAC;AACL;","names":["score"]}
+{"version":3,"file":"efficiency.js","names":[],"sources":["../../../src/evals/scorers/efficiency.ts"],"sourcesContent":["/**\n * MastraCode Efficiency Scorer (Code-based, Agent type, 30% sampling)\n *\n * Measures how efficiently the agent completes work through quantitative ratios.\n * Unlike the Outcome Scorer (which checks *what* happened), this checks\n * *how much work* it took.\n *\n * This scorer produces a score from 0-1 but also exposes raw metrics in the\n * reason text for trend analysis. The score represents \"how cleanly did the\n * agent work?\" — not whether it got the right answer.\n *\n * Sampled at 30% because it's slightly more expensive (analyzes message sequences).\n */\n\nimport type { MastraDBMessage } from '@mastra/core/agent';\nimport { createScorer, filterRun } from '@mastra/core/evals';\n\nimport type { ExtractedToolCall } from './extract-tools.js';\nimport { extractToolCalls } from './extract-tools.js';\n\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n// CONFIGURATION — Adjust weights and thresholds here\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n/**\n * Dimension weights (must sum to 1.0).\n *\n * - redundancy:     Duplicate mutation tool calls with identical args.\n * - turnCount:      Total assistant turns to complete the task.\n * - retryEfficiency: How quickly the agent recovers from tool failures.\n * - readBeforeEdit: Whether files are read before being edited.\n */\nconst WEIGHTS = {\n  redundancy: 0.35,\n  turnCount: 0.3,\n  retryEfficiency: 0.2,\n  readBeforeEdit: 0.15,\n} as const;\n\n/**\n * Scoring thresholds and multipliers.\n */\nconst THRESHOLDS = {\n  /** Redundancy ratio below this is considered acceptable (no penalty). */\n  redundancyAcceptableRate: 0.05,\n\n  /** Penalty multiplier applied to redundancy ratio above the acceptable rate. */\n  redundancyPenaltyMultiplier: 3,\n\n  /** Turns at or below this is considered normal (score 1.0). */\n  turnCountNormalMax: 8,\n\n  /** Penalty per turn beyond the normal maximum (for turns 9–15). */\n  turnCountPenaltyPerTurn: 0.04,\n\n  /** Penalty per turn beyond 15. */\n  turnCountExtendedPenaltyPerTurn: 0.03,\n\n  /** Number of consecutive failures before a retry chain is \"excessive\". */\n  retryExcessiveThreshold: 3,\n\n  /** Tools considered \"edit\" tools that should be preceded by a read. */\n  editTools: ['string_replace_lsp', 'ast_smart_edit'] as string[],\n\n  /** Tools that satisfy read-before-edit (only `view` truly reads a specific file). */\n  readTools: ['view'] as string[],\n\n  /** Tools whose repeated identical calls are intentional (idempotent reads). */\n  redundancyWhitelist: ['view', 'search_content', 'find_files', 'lsp_inspect', 'web_search', 'web_extract'] as string[],\n\n  /** Minimum tool calls required to produce a meaningful score. */\n  minToolCalls: 2,\n} as const;\n\n/**\n * Count assistant turns (messages with role === 'assistant').\n */\nfunction countAssistantTurns(messages: MastraDBMessage[]): number {\n  return messages.filter(m => m.role === 'assistant').length;\n}\n\n/**\n * Calculate redundant call ratio.\n * Only counts mutation/action tools as redundant — reads are whitelisted.\n */\nfunction scoreRedundancy(calls: ExtractedToolCall[]): { score: number; ratio: number; detail: string } {\n  if (calls.length < 2) return { score: 1, ratio: 0, detail: 'Too few calls' };\n\n  // Only consider non-whitelisted (mutation) tools for redundancy\n  const mutationCalls = calls.filter(c => !THRESHOLDS.redundancyWhitelist.includes(c.toolName));\n  if (mutationCalls.length < 2) return { score: 1, ratio: 0, detail: 'No mutation tools to check' };\n\n  const fingerprints = new Map<string, { count: number; successCount: number }>();\n  for (const call of mutationCalls) {\n    const fp = `${call.toolName}:${JSON.stringify(call.args)}`;\n    const existing = fingerprints.get(fp) ?? { count: 0, successCount: 0 };\n    existing.count++;\n    if (!call.isError) existing.successCount++;\n    fingerprints.set(fp, existing);\n  }\n\n  // Only count as redundant if the same successful call happened multiple times\n  let redundantCount = 0;\n  for (const { successCount } of fingerprints.values()) {\n    if (successCount > 1) redundantCount += successCount - 1;\n  }\n\n  const ratio = redundantCount / mutationCalls.length;\n  const acceptable = THRESHOLDS.redundancyAcceptableRate;\n  const score =\n    ratio <= acceptable ? 1 : Math.max(0, 1 - (ratio - acceptable) * THRESHOLDS.redundancyPenaltyMultiplier);\n  return {\n    score,\n    ratio,\n    detail: `${redundantCount}/${mutationCalls.length} redundant mutations (${(ratio * 100).toFixed(0)}%)`,\n  };\n}\n\n/**\n * Score turn count efficiency.\n * Fewer turns to accomplish the task = more efficient.\n */\nfunction scoreTurnCount(turns: number): { score: number; detail: string } {\n  const normalMax = THRESHOLDS.turnCountNormalMax;\n  const penaltyPerTurn = THRESHOLDS.turnCountPenaltyPerTurn;\n  const extendedPenalty = THRESHOLDS.turnCountExtendedPenaltyPerTurn;\n\n  if (turns <= 1) return { score: 1, detail: `${turns} turn` };\n  if (turns <= normalMax) return { score: 1, detail: `${turns} turns (within normal range)` };\n  if (turns <= 15) {\n    const score = 1 - (turns - normalMax) * penaltyPerTurn;\n    return { score, detail: `${turns} turns (slightly extended)` };\n  }\n\n  const midPenalty = (15 - normalMax) * penaltyPerTurn;\n  const score = Math.max(0, 1 - midPenalty - (turns - 15) * extendedPenalty);\n  return { score, detail: `${turns} turns (extended session)` };\n}\n\n/**\n * Score failed-then-succeeded ratio.\n * Groups retry chains by tool name + target file path for more accurate detection.\n */\nfunction scoreRetryEfficiency(calls: ExtractedToolCall[]): { score: number; detail: string } {\n  if (calls.length < 2) return { score: 1, detail: 'Too few calls' };\n\n  // Group calls by tool name + target file path (if available)\n  const groupKey = (call: ExtractedToolCall): string => {\n    const path = String(call.args.path ?? call.args.command ?? '');\n    return `${call.toolName}:${path}`;\n  };\n\n  const byGroup = new Map<string, ExtractedToolCall[]>();\n  for (const call of calls) {\n    const key = groupKey(call);\n    const arr = byGroup.get(key) ?? [];\n    arr.push(call);\n    byGroup.set(key, arr);\n  }\n\n  let totalRetryChains = 0;\n  let excessiveRetries = 0;\n  const threshold = THRESHOLDS.retryExcessiveThreshold;\n\n  for (const [, toolCalls] of byGroup) {\n    let consecutiveFailures = 0;\n    for (const call of toolCalls) {\n      if (call.isError) {\n        consecutiveFailures++;\n      } else {\n        if (consecutiveFailures > 0) {\n          totalRetryChains++;\n          if (consecutiveFailures >= threshold) excessiveRetries++;\n        }\n        consecutiveFailures = 0;\n      }\n    }\n    // Count unrecovered retry chains (failures at end of group with no success)\n    if (consecutiveFailures > 0) {\n      totalRetryChains++;\n      if (consecutiveFailures >= threshold) excessiveRetries++;\n    }\n  }\n\n  if (totalRetryChains === 0) return { score: 1, detail: 'No retry chains' };\n  if (excessiveRetries === 0) return { score: 0.9, detail: `${totalRetryChains} retry chain(s), all resolved quickly` };\n\n  const score = Math.max(0, 0.8 - excessiveRetries * 0.2);\n  return {\n    score,\n    detail: `${excessiveRetries} excessive retry chain(s) (${threshold}+ failures before success)`,\n  };\n}\n\n/**\n * Score read-before-edit compliance.\n * Only `view` satisfies a read — search/find don't provide specific file content.\n */\nfunction scoreReadBeforeEdit(calls: ExtractedToolCall[]): { score: number; detail: string } {\n  const editTools = THRESHOLDS.editTools;\n  const readTools = THRESHOLDS.readTools;\n\n  const edits = calls.filter(c => editTools.includes(c.toolName));\n  if (edits.length === 0) return { score: 1, detail: 'No edits to check' };\n\n  // Build set of files that were read before each edit.\n  // Use suffix-normalized paths so that \"/abs/path/src/foo.ts\" and \"src/foo.ts\" match.\n  const readPaths = new Set<string>();\n  let compliant = 0;\n  let total = 0;\n\n  for (const call of calls) {\n    const rawPath = String(call.args.path ?? '');\n    if (!rawPath) continue;\n\n    if (readTools.includes(call.toolName)) {\n      readPaths.add(rawPath);\n    } else if (editTools.includes(call.toolName)) {\n      total++;\n      if (hasReadPath(readPaths, rawPath)) {\n        compliant++;\n      }\n    }\n  }\n\n  if (total === 0) return { score: 1, detail: 'No path-targeted edits' };\n\n  const ratio = compliant / total;\n  const violations = total - compliant;\n  if (violations === 0) return { score: 1, detail: `All ${total} edits had prior reads` };\n  return { score: ratio, detail: `${violations}/${total} edits without prior read` };\n}\n\n/** Check if any read path matches the edit path, using suffix matching for mixed abs/relative paths. */\nfunction hasReadPath(readPaths: Set<string>, editPath: string): boolean {\n  if (readPaths.has(editPath)) return true;\n  for (const rp of readPaths) {\n    if (rp.endsWith(`/${editPath}`) || editPath.endsWith(`/${rp}`)) return true;\n  }\n  return false;\n}\n\nexport function createEfficiencyScorer() {\n  return createScorer({\n    id: 'mastracode-efficiency',\n    name: 'MastraCode Efficiency',\n    description:\n      'Measures coding session efficiency: redundancy, turn count, retry patterns, read-before-edit. 30% sampled.',\n    type: 'agent',\n    prepareRun: filterRun({ partTypes: ['tool-invocation', 'text'] }),\n  })\n    .preprocess(async ({ run }) => {\n      const messages = (run.output ?? []) as MastraDBMessage[];\n      const calls = extractToolCalls(messages);\n      const turns = countAssistantTurns(messages);\n\n      // H1: Empty sessions → score 0\n      if (calls.length < THRESHOLDS.minToolCalls) {\n        return { empty: true as const, totalCalls: calls.length, totalTurns: turns };\n      }\n\n      const redundancy = scoreRedundancy(calls);\n      const turnCount = scoreTurnCount(turns);\n      const retryEfficiency = scoreRetryEfficiency(calls);\n      const readBeforeEdit = scoreReadBeforeEdit(calls);\n\n      return {\n        empty: false as const,\n        redundancy,\n        turnCount,\n        retryEfficiency,\n        readBeforeEdit,\n        totalCalls: calls.length,\n        totalTurns: turns,\n      };\n    })\n    .generateScore(({ results }) => {\n      const p = results.preprocessStepResult;\n      if (p.empty) return 0;\n\n      const score =\n        p.redundancy.score * WEIGHTS.redundancy +\n        p.turnCount.score * WEIGHTS.turnCount +\n        p.retryEfficiency.score * WEIGHTS.retryEfficiency +\n        p.readBeforeEdit.score * WEIGHTS.readBeforeEdit;\n\n      return Math.round(score * 100) / 100;\n    })\n    .generateReason(({ results, score }) => {\n      const p = results.preprocessStepResult;\n      if (p.empty) {\n        return `Score: 0 (${p.totalCalls} tool calls — below minimum threshold for scoring)`;\n      }\n\n      const pct = (w: number) => `${(w * 100).toFixed(0)}%`;\n      const parts: string[] = [`Score: ${score} (${p.totalCalls} calls, ${p.totalTurns} turns)`];\n\n      parts.push(`Redundancy (${pct(WEIGHTS.redundancy)}): ${p.redundancy.detail} [${p.redundancy.score}]`);\n      parts.push(`Turn count (${pct(WEIGHTS.turnCount)}): ${p.turnCount.detail} [${p.turnCount.score}]`);\n      parts.push(\n        `Retry efficiency (${pct(WEIGHTS.retryEfficiency)}): ${p.retryEfficiency.detail} [${p.retryEfficiency.score}]`,\n      );\n      parts.push(\n        `Read-before-edit (${pct(WEIGHTS.readBeforeEdit)}): ${p.readBeforeEdit.detail} [${p.readBeforeEdit.score}]`,\n      );\n\n      return parts.join('\\n');\n    });\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAM,UAAU;CACd,YAAY;CACZ,WAAW;CACX,iBAAiB;CACjB,gBAAgB;AAClB;;;;AAKA,MAAM,aAAa;;CAEjB,0BAA0B;;CAG1B,6BAA6B;;CAG7B,oBAAoB;;CAGpB,yBAAyB;;CAGzB,iCAAiC;;CAGjC,yBAAyB;;CAGzB,WAAW,CAAC,sBAAsB,gBAAgB;;CAGlD,WAAW,CAAC,MAAM;;CAGlB,qBAAqB;EAAC;EAAQ;EAAkB;EAAc;EAAe;EAAc;CAAa;;CAGxG,cAAc;AAChB;;;;AAKA,SAAS,oBAAoB,UAAqC;CAChE,OAAO,SAAS,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CAAC;AACtD;;;;;AAMA,SAAS,gBAAgB,OAA8E;CACrG,IAAI,MAAM,SAAS,GAAG,OAAO;EAAE,OAAO;EAAG,OAAO;EAAG,QAAQ;CAAgB;CAG3E,MAAM,gBAAgB,MAAM,QAAO,MAAK,CAAC,WAAW,oBAAoB,SAAS,EAAE,QAAQ,CAAC;CAC5F,IAAI,cAAc,SAAS,GAAG,OAAO;EAAE,OAAO;EAAG,OAAO;EAAG,QAAQ;CAA6B;CAEhG,MAAM,+BAAe,IAAI,IAAqD;CAC9E,KAAK,MAAM,QAAQ,eAAe;EAChC,MAAM,KAAK,GAAG,KAAK,SAAS,GAAG,KAAK,UAAU,KAAK,IAAI;EACvD,MAAM,WAAW,aAAa,IAAI,EAAE,KAAK;GAAE,OAAO;GAAG,cAAc;EAAE;EACrE,SAAS;EACT,IAAI,CAAC,KAAK,SAAS,SAAS;EAC5B,aAAa,IAAI,IAAI,QAAQ;CAC/B;CAGA,IAAI,iBAAiB;CACrB,KAAK,MAAM,EAAE,kBAAkB,aAAa,OAAO,GACjD,IAAI,eAAe,GAAG,kBAAkB,eAAe;CAGzD,MAAM,QAAQ,iBAAiB,cAAc;CAC7C,MAAM,aAAa,WAAW;CAG9B,OAAO;EACL,OAFA,SAAS,aAAa,IAAI,KAAK,IAAI,GAAG,KAAK,QAAQ,cAAc,WAAW,2BAA2B;EAGvG;EACA,QAAQ,GAAG,eAAe,GAAG,cAAc,OAAO,yBAAyB,QAAQ,IAAA,CAAK,QAAQ,CAAC,EAAE;CACrG;AACF;;;;;AAMA,SAAS,eAAe,OAAkD;CACxE,MAAM,YAAY,WAAW;CAC7B,MAAM,iBAAiB,WAAW;CAClC,MAAM,kBAAkB,WAAW;CAEnC,IAAI,SAAS,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ,GAAG,MAAM;CAAO;CAC3D,IAAI,SAAS,WAAW,OAAO;EAAE,OAAO;EAAG,QAAQ,GAAG,MAAM;CAA8B;CAC1F,IAAI,SAAS,IAEX,OAAO;EAAE,OADK,KAAK,QAAQ,aAAa;EACxB,QAAQ,GAAG,MAAM;CAA4B;CAG/D,MAAM,cAAc,KAAK,aAAa;CAEtC,OAAO;EAAE,OADK,KAAK,IAAI,GAAG,IAAI,cAAc,QAAQ,MAAM,eAC7C;EAAG,QAAQ,GAAG,MAAM;CAA2B;AAC9D;;;;;AAMA,SAAS,qBAAqB,OAA+D;CAC3F,IAAI,MAAM,SAAS,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ;CAAgB;CAGjE,MAAM,YAAY,SAAoC;EACpD,MAAM,OAAO,OAAO,KAAK,KAAK,QAAQ,KAAK,KAAK,WAAW,EAAE;EAC7D,OAAO,GAAG,KAAK,SAAS,GAAG;CAC7B;CAEA,MAAM,0BAAU,IAAI,IAAiC;CACrD,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,MAAM,SAAS,IAAI;EACzB,MAAM,MAAM,QAAQ,IAAI,GAAG,KAAK,CAAC;EACjC,IAAI,KAAK,IAAI;EACb,QAAQ,IAAI,KAAK,GAAG;CACtB;CAEA,IAAI,mBAAmB;CACvB,IAAI,mBAAmB;CACvB,MAAM,YAAY,WAAW;CAE7B,KAAK,MAAM,GAAG,cAAc,SAAS;EACnC,IAAI,sBAAsB;EAC1B,KAAK,MAAM,QAAQ,WACjB,IAAI,KAAK,SACP;OACK;GACL,IAAI,sBAAsB,GAAG;IAC3B;IACA,IAAI,uBAAuB,WAAW;GACxC;GACA,sBAAsB;EACxB;EAGF,IAAI,sBAAsB,GAAG;GAC3B;GACA,IAAI,uBAAuB,WAAW;EACxC;CACF;CAEA,IAAI,qBAAqB,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ;CAAkB;CACzE,IAAI,qBAAqB,GAAG,OAAO;EAAE,OAAO;EAAK,QAAQ,GAAG,iBAAiB;CAAuC;CAGpH,OAAO;EACL,OAFY,KAAK,IAAI,GAAG,KAAM,mBAAmB,EAE7C;EACJ,QAAQ,GAAG,iBAAiB,6BAA6B,UAAU;CACrE;AACF;;;;;AAMA,SAAS,oBAAoB,OAA+D;CAC1F,MAAM,YAAY,WAAW;CAC7B,MAAM,YAAY,WAAW;CAG7B,IADc,MAAM,QAAO,MAAK,UAAU,SAAS,EAAE,QAAQ,CACrD,CAAC,CAAC,WAAW,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ;CAAoB;CAIvE,MAAM,4BAAY,IAAI,IAAY;CAClC,IAAI,YAAY;CAChB,IAAI,QAAQ;CAEZ,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,UAAU,OAAO,KAAK,KAAK,QAAQ,EAAE;EAC3C,IAAI,CAAC,SAAS;EAEd,IAAI,UAAU,SAAS,KAAK,QAAQ,GAClC,UAAU,IAAI,OAAO;OAChB,IAAI,UAAU,SAAS,KAAK,QAAQ,GAAG;GAC5C;GACA,IAAI,YAAY,WAAW,OAAO,GAChC;EAEJ;CACF;CAEA,IAAI,UAAU,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ;CAAyB;CAErE,MAAM,QAAQ,YAAY;CAC1B,MAAM,aAAa,QAAQ;CAC3B,IAAI,eAAe,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ,OAAO,MAAM;CAAwB;CACtF,OAAO;EAAE,OAAO;EAAO,QAAQ,GAAG,WAAW,GAAG,MAAM;CAA2B;AACnF;;AAGA,SAAS,YAAY,WAAwB,UAA2B;CACtE,IAAI,UAAU,IAAI,QAAQ,GAAG,OAAO;CACpC,KAAK,MAAM,MAAM,WACf,IAAI,GAAG,SAAS,IAAI,UAAU,KAAK,SAAS,SAAS,IAAI,IAAI,GAAG,OAAO;CAEzE,OAAO;AACT;AAEA,SAAgB,yBAAyB;CACvC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aACE;EACF,MAAM;EACN,YAAY,UAAU,EAAE,WAAW,CAAC,mBAAmB,MAAM,EAAE,CAAC;CAClE,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,WAAY,IAAI,UAAU,CAAC;EACjC,MAAM,QAAQ,iBAAiB,QAAQ;EACvC,MAAM,QAAQ,oBAAoB,QAAQ;EAG1C,IAAI,MAAM,SAAS,WAAW,cAC5B,OAAO;GAAE,OAAO;GAAe,YAAY,MAAM;GAAQ,YAAY;EAAM;EAQ7E,OAAO;GACL,OAAO;GACP,YAPiB,gBAAgB,KAOxB;GACT,WAPgB,eAAe,KAOvB;GACR,iBAPsB,qBAAqB,KAO7B;GACd,gBAPqB,oBAAoB,KAO5B;GACb,YAAY,MAAM;GAClB,YAAY;EACd;CACF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,MAAM,IAAI,QAAQ;EAClB,IAAI,EAAE,OAAO,OAAO;EAEpB,MAAM,QACJ,EAAE,WAAW,QAAQ,QAAQ,aAC7B,EAAE,UAAU,QAAQ,QAAQ,YAC5B,EAAE,gBAAgB,QAAQ,QAAQ,kBAClC,EAAE,eAAe,QAAQ,QAAQ;EAEnC,OAAO,KAAK,MAAM,QAAQ,GAAG,IAAI;CACnC,CAAC,CAAC,CACD,gBAAgB,EAAE,SAAS,YAAY;EACtC,MAAM,IAAI,QAAQ;EAClB,IAAI,EAAE,OACJ,OAAO,aAAa,EAAE,WAAW;EAGnC,MAAM,OAAO,MAAc,IAAI,IAAI,IAAA,CAAK,QAAQ,CAAC,EAAE;EACnD,MAAM,QAAkB,CAAC,UAAU,MAAM,IAAI,EAAE,WAAW,UAAU,EAAE,WAAW,QAAQ;EAEzF,MAAM,KAAK,eAAe,IAAI,QAAQ,UAAU,EAAE,KAAK,EAAE,WAAW,OAAO,IAAI,EAAE,WAAW,MAAM,EAAE;EACpG,MAAM,KAAK,eAAe,IAAI,QAAQ,SAAS,EAAE,KAAK,EAAE,UAAU,OAAO,IAAI,EAAE,UAAU,MAAM,EAAE;EACjG,MAAM,KACJ,qBAAqB,IAAI,QAAQ,eAAe,EAAE,KAAK,EAAE,gBAAgB,OAAO,IAAI,EAAE,gBAAgB,MAAM,EAC9G;EACA,MAAM,KACJ,qBAAqB,IAAI,QAAQ,cAAc,EAAE,KAAK,EAAE,eAAe,OAAO,IAAI,EAAE,eAAe,MAAM,EAC3G;EAEA,OAAO,MAAM,KAAK,IAAI;CACxB,CAAC;AACL"}

package/dist/evals/scorers/extract-tools.js

@@ -1,56 +1,60 @@
+//#region src/evals/scorers/extract-tools.ts
 function isToolInvocationPart(part) {
-  return "type" in part && part.type === "tool-invocation" && "toolInvocation" in part;
+	return "type" in part && part.type === "tool-invocation" && "toolInvocation" in part;
 }
 function isCompleted(inv) {
-  return inv.state === "result" || inv.state === "error" || inv.state === "output-error" || inv.state === "output-denied";
+	return inv.state === "result" || inv.state === "error" || inv.state === "output-error" || inv.state === "output-denied";
 }
 function isErrorState(inv) {
-  return inv.state === "error" || inv.state === "output-error" || inv.state === "output-denied";
+	return inv.state === "error" || inv.state === "output-error" || inv.state === "output-denied";
 }
+/**
+* Extract completed tool invocations from MastraDBMessages.
+*
+* Handles both:
+* - `content.parts` (primary format, MastraToolInvocationPart)
+* - `content.toolInvocations` (legacy format, AI SDK ToolInvocation[])
+*/
 function extractToolCalls(messages) {
-  const results = [];
-  const seenToolCallIds = /* @__PURE__ */ new Set();
-  let index = 0;
-  for (const msg of messages) {
-    if (!msg.content) continue;
-    for (const part of msg.content.parts ?? []) {
-      if (isToolInvocationPart(part)) {
-        const inv = part.toolInvocation;
-        if (isCompleted(inv)) {
-          const toolCallId = "toolCallId" in inv ? inv.toolCallId : void 0;
-          if (toolCallId) seenToolCallIds.add(toolCallId);
-          results.push({
-            toolCallId,
-            toolName: inv.toolName ?? "",
-            args: inv.args ?? {},
-            result: inv.result,
-            isError: isErrorState(inv),
-            index: index++
-          });
-        }
-      }
-    }
-    if (msg.content.toolInvocations) {
-      for (const inv of msg.content.toolInvocations) {
-        if (inv.state === "result") {
-          const toolCallId = "toolCallId" in inv ? inv.toolCallId : void 0;
-          if (toolCallId && seenToolCallIds.has(toolCallId)) continue;
-          if (toolCallId) seenToolCallIds.add(toolCallId);
-          results.push({
-            toolCallId,
-            toolName: inv.toolName ?? "",
-            args: inv.args ?? {},
-            result: inv.result,
-            isError: false,
-            index: index++
-          });
-        }
-      }
-    }
-  }
-  return results;
+	const results = [];
+	const seenToolCallIds = /* @__PURE__ */ new Set();
+	let index = 0;
+	for (const msg of messages) {
+		if (!msg.content) continue;
+		for (const part of msg.content.parts ?? []) if (isToolInvocationPart(part)) {
+			const inv = part.toolInvocation;
+			if (isCompleted(inv)) {
+				const toolCallId = "toolCallId" in inv ? inv.toolCallId : void 0;
+				if (toolCallId) seenToolCallIds.add(toolCallId);
+				results.push({
+					toolCallId,
+					toolName: inv.toolName ?? "",
+					args: inv.args ?? {},
+					result: inv.result,
+					isError: isErrorState(inv),
+					index: index++
+				});
+			}
+		}
+		if (msg.content.toolInvocations) {
+			for (const inv of msg.content.toolInvocations) if (inv.state === "result") {
+				const toolCallId = "toolCallId" in inv ? inv.toolCallId : void 0;
+				if (toolCallId && seenToolCallIds.has(toolCallId)) continue;
+				if (toolCallId) seenToolCallIds.add(toolCallId);
+				results.push({
+					toolCallId,
+					toolName: inv.toolName ?? "",
+					args: inv.args ?? {},
+					result: inv.result,
+					isError: false,
+					index: index++
+				});
+			}
+		}
+	}
+	return results;
 }
-export {
-  extractToolCalls
-};
+//#endregion
+export { extractToolCalls };
+
 //# sourceMappingURL=extract-tools.js.map