@mastra/code-sdk 0.0.0-esbuild-bundle-worker-20260807173433

package/dist/evals/scorers/classify-command.js

@@ -0,0 +1,73 @@
+//#region src/evals/scorers/classify-command.ts
+/**
+* Shared command classification utilities for scoring.
+*
+* Used by both live scorers (outcome.ts) and offline scorers (outcome-match.ts)
+* to consistently identify build, test, and other command types.
+*/
+/**
+* Check if a command string is a build/typecheck command.
+* Handles compound commands (e.g. `pnpm build && pnpm test`) by checking each segment.
+*/
+function isBuildCommand(command) {
+	return command.split(/\s*(?:&&|\|\||;)\s*/).some((seg) => {
+		const trimmed = seg.trim();
+		if (!/\b(tsc|typecheck|type-check|build)\b/.test(trimmed)) return false;
+		const primaryVerb = trimmed.replace(/^(?:pnpm|npm|yarn|npx|bunx?|turbo)\s+(?:run\s+)?/, "").split(/\s/)[0] ?? "";
+		return !/^(test|vitest|jest|mocha|pytest)$/.test(primaryVerb);
+	});
+}
+/**
+* Check if a command string is a test command.
+*/
+function isTestCommand(command) {
+	return /\b(test|vitest|jest|mocha|pytest|spec)\b/.test(command);
+}
+/**
+* Extract exit code from an `execute_command` result.
+* Only trusts explicit exitCode/code fields — no heuristic fallbacks.
+*/
+function getExitCode(result) {
+	if (result == null) return null;
+	if (typeof result === "string") {
+		const exitMatch = result.match(/exit(?:ed)?\s+(?:with\s+)?(?:code\s+)?(\d+)/i);
+		if (exitMatch) return parseInt(exitMatch[1], 10);
+		return null;
+	}
+	if (typeof result === "object") {
+		const obj = result;
+		if ("exitCode" in obj && typeof obj.exitCode === "number") return obj.exitCode;
+		if ("code" in obj && typeof obj.code === "number") return obj.code;
+	}
+	return null;
+}
+/**
+* Determine if a tool call result indicates success (exit code 0, no explicit error).
+* Works with both structured result objects and string results.
+*/
+function isSuccessResult(result, error) {
+	if (error) return false;
+	if (!result) return false;
+	if (typeof result === "object") {
+		const obj = result;
+		const exitCode = obj.exitCode ?? obj.code;
+		if (typeof exitCode === "number") return exitCode === 0;
+	}
+	if (typeof result === "string") {
+		const exitMatch = result.match(/exit(?:ed)?\s+(?:with\s+)?(?:code\s+)?(\d+)/i);
+		if (exitMatch) return exitMatch[1] === "0";
+	}
+	return false;
+}
+/**
+* Match a file path from tool output against an expected path.
+* Supports exact match and suffix match (e.g. "src/foo.ts" matches "/abs/path/src/foo.ts").
+*/
+function matchFilePath(actual, expected) {
+	if (actual === expected) return true;
+	return actual.endsWith(`/${expected}`) || expected.endsWith(`/${actual}`);
+}
+//#endregion
+export { getExitCode, isBuildCommand, isSuccessResult, isTestCommand, matchFilePath };
+
+//# sourceMappingURL=classify-command.js.map

package/dist/evals/scorers/classify-command.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classify-command.js","names":[],"sources":["../../../src/evals/scorers/classify-command.ts"],"sourcesContent":["/**\n * Shared command classification utilities for scoring.\n *\n * Used by both live scorers (outcome.ts) and offline scorers (outcome-match.ts)\n * to consistently identify build, test, and other command types.\n */\n\n/**\n * Check if a command string is a build/typecheck command.\n * Handles compound commands (e.g. `pnpm build && pnpm test`) by checking each segment.\n */\nexport function isBuildCommand(command: string): boolean {\n  const segments = command.split(/\\s*(?:&&|\\|\\||;)\\s*/);\n  return segments.some(seg => {\n    const trimmed = seg.trim();\n    if (!/\\b(tsc|typecheck|type-check|build)\\b/.test(trimmed)) return false;\n    // Don't reject if \"test\" only appears as a substring (e.g. \"build-test-output\").\n    // Only reject if a test runner is the primary verb of this segment.\n    const primaryVerb = trimmed.replace(/^(?:pnpm|npm|yarn|npx|bunx?|turbo)\\s+(?:run\\s+)?/, '').split(/\\s/)[0] ?? '';\n    return !/^(test|vitest|jest|mocha|pytest)$/.test(primaryVerb);\n  });\n}\n\n/**\n * Check if a command string is a test command.\n */\nexport function isTestCommand(command: string): boolean {\n  return /\\b(test|vitest|jest|mocha|pytest|spec)\\b/.test(command);\n}\n\n/**\n * Extract exit code from an `execute_command` result.\n * Only trusts explicit exitCode/code fields — no heuristic fallbacks.\n */\nexport function getExitCode(result: unknown): number | null {\n  if (result == null) return null;\n\n  if (typeof result === 'string') {\n    const exitMatch = result.match(/exit(?:ed)?\\s+(?:with\\s+)?(?:code\\s+)?(\\d+)/i);\n    if (exitMatch) return parseInt(exitMatch[1]!, 10);\n    return null;\n  }\n\n  if (typeof result === 'object') {\n    const obj = result as Record<string, unknown>;\n    if ('exitCode' in obj && typeof obj.exitCode === 'number') return obj.exitCode;\n    if ('code' in obj && typeof obj.code === 'number') return obj.code;\n  }\n\n  return null;\n}\n\n/**\n * Determine if a tool call result indicates success (exit code 0, no explicit error).\n * Works with both structured result objects and string results.\n */\nexport function isSuccessResult(result: unknown, error?: unknown): boolean {\n  if (error) return false;\n  if (!result) return false;\n\n  if (typeof result === 'object') {\n    const obj = result as Record<string, unknown>;\n    const exitCode = (obj.exitCode as number | undefined) ?? (obj.code as number | undefined);\n    if (typeof exitCode === 'number') return exitCode === 0;\n  }\n\n  if (typeof result === 'string') {\n    const exitMatch = result.match(/exit(?:ed)?\\s+(?:with\\s+)?(?:code\\s+)?(\\d+)/i);\n    if (exitMatch) return exitMatch[1] === '0';\n  }\n\n  // No explicit exit code — can't determine\n  return false;\n}\n\n/**\n * Match a file path from tool output against an expected path.\n * Supports exact match and suffix match (e.g. \"src/foo.ts\" matches \"/abs/path/src/foo.ts\").\n */\nexport function matchFilePath(actual: string, expected: string): boolean {\n  if (actual === expected) return true;\n  return actual.endsWith(`/${expected}`) || expected.endsWith(`/${actual}`);\n}\n"],"mappings":";;;;;;;;;;;AAWA,SAAgB,eAAe,SAA0B;CAEvD,OADiB,QAAQ,MAAM,qBACjB,CAAC,CAAC,MAAK,QAAO;EAC1B,MAAM,UAAU,IAAI,KAAK;EACzB,IAAI,CAAC,uCAAuC,KAAK,OAAO,GAAG,OAAO;EAGlE,MAAM,cAAc,QAAQ,QAAQ,oDAAoD,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM;EAC9G,OAAO,CAAC,oCAAoC,KAAK,WAAW;CAC9D,CAAC;AACH;;;;AAKA,SAAgB,cAAc,SAA0B;CACtD,OAAO,2CAA2C,KAAK,OAAO;AAChE;;;;;AAMA,SAAgB,YAAY,QAAgC;CAC1D,IAAI,UAAU,MAAM,OAAO;CAE3B,IAAI,OAAO,WAAW,UAAU;EAC9B,MAAM,YAAY,OAAO,MAAM,8CAA8C;EAC7E,IAAI,WAAW,OAAO,SAAS,UAAU,IAAK,EAAE;EAChD,OAAO;CACT;CAEA,IAAI,OAAO,WAAW,UAAU;EAC9B,MAAM,MAAM;EACZ,IAAI,cAAc,OAAO,OAAO,IAAI,aAAa,UAAU,OAAO,IAAI;EACtE,IAAI,UAAU,OAAO,OAAO,IAAI,SAAS,UAAU,OAAO,IAAI;CAChE;CAEA,OAAO;AACT;;;;;AAMA,SAAgB,gBAAgB,QAAiB,OAA0B;CACzE,IAAI,OAAO,OAAO;CAClB,IAAI,CAAC,QAAQ,OAAO;CAEpB,IAAI,OAAO,WAAW,UAAU;EAC9B,MAAM,MAAM;EACZ,MAAM,WAAY,IAAI,YAAoC,IAAI;EAC9D,IAAI,OAAO,aAAa,UAAU,OAAO,aAAa;CACxD;CAEA,IAAI,OAAO,WAAW,UAAU;EAC9B,MAAM,YAAY,OAAO,MAAM,8CAA8C;EAC7E,IAAI,WAAW,OAAO,UAAU,OAAO;CACzC;CAGA,OAAO;AACT;;;;;AAMA,SAAgB,cAAc,QAAgB,UAA2B;CACvE,IAAI,WAAW,UAAU,OAAO;CAChC,OAAO,OAAO,SAAS,IAAI,UAAU,KAAK,SAAS,SAAS,IAAI,QAAQ;AAC1E"}

package/dist/evals/scorers/efficiency.d.ts

@@ -0,0 +1,44 @@
+/**
+ * MastraCode Efficiency Scorer (Code-based, Agent type, 30% sampling)
+ *
+ * Measures how efficiently the agent completes work through quantitative ratios.
+ * Unlike the Outcome Scorer (which checks *what* happened), this checks
+ * *how much work* it took.
+ *
+ * This scorer produces a score from 0-1 but also exposes raw metrics in the
+ * reason text for trend analysis. The score represents "how cleanly did the
+ * agent work?" — not whether it got the right answer.
+ *
+ * Sampled at 30% because it's slightly more expensive (analyzes message sequences).
+ */
+export declare function createEfficiencyScorer(): import("@mastra/core/evals").MastraScorer<"mastracode-efficiency", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    empty: true;
+    totalCalls: number;
+    totalTurns: number;
+    redundancy?: undefined;
+    turnCount?: undefined;
+    retryEfficiency?: undefined;
+    readBeforeEdit?: undefined;
+} | {
+    empty: false;
+    redundancy: {
+        score: number;
+        ratio: number;
+        detail: string;
+    };
+    turnCount: {
+        score: number;
+        detail: string;
+    };
+    retryEfficiency: {
+        score: number;
+        detail: string;
+    };
+    readBeforeEdit: {
+        score: number;
+        detail: string;
+    };
+    totalCalls: number;
+    totalTurns: number;
+}> & Record<"generateScoreStepResult", number> & Record<"generateReasonStepResult", string>>;
+//# sourceMappingURL=efficiency.d.ts.map

package/dist/evals/scorers/efficiency.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"efficiency.d.ts","sourceRoot":"","sources":["../../../src/evals/scorers/efficiency.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAsOH,wBAAgB,sBAAsB;;;;;;;;;;;eA7JyB,MAAM;eAAS,MAAM;gBAAU,MAAM;;;eAqCnD,MAAM;gBAAU,MAAM;;;eAqBH,MAAM;gBAAU,MAAM;;;eAuDvB,MAAM;gBAAU,MAAM;;;;6FA8GxF"}

package/dist/evals/scorers/efficiency.js

@@ -0,0 +1,259 @@
+import { extractToolCalls } from "./extract-tools.js";
+import { createScorer, filterRun } from "@mastra/core/evals";
+//#region src/evals/scorers/efficiency.ts
+/**
+* Dimension weights (must sum to 1.0).
+*
+* - redundancy:     Duplicate mutation tool calls with identical args.
+* - turnCount:      Total assistant turns to complete the task.
+* - retryEfficiency: How quickly the agent recovers from tool failures.
+* - readBeforeEdit: Whether files are read before being edited.
+*/
+const WEIGHTS = {
+	redundancy: .35,
+	turnCount: .3,
+	retryEfficiency: .2,
+	readBeforeEdit: .15
+};
+/**
+* Scoring thresholds and multipliers.
+*/
+const THRESHOLDS = {
+	/** Redundancy ratio below this is considered acceptable (no penalty). */
+	redundancyAcceptableRate: .05,
+	/** Penalty multiplier applied to redundancy ratio above the acceptable rate. */
+	redundancyPenaltyMultiplier: 3,
+	/** Turns at or below this is considered normal (score 1.0). */
+	turnCountNormalMax: 8,
+	/** Penalty per turn beyond the normal maximum (for turns 9–15). */
+	turnCountPenaltyPerTurn: .04,
+	/** Penalty per turn beyond 15. */
+	turnCountExtendedPenaltyPerTurn: .03,
+	/** Number of consecutive failures before a retry chain is "excessive". */
+	retryExcessiveThreshold: 3,
+	/** Tools considered "edit" tools that should be preceded by a read. */
+	editTools: ["string_replace_lsp", "ast_smart_edit"],
+	/** Tools that satisfy read-before-edit (only `view` truly reads a specific file). */
+	readTools: ["view"],
+	/** Tools whose repeated identical calls are intentional (idempotent reads). */
+	redundancyWhitelist: [
+		"view",
+		"search_content",
+		"find_files",
+		"lsp_inspect",
+		"web_search",
+		"web_extract"
+	],
+	/** Minimum tool calls required to produce a meaningful score. */
+	minToolCalls: 2
+};
+/**
+* Count assistant turns (messages with role === 'assistant').
+*/
+function countAssistantTurns(messages) {
+	return messages.filter((m) => m.role === "assistant").length;
+}
+/**
+* Calculate redundant call ratio.
+* Only counts mutation/action tools as redundant — reads are whitelisted.
+*/
+function scoreRedundancy(calls) {
+	if (calls.length < 2) return {
+		score: 1,
+		ratio: 0,
+		detail: "Too few calls"
+	};
+	const mutationCalls = calls.filter((c) => !THRESHOLDS.redundancyWhitelist.includes(c.toolName));
+	if (mutationCalls.length < 2) return {
+		score: 1,
+		ratio: 0,
+		detail: "No mutation tools to check"
+	};
+	const fingerprints = /* @__PURE__ */ new Map();
+	for (const call of mutationCalls) {
+		const fp = `${call.toolName}:${JSON.stringify(call.args)}`;
+		const existing = fingerprints.get(fp) ?? {
+			count: 0,
+			successCount: 0
+		};
+		existing.count++;
+		if (!call.isError) existing.successCount++;
+		fingerprints.set(fp, existing);
+	}
+	let redundantCount = 0;
+	for (const { successCount } of fingerprints.values()) if (successCount > 1) redundantCount += successCount - 1;
+	const ratio = redundantCount / mutationCalls.length;
+	const acceptable = THRESHOLDS.redundancyAcceptableRate;
+	return {
+		score: ratio <= acceptable ? 1 : Math.max(0, 1 - (ratio - acceptable) * THRESHOLDS.redundancyPenaltyMultiplier),
+		ratio,
+		detail: `${redundantCount}/${mutationCalls.length} redundant mutations (${(ratio * 100).toFixed(0)}%)`
+	};
+}
+/**
+* Score turn count efficiency.
+* Fewer turns to accomplish the task = more efficient.
+*/
+function scoreTurnCount(turns) {
+	const normalMax = THRESHOLDS.turnCountNormalMax;
+	const penaltyPerTurn = THRESHOLDS.turnCountPenaltyPerTurn;
+	const extendedPenalty = THRESHOLDS.turnCountExtendedPenaltyPerTurn;
+	if (turns <= 1) return {
+		score: 1,
+		detail: `${turns} turn`
+	};
+	if (turns <= normalMax) return {
+		score: 1,
+		detail: `${turns} turns (within normal range)`
+	};
+	if (turns <= 15) return {
+		score: 1 - (turns - normalMax) * penaltyPerTurn,
+		detail: `${turns} turns (slightly extended)`
+	};
+	const midPenalty = (15 - normalMax) * penaltyPerTurn;
+	return {
+		score: Math.max(0, 1 - midPenalty - (turns - 15) * extendedPenalty),
+		detail: `${turns} turns (extended session)`
+	};
+}
+/**
+* Score failed-then-succeeded ratio.
+* Groups retry chains by tool name + target file path for more accurate detection.
+*/
+function scoreRetryEfficiency(calls) {
+	if (calls.length < 2) return {
+		score: 1,
+		detail: "Too few calls"
+	};
+	const groupKey = (call) => {
+		const path = String(call.args.path ?? call.args.command ?? "");
+		return `${call.toolName}:${path}`;
+	};
+	const byGroup = /* @__PURE__ */ new Map();
+	for (const call of calls) {
+		const key = groupKey(call);
+		const arr = byGroup.get(key) ?? [];
+		arr.push(call);
+		byGroup.set(key, arr);
+	}
+	let totalRetryChains = 0;
+	let excessiveRetries = 0;
+	const threshold = THRESHOLDS.retryExcessiveThreshold;
+	for (const [, toolCalls] of byGroup) {
+		let consecutiveFailures = 0;
+		for (const call of toolCalls) if (call.isError) consecutiveFailures++;
+		else {
+			if (consecutiveFailures > 0) {
+				totalRetryChains++;
+				if (consecutiveFailures >= threshold) excessiveRetries++;
+			}
+			consecutiveFailures = 0;
+		}
+		if (consecutiveFailures > 0) {
+			totalRetryChains++;
+			if (consecutiveFailures >= threshold) excessiveRetries++;
+		}
+	}
+	if (totalRetryChains === 0) return {
+		score: 1,
+		detail: "No retry chains"
+	};
+	if (excessiveRetries === 0) return {
+		score: .9,
+		detail: `${totalRetryChains} retry chain(s), all resolved quickly`
+	};
+	return {
+		score: Math.max(0, .8 - excessiveRetries * .2),
+		detail: `${excessiveRetries} excessive retry chain(s) (${threshold}+ failures before success)`
+	};
+}
+/**
+* Score read-before-edit compliance.
+* Only `view` satisfies a read — search/find don't provide specific file content.
+*/
+function scoreReadBeforeEdit(calls) {
+	const editTools = THRESHOLDS.editTools;
+	const readTools = THRESHOLDS.readTools;
+	if (calls.filter((c) => editTools.includes(c.toolName)).length === 0) return {
+		score: 1,
+		detail: "No edits to check"
+	};
+	const readPaths = /* @__PURE__ */ new Set();
+	let compliant = 0;
+	let total = 0;
+	for (const call of calls) {
+		const rawPath = String(call.args.path ?? "");
+		if (!rawPath) continue;
+		if (readTools.includes(call.toolName)) readPaths.add(rawPath);
+		else if (editTools.includes(call.toolName)) {
+			total++;
+			if (hasReadPath(readPaths, rawPath)) compliant++;
+		}
+	}
+	if (total === 0) return {
+		score: 1,
+		detail: "No path-targeted edits"
+	};
+	const ratio = compliant / total;
+	const violations = total - compliant;
+	if (violations === 0) return {
+		score: 1,
+		detail: `All ${total} edits had prior reads`
+	};
+	return {
+		score: ratio,
+		detail: `${violations}/${total} edits without prior read`
+	};
+}
+/** Check if any read path matches the edit path, using suffix matching for mixed abs/relative paths. */
+function hasReadPath(readPaths, editPath) {
+	if (readPaths.has(editPath)) return true;
+	for (const rp of readPaths) if (rp.endsWith(`/${editPath}`) || editPath.endsWith(`/${rp}`)) return true;
+	return false;
+}
+function createEfficiencyScorer() {
+	return createScorer({
+		id: "mastracode-efficiency",
+		name: "MastraCode Efficiency",
+		description: "Measures coding session efficiency: redundancy, turn count, retry patterns, read-before-edit. 30% sampled.",
+		type: "agent",
+		prepareRun: filterRun({ partTypes: ["tool-invocation", "text"] })
+	}).preprocess(async ({ run }) => {
+		const messages = run.output ?? [];
+		const calls = extractToolCalls(messages);
+		const turns = countAssistantTurns(messages);
+		if (calls.length < THRESHOLDS.minToolCalls) return {
+			empty: true,
+			totalCalls: calls.length,
+			totalTurns: turns
+		};
+		return {
+			empty: false,
+			redundancy: scoreRedundancy(calls),
+			turnCount: scoreTurnCount(turns),
+			retryEfficiency: scoreRetryEfficiency(calls),
+			readBeforeEdit: scoreReadBeforeEdit(calls),
+			totalCalls: calls.length,
+			totalTurns: turns
+		};
+	}).generateScore(({ results }) => {
+		const p = results.preprocessStepResult;
+		if (p.empty) return 0;
+		const score = p.redundancy.score * WEIGHTS.redundancy + p.turnCount.score * WEIGHTS.turnCount + p.retryEfficiency.score * WEIGHTS.retryEfficiency + p.readBeforeEdit.score * WEIGHTS.readBeforeEdit;
+		return Math.round(score * 100) / 100;
+	}).generateReason(({ results, score }) => {
+		const p = results.preprocessStepResult;
+		if (p.empty) return `Score: 0 (${p.totalCalls} tool calls — below minimum threshold for scoring)`;
+		const pct = (w) => `${(w * 100).toFixed(0)}%`;
+		const parts = [`Score: ${score} (${p.totalCalls} calls, ${p.totalTurns} turns)`];
+		parts.push(`Redundancy (${pct(WEIGHTS.redundancy)}): ${p.redundancy.detail} [${p.redundancy.score}]`);
+		parts.push(`Turn count (${pct(WEIGHTS.turnCount)}): ${p.turnCount.detail} [${p.turnCount.score}]`);
+		parts.push(`Retry efficiency (${pct(WEIGHTS.retryEfficiency)}): ${p.retryEfficiency.detail} [${p.retryEfficiency.score}]`);
+		parts.push(`Read-before-edit (${pct(WEIGHTS.readBeforeEdit)}): ${p.readBeforeEdit.detail} [${p.readBeforeEdit.score}]`);
+		return parts.join("\n");
+	});
+}
+//#endregion
+export { createEfficiencyScorer };
+
+//# sourceMappingURL=efficiency.js.map

package/dist/evals/scorers/efficiency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"efficiency.js","names":[],"sources":["../../../src/evals/scorers/efficiency.ts"],"sourcesContent":["/**\n * MastraCode Efficiency Scorer (Code-based, Agent type, 30% sampling)\n *\n * Measures how efficiently the agent completes work through quantitative ratios.\n * Unlike the Outcome Scorer (which checks *what* happened), this checks\n * *how much work* it took.\n *\n * This scorer produces a score from 0-1 but also exposes raw metrics in the\n * reason text for trend analysis. The score represents \"how cleanly did the\n * agent work?\" — not whether it got the right answer.\n *\n * Sampled at 30% because it's slightly more expensive (analyzes message sequences).\n */\n\nimport type { MastraDBMessage } from '@mastra/core/agent';\nimport { createScorer, filterRun } from '@mastra/core/evals';\n\nimport type { ExtractedToolCall } from './extract-tools.js';\nimport { extractToolCalls } from './extract-tools.js';\n\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n// CONFIGURATION — Adjust weights and thresholds here\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n/**\n * Dimension weights (must sum to 1.0).\n *\n * - redundancy:     Duplicate mutation tool calls with identical args.\n * - turnCount:      Total assistant turns to complete the task.\n * - retryEfficiency: How quickly the agent recovers from tool failures.\n * - readBeforeEdit: Whether files are read before being edited.\n */\nconst WEIGHTS = {\n  redundancy: 0.35,\n  turnCount: 0.3,\n  retryEfficiency: 0.2,\n  readBeforeEdit: 0.15,\n} as const;\n\n/**\n * Scoring thresholds and multipliers.\n */\nconst THRESHOLDS = {\n  /** Redundancy ratio below this is considered acceptable (no penalty). */\n  redundancyAcceptableRate: 0.05,\n\n  /** Penalty multiplier applied to redundancy ratio above the acceptable rate. */\n  redundancyPenaltyMultiplier: 3,\n\n  /** Turns at or below this is considered normal (score 1.0). */\n  turnCountNormalMax: 8,\n\n  /** Penalty per turn beyond the normal maximum (for turns 9–15). */\n  turnCountPenaltyPerTurn: 0.04,\n\n  /** Penalty per turn beyond 15. */\n  turnCountExtendedPenaltyPerTurn: 0.03,\n\n  /** Number of consecutive failures before a retry chain is \"excessive\". */\n  retryExcessiveThreshold: 3,\n\n  /** Tools considered \"edit\" tools that should be preceded by a read. */\n  editTools: ['string_replace_lsp', 'ast_smart_edit'] as string[],\n\n  /** Tools that satisfy read-before-edit (only `view` truly reads a specific file). */\n  readTools: ['view'] as string[],\n\n  /** Tools whose repeated identical calls are intentional (idempotent reads). */\n  redundancyWhitelist: ['view', 'search_content', 'find_files', 'lsp_inspect', 'web_search', 'web_extract'] as string[],\n\n  /** Minimum tool calls required to produce a meaningful score. */\n  minToolCalls: 2,\n} as const;\n\n/**\n * Count assistant turns (messages with role === 'assistant').\n */\nfunction countAssistantTurns(messages: MastraDBMessage[]): number {\n  return messages.filter(m => m.role === 'assistant').length;\n}\n\n/**\n * Calculate redundant call ratio.\n * Only counts mutation/action tools as redundant — reads are whitelisted.\n */\nfunction scoreRedundancy(calls: ExtractedToolCall[]): { score: number; ratio: number; detail: string } {\n  if (calls.length < 2) return { score: 1, ratio: 0, detail: 'Too few calls' };\n\n  // Only consider non-whitelisted (mutation) tools for redundancy\n  const mutationCalls = calls.filter(c => !THRESHOLDS.redundancyWhitelist.includes(c.toolName));\n  if (mutationCalls.length < 2) return { score: 1, ratio: 0, detail: 'No mutation tools to check' };\n\n  const fingerprints = new Map<string, { count: number; successCount: number }>();\n  for (const call of mutationCalls) {\n    const fp = `${call.toolName}:${JSON.stringify(call.args)}`;\n    const existing = fingerprints.get(fp) ?? { count: 0, successCount: 0 };\n    existing.count++;\n    if (!call.isError) existing.successCount++;\n    fingerprints.set(fp, existing);\n  }\n\n  // Only count as redundant if the same successful call happened multiple times\n  let redundantCount = 0;\n  for (const { successCount } of fingerprints.values()) {\n    if (successCount > 1) redundantCount += successCount - 1;\n  }\n\n  const ratio = redundantCount / mutationCalls.length;\n  const acceptable = THRESHOLDS.redundancyAcceptableRate;\n  const score =\n    ratio <= acceptable ? 1 : Math.max(0, 1 - (ratio - acceptable) * THRESHOLDS.redundancyPenaltyMultiplier);\n  return {\n    score,\n    ratio,\n    detail: `${redundantCount}/${mutationCalls.length} redundant mutations (${(ratio * 100).toFixed(0)}%)`,\n  };\n}\n\n/**\n * Score turn count efficiency.\n * Fewer turns to accomplish the task = more efficient.\n */\nfunction scoreTurnCount(turns: number): { score: number; detail: string } {\n  const normalMax = THRESHOLDS.turnCountNormalMax;\n  const penaltyPerTurn = THRESHOLDS.turnCountPenaltyPerTurn;\n  const extendedPenalty = THRESHOLDS.turnCountExtendedPenaltyPerTurn;\n\n  if (turns <= 1) return { score: 1, detail: `${turns} turn` };\n  if (turns <= normalMax) return { score: 1, detail: `${turns} turns (within normal range)` };\n  if (turns <= 15) {\n    const score = 1 - (turns - normalMax) * penaltyPerTurn;\n    return { score, detail: `${turns} turns (slightly extended)` };\n  }\n\n  const midPenalty = (15 - normalMax) * penaltyPerTurn;\n  const score = Math.max(0, 1 - midPenalty - (turns - 15) * extendedPenalty);\n  return { score, detail: `${turns} turns (extended session)` };\n}\n\n/**\n * Score failed-then-succeeded ratio.\n * Groups retry chains by tool name + target file path for more accurate detection.\n */\nfunction scoreRetryEfficiency(calls: ExtractedToolCall[]): { score: number; detail: string } {\n  if (calls.length < 2) return { score: 1, detail: 'Too few calls' };\n\n  // Group calls by tool name + target file path (if available)\n  const groupKey = (call: ExtractedToolCall): string => {\n    const path = String(call.args.path ?? call.args.command ?? '');\n    return `${call.toolName}:${path}`;\n  };\n\n  const byGroup = new Map<string, ExtractedToolCall[]>();\n  for (const call of calls) {\n    const key = groupKey(call);\n    const arr = byGroup.get(key) ?? [];\n    arr.push(call);\n    byGroup.set(key, arr);\n  }\n\n  let totalRetryChains = 0;\n  let excessiveRetries = 0;\n  const threshold = THRESHOLDS.retryExcessiveThreshold;\n\n  for (const [, toolCalls] of byGroup) {\n    let consecutiveFailures = 0;\n    for (const call of toolCalls) {\n      if (call.isError) {\n        consecutiveFailures++;\n      } else {\n        if (consecutiveFailures > 0) {\n          totalRetryChains++;\n          if (consecutiveFailures >= threshold) excessiveRetries++;\n        }\n        consecutiveFailures = 0;\n      }\n    }\n    // Count unrecovered retry chains (failures at end of group with no success)\n    if (consecutiveFailures > 0) {\n      totalRetryChains++;\n      if (consecutiveFailures >= threshold) excessiveRetries++;\n    }\n  }\n\n  if (totalRetryChains === 0) return { score: 1, detail: 'No retry chains' };\n  if (excessiveRetries === 0) return { score: 0.9, detail: `${totalRetryChains} retry chain(s), all resolved quickly` };\n\n  const score = Math.max(0, 0.8 - excessiveRetries * 0.2);\n  return {\n    score,\n    detail: `${excessiveRetries} excessive retry chain(s) (${threshold}+ failures before success)`,\n  };\n}\n\n/**\n * Score read-before-edit compliance.\n * Only `view` satisfies a read — search/find don't provide specific file content.\n */\nfunction scoreReadBeforeEdit(calls: ExtractedToolCall[]): { score: number; detail: string } {\n  const editTools = THRESHOLDS.editTools;\n  const readTools = THRESHOLDS.readTools;\n\n  const edits = calls.filter(c => editTools.includes(c.toolName));\n  if (edits.length === 0) return { score: 1, detail: 'No edits to check' };\n\n  // Build set of files that were read before each edit.\n  // Use suffix-normalized paths so that \"/abs/path/src/foo.ts\" and \"src/foo.ts\" match.\n  const readPaths = new Set<string>();\n  let compliant = 0;\n  let total = 0;\n\n  for (const call of calls) {\n    const rawPath = String(call.args.path ?? '');\n    if (!rawPath) continue;\n\n    if (readTools.includes(call.toolName)) {\n      readPaths.add(rawPath);\n    } else if (editTools.includes(call.toolName)) {\n      total++;\n      if (hasReadPath(readPaths, rawPath)) {\n        compliant++;\n      }\n    }\n  }\n\n  if (total === 0) return { score: 1, detail: 'No path-targeted edits' };\n\n  const ratio = compliant / total;\n  const violations = total - compliant;\n  if (violations === 0) return { score: 1, detail: `All ${total} edits had prior reads` };\n  return { score: ratio, detail: `${violations}/${total} edits without prior read` };\n}\n\n/** Check if any read path matches the edit path, using suffix matching for mixed abs/relative paths. */\nfunction hasReadPath(readPaths: Set<string>, editPath: string): boolean {\n  if (readPaths.has(editPath)) return true;\n  for (const rp of readPaths) {\n    if (rp.endsWith(`/${editPath}`) || editPath.endsWith(`/${rp}`)) return true;\n  }\n  return false;\n}\n\nexport function createEfficiencyScorer() {\n  return createScorer({\n    id: 'mastracode-efficiency',\n    name: 'MastraCode Efficiency',\n    description:\n      'Measures coding session efficiency: redundancy, turn count, retry patterns, read-before-edit. 30% sampled.',\n    type: 'agent',\n    prepareRun: filterRun({ partTypes: ['tool-invocation', 'text'] }),\n  })\n    .preprocess(async ({ run }) => {\n      const messages = (run.output ?? []) as MastraDBMessage[];\n      const calls = extractToolCalls(messages);\n      const turns = countAssistantTurns(messages);\n\n      // H1: Empty sessions → score 0\n      if (calls.length < THRESHOLDS.minToolCalls) {\n        return { empty: true as const, totalCalls: calls.length, totalTurns: turns };\n      }\n\n      const redundancy = scoreRedundancy(calls);\n      const turnCount = scoreTurnCount(turns);\n      const retryEfficiency = scoreRetryEfficiency(calls);\n      const readBeforeEdit = scoreReadBeforeEdit(calls);\n\n      return {\n        empty: false as const,\n        redundancy,\n        turnCount,\n        retryEfficiency,\n        readBeforeEdit,\n        totalCalls: calls.length,\n        totalTurns: turns,\n      };\n    })\n    .generateScore(({ results }) => {\n      const p = results.preprocessStepResult;\n      if (p.empty) return 0;\n\n      const score =\n        p.redundancy.score * WEIGHTS.redundancy +\n        p.turnCount.score * WEIGHTS.turnCount +\n        p.retryEfficiency.score * WEIGHTS.retryEfficiency +\n        p.readBeforeEdit.score * WEIGHTS.readBeforeEdit;\n\n      return Math.round(score * 100) / 100;\n    })\n    .generateReason(({ results, score }) => {\n      const p = results.preprocessStepResult;\n      if (p.empty) {\n        return `Score: 0 (${p.totalCalls} tool calls — below minimum threshold for scoring)`;\n      }\n\n      const pct = (w: number) => `${(w * 100).toFixed(0)}%`;\n      const parts: string[] = [`Score: ${score} (${p.totalCalls} calls, ${p.totalTurns} turns)`];\n\n      parts.push(`Redundancy (${pct(WEIGHTS.redundancy)}): ${p.redundancy.detail} [${p.redundancy.score}]`);\n      parts.push(`Turn count (${pct(WEIGHTS.turnCount)}): ${p.turnCount.detail} [${p.turnCount.score}]`);\n      parts.push(\n        `Retry efficiency (${pct(WEIGHTS.retryEfficiency)}): ${p.retryEfficiency.detail} [${p.retryEfficiency.score}]`,\n      );\n      parts.push(\n        `Read-before-edit (${pct(WEIGHTS.readBeforeEdit)}): ${p.readBeforeEdit.detail} [${p.readBeforeEdit.score}]`,\n      );\n\n      return parts.join('\\n');\n    });\n}\n"],"mappings":";;;;;;;;;;;AAgCA,MAAM,UAAU;CACd,YAAY;CACZ,WAAW;CACX,iBAAiB;CACjB,gBAAgB;AAClB;;;;AAKA,MAAM,aAAa;;CAEjB,0BAA0B;;CAG1B,6BAA6B;;CAG7B,oBAAoB;;CAGpB,yBAAyB;;CAGzB,iCAAiC;;CAGjC,yBAAyB;;CAGzB,WAAW,CAAC,sBAAsB,gBAAgB;;CAGlD,WAAW,CAAC,MAAM;;CAGlB,qBAAqB;EAAC;EAAQ;EAAkB;EAAc;EAAe;EAAc;CAAa;;CAGxG,cAAc;AAChB;;;;AAKA,SAAS,oBAAoB,UAAqC;CAChE,OAAO,SAAS,QAAO,MAAK,EAAE,SAAS,WAAW,CAAC,CAAC;AACtD;;;;;AAMA,SAAS,gBAAgB,OAA8E;CACrG,IAAI,MAAM,SAAS,GAAG,OAAO;EAAE,OAAO;EAAG,OAAO;EAAG,QAAQ;CAAgB;CAG3E,MAAM,gBAAgB,MAAM,QAAO,MAAK,CAAC,WAAW,oBAAoB,SAAS,EAAE,QAAQ,CAAC;CAC5F,IAAI,cAAc,SAAS,GAAG,OAAO;EAAE,OAAO;EAAG,OAAO;EAAG,QAAQ;CAA6B;CAEhG,MAAM,+BAAe,IAAI,IAAqD;CAC9E,KAAK,MAAM,QAAQ,eAAe;EAChC,MAAM,KAAK,GAAG,KAAK,SAAS,GAAG,KAAK,UAAU,KAAK,IAAI;EACvD,MAAM,WAAW,aAAa,IAAI,EAAE,KAAK;GAAE,OAAO;GAAG,cAAc;EAAE;EACrE,SAAS;EACT,IAAI,CAAC,KAAK,SAAS,SAAS;EAC5B,aAAa,IAAI,IAAI,QAAQ;CAC/B;CAGA,IAAI,iBAAiB;CACrB,KAAK,MAAM,EAAE,kBAAkB,aAAa,OAAO,GACjD,IAAI,eAAe,GAAG,kBAAkB,eAAe;CAGzD,MAAM,QAAQ,iBAAiB,cAAc;CAC7C,MAAM,aAAa,WAAW;CAG9B,OAAO;EACL,OAFA,SAAS,aAAa,IAAI,KAAK,IAAI,GAAG,KAAK,QAAQ,cAAc,WAAW,2BAA2B;EAGvG;EACA,QAAQ,GAAG,eAAe,GAAG,cAAc,OAAO,yBAAyB,QAAQ,IAAA,CAAK,QAAQ,CAAC,EAAE;CACrG;AACF;;;;;AAMA,SAAS,eAAe,OAAkD;CACxE,MAAM,YAAY,WAAW;CAC7B,MAAM,iBAAiB,WAAW;CAClC,MAAM,kBAAkB,WAAW;CAEnC,IAAI,SAAS,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ,GAAG,MAAM;CAAO;CAC3D,IAAI,SAAS,WAAW,OAAO;EAAE,OAAO;EAAG,QAAQ,GAAG,MAAM;CAA8B;CAC1F,IAAI,SAAS,IAEX,OAAO;EAAE,OADK,KAAK,QAAQ,aAAa;EACxB,QAAQ,GAAG,MAAM;CAA4B;CAG/D,MAAM,cAAc,KAAK,aAAa;CAEtC,OAAO;EAAE,OADK,KAAK,IAAI,GAAG,IAAI,cAAc,QAAQ,MAAM,eAC7C;EAAG,QAAQ,GAAG,MAAM;CAA2B;AAC9D;;;;;AAMA,SAAS,qBAAqB,OAA+D;CAC3F,IAAI,MAAM,SAAS,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ;CAAgB;CAGjE,MAAM,YAAY,SAAoC;EACpD,MAAM,OAAO,OAAO,KAAK,KAAK,QAAQ,KAAK,KAAK,WAAW,EAAE;EAC7D,OAAO,GAAG,KAAK,SAAS,GAAG;CAC7B;CAEA,MAAM,0BAAU,IAAI,IAAiC;CACrD,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,MAAM,SAAS,IAAI;EACzB,MAAM,MAAM,QAAQ,IAAI,GAAG,KAAK,CAAC;EACjC,IAAI,KAAK,IAAI;EACb,QAAQ,IAAI,KAAK,GAAG;CACtB;CAEA,IAAI,mBAAmB;CACvB,IAAI,mBAAmB;CACvB,MAAM,YAAY,WAAW;CAE7B,KAAK,MAAM,GAAG,cAAc,SAAS;EACnC,IAAI,sBAAsB;EAC1B,KAAK,MAAM,QAAQ,WACjB,IAAI,KAAK,SACP;OACK;GACL,IAAI,sBAAsB,GAAG;IAC3B;IACA,IAAI,uBAAuB,WAAW;GACxC;GACA,sBAAsB;EACxB;EAGF,IAAI,sBAAsB,GAAG;GAC3B;GACA,IAAI,uBAAuB,WAAW;EACxC;CACF;CAEA,IAAI,qBAAqB,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ;CAAkB;CACzE,IAAI,qBAAqB,GAAG,OAAO;EAAE,OAAO;EAAK,QAAQ,GAAG,iBAAiB;CAAuC;CAGpH,OAAO;EACL,OAFY,KAAK,IAAI,GAAG,KAAM,mBAAmB,EAE7C;EACJ,QAAQ,GAAG,iBAAiB,6BAA6B,UAAU;CACrE;AACF;;;;;AAMA,SAAS,oBAAoB,OAA+D;CAC1F,MAAM,YAAY,WAAW;CAC7B,MAAM,YAAY,WAAW;CAG7B,IADc,MAAM,QAAO,MAAK,UAAU,SAAS,EAAE,QAAQ,CACrD,CAAC,CAAC,WAAW,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ;CAAoB;CAIvE,MAAM,4BAAY,IAAI,IAAY;CAClC,IAAI,YAAY;CAChB,IAAI,QAAQ;CAEZ,KAAK,MAAM,QAAQ,OAAO;EACxB,MAAM,UAAU,OAAO,KAAK,KAAK,QAAQ,EAAE;EAC3C,IAAI,CAAC,SAAS;EAEd,IAAI,UAAU,SAAS,KAAK,QAAQ,GAClC,UAAU,IAAI,OAAO;OAChB,IAAI,UAAU,SAAS,KAAK,QAAQ,GAAG;GAC5C;GACA,IAAI,YAAY,WAAW,OAAO,GAChC;EAEJ;CACF;CAEA,IAAI,UAAU,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ;CAAyB;CAErE,MAAM,QAAQ,YAAY;CAC1B,MAAM,aAAa,QAAQ;CAC3B,IAAI,eAAe,GAAG,OAAO;EAAE,OAAO;EAAG,QAAQ,OAAO,MAAM;CAAwB;CACtF,OAAO;EAAE,OAAO;EAAO,QAAQ,GAAG,WAAW,GAAG,MAAM;CAA2B;AACnF;;AAGA,SAAS,YAAY,WAAwB,UAA2B;CACtE,IAAI,UAAU,IAAI,QAAQ,GAAG,OAAO;CACpC,KAAK,MAAM,MAAM,WACf,IAAI,GAAG,SAAS,IAAI,UAAU,KAAK,SAAS,SAAS,IAAI,IAAI,GAAG,OAAO;CAEzE,OAAO;AACT;AAEA,SAAgB,yBAAyB;CACvC,OAAO,aAAa;EAClB,IAAI;EACJ,MAAM;EACN,aACE;EACF,MAAM;EACN,YAAY,UAAU,EAAE,WAAW,CAAC,mBAAmB,MAAM,EAAE,CAAC;CAClE,CAAC,CAAC,CACC,WAAW,OAAO,EAAE,UAAU;EAC7B,MAAM,WAAY,IAAI,UAAU,CAAC;EACjC,MAAM,QAAQ,iBAAiB,QAAQ;EACvC,MAAM,QAAQ,oBAAoB,QAAQ;EAG1C,IAAI,MAAM,SAAS,WAAW,cAC5B,OAAO;GAAE,OAAO;GAAe,YAAY,MAAM;GAAQ,YAAY;EAAM;EAQ7E,OAAO;GACL,OAAO;GACP,YAPiB,gBAAgB,KAOxB;GACT,WAPgB,eAAe,KAOvB;GACR,iBAPsB,qBAAqB,KAO7B;GACd,gBAPqB,oBAAoB,KAO5B;GACb,YAAY,MAAM;GAClB,YAAY;EACd;CACF,CAAC,CAAC,CACD,eAAe,EAAE,cAAc;EAC9B,MAAM,IAAI,QAAQ;EAClB,IAAI,EAAE,OAAO,OAAO;EAEpB,MAAM,QACJ,EAAE,WAAW,QAAQ,QAAQ,aAC7B,EAAE,UAAU,QAAQ,QAAQ,YAC5B,EAAE,gBAAgB,QAAQ,QAAQ,kBAClC,EAAE,eAAe,QAAQ,QAAQ;EAEnC,OAAO,KAAK,MAAM,QAAQ,GAAG,IAAI;CACnC,CAAC,CAAC,CACD,gBAAgB,EAAE,SAAS,YAAY;EACtC,MAAM,IAAI,QAAQ;EAClB,IAAI,EAAE,OACJ,OAAO,aAAa,EAAE,WAAW;EAGnC,MAAM,OAAO,MAAc,IAAI,IAAI,IAAA,CAAK,QAAQ,CAAC,EAAE;EACnD,MAAM,QAAkB,CAAC,UAAU,MAAM,IAAI,EAAE,WAAW,UAAU,EAAE,WAAW,QAAQ;EAEzF,MAAM,KAAK,eAAe,IAAI,QAAQ,UAAU,EAAE,KAAK,EAAE,WAAW,OAAO,IAAI,EAAE,WAAW,MAAM,EAAE;EACpG,MAAM,KAAK,eAAe,IAAI,QAAQ,SAAS,EAAE,KAAK,EAAE,UAAU,OAAO,IAAI,EAAE,UAAU,MAAM,EAAE;EACjG,MAAM,KACJ,qBAAqB,IAAI,QAAQ,eAAe,EAAE,KAAK,EAAE,gBAAgB,OAAO,IAAI,EAAE,gBAAgB,MAAM,EAC9G;EACA,MAAM,KACJ,qBAAqB,IAAI,QAAQ,cAAc,EAAE,KAAK,EAAE,eAAe,OAAO,IAAI,EAAE,eAAe,MAAM,EAC3G;EAEA,OAAO,MAAM,KAAK,IAAI;CACxB,CAAC;AACL"}

package/dist/evals/scorers/extract-tools.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Shared tool invocation extraction from MastraDBMessages.
+ *
+ * Used by both the Outcome and Efficiency scorers to extract completed
+ * tool calls from message content, handling both the `parts` array (primary)
+ * and the legacy `toolInvocations` array.
+ */
+import type { MastraDBMessage } from '@mastra/core/agent';
+export type ExtractedToolCall = {
+    toolCallId?: string;
+    toolName: string;
+    args: Record<string, unknown>;
+    result: unknown;
+    isError: boolean;
+    index: number;
+};
+/**
+ * Extract completed tool invocations from MastraDBMessages.
+ *
+ * Handles both:
+ * - `content.parts` (primary format, MastraToolInvocationPart)
+ * - `content.toolInvocations` (legacy format, AI SDK ToolInvocation[])
+ */
+export declare function extractToolCalls(messages: MastraDBMessage[]): ExtractedToolCall[];
+//# sourceMappingURL=extract-tools.d.ts.map

package/dist/evals/scorers/extract-tools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-tools.d.ts","sourceRoot":"","sources":["../../../src/evals/scorers/extract-tools.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAG1D,MAAM,MAAM,iBAAiB,GAAG;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAmBF;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,iBAAiB,EAAE,CAoDjF"}

package/dist/evals/scorers/extract-tools.js

@@ -0,0 +1,60 @@
+//#region src/evals/scorers/extract-tools.ts
+function isToolInvocationPart(part) {
+	return "type" in part && part.type === "tool-invocation" && "toolInvocation" in part;
+}
+function isCompleted(inv) {
+	return inv.state === "result" || inv.state === "error" || inv.state === "output-error" || inv.state === "output-denied";
+}
+function isErrorState(inv) {
+	return inv.state === "error" || inv.state === "output-error" || inv.state === "output-denied";
+}
+/**
+* Extract completed tool invocations from MastraDBMessages.
+*
+* Handles both:
+* - `content.parts` (primary format, MastraToolInvocationPart)
+* - `content.toolInvocations` (legacy format, AI SDK ToolInvocation[])
+*/
+function extractToolCalls(messages) {
+	const results = [];
+	const seenToolCallIds = /* @__PURE__ */ new Set();
+	let index = 0;
+	for (const msg of messages) {
+		if (!msg.content) continue;
+		for (const part of msg.content.parts ?? []) if (isToolInvocationPart(part)) {
+			const inv = part.toolInvocation;
+			if (isCompleted(inv)) {
+				const toolCallId = "toolCallId" in inv ? inv.toolCallId : void 0;
+				if (toolCallId) seenToolCallIds.add(toolCallId);
+				results.push({
+					toolCallId,
+					toolName: inv.toolName ?? "",
+					args: inv.args ?? {},
+					result: inv.result,
+					isError: isErrorState(inv),
+					index: index++
+				});
+			}
+		}
+		if (msg.content.toolInvocations) {
+			for (const inv of msg.content.toolInvocations) if (inv.state === "result") {
+				const toolCallId = "toolCallId" in inv ? inv.toolCallId : void 0;
+				if (toolCallId && seenToolCallIds.has(toolCallId)) continue;
+				if (toolCallId) seenToolCallIds.add(toolCallId);
+				results.push({
+					toolCallId,
+					toolName: inv.toolName ?? "",
+					args: inv.args ?? {},
+					result: inv.result,
+					isError: false,
+					index: index++
+				});
+			}
+		}
+	}
+	return results;
+}
+//#endregion
+export { extractToolCalls };
+
+//# sourceMappingURL=extract-tools.js.map

package/dist/evals/scorers/extract-tools.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-tools.js","names":[],"sources":["../../../src/evals/scorers/extract-tools.ts"],"sourcesContent":["/**\n * Shared tool invocation extraction from MastraDBMessages.\n *\n * Used by both the Outcome and Efficiency scorers to extract completed\n * tool calls from message content, handling both the `parts` array (primary)\n * and the legacy `toolInvocations` array.\n */\n\nimport type { MastraDBMessage } from '@mastra/core/agent';\nimport type { MastraMessagePart, MastraToolInvocation } from '@mastra/core/agent/message-list';\n\nexport type ExtractedToolCall = {\n  toolCallId?: string;\n  toolName: string;\n  args: Record<string, unknown>;\n  result: unknown;\n  isError: boolean;\n  index: number;\n};\n\nfunction isToolInvocationPart(part: MastraMessagePart): part is MastraMessagePart & {\n  type: 'tool-invocation';\n  toolInvocation: MastraToolInvocation;\n} {\n  return 'type' in part && part.type === 'tool-invocation' && 'toolInvocation' in part;\n}\n\nfunction isCompleted(inv: { state: string }): boolean {\n  return (\n    inv.state === 'result' || inv.state === 'error' || inv.state === 'output-error' || inv.state === 'output-denied'\n  );\n}\n\nfunction isErrorState(inv: { state: string }): boolean {\n  return inv.state === 'error' || inv.state === 'output-error' || inv.state === 'output-denied';\n}\n\n/**\n * Extract completed tool invocations from MastraDBMessages.\n *\n * Handles both:\n * - `content.parts` (primary format, MastraToolInvocationPart)\n * - `content.toolInvocations` (legacy format, AI SDK ToolInvocation[])\n */\nexport function extractToolCalls(messages: MastraDBMessage[]): ExtractedToolCall[] {\n  const results: ExtractedToolCall[] = [];\n  const seenToolCallIds = new Set<string>();\n  let index = 0;\n\n  for (const msg of messages) {\n    if (!msg.content) continue;\n\n    // Primary format: parts array with tool-invocation parts\n    for (const part of msg.content.parts ?? []) {\n      if (isToolInvocationPart(part)) {\n        const inv = part.toolInvocation;\n        if (isCompleted(inv)) {\n          const toolCallId = 'toolCallId' in inv ? (inv.toolCallId as string) : undefined;\n          if (toolCallId) seenToolCallIds.add(toolCallId);\n\n          results.push({\n            toolCallId,\n            toolName: inv.toolName ?? '',\n            args: (inv.args ?? {}) as Record<string, unknown>,\n            result: inv.result,\n            isError: isErrorState(inv),\n            index: index++,\n          });\n        }\n      }\n    }\n\n    // Legacy format: toolInvocations array (AI SDK ToolInvocation)\n    // Only states here are 'partial-call' | 'call' | 'result' — no error state.\n    if (msg.content.toolInvocations) {\n      for (const inv of msg.content.toolInvocations) {\n        if (inv.state === 'result') {\n          // Skip if we already extracted this from parts (deduplicate by toolCallId)\n          const toolCallId = 'toolCallId' in inv ? inv.toolCallId : undefined;\n          if (toolCallId && seenToolCallIds.has(toolCallId)) continue;\n          if (toolCallId) seenToolCallIds.add(toolCallId);\n\n          results.push({\n            toolCallId,\n            toolName: inv.toolName ?? '',\n            args: (inv.args ?? {}) as Record<string, unknown>,\n            result: inv.result,\n            isError: false,\n            index: index++,\n          });\n        }\n      }\n    }\n  }\n\n  return results;\n}\n"],"mappings":";AAoBA,SAAS,qBAAqB,MAG5B;CACA,OAAO,UAAU,QAAQ,KAAK,SAAS,qBAAqB,oBAAoB;AAClF;AAEA,SAAS,YAAY,KAAiC;CACpD,OACE,IAAI,UAAU,YAAY,IAAI,UAAU,WAAW,IAAI,UAAU,kBAAkB,IAAI,UAAU;AAErG;AAEA,SAAS,aAAa,KAAiC;CACrD,OAAO,IAAI,UAAU,WAAW,IAAI,UAAU,kBAAkB,IAAI,UAAU;AAChF;;;;;;;;AASA,SAAgB,iBAAiB,UAAkD;CACjF,MAAM,UAA+B,CAAC;CACtC,MAAM,kCAAkB,IAAI,IAAY;CACxC,IAAI,QAAQ;CAEZ,KAAK,MAAM,OAAO,UAAU;EAC1B,IAAI,CAAC,IAAI,SAAS;EAGlB,KAAK,MAAM,QAAQ,IAAI,QAAQ,SAAS,CAAC,GACvC,IAAI,qBAAqB,IAAI,GAAG;GAC9B,MAAM,MAAM,KAAK;GACjB,IAAI,YAAY,GAAG,GAAG;IACpB,MAAM,aAAa,gBAAgB,MAAO,IAAI,aAAwB,KAAA;IACtE,IAAI,YAAY,gBAAgB,IAAI,UAAU;IAE9C,QAAQ,KAAK;KACX;KACA,UAAU,IAAI,YAAY;KAC1B,MAAO,IAAI,QAAQ,CAAC;KACpB,QAAQ,IAAI;KACZ,SAAS,aAAa,GAAG;KACzB,OAAO;IACT,CAAC;GACH;EACF;EAKF,IAAI,IAAI,QAAQ,iBACT;QAAA,MAAM,OAAO,IAAI,QAAQ,iBAC5B,IAAI,IAAI,UAAU,UAAU;IAE1B,MAAM,aAAa,gBAAgB,MAAM,IAAI,aAAa,KAAA;IAC1D,IAAI,cAAc,gBAAgB,IAAI,UAAU,GAAG;IACnD,IAAI,YAAY,gBAAgB,IAAI,UAAU;IAE9C,QAAQ,KAAK;KACX;KACA,UAAU,IAAI,YAAY;KAC1B,MAAO,IAAI,QAAQ,CAAC;KACpB,QAAQ,IAAI;KACZ,SAAS;KACT,OAAO;IACT,CAAC;GACH;;CAGN;CAEA,OAAO;AACT"}

package/dist/evals/scorers/index.d.ts

@@ -0,0 +1,3 @@
+export { createOutcomeScorer } from './outcome.js';
+export { createEfficiencyScorer } from './efficiency.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/evals/scorers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/evals/scorers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/evals/scorers/index.js

@@ -0,0 +1,3 @@
+import { createOutcomeScorer } from "./outcome.js";
+import { createEfficiencyScorer } from "./efficiency.js";
+export { createEfficiencyScorer, createOutcomeScorer };

package/dist/evals/scorers/outcome.d.ts

@@ -0,0 +1,49 @@
+/**
+ * MastraCode Outcome Scorer (Code-based, Agent type, always-on)
+ *
+ * Grades the objective outcome of a coding session by checking hard facts:
+ * - Did the build/typecheck pass?
+ * - Did tests pass?
+ * - What was the tool error rate?
+ * - Did the agent get stuck in a loop?
+ * - Was there a regression (build passed earlier, failed later)?
+ * - Did the agent use ask_user unnecessarily? (autonomy)
+ *
+ * This scorer is always-on (no sampling) because it's cheap — it only
+ * reads exit codes and error states from tool results already in the messages.
+ *
+ * Scoring philosophy (from Anthropic's eval guidance):
+ * "Grade outcomes, not paths" — we check what happened, not how.
+ * Partial credit: a session that passes build but fails tests scores
+ * higher than one that fails build entirely.
+ *
+ * Dimensions that don't apply to a session (e.g. build for a docs-only task)
+ * are excluded from the weighted average, so sessions aren't penalized for
+ * irrelevant criteria.
+ */
+type DimensionResult = {
+    score: number;
+    detail: string;
+    applicable: boolean;
+};
+export declare function createOutcomeScorer(): import("@mastra/core/evals").MastraScorer<"mastracode-outcome", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    empty: true;
+    build: DimensionResult;
+    tests: DimensionResult;
+    toolErrors: DimensionResult;
+    loops: DimensionResult;
+    regression: DimensionResult;
+    autonomy: DimensionResult;
+    totalCalls: number;
+} | {
+    empty: false;
+    build: DimensionResult;
+    tests: DimensionResult;
+    toolErrors: DimensionResult;
+    loops: DimensionResult;
+    regression: DimensionResult;
+    autonomy: DimensionResult;
+    totalCalls: number;
+}> & Record<"generateScoreStepResult", number> & Record<"generateReasonStepResult", string>>;
+export {};
+//# sourceMappingURL=outcome.d.ts.map

package/dist/evals/scorers/outcome.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outcome.d.ts","sourceRoot":"","sources":["../../../src/evals/scorers/outcome.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAsDH,KAAK,eAAe,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,OAAO,CAAA;CAAE,CAAC;AAuN9E,wBAAgB,mBAAmB;;;;;;;;;;;;;;;;;;6FAkFlC"}