@mastra/code-sdk 1.0.3-alpha.0 → 1.1.0-alpha.2

package/dist/evals/scorers/extract-tools.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/evals/scorers/extract-tools.ts"],"sourcesContent":["/**\n * Shared tool invocation extraction from MastraDBMessages.\n *\n * Used by both the Outcome and Efficiency scorers to extract completed\n * tool calls from message content, handling both the `parts` array (primary)\n * and the legacy `toolInvocations` array.\n */\n\nimport type { MastraDBMessage } from '@mastra/core/agent';\nimport type { MastraMessagePart, MastraToolInvocation } from '@mastra/core/agent/message-list';\n\nexport type ExtractedToolCall = {\n  toolCallId?: string;\n  toolName: string;\n  args: Record<string, unknown>;\n  result: unknown;\n  isError: boolean;\n  index: number;\n};\n\nfunction isToolInvocationPart(part: MastraMessagePart): part is MastraMessagePart & {\n  type: 'tool-invocation';\n  toolInvocation: MastraToolInvocation;\n} {\n  return 'type' in part && part.type === 'tool-invocation' && 'toolInvocation' in part;\n}\n\nfunction isCompleted(inv: { state: string }): boolean {\n  return (\n    inv.state === 'result' || inv.state === 'error' || inv.state === 'output-error' || inv.state === 'output-denied'\n  );\n}\n\nfunction isErrorState(inv: { state: string }): boolean {\n  return inv.state === 'error' || inv.state === 'output-error' || inv.state === 'output-denied';\n}\n\n/**\n * Extract completed tool invocations from MastraDBMessages.\n *\n * Handles both:\n * - `content.parts` (primary format, MastraToolInvocationPart)\n * - `content.toolInvocations` (legacy format, AI SDK ToolInvocation[])\n */\nexport function extractToolCalls(messages: MastraDBMessage[]): ExtractedToolCall[] {\n  const results: ExtractedToolCall[] = [];\n  const seenToolCallIds = new Set<string>();\n  let index = 0;\n\n  for (const msg of messages) {\n    if (!msg.content) continue;\n\n    // Primary format: parts array with tool-invocation parts\n    for (const part of msg.content.parts ?? []) {\n      if (isToolInvocationPart(part)) {\n        const inv = part.toolInvocation;\n        if (isCompleted(inv)) {\n          const toolCallId = 'toolCallId' in inv ? (inv.toolCallId as string) : undefined;\n          if (toolCallId) seenToolCallIds.add(toolCallId);\n\n          results.push({\n            toolCallId,\n            toolName: inv.toolName ?? '',\n            args: (inv.args ?? {}) as Record<string, unknown>,\n            result: inv.result,\n            isError: isErrorState(inv),\n            index: index++,\n          });\n        }\n      }\n    }\n\n    // Legacy format: toolInvocations array (AI SDK ToolInvocation)\n    // Only states here are 'partial-call' | 'call' | 'result' — no error state.\n    if (msg.content.toolInvocations) {\n      for (const inv of msg.content.toolInvocations) {\n        if (inv.state === 'result') {\n          // Skip if we already extracted this from parts (deduplicate by toolCallId)\n          const toolCallId = 'toolCallId' in inv ? inv.toolCallId : undefined;\n          if (toolCallId && seenToolCallIds.has(toolCallId)) continue;\n          if (toolCallId) seenToolCallIds.add(toolCallId);\n\n          results.push({\n            toolCallId,\n            toolName: inv.toolName ?? '',\n            args: (inv.args ?? {}) as Record<string, unknown>,\n            result: inv.result,\n            isError: false,\n            index: index++,\n          });\n        }\n      }\n    }\n  }\n\n  return results;\n}\n"],"mappings":"AAoBA,SAAS,qBAAqB,MAG5B;AACA,SAAO,UAAU,QAAQ,KAAK,SAAS,qBAAqB,oBAAoB;AAClF;AAEA,SAAS,YAAY,KAAiC;AACpD,SACE,IAAI,UAAU,YAAY,IAAI,UAAU,WAAW,IAAI,UAAU,kBAAkB,IAAI,UAAU;AAErG;AAEA,SAAS,aAAa,KAAiC;AACrD,SAAO,IAAI,UAAU,WAAW,IAAI,UAAU,kBAAkB,IAAI,UAAU;AAChF;AASO,SAAS,iBAAiB,UAAkD;AACjF,QAAM,UAA+B,CAAC;AACtC,QAAM,kBAAkB,oBAAI,IAAY;AACxC,MAAI,QAAQ;AAEZ,aAAW,OAAO,UAAU;AAC1B,QAAI,CAAC,IAAI,QAAS;AAGlB,eAAW,QAAQ,IAAI,QAAQ,SAAS,CAAC,GAAG;AAC1C,UAAI,qBAAqB,IAAI,GAAG;AAC9B,cAAM,MAAM,KAAK;AACjB,YAAI,YAAY,GAAG,GAAG;AACpB,gBAAM,aAAa,gBAAgB,MAAO,IAAI,aAAwB;AACtE,cAAI,WAAY,iBAAgB,IAAI,UAAU;AAE9C,kBAAQ,KAAK;AAAA,YACX;AAAA,YACA,UAAU,IAAI,YAAY;AAAA,YAC1B,MAAO,IAAI,QAAQ,CAAC;AAAA,YACpB,QAAQ,IAAI;AAAA,YACZ,SAAS,aAAa,GAAG;AAAA,YACzB,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAIA,QAAI,IAAI,QAAQ,iBAAiB;AAC/B,iBAAW,OAAO,IAAI,QAAQ,iBAAiB;AAC7C,YAAI,IAAI,UAAU,UAAU;AAE1B,gBAAM,aAAa,gBAAgB,MAAM,IAAI,aAAa;AAC1D,cAAI,cAAc,gBAAgB,IAAI,UAAU,EAAG;AACnD,cAAI,WAAY,iBAAgB,IAAI,UAAU;AAE9C,kBAAQ,KAAK;AAAA,YACX;AAAA,YACA,UAAU,IAAI,YAAY;AAAA,YAC1B,MAAO,IAAI,QAAQ,CAAC;AAAA,YACpB,QAAQ,IAAI;AAAA,YACZ,SAAS;AAAA,YACT,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;","names":[]}
+{"version":3,"file":"extract-tools.js","names":[],"sources":["../../../src/evals/scorers/extract-tools.ts"],"sourcesContent":["/**\n * Shared tool invocation extraction from MastraDBMessages.\n *\n * Used by both the Outcome and Efficiency scorers to extract completed\n * tool calls from message content, handling both the `parts` array (primary)\n * and the legacy `toolInvocations` array.\n */\n\nimport type { MastraDBMessage } from '@mastra/core/agent';\nimport type { MastraMessagePart, MastraToolInvocation } from '@mastra/core/agent/message-list';\n\nexport type ExtractedToolCall = {\n  toolCallId?: string;\n  toolName: string;\n  args: Record<string, unknown>;\n  result: unknown;\n  isError: boolean;\n  index: number;\n};\n\nfunction isToolInvocationPart(part: MastraMessagePart): part is MastraMessagePart & {\n  type: 'tool-invocation';\n  toolInvocation: MastraToolInvocation;\n} {\n  return 'type' in part && part.type === 'tool-invocation' && 'toolInvocation' in part;\n}\n\nfunction isCompleted(inv: { state: string }): boolean {\n  return (\n    inv.state === 'result' || inv.state === 'error' || inv.state === 'output-error' || inv.state === 'output-denied'\n  );\n}\n\nfunction isErrorState(inv: { state: string }): boolean {\n  return inv.state === 'error' || inv.state === 'output-error' || inv.state === 'output-denied';\n}\n\n/**\n * Extract completed tool invocations from MastraDBMessages.\n *\n * Handles both:\n * - `content.parts` (primary format, MastraToolInvocationPart)\n * - `content.toolInvocations` (legacy format, AI SDK ToolInvocation[])\n */\nexport function extractToolCalls(messages: MastraDBMessage[]): ExtractedToolCall[] {\n  const results: ExtractedToolCall[] = [];\n  const seenToolCallIds = new Set<string>();\n  let index = 0;\n\n  for (const msg of messages) {\n    if (!msg.content) continue;\n\n    // Primary format: parts array with tool-invocation parts\n    for (const part of msg.content.parts ?? []) {\n      if (isToolInvocationPart(part)) {\n        const inv = part.toolInvocation;\n        if (isCompleted(inv)) {\n          const toolCallId = 'toolCallId' in inv ? (inv.toolCallId as string) : undefined;\n          if (toolCallId) seenToolCallIds.add(toolCallId);\n\n          results.push({\n            toolCallId,\n            toolName: inv.toolName ?? '',\n            args: (inv.args ?? {}) as Record<string, unknown>,\n            result: inv.result,\n            isError: isErrorState(inv),\n            index: index++,\n          });\n        }\n      }\n    }\n\n    // Legacy format: toolInvocations array (AI SDK ToolInvocation)\n    // Only states here are 'partial-call' | 'call' | 'result' — no error state.\n    if (msg.content.toolInvocations) {\n      for (const inv of msg.content.toolInvocations) {\n        if (inv.state === 'result') {\n          // Skip if we already extracted this from parts (deduplicate by toolCallId)\n          const toolCallId = 'toolCallId' in inv ? inv.toolCallId : undefined;\n          if (toolCallId && seenToolCallIds.has(toolCallId)) continue;\n          if (toolCallId) seenToolCallIds.add(toolCallId);\n\n          results.push({\n            toolCallId,\n            toolName: inv.toolName ?? '',\n            args: (inv.args ?? {}) as Record<string, unknown>,\n            result: inv.result,\n            isError: false,\n            index: index++,\n          });\n        }\n      }\n    }\n  }\n\n  return results;\n}\n"],"mappings":";AAoBA,SAAS,qBAAqB,MAG5B;CACA,OAAO,UAAU,QAAQ,KAAK,SAAS,qBAAqB,oBAAoB;AAClF;AAEA,SAAS,YAAY,KAAiC;CACpD,OACE,IAAI,UAAU,YAAY,IAAI,UAAU,WAAW,IAAI,UAAU,kBAAkB,IAAI,UAAU;AAErG;AAEA,SAAS,aAAa,KAAiC;CACrD,OAAO,IAAI,UAAU,WAAW,IAAI,UAAU,kBAAkB,IAAI,UAAU;AAChF;;;;;;;;AASA,SAAgB,iBAAiB,UAAkD;CACjF,MAAM,UAA+B,CAAC;CACtC,MAAM,kCAAkB,IAAI,IAAY;CACxC,IAAI,QAAQ;CAEZ,KAAK,MAAM,OAAO,UAAU;EAC1B,IAAI,CAAC,IAAI,SAAS;EAGlB,KAAK,MAAM,QAAQ,IAAI,QAAQ,SAAS,CAAC,GACvC,IAAI,qBAAqB,IAAI,GAAG;GAC9B,MAAM,MAAM,KAAK;GACjB,IAAI,YAAY,GAAG,GAAG;IACpB,MAAM,aAAa,gBAAgB,MAAO,IAAI,aAAwB,KAAA;IACtE,IAAI,YAAY,gBAAgB,IAAI,UAAU;IAE9C,QAAQ,KAAK;KACX;KACA,UAAU,IAAI,YAAY;KAC1B,MAAO,IAAI,QAAQ,CAAC;KACpB,QAAQ,IAAI;KACZ,SAAS,aAAa,GAAG;KACzB,OAAO;IACT,CAAC;GACH;EACF;EAKF,IAAI,IAAI,QAAQ,iBACT;QAAA,MAAM,OAAO,IAAI,QAAQ,iBAC5B,IAAI,IAAI,UAAU,UAAU;IAE1B,MAAM,aAAa,gBAAgB,MAAM,IAAI,aAAa,KAAA;IAC1D,IAAI,cAAc,gBAAgB,IAAI,UAAU,GAAG;IACnD,IAAI,YAAY,gBAAgB,IAAI,UAAU;IAE9C,QAAQ,KAAK;KACX;KACA,UAAU,IAAI,YAAY;KAC1B,MAAO,IAAI,QAAQ,CAAC;KACpB,QAAQ,IAAI;KACZ,SAAS;KACT,OAAO;IACT,CAAC;GACH;;CAGN;CAEA,OAAO;AACT"}

package/dist/evals/scorers/index.js

@@ -1,7 +1,3 @@
 import { createOutcomeScorer } from "./outcome.js";
 import { createEfficiencyScorer } from "./efficiency.js";
-export {
-  createEfficiencyScorer,
-  createOutcomeScorer
-};
-//# sourceMappingURL=index.js.map
+export { createEfficiencyScorer, createOutcomeScorer };

package/dist/evals/scorers/outcome.js

@@ -1,238 +1,347 @@
-import { createScorer, filterRun } from "@mastra/core/evals";
-import { isBuildCommand, isTestCommand, getExitCode } from "./classify-command.js";
+import { getExitCode, isBuildCommand, isTestCommand } from "./classify-command.js";
 import { extractToolCalls } from "./extract-tools.js";
+import { createScorer, filterRun } from "@mastra/core/evals";
+//#region src/evals/scorers/outcome.ts
+/**
+* Dimension weights. Used to compute the weighted average of applicable dimensions.
+* Dimensions that don't apply (e.g. no build ran) are excluded from the average.
+*/
 const WEIGHTS = {
-  build: 0.3,
-  tests: 0.25,
-  toolErrors: 0.2,
-  loops: 0.1,
-  regression: 0.1,
-  autonomy: 0.05
+	build: .3,
+	tests: .25,
+	toolErrors: .2,
+	loops: .1,
+	regression: .1,
+	autonomy: .05
 };
+/**
+* Scoring thresholds and multipliers.
+*/
 const THRESHOLDS = {
-  /** Multiplier for tool error rate penalty. Score = max(0, 1 - rate × this). */
-  toolErrorPenaltyMultiplier: 1,
-  /** How many consecutive identical calls (or same-error repeats) before it's a "loop". */
-  loopMinRepetitions: 3,
-  /** Score penalty per detected loop. */
-  loopPenaltyPerOccurrence: 0.3,
-  /** Score deducted per ask_user call. */
-  autonomyPenaltyPerAsk: 0.25,
-  /** Score when a command ran but exit code is ambiguous. */
-  ambiguousExitScore: 0.75,
-  /** Minimum tool calls required to produce a meaningful score. */
-  minToolCalls: 1,
-  /** Tool names whose errors are expected/benign (not penalized). */
-  benignErrorTools: ["search_content", "find_files", "lsp_inspect"]
+	/** Multiplier for tool error rate penalty. Score = max(0, 1 - rate × this). */
+	toolErrorPenaltyMultiplier: 1,
+	/** How many consecutive identical calls (or same-error repeats) before it's a "loop". */
+	loopMinRepetitions: 3,
+	/** Score penalty per detected loop. */
+	loopPenaltyPerOccurrence: .3,
+	/** Score deducted per ask_user call. */
+	autonomyPenaltyPerAsk: .25,
+	/** Score when a command ran but exit code is ambiguous. */
+	ambiguousExitScore: .75,
+	/** Minimum tool calls required to produce a meaningful score. */
+	minToolCalls: 1,
+	/** Tool names whose errors are expected/benign (not penalized). */
+	benignErrorTools: [
+		"search_content",
+		"find_files",
+		"lsp_inspect"
+	]
 };
+/**
+* Check if the primary verb of an execute_command is a build/typecheck command.
+* Splits compound commands (&&, ;) and checks the first segment.
+*/
+/** Extract command string from tool call args, then delegate to shared classifier. */
 function isBuildArgs(args) {
-  return isBuildCommand(String(args.command ?? ""));
+	return isBuildCommand(String(args.command ?? ""));
 }
 function isTestArgs(args) {
-  return isTestCommand(String(args.command ?? ""));
+	return isTestCommand(String(args.command ?? ""));
 }
+/**
+* Score build/typecheck outcome (final state).
+*/
 function scoreBuild(execResults) {
-  const builds = execResults.filter((r) => r.toolName === "execute_command" && isBuildArgs(r.args));
-  if (builds.length === 0) return { score: 0, detail: "No build/typecheck ran", applicable: false };
-  const last = builds[builds.length - 1];
-  if (last.isError) return { score: 0, detail: "Build/typecheck tool errored", applicable: true };
-  const exitCode = getExitCode(last.result);
-  if (exitCode === 0) return { score: 1, detail: "Build/typecheck passed", applicable: true };
-  if (exitCode !== null && exitCode !== 0)
-    return { score: 0, detail: `Build failed (exit ${exitCode})`, applicable: true };
-  const text = typeof last.result === "string" ? last.result : JSON.stringify(last.result ?? "");
-  if (/error TS\d+|Cannot find module|is not assignable/i.test(text)) {
-    return { score: 0, detail: "Build failed (TypeScript errors in output)", applicable: true };
-  }
-  return { score: THRESHOLDS.ambiguousExitScore, detail: "Build ran, outcome unclear", applicable: true };
+	const builds = execResults.filter((r) => r.toolName === "execute_command" && isBuildArgs(r.args));
+	if (builds.length === 0) return {
+		score: 0,
+		detail: "No build/typecheck ran",
+		applicable: false
+	};
+	const last = builds[builds.length - 1];
+	if (last.isError) return {
+		score: 0,
+		detail: "Build/typecheck tool errored",
+		applicable: true
+	};
+	const exitCode = getExitCode(last.result);
+	if (exitCode === 0) return {
+		score: 1,
+		detail: "Build/typecheck passed",
+		applicable: true
+	};
+	if (exitCode !== null && exitCode !== 0) return {
+		score: 0,
+		detail: `Build failed (exit ${exitCode})`,
+		applicable: true
+	};
+	const text = typeof last.result === "string" ? last.result : JSON.stringify(last.result ?? "");
+	if (/error TS\d+|Cannot find module|is not assignable/i.test(text)) return {
+		score: 0,
+		detail: "Build failed (TypeScript errors in output)",
+		applicable: true
+	};
+	return {
+		score: THRESHOLDS.ambiguousExitScore,
+		detail: "Build ran, outcome unclear",
+		applicable: true
+	};
 }
+/**
+* Score test outcome (final state).
+*/
 function scoreTests(execResults) {
-  const tests = execResults.filter((r) => r.toolName === "execute_command" && isTestArgs(r.args));
-  if (tests.length === 0) return { score: 0, detail: "No tests ran", applicable: false };
-  const last = tests[tests.length - 1];
-  if (last.isError) return { score: 0, detail: "Test command errored", applicable: true };
-  const exitCode = getExitCode(last.result);
-  if (exitCode === 0) return { score: 1, detail: "Tests passed", applicable: true };
-  if (exitCode !== null && exitCode !== 0)
-    return { score: 0, detail: `Tests failed (exit ${exitCode})`, applicable: true };
-  const text = typeof last.result === "string" ? last.result : JSON.stringify(last.result ?? "");
-  if (/\d+ (?:tests? )?passed|✓|PASS/i.test(text) && !/\bfail(?:ed|ure)?\b|\berror\b/i.test(text)) {
-    return { score: 1, detail: "Tests passed (inferred)", applicable: true };
-  }
-  if (/\bFAIL\b|failed|✗|✘/i.test(text)) {
-    return { score: 0, detail: "Tests failed (inferred)", applicable: true };
-  }
-  return { score: THRESHOLDS.ambiguousExitScore, detail: "Tests ran, outcome unclear", applicable: true };
+	const tests = execResults.filter((r) => r.toolName === "execute_command" && isTestArgs(r.args));
+	if (tests.length === 0) return {
+		score: 0,
+		detail: "No tests ran",
+		applicable: false
+	};
+	const last = tests[tests.length - 1];
+	if (last.isError) return {
+		score: 0,
+		detail: "Test command errored",
+		applicable: true
+	};
+	const exitCode = getExitCode(last.result);
+	if (exitCode === 0) return {
+		score: 1,
+		detail: "Tests passed",
+		applicable: true
+	};
+	if (exitCode !== null && exitCode !== 0) return {
+		score: 0,
+		detail: `Tests failed (exit ${exitCode})`,
+		applicable: true
+	};
+	const text = typeof last.result === "string" ? last.result : JSON.stringify(last.result ?? "");
+	if (/\d+ (?:tests? )?passed|✓|PASS/i.test(text) && !/\bfail(?:ed|ure)?\b|\berror\b/i.test(text)) return {
+		score: 1,
+		detail: "Tests passed (inferred)",
+		applicable: true
+	};
+	if (/\bFAIL\b|failed|✗|✘/i.test(text)) return {
+		score: 0,
+		detail: "Tests failed (inferred)",
+		applicable: true
+	};
+	return {
+		score: THRESHOLDS.ambiguousExitScore,
+		detail: "Tests ran, outcome unclear",
+		applicable: true
+	};
 }
+/**
+* Score tool error rate, excluding benign errors from expected-to-fail tools.
+*/
 function scoreToolErrors(results) {
-  if (results.length === 0) return { score: 1, detail: "No tool calls", applicable: false };
-  const nonBenign = results.filter((r) => !THRESHOLDS.benignErrorTools.includes(r.toolName));
-  if (nonBenign.length === 0) return { score: 1, detail: "All tool calls are benign-error tools", applicable: true };
-  const errors = nonBenign.filter((r) => r.isError);
-  const rate = errors.length / nonBenign.length;
-  if (rate === 0) return { score: 1, detail: "No tool errors", applicable: true };
-  const score = Math.max(0, 1 - rate * THRESHOLDS.toolErrorPenaltyMultiplier);
-  return {
-    score,
-    detail: `${errors.length}/${nonBenign.length} tools errored (${(rate * 100).toFixed(0)}%)`,
-    applicable: true
-  };
+	if (results.length === 0) return {
+		score: 1,
+		detail: "No tool calls",
+		applicable: false
+	};
+	const nonBenign = results.filter((r) => !THRESHOLDS.benignErrorTools.includes(r.toolName));
+	if (nonBenign.length === 0) return {
+		score: 1,
+		detail: "All tool calls are benign-error tools",
+		applicable: true
+	};
+	const errors = nonBenign.filter((r) => r.isError);
+	const rate = errors.length / nonBenign.length;
+	if (rate === 0) return {
+		score: 1,
+		detail: "No tool errors",
+		applicable: true
+	};
+	return {
+		score: Math.max(0, 1 - rate * THRESHOLDS.toolErrorPenaltyMultiplier),
+		detail: `${errors.length}/${nonBenign.length} tools errored (${(rate * 100).toFixed(0)}%)`,
+		applicable: true
+	};
 }
+/**
+* Detect stuck loops: same tool + same args appearing 3+ times consecutively,
+* or same tool + same args erroring 3+ times total.
+*/
 function scoreStuckLoops(results) {
-  if (results.length < 3) return { score: 1, detail: "Too few calls for loop detection", applicable: true };
-  const fingerprint = (r) => `${r.toolName}:${JSON.stringify(r.args)}`;
-  let maxConsecutive = 1;
-  let currentRun = 1;
-  for (let i = 1; i < results.length; i++) {
-    if (fingerprint(results[i]) === fingerprint(results[i - 1])) {
-      currentRun++;
-      maxConsecutive = Math.max(maxConsecutive, currentRun);
-    } else {
-      currentRun = 1;
-    }
-  }
-  const errorCounts = /* @__PURE__ */ new Map();
-  for (const r of results) {
-    if (!r.isError) continue;
-    const fp = fingerprint(r);
-    errorCounts.set(fp, (errorCounts.get(fp) ?? 0) + 1);
-  }
-  const maxErrorRepeat = Math.max(0, ...[...errorCounts.values()]);
-  const minReps = THRESHOLDS.loopMinRepetitions;
-  const loopSeverity = Math.max(
-    maxConsecutive >= minReps ? maxConsecutive - (minReps - 1) : 0,
-    maxErrorRepeat >= minReps ? maxErrorRepeat - (minReps - 1) : 0
-  );
-  if (loopSeverity === 0) return { score: 1, detail: "No stuck loops", applicable: true };
-  const score = Math.max(0, 1 - loopSeverity * THRESHOLDS.loopPenaltyPerOccurrence);
-  const details = [];
-  if (maxConsecutive >= minReps) details.push(`${maxConsecutive} consecutive identical calls`);
-  if (maxErrorRepeat >= minReps) details.push(`same error repeated ${maxErrorRepeat}x`);
-  return { score, detail: details.join("; "), applicable: true };
+	if (results.length < 3) return {
+		score: 1,
+		detail: "Too few calls for loop detection",
+		applicable: true
+	};
+	const fingerprint = (r) => `${r.toolName}:${JSON.stringify(r.args)}`;
+	let maxConsecutive = 1;
+	let currentRun = 1;
+	for (let i = 1; i < results.length; i++) if (fingerprint(results[i]) === fingerprint(results[i - 1])) {
+		currentRun++;
+		maxConsecutive = Math.max(maxConsecutive, currentRun);
+	} else currentRun = 1;
+	const errorCounts = /* @__PURE__ */ new Map();
+	for (const r of results) {
+		if (!r.isError) continue;
+		const fp = fingerprint(r);
+		errorCounts.set(fp, (errorCounts.get(fp) ?? 0) + 1);
+	}
+	const maxErrorRepeat = Math.max(0, ...[...errorCounts.values()]);
+	const minReps = THRESHOLDS.loopMinRepetitions;
+	const loopSeverity = Math.max(maxConsecutive >= minReps ? maxConsecutive - (minReps - 1) : 0, maxErrorRepeat >= minReps ? maxErrorRepeat - (minReps - 1) : 0);
+	if (loopSeverity === 0) return {
+		score: 1,
+		detail: "No stuck loops",
+		applicable: true
+	};
+	const score = Math.max(0, 1 - loopSeverity * THRESHOLDS.loopPenaltyPerOccurrence);
+	const details = [];
+	if (maxConsecutive >= minReps) details.push(`${maxConsecutive} consecutive identical calls`);
+	if (maxErrorRepeat >= minReps) details.push(`same error repeated ${maxErrorRepeat}x`);
+	return {
+		score,
+		detail: details.join("; "),
+		applicable: true
+	};
 }
+/**
+* Detect regressions: build/test passed at some point, then failed later.
+* Gives partial credit if the agent recovered (final state passes).
+*/
 function scoreRegression(execResults) {
-  const regressions = [];
-  for (const [label, filter] of [
-    ["Build", isBuildArgs],
-    ["Tests", isTestArgs]
-  ]) {
-    const cmds = execResults.filter((r) => r.toolName === "execute_command" && filter(r.args));
-    if (cmds.length < 2) continue;
-    let sawPass = false;
-    let sawRegression = false;
-    for (const cmd of cmds) {
-      const exit = getExitCode(cmd.result);
-      if (exit === 0) {
-        if (sawRegression) {
-          regressions.push({ label, recovered: true });
-          sawRegression = false;
-        }
-        sawPass = true;
-      }
-      if (sawPass && exit !== null && exit !== 0) {
-        sawRegression = true;
-      }
-    }
-    if (sawRegression) {
-      regressions.push({ label, recovered: false });
-    }
-  }
-  if (regressions.length === 0) return { score: 1, detail: "No regressions", applicable: true };
-  const persistentCount = regressions.filter((r) => !r.recovered).length;
-  const recoveredCount = regressions.filter((r) => r.recovered).length;
-  const score = Math.max(0, 1 - persistentCount * 0.5 - recoveredCount * 0.1);
-  const details = regressions.map(
-    (r) => `${r.label} ${r.recovered ? "regressed then recovered" : "regressed (persisted)"}`
-  );
-  return { score, detail: details.join("; "), applicable: true };
+	const regressions = [];
+	for (const [label, filter] of [["Build", isBuildArgs], ["Tests", isTestArgs]]) {
+		const cmds = execResults.filter((r) => r.toolName === "execute_command" && filter(r.args));
+		if (cmds.length < 2) continue;
+		let sawPass = false;
+		let sawRegression = false;
+		for (const cmd of cmds) {
+			const exit = getExitCode(cmd.result);
+			if (exit === 0) {
+				if (sawRegression) {
+					regressions.push({
+						label,
+						recovered: true
+					});
+					sawRegression = false;
+				}
+				sawPass = true;
+			}
+			if (sawPass && exit !== null && exit !== 0) sawRegression = true;
+		}
+		if (sawRegression) regressions.push({
+			label,
+			recovered: false
+		});
+	}
+	if (regressions.length === 0) return {
+		score: 1,
+		detail: "No regressions",
+		applicable: true
+	};
+	const persistentCount = regressions.filter((r) => !r.recovered).length;
+	const recoveredCount = regressions.filter((r) => r.recovered).length;
+	return {
+		score: Math.max(0, 1 - persistentCount * .5 - recoveredCount * .1),
+		detail: regressions.map((r) => `${r.label} ${r.recovered ? "regressed then recovered" : "regressed (persisted)"}`).join("; "),
+		applicable: true
+	};
 }
+/**
+* Score autonomy: penalize ask_user calls.
+* 0 calls = 1.0, each call reduces by 0.25 (up to a floor of 0).
+*/
 function scoreAutonomy(results) {
-  const askUserCalls = results.filter((r) => r.toolName === "ask_user");
-  const count = askUserCalls.length;
-  if (count === 0) return { score: 1, detail: "No ask_user calls (fully autonomous)", applicable: true };
-  const score = Math.max(0, 1 - count * THRESHOLDS.autonomyPenaltyPerAsk);
-  return { score, detail: `${count} ask_user call${count > 1 ? "s" : ""}`, applicable: true };
+	const count = results.filter((r) => r.toolName === "ask_user").length;
+	if (count === 0) return {
+		score: 1,
+		detail: "No ask_user calls (fully autonomous)",
+		applicable: true
+	};
+	return {
+		score: Math.max(0, 1 - count * THRESHOLDS.autonomyPenaltyPerAsk),
+		detail: `${count} ask_user call${count > 1 ? "s" : ""}`,
+		applicable: true
+	};
 }
+/**
+* Compute weighted average of only applicable dimensions.
+* Returns null if no dimensions are applicable.
+*/
 function weightedAverage(dimensions, weights) {
-  let totalWeight = 0;
-  let weightedSum = 0;
-  for (const [key, result] of Object.entries(dimensions)) {
-    if (!result.applicable) continue;
-    const weight = weights[key] ?? 0;
-    totalWeight += weight;
-    weightedSum += result.score * weight;
-  }
-  if (totalWeight === 0) return null;
-  return weightedSum / totalWeight;
+	let totalWeight = 0;
+	let weightedSum = 0;
+	for (const [key, result] of Object.entries(dimensions)) {
+		if (!result.applicable) continue;
+		const weight = weights[key] ?? 0;
+		totalWeight += weight;
+		weightedSum += result.score * weight;
+	}
+	if (totalWeight === 0) return null;
+	return weightedSum / totalWeight;
 }
 function createOutcomeScorer() {
-  return createScorer({
-    id: "mastracode-outcome",
-    name: "MastraCode Outcome",
-    description: "Grades coding session outcomes: build/test pass, tool errors, stuck loops, regressions, autonomy. Always-on.",
-    type: "agent",
-    prepareRun: filterRun({ partTypes: ["tool-invocation", "text"] })
-  }).preprocess(async ({ run }) => {
-    const messages = run.output ?? [];
-    const allResults = extractToolCalls(messages);
-    const totalCalls = allResults.length;
-    if (totalCalls < THRESHOLDS.minToolCalls) {
-      const empty = { score: 0, detail: "No tool calls", applicable: false };
-      return {
-        empty: true,
-        build: empty,
-        tests: empty,
-        toolErrors: empty,
-        loops: empty,
-        regression: empty,
-        autonomy: empty,
-        totalCalls
-      };
-    }
-    const execResults = allResults.filter((r) => r.toolName === "execute_command");
-    return {
-      empty: false,
-      build: scoreBuild(execResults),
-      tests: scoreTests(execResults),
-      toolErrors: scoreToolErrors(allResults),
-      loops: scoreStuckLoops(allResults),
-      regression: scoreRegression(execResults),
-      autonomy: scoreAutonomy(allResults),
-      totalCalls
-    };
-  }).generateScore(({ results }) => {
-    const p = results.preprocessStepResult;
-    if (p.empty) return 0;
-    const score = weightedAverage(
-      {
-        build: p.build,
-        tests: p.tests,
-        toolErrors: p.toolErrors,
-        loops: p.loops,
-        regression: p.regression,
-        autonomy: p.autonomy
-      },
-      WEIGHTS
-    );
-    return score === null ? 0 : Math.round(score * 100) / 100;
-  }).generateReason(({ results, score }) => {
-    const p = results.preprocessStepResult;
-    if (p.empty) {
-      return `Score: 0 (${p.totalCalls} tool calls \u2014 below minimum threshold for scoring)`;
-    }
-    const pct = (w) => `${(w * 100).toFixed(0)}%`;
-    const dimLine = (name, weight, r) => r.applicable ? `${name} (${pct(weight)}): ${r.detail} [${r.score}]` : `${name}: ${r.detail} [N/A \u2014 excluded from average]`;
-    const parts = [`Score: ${score} (${p.totalCalls} tool calls total)`];
-    parts.push(dimLine("Build", WEIGHTS.build, p.build));
-    parts.push(dimLine("Tests", WEIGHTS.tests, p.tests));
-    parts.push(dimLine("Tool errors", WEIGHTS.toolErrors, p.toolErrors));
-    parts.push(dimLine("Loops", WEIGHTS.loops, p.loops));
-    parts.push(dimLine("Regression", WEIGHTS.regression, p.regression));
-    parts.push(dimLine("Autonomy", WEIGHTS.autonomy, p.autonomy));
-    return parts.join("\n");
-  });
+	return createScorer({
+		id: "mastracode-outcome",
+		name: "MastraCode Outcome",
+		description: "Grades coding session outcomes: build/test pass, tool errors, stuck loops, regressions, autonomy. Always-on.",
+		type: "agent",
+		prepareRun: filterRun({ partTypes: ["tool-invocation", "text"] })
+	}).preprocess(async ({ run }) => {
+		const allResults = extractToolCalls(run.output ?? []);
+		const totalCalls = allResults.length;
+		if (totalCalls < THRESHOLDS.minToolCalls) {
+			const empty = {
+				score: 0,
+				detail: "No tool calls",
+				applicable: false
+			};
+			return {
+				empty: true,
+				build: empty,
+				tests: empty,
+				toolErrors: empty,
+				loops: empty,
+				regression: empty,
+				autonomy: empty,
+				totalCalls
+			};
+		}
+		const execResults = allResults.filter((r) => r.toolName === "execute_command");
+		return {
+			empty: false,
+			build: scoreBuild(execResults),
+			tests: scoreTests(execResults),
+			toolErrors: scoreToolErrors(allResults),
+			loops: scoreStuckLoops(allResults),
+			regression: scoreRegression(execResults),
+			autonomy: scoreAutonomy(allResults),
+			totalCalls
+		};
+	}).generateScore(({ results }) => {
+		const p = results.preprocessStepResult;
+		if (p.empty) return 0;
+		const score = weightedAverage({
+			build: p.build,
+			tests: p.tests,
+			toolErrors: p.toolErrors,
+			loops: p.loops,
+			regression: p.regression,
+			autonomy: p.autonomy
+		}, WEIGHTS);
+		return score === null ? 0 : Math.round(score * 100) / 100;
+	}).generateReason(({ results, score }) => {
+		const p = results.preprocessStepResult;
+		if (p.empty) return `Score: 0 (${p.totalCalls} tool calls — below minimum threshold for scoring)`;
+		const pct = (w) => `${(w * 100).toFixed(0)}%`;
+		const dimLine = (name, weight, r) => r.applicable ? `${name} (${pct(weight)}): ${r.detail} [${r.score}]` : `${name}: ${r.detail} [N/A — excluded from average]`;
+		const parts = [`Score: ${score} (${p.totalCalls} tool calls total)`];
+		parts.push(dimLine("Build", WEIGHTS.build, p.build));
+		parts.push(dimLine("Tests", WEIGHTS.tests, p.tests));
+		parts.push(dimLine("Tool errors", WEIGHTS.toolErrors, p.toolErrors));
+		parts.push(dimLine("Loops", WEIGHTS.loops, p.loops));
+		parts.push(dimLine("Regression", WEIGHTS.regression, p.regression));
+		parts.push(dimLine("Autonomy", WEIGHTS.autonomy, p.autonomy));
+		return parts.join("\n");
+	});
 }
-export {
-  createOutcomeScorer
-};
+//#endregion
+export { createOutcomeScorer };
+
 //# sourceMappingURL=outcome.js.map