@mastra/code-sdk 0.0.0-livekit-beta-v3-20260709031728

package/dist/evals/scorers/extract-tools.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Shared tool invocation extraction from MastraDBMessages.
+ *
+ * Used by both the Outcome and Efficiency scorers to extract completed
+ * tool calls from message content, handling both the `parts` array (primary)
+ * and the legacy `toolInvocations` array.
+ */
+import type { MastraDBMessage } from '@mastra/core/agent';
+export type ExtractedToolCall = {
+    toolCallId?: string;
+    toolName: string;
+    args: Record<string, unknown>;
+    result: unknown;
+    isError: boolean;
+    index: number;
+};
+/**
+ * Extract completed tool invocations from MastraDBMessages.
+ *
+ * Handles both:
+ * - `content.parts` (primary format, MastraToolInvocationPart)
+ * - `content.toolInvocations` (legacy format, AI SDK ToolInvocation[])
+ */
+export declare function extractToolCalls(messages: MastraDBMessage[]): ExtractedToolCall[];
+//# sourceMappingURL=extract-tools.d.ts.map

package/dist/evals/scorers/extract-tools.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract-tools.d.ts","sourceRoot":"","sources":["../../../src/evals/scorers/extract-tools.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAG1D,MAAM,MAAM,iBAAiB,GAAG;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAmBF;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,eAAe,EAAE,GAAG,iBAAiB,EAAE,CAoDjF"}

package/dist/evals/scorers/extract-tools.js

@@ -0,0 +1,56 @@
+function isToolInvocationPart(part) {
+  return "type" in part && part.type === "tool-invocation" && "toolInvocation" in part;
+}
+function isCompleted(inv) {
+  return inv.state === "result" || inv.state === "error" || inv.state === "output-error" || inv.state === "output-denied";
+}
+function isErrorState(inv) {
+  return inv.state === "error" || inv.state === "output-error" || inv.state === "output-denied";
+}
+function extractToolCalls(messages) {
+  const results = [];
+  const seenToolCallIds = /* @__PURE__ */ new Set();
+  let index = 0;
+  for (const msg of messages) {
+    if (!msg.content) continue;
+    for (const part of msg.content.parts ?? []) {
+      if (isToolInvocationPart(part)) {
+        const inv = part.toolInvocation;
+        if (isCompleted(inv)) {
+          const toolCallId = "toolCallId" in inv ? inv.toolCallId : void 0;
+          if (toolCallId) seenToolCallIds.add(toolCallId);
+          results.push({
+            toolCallId,
+            toolName: inv.toolName ?? "",
+            args: inv.args ?? {},
+            result: inv.result,
+            isError: isErrorState(inv),
+            index: index++
+          });
+        }
+      }
+    }
+    if (msg.content.toolInvocations) {
+      for (const inv of msg.content.toolInvocations) {
+        if (inv.state === "result") {
+          const toolCallId = "toolCallId" in inv ? inv.toolCallId : void 0;
+          if (toolCallId && seenToolCallIds.has(toolCallId)) continue;
+          if (toolCallId) seenToolCallIds.add(toolCallId);
+          results.push({
+            toolCallId,
+            toolName: inv.toolName ?? "",
+            args: inv.args ?? {},
+            result: inv.result,
+            isError: false,
+            index: index++
+          });
+        }
+      }
+    }
+  }
+  return results;
+}
+export {
+  extractToolCalls
+};
+//# sourceMappingURL=extract-tools.js.map

package/dist/evals/scorers/extract-tools.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/evals/scorers/extract-tools.ts"],"sourcesContent":["/**\n * Shared tool invocation extraction from MastraDBMessages.\n *\n * Used by both the Outcome and Efficiency scorers to extract completed\n * tool calls from message content, handling both the `parts` array (primary)\n * and the legacy `toolInvocations` array.\n */\n\nimport type { MastraDBMessage } from '@mastra/core/agent';\nimport type { MastraMessagePart, MastraToolInvocation } from '@mastra/core/agent/message-list';\n\nexport type ExtractedToolCall = {\n  toolCallId?: string;\n  toolName: string;\n  args: Record<string, unknown>;\n  result: unknown;\n  isError: boolean;\n  index: number;\n};\n\nfunction isToolInvocationPart(part: MastraMessagePart): part is MastraMessagePart & {\n  type: 'tool-invocation';\n  toolInvocation: MastraToolInvocation;\n} {\n  return 'type' in part && part.type === 'tool-invocation' && 'toolInvocation' in part;\n}\n\nfunction isCompleted(inv: { state: string }): boolean {\n  return (\n    inv.state === 'result' || inv.state === 'error' || inv.state === 'output-error' || inv.state === 'output-denied'\n  );\n}\n\nfunction isErrorState(inv: { state: string }): boolean {\n  return inv.state === 'error' || inv.state === 'output-error' || inv.state === 'output-denied';\n}\n\n/**\n * Extract completed tool invocations from MastraDBMessages.\n *\n * Handles both:\n * - `content.parts` (primary format, MastraToolInvocationPart)\n * - `content.toolInvocations` (legacy format, AI SDK ToolInvocation[])\n */\nexport function extractToolCalls(messages: MastraDBMessage[]): ExtractedToolCall[] {\n  const results: ExtractedToolCall[] = [];\n  const seenToolCallIds = new Set<string>();\n  let index = 0;\n\n  for (const msg of messages) {\n    if (!msg.content) continue;\n\n    // Primary format: parts array with tool-invocation parts\n    for (const part of msg.content.parts ?? []) {\n      if (isToolInvocationPart(part)) {\n        const inv = part.toolInvocation;\n        if (isCompleted(inv)) {\n          const toolCallId = 'toolCallId' in inv ? (inv.toolCallId as string) : undefined;\n          if (toolCallId) seenToolCallIds.add(toolCallId);\n\n          results.push({\n            toolCallId,\n            toolName: inv.toolName ?? '',\n            args: (inv.args ?? {}) as Record<string, unknown>,\n            result: inv.result,\n            isError: isErrorState(inv),\n            index: index++,\n          });\n        }\n      }\n    }\n\n    // Legacy format: toolInvocations array (AI SDK ToolInvocation)\n    // Only states here are 'partial-call' | 'call' | 'result' — no error state.\n    if (msg.content.toolInvocations) {\n      for (const inv of msg.content.toolInvocations) {\n        if (inv.state === 'result') {\n          // Skip if we already extracted this from parts (deduplicate by toolCallId)\n          const toolCallId = 'toolCallId' in inv ? inv.toolCallId : undefined;\n          if (toolCallId && seenToolCallIds.has(toolCallId)) continue;\n          if (toolCallId) seenToolCallIds.add(toolCallId);\n\n          results.push({\n            toolCallId,\n            toolName: inv.toolName ?? '',\n            args: (inv.args ?? {}) as Record<string, unknown>,\n            result: inv.result,\n            isError: false,\n            index: index++,\n          });\n        }\n      }\n    }\n  }\n\n  return results;\n}\n"],"mappings":"AAoBA,SAAS,qBAAqB,MAG5B;AACA,SAAO,UAAU,QAAQ,KAAK,SAAS,qBAAqB,oBAAoB;AAClF;AAEA,SAAS,YAAY,KAAiC;AACpD,SACE,IAAI,UAAU,YAAY,IAAI,UAAU,WAAW,IAAI,UAAU,kBAAkB,IAAI,UAAU;AAErG;AAEA,SAAS,aAAa,KAAiC;AACrD,SAAO,IAAI,UAAU,WAAW,IAAI,UAAU,kBAAkB,IAAI,UAAU;AAChF;AASO,SAAS,iBAAiB,UAAkD;AACjF,QAAM,UAA+B,CAAC;AACtC,QAAM,kBAAkB,oBAAI,IAAY;AACxC,MAAI,QAAQ;AAEZ,aAAW,OAAO,UAAU;AAC1B,QAAI,CAAC,IAAI,QAAS;AAGlB,eAAW,QAAQ,IAAI,QAAQ,SAAS,CAAC,GAAG;AAC1C,UAAI,qBAAqB,IAAI,GAAG;AAC9B,cAAM,MAAM,KAAK;AACjB,YAAI,YAAY,GAAG,GAAG;AACpB,gBAAM,aAAa,gBAAgB,MAAO,IAAI,aAAwB;AACtE,cAAI,WAAY,iBAAgB,IAAI,UAAU;AAE9C,kBAAQ,KAAK;AAAA,YACX;AAAA,YACA,UAAU,IAAI,YAAY;AAAA,YAC1B,MAAO,IAAI,QAAQ,CAAC;AAAA,YACpB,QAAQ,IAAI;AAAA,YACZ,SAAS,aAAa,GAAG;AAAA,YACzB,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAIA,QAAI,IAAI,QAAQ,iBAAiB;AAC/B,iBAAW,OAAO,IAAI,QAAQ,iBAAiB;AAC7C,YAAI,IAAI,UAAU,UAAU;AAE1B,gBAAM,aAAa,gBAAgB,MAAM,IAAI,aAAa;AAC1D,cAAI,cAAc,gBAAgB,IAAI,UAAU,EAAG;AACnD,cAAI,WAAY,iBAAgB,IAAI,UAAU;AAE9C,kBAAQ,KAAK;AAAA,YACX;AAAA,YACA,UAAU,IAAI,YAAY;AAAA,YAC1B,MAAO,IAAI,QAAQ,CAAC;AAAA,YACpB,QAAQ,IAAI;AAAA,YACZ,SAAS;AAAA,YACT,OAAO;AAAA,UACT,CAAC;AAAA,QACH;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,SAAO;AACT;","names":[]}

package/dist/evals/scorers/index.d.ts

@@ -0,0 +1,3 @@
+export { createOutcomeScorer } from './outcome.js';
+export { createEfficiencyScorer } from './efficiency.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/evals/scorers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/evals/scorers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/evals/scorers/index.js

@@ -0,0 +1,7 @@
+import { createOutcomeScorer } from "./outcome.js";
+import { createEfficiencyScorer } from "./efficiency.js";
+export {
+  createEfficiencyScorer,
+  createOutcomeScorer
+};
+//# sourceMappingURL=index.js.map

package/dist/evals/scorers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/evals/scorers/index.ts"],"sourcesContent":["export { createOutcomeScorer } from './outcome.js';\nexport { createEfficiencyScorer } from './efficiency.js';\n"],"mappings":"AAAA,SAAS,2BAA2B;AACpC,SAAS,8BAA8B;","names":[]}

package/dist/evals/scorers/outcome.d.ts

@@ -0,0 +1,49 @@
+/**
+ * MastraCode Outcome Scorer (Code-based, Agent type, always-on)
+ *
+ * Grades the objective outcome of a coding session by checking hard facts:
+ * - Did the build/typecheck pass?
+ * - Did tests pass?
+ * - What was the tool error rate?
+ * - Did the agent get stuck in a loop?
+ * - Was there a regression (build passed earlier, failed later)?
+ * - Did the agent use ask_user unnecessarily? (autonomy)
+ *
+ * This scorer is always-on (no sampling) because it's cheap — it only
+ * reads exit codes and error states from tool results already in the messages.
+ *
+ * Scoring philosophy (from Anthropic's eval guidance):
+ * "Grade outcomes, not paths" — we check what happened, not how.
+ * Partial credit: a session that passes build but fails tests scores
+ * higher than one that fails build entirely.
+ *
+ * Dimensions that don't apply to a session (e.g. build for a docs-only task)
+ * are excluded from the weighted average, so sessions aren't penalized for
+ * irrelevant criteria.
+ */
+type DimensionResult = {
+    score: number;
+    detail: string;
+    applicable: boolean;
+};
+export declare function createOutcomeScorer(): import("@mastra/core/evals").MastraScorer<"mastracode-outcome", import("@mastra/core/evals").ScorerRunInputForAgent, import("@mastra/core/evals").ScorerRunOutputForAgent, Record<"preprocessStepResult", {
+    empty: true;
+    build: DimensionResult;
+    tests: DimensionResult;
+    toolErrors: DimensionResult;
+    loops: DimensionResult;
+    regression: DimensionResult;
+    autonomy: DimensionResult;
+    totalCalls: number;
+} | {
+    empty: false;
+    build: DimensionResult;
+    tests: DimensionResult;
+    toolErrors: DimensionResult;
+    loops: DimensionResult;
+    regression: DimensionResult;
+    autonomy: DimensionResult;
+    totalCalls: number;
+}> & Record<"generateScoreStepResult", number> & Record<"generateReasonStepResult", string>>;
+export {};
+//# sourceMappingURL=outcome.d.ts.map

package/dist/evals/scorers/outcome.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outcome.d.ts","sourceRoot":"","sources":["../../../src/evals/scorers/outcome.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAsDH,KAAK,eAAe,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,OAAO,CAAA;CAAE,CAAC;AAuN9E,wBAAgB,mBAAmB;;;;;;;;;;;;;;;;;;6FAkFlC"}

package/dist/evals/scorers/outcome.js

@@ -0,0 +1,238 @@
+import { createScorer, filterRun } from "@mastra/core/evals";
+import { isBuildCommand, isTestCommand, getExitCode } from "./classify-command.js";
+import { extractToolCalls } from "./extract-tools.js";
+const WEIGHTS = {
+  build: 0.3,
+  tests: 0.25,
+  toolErrors: 0.2,
+  loops: 0.1,
+  regression: 0.1,
+  autonomy: 0.05
+};
+const THRESHOLDS = {
+  /** Multiplier for tool error rate penalty. Score = max(0, 1 - rate × this). */
+  toolErrorPenaltyMultiplier: 1,
+  /** How many consecutive identical calls (or same-error repeats) before it's a "loop". */
+  loopMinRepetitions: 3,
+  /** Score penalty per detected loop. */
+  loopPenaltyPerOccurrence: 0.3,
+  /** Score deducted per ask_user call. */
+  autonomyPenaltyPerAsk: 0.25,
+  /** Score when a command ran but exit code is ambiguous. */
+  ambiguousExitScore: 0.75,
+  /** Minimum tool calls required to produce a meaningful score. */
+  minToolCalls: 1,
+  /** Tool names whose errors are expected/benign (not penalized). */
+  benignErrorTools: ["search_content", "find_files", "lsp_inspect"]
+};
+function isBuildArgs(args) {
+  return isBuildCommand(String(args.command ?? ""));
+}
+function isTestArgs(args) {
+  return isTestCommand(String(args.command ?? ""));
+}
+function scoreBuild(execResults) {
+  const builds = execResults.filter((r) => r.toolName === "execute_command" && isBuildArgs(r.args));
+  if (builds.length === 0) return { score: 0, detail: "No build/typecheck ran", applicable: false };
+  const last = builds[builds.length - 1];
+  if (last.isError) return { score: 0, detail: "Build/typecheck tool errored", applicable: true };
+  const exitCode = getExitCode(last.result);
+  if (exitCode === 0) return { score: 1, detail: "Build/typecheck passed", applicable: true };
+  if (exitCode !== null && exitCode !== 0)
+    return { score: 0, detail: `Build failed (exit ${exitCode})`, applicable: true };
+  const text = typeof last.result === "string" ? last.result : JSON.stringify(last.result ?? "");
+  if (/error TS\d+|Cannot find module|is not assignable/i.test(text)) {
+    return { score: 0, detail: "Build failed (TypeScript errors in output)", applicable: true };
+  }
+  return { score: THRESHOLDS.ambiguousExitScore, detail: "Build ran, outcome unclear", applicable: true };
+}
+function scoreTests(execResults) {
+  const tests = execResults.filter((r) => r.toolName === "execute_command" && isTestArgs(r.args));
+  if (tests.length === 0) return { score: 0, detail: "No tests ran", applicable: false };
+  const last = tests[tests.length - 1];
+  if (last.isError) return { score: 0, detail: "Test command errored", applicable: true };
+  const exitCode = getExitCode(last.result);
+  if (exitCode === 0) return { score: 1, detail: "Tests passed", applicable: true };
+  if (exitCode !== null && exitCode !== 0)
+    return { score: 0, detail: `Tests failed (exit ${exitCode})`, applicable: true };
+  const text = typeof last.result === "string" ? last.result : JSON.stringify(last.result ?? "");
+  if (/\d+ (?:tests? )?passed|✓|PASS/i.test(text) && !/\bfail(?:ed|ure)?\b|\berror\b/i.test(text)) {
+    return { score: 1, detail: "Tests passed (inferred)", applicable: true };
+  }
+  if (/\bFAIL\b|failed|✗|✘/i.test(text)) {
+    return { score: 0, detail: "Tests failed (inferred)", applicable: true };
+  }
+  return { score: THRESHOLDS.ambiguousExitScore, detail: "Tests ran, outcome unclear", applicable: true };
+}
+function scoreToolErrors(results) {
+  if (results.length === 0) return { score: 1, detail: "No tool calls", applicable: false };
+  const nonBenign = results.filter((r) => !THRESHOLDS.benignErrorTools.includes(r.toolName));
+  if (nonBenign.length === 0) return { score: 1, detail: "All tool calls are benign-error tools", applicable: true };
+  const errors = nonBenign.filter((r) => r.isError);
+  const rate = errors.length / nonBenign.length;
+  if (rate === 0) return { score: 1, detail: "No tool errors", applicable: true };
+  const score = Math.max(0, 1 - rate * THRESHOLDS.toolErrorPenaltyMultiplier);
+  return {
+    score,
+    detail: `${errors.length}/${nonBenign.length} tools errored (${(rate * 100).toFixed(0)}%)`,
+    applicable: true
+  };
+}
+function scoreStuckLoops(results) {
+  if (results.length < 3) return { score: 1, detail: "Too few calls for loop detection", applicable: true };
+  const fingerprint = (r) => `${r.toolName}:${JSON.stringify(r.args)}`;
+  let maxConsecutive = 1;
+  let currentRun = 1;
+  for (let i = 1; i < results.length; i++) {
+    if (fingerprint(results[i]) === fingerprint(results[i - 1])) {
+      currentRun++;
+      maxConsecutive = Math.max(maxConsecutive, currentRun);
+    } else {
+      currentRun = 1;
+    }
+  }
+  const errorCounts = /* @__PURE__ */ new Map();
+  for (const r of results) {
+    if (!r.isError) continue;
+    const fp = fingerprint(r);
+    errorCounts.set(fp, (errorCounts.get(fp) ?? 0) + 1);
+  }
+  const maxErrorRepeat = Math.max(0, ...[...errorCounts.values()]);
+  const minReps = THRESHOLDS.loopMinRepetitions;
+  const loopSeverity = Math.max(
+    maxConsecutive >= minReps ? maxConsecutive - (minReps - 1) : 0,
+    maxErrorRepeat >= minReps ? maxErrorRepeat - (minReps - 1) : 0
+  );
+  if (loopSeverity === 0) return { score: 1, detail: "No stuck loops", applicable: true };
+  const score = Math.max(0, 1 - loopSeverity * THRESHOLDS.loopPenaltyPerOccurrence);
+  const details = [];
+  if (maxConsecutive >= minReps) details.push(`${maxConsecutive} consecutive identical calls`);
+  if (maxErrorRepeat >= minReps) details.push(`same error repeated ${maxErrorRepeat}x`);
+  return { score, detail: details.join("; "), applicable: true };
+}
+function scoreRegression(execResults) {
+  const regressions = [];
+  for (const [label, filter] of [
+    ["Build", isBuildArgs],
+    ["Tests", isTestArgs]
+  ]) {
+    const cmds = execResults.filter((r) => r.toolName === "execute_command" && filter(r.args));
+    if (cmds.length < 2) continue;
+    let sawPass = false;
+    let sawRegression = false;
+    for (const cmd of cmds) {
+      const exit = getExitCode(cmd.result);
+      if (exit === 0) {
+        if (sawRegression) {
+          regressions.push({ label, recovered: true });
+          sawRegression = false;
+        }
+        sawPass = true;
+      }
+      if (sawPass && exit !== null && exit !== 0) {
+        sawRegression = true;
+      }
+    }
+    if (sawRegression) {
+      regressions.push({ label, recovered: false });
+    }
+  }
+  if (regressions.length === 0) return { score: 1, detail: "No regressions", applicable: true };
+  const persistentCount = regressions.filter((r) => !r.recovered).length;
+  const recoveredCount = regressions.filter((r) => r.recovered).length;
+  const score = Math.max(0, 1 - persistentCount * 0.5 - recoveredCount * 0.1);
+  const details = regressions.map(
+    (r) => `${r.label} ${r.recovered ? "regressed then recovered" : "regressed (persisted)"}`
+  );
+  return { score, detail: details.join("; "), applicable: true };
+}
+function scoreAutonomy(results) {
+  const askUserCalls = results.filter((r) => r.toolName === "ask_user");
+  const count = askUserCalls.length;
+  if (count === 0) return { score: 1, detail: "No ask_user calls (fully autonomous)", applicable: true };
+  const score = Math.max(0, 1 - count * THRESHOLDS.autonomyPenaltyPerAsk);
+  return { score, detail: `${count} ask_user call${count > 1 ? "s" : ""}`, applicable: true };
+}
+function weightedAverage(dimensions, weights) {
+  let totalWeight = 0;
+  let weightedSum = 0;
+  for (const [key, result] of Object.entries(dimensions)) {
+    if (!result.applicable) continue;
+    const weight = weights[key] ?? 0;
+    totalWeight += weight;
+    weightedSum += result.score * weight;
+  }
+  if (totalWeight === 0) return null;
+  return weightedSum / totalWeight;
+}
+function createOutcomeScorer() {
+  return createScorer({
+    id: "mastracode-outcome",
+    name: "MastraCode Outcome",
+    description: "Grades coding session outcomes: build/test pass, tool errors, stuck loops, regressions, autonomy. Always-on.",
+    type: "agent",
+    prepareRun: filterRun({ partTypes: ["tool-invocation", "text"] })
+  }).preprocess(async ({ run }) => {
+    const messages = run.output ?? [];
+    const allResults = extractToolCalls(messages);
+    const totalCalls = allResults.length;
+    if (totalCalls < THRESHOLDS.minToolCalls) {
+      const empty = { score: 0, detail: "No tool calls", applicable: false };
+      return {
+        empty: true,
+        build: empty,
+        tests: empty,
+        toolErrors: empty,
+        loops: empty,
+        regression: empty,
+        autonomy: empty,
+        totalCalls
+      };
+    }
+    const execResults = allResults.filter((r) => r.toolName === "execute_command");
+    return {
+      empty: false,
+      build: scoreBuild(execResults),
+      tests: scoreTests(execResults),
+      toolErrors: scoreToolErrors(allResults),
+      loops: scoreStuckLoops(allResults),
+      regression: scoreRegression(execResults),
+      autonomy: scoreAutonomy(allResults),
+      totalCalls
+    };
+  }).generateScore(({ results }) => {
+    const p = results.preprocessStepResult;
+    if (p.empty) return 0;
+    const score = weightedAverage(
+      {
+        build: p.build,
+        tests: p.tests,
+        toolErrors: p.toolErrors,
+        loops: p.loops,
+        regression: p.regression,
+        autonomy: p.autonomy
+      },
+      WEIGHTS
+    );
+    return score === null ? 0 : Math.round(score * 100) / 100;
+  }).generateReason(({ results, score }) => {
+    const p = results.preprocessStepResult;
+    if (p.empty) {
+      return `Score: 0 (${p.totalCalls} tool calls \u2014 below minimum threshold for scoring)`;
+    }
+    const pct = (w) => `${(w * 100).toFixed(0)}%`;
+    const dimLine = (name, weight, r) => r.applicable ? `${name} (${pct(weight)}): ${r.detail} [${r.score}]` : `${name}: ${r.detail} [N/A \u2014 excluded from average]`;
+    const parts = [`Score: ${score} (${p.totalCalls} tool calls total)`];
+    parts.push(dimLine("Build", WEIGHTS.build, p.build));
+    parts.push(dimLine("Tests", WEIGHTS.tests, p.tests));
+    parts.push(dimLine("Tool errors", WEIGHTS.toolErrors, p.toolErrors));
+    parts.push(dimLine("Loops", WEIGHTS.loops, p.loops));
+    parts.push(dimLine("Regression", WEIGHTS.regression, p.regression));
+    parts.push(dimLine("Autonomy", WEIGHTS.autonomy, p.autonomy));
+    return parts.join("\n");
+  });
+}
+export {
+  createOutcomeScorer
+};
+//# sourceMappingURL=outcome.js.map

package/dist/evals/scorers/outcome.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/evals/scorers/outcome.ts"],"sourcesContent":["/**\n * MastraCode Outcome Scorer (Code-based, Agent type, always-on)\n *\n * Grades the objective outcome of a coding session by checking hard facts:\n * - Did the build/typecheck pass?\n * - Did tests pass?\n * - What was the tool error rate?\n * - Did the agent get stuck in a loop?\n * - Was there a regression (build passed earlier, failed later)?\n * - Did the agent use ask_user unnecessarily? (autonomy)\n *\n * This scorer is always-on (no sampling) because it's cheap — it only\n * reads exit codes and error states from tool results already in the messages.\n *\n * Scoring philosophy (from Anthropic's eval guidance):\n * \"Grade outcomes, not paths\" — we check what happened, not how.\n * Partial credit: a session that passes build but fails tests scores\n * higher than one that fails build entirely.\n *\n * Dimensions that don't apply to a session (e.g. build for a docs-only task)\n * are excluded from the weighted average, so sessions aren't penalized for\n * irrelevant criteria.\n */\n\nimport type { MastraDBMessage } from '@mastra/core/agent';\nimport { createScorer, filterRun } from '@mastra/core/evals';\n\nimport { isBuildCommand, isTestCommand, getExitCode } from './classify-command.js';\nimport type { ExtractedToolCall } from './extract-tools.js';\nimport { extractToolCalls } from './extract-tools.js';\n\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n// CONFIGURATION — Adjust weights and thresholds here\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n/**\n * Dimension weights. Used to compute the weighted average of applicable dimensions.\n * Dimensions that don't apply (e.g. no build ran) are excluded from the average.\n */\nconst WEIGHTS = {\n  build: 0.3,\n  tests: 0.25,\n  toolErrors: 0.2,\n  loops: 0.1,\n  regression: 0.1,\n  autonomy: 0.05,\n} as const;\n\n/**\n * Scoring thresholds and multipliers.\n */\nconst THRESHOLDS = {\n  /** Multiplier for tool error rate penalty. Score = max(0, 1 - rate × this). */\n  toolErrorPenaltyMultiplier: 1.0,\n\n  /** How many consecutive identical calls (or same-error repeats) before it's a \"loop\". */\n  loopMinRepetitions: 3,\n\n  /** Score penalty per detected loop. */\n  loopPenaltyPerOccurrence: 0.3,\n\n  /** Score deducted per ask_user call. */\n  autonomyPenaltyPerAsk: 0.25,\n\n  /** Score when a command ran but exit code is ambiguous. */\n  ambiguousExitScore: 0.75,\n\n  /** Minimum tool calls required to produce a meaningful score. */\n  minToolCalls: 1,\n\n  /** Tool names whose errors are expected/benign (not penalized). */\n  benignErrorTools: ['search_content', 'find_files', 'lsp_inspect'] as string[],\n} as const;\n\n// ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\ntype DimensionResult = { score: number; detail: string; applicable: boolean };\n\n/**\n * Check if the primary verb of an execute_command is a build/typecheck command.\n * Splits compound commands (&&, ;) and checks the first segment.\n */\n/** Extract command string from tool call args, then delegate to shared classifier. */\nfunction isBuildArgs(args: Record<string, unknown>): boolean {\n  return isBuildCommand(String(args.command ?? ''));\n}\n\nfunction isTestArgs(args: Record<string, unknown>): boolean {\n  return isTestCommand(String(args.command ?? ''));\n}\n\n/**\n * Score build/typecheck outcome (final state).\n */\nfunction scoreBuild(execResults: ExtractedToolCall[]): DimensionResult {\n  const builds = execResults.filter(r => r.toolName === 'execute_command' && isBuildArgs(r.args));\n  if (builds.length === 0) return { score: 0, detail: 'No build/typecheck ran', applicable: false };\n\n  const last = builds[builds.length - 1]!;\n  if (last.isError) return { score: 0, detail: 'Build/typecheck tool errored', applicable: true };\n\n  const exitCode = getExitCode(last.result);\n  if (exitCode === 0) return { score: 1, detail: 'Build/typecheck passed', applicable: true };\n  if (exitCode !== null && exitCode !== 0)\n    return { score: 0, detail: `Build failed (exit ${exitCode})`, applicable: true };\n\n  // Fallback: check result text for TS errors\n  const text = typeof last.result === 'string' ? last.result : JSON.stringify(last.result ?? '');\n  if (/error TS\\d+|Cannot find module|is not assignable/i.test(text)) {\n    return { score: 0, detail: 'Build failed (TypeScript errors in output)', applicable: true };\n  }\n\n  return { score: THRESHOLDS.ambiguousExitScore, detail: 'Build ran, outcome unclear', applicable: true };\n}\n\n/**\n * Score test outcome (final state).\n */\nfunction scoreTests(execResults: ExtractedToolCall[]): DimensionResult {\n  const tests = execResults.filter(r => r.toolName === 'execute_command' && isTestArgs(r.args));\n  if (tests.length === 0) return { score: 0, detail: 'No tests ran', applicable: false };\n\n  const last = tests[tests.length - 1]!;\n  if (last.isError) return { score: 0, detail: 'Test command errored', applicable: true };\n\n  const exitCode = getExitCode(last.result);\n  if (exitCode === 0) return { score: 1, detail: 'Tests passed', applicable: true };\n  if (exitCode !== null && exitCode !== 0)\n    return { score: 0, detail: `Tests failed (exit ${exitCode})`, applicable: true };\n\n  const text = typeof last.result === 'string' ? last.result : JSON.stringify(last.result ?? '');\n  if (/\\d+ (?:tests? )?passed|✓|PASS/i.test(text) && !/\\bfail(?:ed|ure)?\\b|\\berror\\b/i.test(text)) {\n    return { score: 1, detail: 'Tests passed (inferred)', applicable: true };\n  }\n  if (/\\bFAIL\\b|failed|✗|✘/i.test(text)) {\n    return { score: 0, detail: 'Tests failed (inferred)', applicable: true };\n  }\n\n  return { score: THRESHOLDS.ambiguousExitScore, detail: 'Tests ran, outcome unclear', applicable: true };\n}\n\n/**\n * Score tool error rate, excluding benign errors from expected-to-fail tools.\n */\nfunction scoreToolErrors(results: ExtractedToolCall[]): DimensionResult {\n  if (results.length === 0) return { score: 1, detail: 'No tool calls', applicable: false };\n\n  // Only count errors from non-benign tools\n  const nonBenign = results.filter(r => !THRESHOLDS.benignErrorTools.includes(r.toolName));\n  if (nonBenign.length === 0) return { score: 1, detail: 'All tool calls are benign-error tools', applicable: true };\n\n  const errors = nonBenign.filter(r => r.isError);\n  const rate = errors.length / nonBenign.length;\n\n  if (rate === 0) return { score: 1, detail: 'No tool errors', applicable: true };\n\n  const score = Math.max(0, 1 - rate * THRESHOLDS.toolErrorPenaltyMultiplier);\n  return {\n    score,\n    detail: `${errors.length}/${nonBenign.length} tools errored (${(rate * 100).toFixed(0)}%)`,\n    applicable: true,\n  };\n}\n\n/**\n * Detect stuck loops: same tool + same args appearing 3+ times consecutively,\n * or same tool + same args erroring 3+ times total.\n */\nfunction scoreStuckLoops(results: ExtractedToolCall[]): DimensionResult {\n  if (results.length < 3) return { score: 1, detail: 'Too few calls for loop detection', applicable: true };\n\n  const fingerprint = (r: ExtractedToolCall) => `${r.toolName}:${JSON.stringify(r.args)}`;\n\n  // Check consecutive identical calls\n  let maxConsecutive = 1;\n  let currentRun = 1;\n  for (let i = 1; i < results.length; i++) {\n    if (fingerprint(results[i]!) === fingerprint(results[i - 1]!)) {\n      currentRun++;\n      maxConsecutive = Math.max(maxConsecutive, currentRun);\n    } else {\n      currentRun = 1;\n    }\n  }\n\n  // Check repeated error fingerprints\n  const errorCounts = new Map<string, number>();\n  for (const r of results) {\n    if (!r.isError) continue;\n    const fp = fingerprint(r);\n    errorCounts.set(fp, (errorCounts.get(fp) ?? 0) + 1);\n  }\n  const maxErrorRepeat = Math.max(0, ...[...errorCounts.values()]);\n\n  const minReps = THRESHOLDS.loopMinRepetitions;\n  const loopSeverity = Math.max(\n    maxConsecutive >= minReps ? maxConsecutive - (minReps - 1) : 0,\n    maxErrorRepeat >= minReps ? maxErrorRepeat - (minReps - 1) : 0,\n  );\n\n  if (loopSeverity === 0) return { score: 1, detail: 'No stuck loops', applicable: true };\n\n  const score = Math.max(0, 1 - loopSeverity * THRESHOLDS.loopPenaltyPerOccurrence);\n  const details: string[] = [];\n  if (maxConsecutive >= minReps) details.push(`${maxConsecutive} consecutive identical calls`);\n  if (maxErrorRepeat >= minReps) details.push(`same error repeated ${maxErrorRepeat}x`);\n  return { score, detail: details.join('; '), applicable: true };\n}\n\n/**\n * Detect regressions: build/test passed at some point, then failed later.\n * Gives partial credit if the agent recovered (final state passes).\n */\nfunction scoreRegression(execResults: ExtractedToolCall[]): DimensionResult {\n  const regressions: { label: string; recovered: boolean }[] = [];\n\n  for (const [label, filter] of [\n    ['Build', isBuildArgs],\n    ['Tests', isTestArgs],\n  ] as const) {\n    const cmds = execResults.filter(r => r.toolName === 'execute_command' && filter(r.args));\n    if (cmds.length < 2) continue;\n\n    let sawPass = false;\n    let sawRegression = false;\n    for (const cmd of cmds) {\n      const exit = getExitCode(cmd.result);\n      if (exit === 0) {\n        if (sawRegression) {\n          // Recovered from regression\n          regressions.push({ label, recovered: true });\n          sawRegression = false;\n        }\n        sawPass = true;\n      }\n      if (sawPass && exit !== null && exit !== 0) {\n        sawRegression = true;\n      }\n    }\n\n    if (sawRegression) {\n      // Regression persisted at end of session\n      regressions.push({ label, recovered: false });\n    }\n  }\n\n  if (regressions.length === 0) return { score: 1, detail: 'No regressions', applicable: true };\n\n  // Persistent regressions are severe (0), recovered regressions get partial credit (0.5)\n  const persistentCount = regressions.filter(r => !r.recovered).length;\n  const recoveredCount = regressions.filter(r => r.recovered).length;\n  const score = Math.max(0, 1 - persistentCount * 0.5 - recoveredCount * 0.1);\n  const details = regressions.map(\n    r => `${r.label} ${r.recovered ? 'regressed then recovered' : 'regressed (persisted)'}`,\n  );\n  return { score, detail: details.join('; '), applicable: true };\n}\n\n/**\n * Score autonomy: penalize ask_user calls.\n * 0 calls = 1.0, each call reduces by 0.25 (up to a floor of 0).\n */\nfunction scoreAutonomy(results: ExtractedToolCall[]): DimensionResult {\n  const askUserCalls = results.filter(r => r.toolName === 'ask_user');\n  const count = askUserCalls.length;\n\n  if (count === 0) return { score: 1, detail: 'No ask_user calls (fully autonomous)', applicable: true };\n\n  const score = Math.max(0, 1 - count * THRESHOLDS.autonomyPenaltyPerAsk);\n  return { score, detail: `${count} ask_user call${count > 1 ? 's' : ''}`, applicable: true };\n}\n\n/**\n * Compute weighted average of only applicable dimensions.\n * Returns null if no dimensions are applicable.\n */\nfunction weightedAverage(dimensions: Record<string, DimensionResult>, weights: Record<string, number>): number | null {\n  let totalWeight = 0;\n  let weightedSum = 0;\n\n  for (const [key, result] of Object.entries(dimensions)) {\n    if (!result.applicable) continue;\n    const weight = weights[key] ?? 0;\n    totalWeight += weight;\n    weightedSum += result.score * weight;\n  }\n\n  if (totalWeight === 0) return null;\n  return weightedSum / totalWeight;\n}\n\nexport function createOutcomeScorer() {\n  return createScorer({\n    id: 'mastracode-outcome',\n    name: 'MastraCode Outcome',\n    description:\n      'Grades coding session outcomes: build/test pass, tool errors, stuck loops, regressions, autonomy. Always-on.',\n    type: 'agent',\n    prepareRun: filterRun({ partTypes: ['tool-invocation', 'text'] }),\n  })\n    .preprocess(async ({ run }) => {\n      const messages = (run.output ?? []) as MastraDBMessage[];\n      const allResults = extractToolCalls(messages);\n      const totalCalls = allResults.length;\n\n      // H1: Empty sessions or sessions with no tool calls → score 0\n      if (totalCalls < THRESHOLDS.minToolCalls) {\n        const empty: DimensionResult = { score: 0, detail: 'No tool calls', applicable: false };\n        return {\n          empty: true as const,\n          build: empty,\n          tests: empty,\n          toolErrors: empty,\n          loops: empty,\n          regression: empty,\n          autonomy: empty,\n          totalCalls,\n        };\n      }\n\n      const execResults = allResults.filter(r => r.toolName === 'execute_command');\n\n      return {\n        empty: false as const,\n        build: scoreBuild(execResults),\n        tests: scoreTests(execResults),\n        toolErrors: scoreToolErrors(allResults),\n        loops: scoreStuckLoops(allResults),\n        regression: scoreRegression(execResults),\n        autonomy: scoreAutonomy(allResults),\n        totalCalls,\n      };\n    })\n    .generateScore(({ results }) => {\n      const p = results.preprocessStepResult;\n      if (p.empty) return 0;\n\n      const score = weightedAverage(\n        {\n          build: p.build,\n          tests: p.tests,\n          toolErrors: p.toolErrors,\n          loops: p.loops,\n          regression: p.regression,\n          autonomy: p.autonomy,\n        },\n        WEIGHTS,\n      );\n\n      return score === null ? 0 : Math.round(score * 100) / 100;\n    })\n    .generateReason(({ results, score }) => {\n      const p = results.preprocessStepResult;\n      if (p.empty) {\n        return `Score: 0 (${p.totalCalls} tool calls — below minimum threshold for scoring)`;\n      }\n\n      const pct = (w: number) => `${(w * 100).toFixed(0)}%`;\n      const dimLine = (name: string, weight: number, r: DimensionResult) =>\n        r.applicable\n          ? `${name} (${pct(weight)}): ${r.detail} [${r.score}]`\n          : `${name}: ${r.detail} [N/A — excluded from average]`;\n\n      const parts: string[] = [`Score: ${score} (${p.totalCalls} tool calls total)`];\n      parts.push(dimLine('Build', WEIGHTS.build, p.build));\n      parts.push(dimLine('Tests', WEIGHTS.tests, p.tests));\n      parts.push(dimLine('Tool errors', WEIGHTS.toolErrors, p.toolErrors));\n      parts.push(dimLine('Loops', WEIGHTS.loops, p.loops));\n      parts.push(dimLine('Regression', WEIGHTS.regression, p.regression));\n      parts.push(dimLine('Autonomy', WEIGHTS.autonomy, p.autonomy));\n\n      return parts.join('\\n');\n    });\n}\n"],"mappings":"AAyBA,SAAS,cAAc,iBAAiB;AAExC,SAAS,gBAAgB,eAAe,mBAAmB;AAE3D,SAAS,wBAAwB;AAUjC,MAAM,UAAU;AAAA,EACd,OAAO;AAAA,EACP,OAAO;AAAA,EACP,YAAY;AAAA,EACZ,OAAO;AAAA,EACP,YAAY;AAAA,EACZ,UAAU;AACZ;AAKA,MAAM,aAAa;AAAA;AAAA,EAEjB,4BAA4B;AAAA;AAAA,EAG5B,oBAAoB;AAAA;AAAA,EAGpB,0BAA0B;AAAA;AAAA,EAG1B,uBAAuB;AAAA;AAAA,EAGvB,oBAAoB;AAAA;AAAA,EAGpB,cAAc;AAAA;AAAA,EAGd,kBAAkB,CAAC,kBAAkB,cAAc,aAAa;AAClE;AAWA,SAAS,YAAY,MAAwC;AAC3D,SAAO,eAAe,OAAO,KAAK,WAAW,EAAE,CAAC;AAClD;AAEA,SAAS,WAAW,MAAwC;AAC1D,SAAO,cAAc,OAAO,KAAK,WAAW,EAAE,CAAC;AACjD;AAKA,SAAS,WAAW,aAAmD;AACrE,QAAM,SAAS,YAAY,OAAO,OAAK,EAAE,aAAa,qBAAqB,YAAY,EAAE,IAAI,CAAC;AAC9F,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,0BAA0B,YAAY,MAAM;AAEhG,QAAM,OAAO,OAAO,OAAO,SAAS,CAAC;AACrC,MAAI,KAAK,QAAS,QAAO,EAAE,OAAO,GAAG,QAAQ,gCAAgC,YAAY,KAAK;AAE9F,QAAM,WAAW,YAAY,KAAK,MAAM;AACxC,MAAI,aAAa,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,0BAA0B,YAAY,KAAK;AAC1F,MAAI,aAAa,QAAQ,aAAa;AACpC,WAAO,EAAE,OAAO,GAAG,QAAQ,sBAAsB,QAAQ,KAAK,YAAY,KAAK;AAGjF,QAAM,OAAO,OAAO,KAAK,WAAW,WAAW,KAAK,SAAS,KAAK,UAAU,KAAK,UAAU,EAAE;AAC7F,MAAI,oDAAoD,KAAK,IAAI,GAAG;AAClE,WAAO,EAAE,OAAO,GAAG,QAAQ,8CAA8C,YAAY,KAAK;AAAA,EAC5F;AAEA,SAAO,EAAE,OAAO,WAAW,oBAAoB,QAAQ,8BAA8B,YAAY,KAAK;AACxG;AAKA,SAAS,WAAW,aAAmD;AACrE,QAAM,QAAQ,YAAY,OAAO,OAAK,EAAE,aAAa,qBAAqB,WAAW,EAAE,IAAI,CAAC;AAC5F,MAAI,MAAM,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,gBAAgB,YAAY,MAAM;AAErF,QAAM,OAAO,MAAM,MAAM,SAAS,CAAC;AACnC,MAAI,KAAK,QAAS,QAAO,EAAE,OAAO,GAAG,QAAQ,wBAAwB,YAAY,KAAK;AAEtF,QAAM,WAAW,YAAY,KAAK,MAAM;AACxC,MAAI,aAAa,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,gBAAgB,YAAY,KAAK;AAChF,MAAI,aAAa,QAAQ,aAAa;AACpC,WAAO,EAAE,OAAO,GAAG,QAAQ,sBAAsB,QAAQ,KAAK,YAAY,KAAK;AAEjF,QAAM,OAAO,OAAO,KAAK,WAAW,WAAW,KAAK,SAAS,KAAK,UAAU,KAAK,UAAU,EAAE;AAC7F,MAAI,iCAAiC,KAAK,IAAI,KAAK,CAAC,iCAAiC,KAAK,IAAI,GAAG;AAC/F,WAAO,EAAE,OAAO,GAAG,QAAQ,2BAA2B,YAAY,KAAK;AAAA,EACzE;AACA,MAAI,uBAAuB,KAAK,IAAI,GAAG;AACrC,WAAO,EAAE,OAAO,GAAG,QAAQ,2BAA2B,YAAY,KAAK;AAAA,EACzE;AAEA,SAAO,EAAE,OAAO,WAAW,oBAAoB,QAAQ,8BAA8B,YAAY,KAAK;AACxG;AAKA,SAAS,gBAAgB,SAA+C;AACtE,MAAI,QAAQ,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,iBAAiB,YAAY,MAAM;AAGxF,QAAM,YAAY,QAAQ,OAAO,OAAK,CAAC,WAAW,iBAAiB,SAAS,EAAE,QAAQ,CAAC;AACvF,MAAI,UAAU,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,yCAAyC,YAAY,KAAK;AAEjH,QAAM,SAAS,UAAU,OAAO,OAAK,EAAE,OAAO;AAC9C,QAAM,OAAO,OAAO,SAAS,UAAU;AAEvC,MAAI,SAAS,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,kBAAkB,YAAY,KAAK;AAE9E,QAAM,QAAQ,KAAK,IAAI,GAAG,IAAI,OAAO,WAAW,0BAA0B;AAC1E,SAAO;AAAA,IACL;AAAA,IACA,QAAQ,GAAG,OAAO,MAAM,IAAI,UAAU,MAAM,oBAAoB,OAAO,KAAK,QAAQ,CAAC,CAAC;AAAA,IACtF,YAAY;AAAA,EACd;AACF;AAMA,SAAS,gBAAgB,SAA+C;AACtE,MAAI,QAAQ,SAAS,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,oCAAoC,YAAY,KAAK;AAExG,QAAM,cAAc,CAAC,MAAyB,GAAG,EAAE,QAAQ,IAAI,KAAK,UAAU,EAAE,IAAI,CAAC;AAGrF,MAAI,iBAAiB;AACrB,MAAI,aAAa;AACjB,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,QAAI,YAAY,QAAQ,CAAC,CAAE,MAAM,YAAY,QAAQ,IAAI,CAAC,CAAE,GAAG;AAC7D;AACA,uBAAiB,KAAK,IAAI,gBAAgB,UAAU;AAAA,IACtD,OAAO;AACL,mBAAa;AAAA,IACf;AAAA,EACF;AAGA,QAAM,cAAc,oBAAI,IAAoB;AAC5C,aAAW,KAAK,SAAS;AACvB,QAAI,CAAC,EAAE,QAAS;AAChB,UAAM,KAAK,YAAY,CAAC;AACxB,gBAAY,IAAI,KAAK,YAAY,IAAI,EAAE,KAAK,KAAK,CAAC;AAAA,EACpD;AACA,QAAM,iBAAiB,KAAK,IAAI,GAAG,GAAG,CAAC,GAAG,YAAY,OAAO,CAAC,CAAC;AAE/D,QAAM,UAAU,WAAW;AAC3B,QAAM,eAAe,KAAK;AAAA,IACxB,kBAAkB,UAAU,kBAAkB,UAAU,KAAK;AAAA,IAC7D,kBAAkB,UAAU,kBAAkB,UAAU,KAAK;AAAA,EAC/D;AAEA,MAAI,iBAAiB,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,kBAAkB,YAAY,KAAK;AAEtF,QAAM,QAAQ,KAAK,IAAI,GAAG,IAAI,eAAe,WAAW,wBAAwB;AAChF,QAAM,UAAoB,CAAC;AAC3B,MAAI,kBAAkB,QAAS,SAAQ,KAAK,GAAG,cAAc,8BAA8B;AAC3F,MAAI,kBAAkB,QAAS,SAAQ,KAAK,uBAAuB,cAAc,GAAG;AACpF,SAAO,EAAE,OAAO,QAAQ,QAAQ,KAAK,IAAI,GAAG,YAAY,KAAK;AAC/D;AAMA,SAAS,gBAAgB,aAAmD;AAC1E,QAAM,cAAuD,CAAC;AAE9D,aAAW,CAAC,OAAO,MAAM,KAAK;AAAA,IAC5B,CAAC,SAAS,WAAW;AAAA,IACrB,CAAC,SAAS,UAAU;AAAA,EACtB,GAAY;AACV,UAAM,OAAO,YAAY,OAAO,OAAK,EAAE,aAAa,qBAAqB,OAAO,EAAE,IAAI,CAAC;AACvF,QAAI,KAAK,SAAS,EAAG;AAErB,QAAI,UAAU;AACd,QAAI,gBAAgB;AACpB,eAAW,OAAO,MAAM;AACtB,YAAM,OAAO,YAAY,IAAI,MAAM;AACnC,UAAI,SAAS,GAAG;AACd,YAAI,eAAe;AAEjB,sBAAY,KAAK,EAAE,OAAO,WAAW,KAAK,CAAC;AAC3C,0BAAgB;AAAA,QAClB;AACA,kBAAU;AAAA,MACZ;AACA,UAAI,WAAW,SAAS,QAAQ,SAAS,GAAG;AAC1C,wBAAgB;AAAA,MAClB;AAAA,IACF;AAEA,QAAI,eAAe;AAEjB,kBAAY,KAAK,EAAE,OAAO,WAAW,MAAM,CAAC;AAAA,IAC9C;AAAA,EACF;AAEA,MAAI,YAAY,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,kBAAkB,YAAY,KAAK;AAG5F,QAAM,kBAAkB,YAAY,OAAO,OAAK,CAAC,EAAE,SAAS,EAAE;AAC9D,QAAM,iBAAiB,YAAY,OAAO,OAAK,EAAE,SAAS,EAAE;AAC5D,QAAM,QAAQ,KAAK,IAAI,GAAG,IAAI,kBAAkB,MAAM,iBAAiB,GAAG;AAC1E,QAAM,UAAU,YAAY;AAAA,IAC1B,OAAK,GAAG,EAAE,KAAK,IAAI,EAAE,YAAY,6BAA6B,uBAAuB;AAAA,EACvF;AACA,SAAO,EAAE,OAAO,QAAQ,QAAQ,KAAK,IAAI,GAAG,YAAY,KAAK;AAC/D;AAMA,SAAS,cAAc,SAA+C;AACpE,QAAM,eAAe,QAAQ,OAAO,OAAK,EAAE,aAAa,UAAU;AAClE,QAAM,QAAQ,aAAa;AAE3B,MAAI,UAAU,EAAG,QAAO,EAAE,OAAO,GAAG,QAAQ,wCAAwC,YAAY,KAAK;AAErG,QAAM,QAAQ,KAAK,IAAI,GAAG,IAAI,QAAQ,WAAW,qBAAqB;AACtE,SAAO,EAAE,OAAO,QAAQ,GAAG,KAAK,iBAAiB,QAAQ,IAAI,MAAM,EAAE,IAAI,YAAY,KAAK;AAC5F;AAMA,SAAS,gBAAgB,YAA6C,SAAgD;AACpH,MAAI,cAAc;AAClB,MAAI,cAAc;AAElB,aAAW,CAAC,KAAK,MAAM,KAAK,OAAO,QAAQ,UAAU,GAAG;AACtD,QAAI,CAAC,OAAO,WAAY;AACxB,UAAM,SAAS,QAAQ,GAAG,KAAK;AAC/B,mBAAe;AACf,mBAAe,OAAO,QAAQ;AAAA,EAChC;AAEA,MAAI,gBAAgB,EAAG,QAAO;AAC9B,SAAO,cAAc;AACvB;AAEO,SAAS,sBAAsB;AACpC,SAAO,aAAa;AAAA,IAClB,IAAI;AAAA,IACJ,MAAM;AAAA,IACN,aACE;AAAA,IACF,MAAM;AAAA,IACN,YAAY,UAAU,EAAE,WAAW,CAAC,mBAAmB,MAAM,EAAE,CAAC;AAAA,EAClE,CAAC,EACE,WAAW,OAAO,EAAE,IAAI,MAAM;AAC7B,UAAM,WAAY,IAAI,UAAU,CAAC;AACjC,UAAM,aAAa,iBAAiB,QAAQ;AAC5C,UAAM,aAAa,WAAW;AAG9B,QAAI,aAAa,WAAW,cAAc;AACxC,YAAM,QAAyB,EAAE,OAAO,GAAG,QAAQ,iBAAiB,YAAY,MAAM;AACtF,aAAO;AAAA,QACL,OAAO;AAAA,QACP,OAAO;AAAA,QACP,OAAO;AAAA,QACP,YAAY;AAAA,QACZ,OAAO;AAAA,QACP,YAAY;AAAA,QACZ,UAAU;AAAA,QACV;AAAA,MACF;AAAA,IACF;AAEA,UAAM,cAAc,WAAW,OAAO,OAAK,EAAE,aAAa,iBAAiB;AAE3E,WAAO;AAAA,MACL,OAAO;AAAA,MACP,OAAO,WAAW,WAAW;AAAA,MAC7B,OAAO,WAAW,WAAW;AAAA,MAC7B,YAAY,gBAAgB,UAAU;AAAA,MACtC,OAAO,gBAAgB,UAAU;AAAA,MACjC,YAAY,gBAAgB,WAAW;AAAA,MACvC,UAAU,cAAc,UAAU;AAAA,MAClC;AAAA,IACF;AAAA,EACF,CAAC,EACA,cAAc,CAAC,EAAE,QAAQ,MAAM;AAC9B,UAAM,IAAI,QAAQ;AAClB,QAAI,EAAE,MAAO,QAAO;AAEpB,UAAM,QAAQ;AAAA,MACZ;AAAA,QACE,OAAO,EAAE;AAAA,QACT,OAAO,EAAE;AAAA,QACT,YAAY,EAAE;AAAA,QACd,OAAO,EAAE;AAAA,QACT,YAAY,EAAE;AAAA,QACd,UAAU,EAAE;AAAA,MACd;AAAA,MACA;AAAA,IACF;AAEA,WAAO,UAAU,OAAO,IAAI,KAAK,MAAM,QAAQ,GAAG,IAAI;AAAA,EACxD,CAAC,EACA,eAAe,CAAC,EAAE,SAAS,MAAM,MAAM;AACtC,UAAM,IAAI,QAAQ;AAClB,QAAI,EAAE,OAAO;AACX,aAAO,aAAa,EAAE,UAAU;AAAA,IAClC;AAEA,UAAM,MAAM,CAAC,MAAc,IAAI,IAAI,KAAK,QAAQ,CAAC,CAAC;AAClD,UAAM,UAAU,CAAC,MAAc,QAAgB,MAC7C,EAAE,aACE,GAAG,IAAI,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,KAAK,EAAE,KAAK,MACjD,GAAG,IAAI,KAAK,EAAE,MAAM;AAE1B,UAAM,QAAkB,CAAC,UAAU,KAAK,KAAK,EAAE,UAAU,oBAAoB;AAC7E,UAAM,KAAK,QAAQ,SAAS,QAAQ,OAAO,EAAE,KAAK,CAAC;AACnD,UAAM,KAAK,QAAQ,SAAS,QAAQ,OAAO,EAAE,KAAK,CAAC;AACnD,UAAM,KAAK,QAAQ,eAAe,QAAQ,YAAY,EAAE,UAAU,CAAC;AACnE,UAAM,KAAK,QAAQ,SAAS,QAAQ,OAAO,EAAE,KAAK,CAAC;AACnD,UAAM,KAAK,QAAQ,cAAc,QAAQ,YAAY,EAAE,UAAU,CAAC;AAClE,UAAM,KAAK,QAAQ,YAAY,QAAQ,UAAU,EAAE,QAAQ,CAAC;AAE5D,WAAO,MAAM,KAAK,IAAI;AAAA,EACxB,CAAC;AACL;","names":[]}

package/dist/headless/cli.d.ts

@@ -0,0 +1,42 @@
+import type { PermissionMode, RunMode, ThinkingLevel } from './types.js';
+/** Consolidated output mode (replaces the old `--format` + `--output-format`). */
+export type OutputMode = 'human' | 'json' | 'jsonl';
+export interface HeadlessArgs {
+    prompt?: string;
+    /** Timeout in seconds (CLI surface); converted to ms before `runMC`. */
+    timeout?: number;
+    output: OutputMode;
+    continue_: boolean;
+    model?: string;
+    mode?: RunMode;
+    thinkingLevel?: ThinkingLevel;
+    settings?: string;
+    thread?: string;
+    title?: string;
+    cloneThread: boolean;
+    resourceId?: string;
+    /** Max agentic turns before the run aborts with exit code 1. */
+    maxTurns?: number;
+    /** Named permission mode resolving to a built-in policy. Defaults to `auto`. */
+    permissionMode?: PermissionMode;
+}
+/**
+ * Returns true if `argv` selects headless mode. This must agree with what
+ * {@link parseHeadlessArgs} (and `runMCCli`) accept as a prompt: `--prompt`/`-p`
+ * or a bare positional prompt (e.g. `mastracode "Fix the bug"`). Note that a
+ * prompt piped via stdin without a flag is handled separately by the caller.
+ */
+export declare function hasHeadlessFlag(argv: string[]): boolean;
+/**
+ * Parse CLI arguments for headless mode. The flag table in `flags.ts` is the
+ * single source of truth: each flag carries its own coercion/validation, so this
+ * function just walks {@link FLAGS} and assembles the typed {@link HeadlessArgs}.
+ */
+export declare function parseHeadlessArgs(argv: string[]): HeadlessArgs;
+export declare function printHeadlessUsage(): void;
+/**
+ * Headless CLI entry point: parse arguments, read stdin, initialize MastraCode,
+ * run via `runMC`, render output, and exit with the mapped code.
+ */
+export declare function runMCCli(predrainedInput?: string | null): Promise<never>;
+//# sourceMappingURL=cli.d.ts.map

package/dist/headless/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../src/headless/cli.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEzE,kFAAkF;AAClF,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,MAAM,GAAG,OAAO,CAAC;AAEpD,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,gFAAgF;IAChF,cAAc,CAAC,EAAE,cAAc,CAAC;CACjC;AAID;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAcvD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,YAAY,CAsC9D;AAED,wBAAgB,kBAAkB,IAAI,IAAI,CA+BzC;AAED;;;GAGG;AACH,wBAAsB,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CA+G9E"}