@agentgrader/core 1.0.0

package/dist/index.js

@@ -0,0 +1,1114 @@
+import { z } from 'zod';
+import { getCachedBaseline, saveCachedBaseline, createRun, updateRun, addTrace } from '@agentgrader/store';
+import { createStep, createWorkflow } from '@mastra/core/workflows';
+import { createHash, randomUUID } from 'crypto';
+import { readFileSync, readdirSync, statSync, existsSync } from 'fs';
+import { relative, sep, join, resolve } from 'path';
+import { parse } from 'yaml';
+
+// src/schema/test-case.ts
+var SuccessCriterionSchema = z.union([
+  z.object({
+    run: z.string(),
+    expect: z.object({
+      exit_code: z.number().default(0)
+    })
+  }),
+  z.object({
+    assert: z.string()
+  })
+]);
+var TestCaseSchema = z.object({
+  id: z.string().optional(),
+  // Can be inferred from file name/folder if omitted
+  name: z.string(),
+  description: z.string().optional(),
+  fixture: z.string(),
+  prompt: z.string(),
+  success: z.array(SuccessCriterionSchema),
+  timeout_seconds: z.number().default(300),
+  // SWE-bench based fields (all optional, backwards compatible)
+  tags: z.array(z.string()).optional(),
+  // shell command used to run the test suite (TAP output expected for node:test)
+  test_command: z.string().optional(),
+  // names of tests that must flip from failing -> passing (FAIL_TO_PASS)
+  fail_to_pass: z.array(z.string()).optional(),
+  // names of tests that must remain passing throughout (PASS_TO_PASS)
+  pass_to_pass: z.array(z.string()).optional(),
+  // glob patterns of files the agent must NOT modify (e.g. test files - tamper guard)
+  forbid_modified: z.array(z.string()).optional(),
+  // glob patterns of files the agent is expected to touch (for localization scoring)
+  expected_files: z.array(z.string()).optional(),
+  // path (relative to the agr.yaml) to a gold-standard patch/diff for this task
+  solution: z.string().optional(),
+  // path (relative to the agr.yaml) to a patch that adds/updates the test suite itself
+  test_patch: z.string().optional(),
+  // original creation date of the underlying issue/PR (contamination/date-cutoff checks)
+  created_at: z.string().optional(),
+  // custom Docker image to run the sandbox from (defaults to the provider's default)
+  image: z.string().optional(),
+  // paths to toolkit directories (custom CLI tools + .claude/skills/) to
+  // inject into the sandbox and surface to the agent via the system prompt,
+  // in addition to any toolkits configured on the agent
+  toolkits: z.array(z.string()).optional()
+});
+var SkillFrontmatterSchema = z.object({
+  /** lowercase letters, numbers, hyphens; max 64 chars */
+  name: z.string().max(64).regex(/^[a-z0-9]+(-[a-z0-9]+)*$/, "name must be lowercase letters, numbers, and hyphens"),
+  /** third-person description of what the skill does and when to use it; max 1024 chars */
+  description: z.string().max(1024),
+  "allowed-tools": z.array(z.string()).optional(),
+  "disallowed-tools": z.array(z.string()).optional(),
+  license: z.string().optional()
+}).passthrough();
+var McpServerConfigSchema = z.union([
+  z.object({
+    command: z.string(),
+    args: z.array(z.string()).optional(),
+    env: z.record(z.string()).optional()
+  }),
+  z.object({
+    type: z.enum(["http", "sse"]).optional(),
+    url: z.string(),
+    headers: z.record(z.string()).optional()
+  })
+]);
+
+// src/schema/agent-config.ts
+var AgentConfigSchema = z.object({
+  id: z.string().optional(),
+  name: z.string(),
+  model: z.string(),
+  max_steps: z.number().default(30),
+  temperature: z.number().optional(),
+  system_prompt: z.string().optional(),
+  tools: z.array(z.string()).optional(),
+  // paths to toolkit directories (custom CLI tools + .claude/skills/) to
+  // inject into the sandbox and surface to the agent via the system prompt
+  toolkits: z.array(z.string()).optional(),
+  // MCP servers to connect to and expose as additional tools, keyed by name
+  mcp_servers: z.record(McpServerConfigSchema).optional()
+});
+var RunSchema = z.object({
+  id: z.string(),
+  testCaseId: z.string(),
+  agentConfigId: z.string(),
+  sandboxProvider: z.string(),
+  status: z.enum(["running", "completed", "failed"]),
+  passed: z.boolean().optional(),
+  score: z.number().optional(),
+  stepsCount: z.number().default(0),
+  tokensIn: z.number().default(0),
+  tokensOut: z.number().default(0),
+  costUsd: z.number().default(0),
+  durationMs: z.number().default(0),
+  error: z.string().optional(),
+  finalDiff: z.string().optional(),
+  // extended scoring metrics: regression (FAIL_TO_PASS/PASS_TO_PASS), diff stats,
+  // localization precision/recall, etc. Stored as JSON.
+  metrics: z.record(z.any()).optional(),
+  createdAt: z.number(),
+  completedAt: z.number().optional()
+});
+var StepEventSchema = z.object({
+  index: z.number(),
+  kind: z.enum(["tool_call", "tool_result", "message", "thinking"]),
+  tool: z.string().optional(),
+  tokensIn: z.number().default(0),
+  tokensOut: z.number().default(0),
+  costUsd: z.number().default(0),
+  timestamp: z.number(),
+  content: z.string().optional()
+});
+var TraceSchema = z.object({
+  runId: z.string(),
+  steps: z.array(StepEventSchema)
+});
+
+// src/adapters/test-result-parser.ts
+var TapTestResultParser = class {
+  name = "tap";
+  parse(output) {
+    const map = {};
+    const lineRe = /^\s*(ok|not ok)\s+\d+\s*-?\s*(.*)$/;
+    for (const rawLine of output.split("\n")) {
+      const line = rawLine.replace(/\r$/, "");
+      const match = line.match(lineRe);
+      if (!match) continue;
+      const [, statusToken, rest] = match;
+      let name = rest.trim();
+      let directive;
+      const hashIdx = name.indexOf("#");
+      if (hashIdx !== -1) {
+        directive = name.slice(hashIdx + 1).trim();
+        name = name.slice(0, hashIdx).trim();
+      }
+      if (!name) continue;
+      if (directive && /^(skip|todo)/i.test(directive)) {
+        map[name] = "SKIP";
+        continue;
+      }
+      map[name] = statusToken === "ok" ? "PASS" : "FAIL";
+    }
+    return map;
+  }
+};
+
+// src/scorers/command-scorer.ts
+var CommandScorer = class {
+  name = "CommandScorer";
+  async score(input) {
+    for (const criterion of input.testCase.success) {
+      if ("run" in criterion) {
+        const cmd = criterion.run;
+        const expectedExitCode = criterion.expect?.exit_code ?? 0;
+        try {
+          const res = await input.sandbox.exec(cmd);
+          if (res.exitCode !== expectedExitCode) {
+            return {
+              passed: false,
+              detail: `Command "${cmd}" exited with code ${res.exitCode}, expected ${expectedExitCode}.
+Stdout: ${res.stdout}
+Stderr: ${res.stderr}`
+            };
+          }
+        } catch (err) {
+          return {
+            passed: false,
+            detail: `Failed to execute command "${cmd}": ${err.message}`
+          };
+        }
+      }
+    }
+    return {
+      passed: true,
+      detail: "All success run commands completed with expected exit codes."
+    };
+  }
+};
+
+// src/scorers/assertion-scorer.ts
+var AssertionScorer = class {
+  name = "AssertionScorer";
+  async score(input) {
+    const steps = input.trace.steps.filter(
+      (s) => s.kind === "tool_call" || s.kind === "message"
+    ).length;
+    let costUsd = 0;
+    for (const step of input.trace.steps) {
+      costUsd += step.costUsd || 0;
+    }
+    for (const criterion of input.testCase.success) {
+      if ("assert" in criterion) {
+        const expression = criterion.assert;
+        try {
+          const sanitizedExpr = expression.replace(/\bcost_usd\b/g, "costUsd").replace(/\btimeout_seconds\b/g, "timeoutSeconds").replace(/\bduration_ms\b/g, "durationMs");
+          const fn = new Function(
+            "steps",
+            "costUsd",
+            `try { return Boolean(${sanitizedExpr}); } catch(e) { return false; }`
+          );
+          const passed = fn(steps, costUsd);
+          if (!passed) {
+            return {
+              passed: false,
+              detail: `Assertion failed: "${expression}" (actual: steps=${steps}, cost_usd=$${costUsd.toFixed(4)})`
+            };
+          }
+        } catch (err) {
+          return {
+            passed: false,
+            detail: `Error evaluating assertion "${expression}": ${err.message}`
+          };
+        }
+      }
+    }
+    return {
+      passed: true,
+      detail: "All assertion checks passed successfully."
+    };
+  }
+};
+
+// src/runner/glob.ts
+function matchGlob(pattern, filePath) {
+  const normalizedPattern = pattern.replace(/^\.\//, "");
+  const normalizedPath = filePath.replace(/^\.\//, "");
+  const regex = new RegExp(`^${globToRegExpSource(normalizedPattern)}$`);
+  return regex.test(normalizedPath);
+}
+function matchAnyGlob(patterns, filePath) {
+  return patterns.some((pattern) => matchGlob(pattern, filePath));
+}
+function globToRegExpSource(pattern) {
+  let result = "";
+  for (let i = 0; i < pattern.length; i++) {
+    const c = pattern[i];
+    if (c === "*") {
+      if (pattern[i + 1] === "*") {
+        result += ".*";
+        i++;
+        if (pattern[i + 1] === "/") {
+          i++;
+        }
+      } else {
+        result += "[^/]*";
+      }
+    } else if (c === "?") {
+      result += "[^/]";
+    } else if (/[.+^${}()|[\]\\]/.test(c)) {
+      result += `\\${c}`;
+    } else {
+      result += c;
+    }
+  }
+  return result;
+}
+
+// src/scorers/regression-scorer.ts
+var RegressionScorer = class {
+  name = "RegressionScorer";
+  async score(input) {
+    const { testCase, sandbox, baseline } = input;
+    const failToPass = testCase.fail_to_pass ?? [];
+    const passToPass = testCase.pass_to_pass ?? [];
+    if (!testCase.test_command || failToPass.length === 0 && passToPass.length === 0) {
+      return {
+        passed: true,
+        score: 1,
+        detail: "No fail_to_pass/pass_to_pass criteria configured; skipping regression check."
+      };
+    }
+    if (testCase.forbid_modified?.length) {
+      const diff = await sandbox.gitDiff();
+      const changedFiles = parseChangedFiles(diff);
+      const tampered = changedFiles.filter((f) => matchAnyGlob(testCase.forbid_modified, f));
+      if (tampered.length > 0) {
+        return {
+          passed: false,
+          score: 0,
+          detail: `Tamper guard triggered: forbidden files were modified: ${tampered.join(", ")}`
+        };
+      }
+    }
+    let output = "";
+    try {
+      const res = await sandbox.exec(testCase.test_command);
+      output = `${res.stdout}
+${res.stderr}`;
+    } catch (err) {
+      output = err?.message ?? String(err);
+    }
+    const statusMap = new TapTestResultParser().parse(output);
+    const failures = [];
+    for (const name of failToPass) {
+      if (statusMap[name] !== "PASS") {
+        failures.push(
+          `FAIL_TO_PASS "${name}" did not pass (status: ${statusMap[name] ?? "not found"})`
+        );
+      }
+    }
+    for (const name of passToPass) {
+      if (statusMap[name] !== "PASS") {
+        if (baseline?.[name] && baseline[name] !== "PASS") {
+          continue;
+        }
+        failures.push(
+          `PASS_TO_PASS "${name}" regressed (status: ${statusMap[name] ?? "not found"})`
+        );
+      }
+    }
+    const total = failToPass.length + passToPass.length;
+    const score = total > 0 ? (total - failures.length) / total : 1;
+    if (failures.length > 0) {
+      return {
+        passed: false,
+        score,
+        detail: `Regression check failed (${total - failures.length}/${total} ok):
+${failures.join("\n")}`
+      };
+    }
+    return {
+      passed: true,
+      score: 1,
+      detail: `All ${total} FAIL_TO_PASS/PASS_TO_PASS checks passed.`
+    };
+  }
+};
+function parseChangedFiles(diff) {
+  const files = /* @__PURE__ */ new Set();
+  const re = /^diff --git a\/(.+?) b\/(.+)$/gm;
+  let match = re.exec(diff);
+  while (match) {
+    files.add(match[1]);
+    files.add(match[2]);
+    match = re.exec(diff);
+  }
+  return Array.from(files);
+}
+
+// src/scorers/diff-scorer.ts
+function parseDiffStats(diff) {
+  const filesChanged = /* @__PURE__ */ new Set();
+  let insertions = 0;
+  let deletions = 0;
+  for (const line of diff.split("\n")) {
+    const fileMatch = line.match(/^diff --git a\/(.+?) b\/(.+)$/);
+    if (fileMatch) {
+      filesChanged.add(fileMatch[2]);
+      continue;
+    }
+    if (line.startsWith("+++") || line.startsWith("---")) continue;
+    if (line.startsWith("+")) insertions++;
+    else if (line.startsWith("-")) deletions++;
+  }
+  return {
+    filesChanged: Array.from(filesChanged),
+    insertions,
+    deletions,
+    linesChanged: insertions + deletions
+  };
+}
+var DiffScorer = class {
+  name = "DiffScorer";
+  async score(input) {
+    const stats = parseDiffStats(input.result.finalDiff ?? "");
+    if (stats.filesChanged.length === 0) {
+      return {
+        passed: false,
+        score: 0,
+        detail: "Agent's diff is empty - no files were changed."
+      };
+    }
+    const details = [
+      `Agent diff: ${stats.filesChanged.length} file(s), +${stats.insertions}/-${stats.deletions} lines.`
+    ];
+    let score = 1;
+    const solutionDiff = input.testCase.solution;
+    if (solutionDiff?.includes("diff --git")) {
+      const goldStats = parseDiffStats(solutionDiff);
+      details.push(
+        `Gold diff: ${goldStats.filesChanged.length} file(s), +${goldStats.insertions}/-${goldStats.deletions} lines.`
+      );
+      const goldLines = Math.max(goldStats.linesChanged, 1);
+      const ratio = stats.linesChanged / goldLines;
+      score = ratio <= 1 ? 1 : Math.max(0.1, 1 / ratio);
+      details.push(
+        `Scope ratio (agent/gold lines changed): ${ratio.toFixed(2)} -> score ${score.toFixed(2)}`
+      );
+    }
+    return {
+      passed: true,
+      score,
+      detail: details.join("\n")
+    };
+  }
+};
+
+// src/scorers/localization-scorer.ts
+var LocalizationScorer = class {
+  name = "LocalizationScorer";
+  async score(input) {
+    const expectedFiles = input.testCase.expected_files;
+    if (!expectedFiles || expectedFiles.length === 0) {
+      return {
+        passed: true,
+        score: 1,
+        detail: "No expected_files configured; skipping localization check."
+      };
+    }
+    const stats = parseDiffStats(input.result.finalDiff ?? "");
+    const touched = stats.filesChanged;
+    if (touched.length === 0) {
+      return {
+        passed: false,
+        score: 0,
+        detail: "Agent did not modify any files."
+      };
+    }
+    const truePositives = touched.filter((f) => matchAnyGlob(expectedFiles, f));
+    const precision = truePositives.length / touched.length;
+    const matchedPatterns = expectedFiles.filter(
+      (pattern) => touched.some((f) => matchAnyGlob([pattern], f))
+    );
+    const recall = matchedPatterns.length / expectedFiles.length;
+    const f1 = precision + recall === 0 ? 0 : 2 * precision * recall / (precision + recall);
+    return {
+      passed: f1 > 0,
+      score: f1,
+      detail: [
+        `Localization: precision=${precision.toFixed(2)}, recall=${recall.toFixed(2)}, f1=${f1.toFixed(2)}`,
+        `Touched files: ${touched.join(", ")}`,
+        `Expected patterns: ${expectedFiles.join(", ")}`
+      ].join("\n")
+    };
+  }
+};
+var IGNORED_DIRS = /* @__PURE__ */ new Set(["node_modules", ".git", "dist", ".turbo"]);
+function hashFixture(fixtureDir) {
+  const hash = createHash("sha256");
+  let files = [];
+  try {
+    files = listFilesRecursive(fixtureDir).sort();
+  } catch {
+    hash.update(fixtureDir);
+    return hash.digest("hex");
+  }
+  for (const file of files) {
+    const rel = relative(fixtureDir, file).split(sep).join("/");
+    hash.update(rel);
+    hash.update(readFileSync(file));
+  }
+  return hash.digest("hex");
+}
+function listFilesRecursive(dir) {
+  const out = [];
+  for (const entry of readdirSync(dir, { withFileTypes: true })) {
+    if (IGNORED_DIRS.has(entry.name)) continue;
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      out.push(...listFilesRecursive(full));
+    } else if (entry.isFile() || entry.isSymbolicLink()) {
+      try {
+        if (statSync(full).isFile()) out.push(full);
+      } catch {
+      }
+    }
+  }
+  return out;
+}
+
+// src/runner/baseline.ts
+async function getOrComputeBaseline(input) {
+  const { testCase, sandboxProvider, db } = input;
+  if (!testCase.test_command) return void 0;
+  const testCaseId = testCase.id || testCase.name;
+  const fixtureHash = hashFixture(testCase.fixture);
+  if (db) {
+    try {
+      const cached = await getCachedBaseline(db, testCaseId, fixtureHash);
+      if (cached && cached.testCommand === testCase.test_command) {
+        return {
+          fixtureHash,
+          statusMap: JSON.parse(cached.statusMap),
+          cached: true
+        };
+      }
+    } catch (err) {
+      console.error(`Failed to read cached baseline: ${err.message}`);
+    }
+  }
+  const sandbox = await sandboxProvider.create({ gitSnapshot: testCase.fixture });
+  let statusMap = {};
+  try {
+    const res = await sandbox.exec(testCase.test_command);
+    statusMap = new TapTestResultParser().parse(`${res.stdout}
+${res.stderr}`);
+  } finally {
+    await sandbox.destroy();
+  }
+  if (db) {
+    try {
+      await saveCachedBaseline(db, {
+        id: `${testCaseId}:${fixtureHash}`,
+        testCaseId,
+        fixtureHash,
+        testCommand: testCase.test_command,
+        statusMap: JSON.stringify(statusMap),
+        createdAt: Math.floor(Date.now() / 1e3)
+      });
+    } catch (err) {
+      console.error(`Failed to persist baseline cache: ${err.message}`);
+    }
+  }
+  return { fixtureHash, statusMap, cached: false };
+}
+var FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/;
+function parseSkillMarkdown(content, path, dir) {
+  const match = content.match(FRONTMATTER_RE);
+  if (!match) {
+    throw new Error(`SKILL.md at "${path}" is missing a YAML frontmatter block ("---" ... "---")`);
+  }
+  const [, frontmatterYaml, body] = match;
+  const raw = parse(frontmatterYaml);
+  let frontmatter;
+  try {
+    frontmatter = SkillFrontmatterSchema.parse(raw);
+  } catch (err) {
+    throw new Error(`Invalid SKILL.md frontmatter in "${path}": ${err.message}`);
+  }
+  return { frontmatter, body: body.trim(), path, dir };
+}
+function discoverSkills(toolkitDir) {
+  const skillsDir = resolve(toolkitDir, ".claude", "skills");
+  if (!existsSync(skillsDir)) return [];
+  const skills = [];
+  for (const entry of readdirSync(skillsDir)) {
+    const dir = join(skillsDir, entry);
+    if (!statSync(dir).isDirectory()) continue;
+    const skillPath = join(dir, "SKILL.md");
+    if (!existsSync(skillPath)) continue;
+    const content = readFileSync(skillPath, "utf-8");
+    skills.push(parseSkillMarkdown(content, skillPath, dir));
+  }
+  return skills;
+}
+function discoverSkillsForToolkits(toolkitDirs) {
+  return toolkitDirs.flatMap((dir) => discoverSkills(dir));
+}
+function buildSkillsPromptAddendum(skills) {
+  if (skills.length === 0) return "";
+  const entries = skills.map((skill) => {
+    const { name, description } = skill.frontmatter;
+    const sandboxPath = `/app/.claude/skills/${name}/SKILL.md`;
+    return `- **${name}**: ${description}
+  Read \`${sandboxPath}\` (e.g. via your readFile tool) for full instructions before using this skill.`;
+  }).join("\n");
+  return [
+    "## Available skills",
+    "",
+    'The sandbox is preloaded with additional tools and documented "skills". Each skill below is available in the sandbox; read its SKILL.md for full usage instructions before relying on it.',
+    "",
+    entries
+  ].join("\n");
+}
+
+// src/runner/run-single.ts
+async function runSingle(input) {
+  const { testCase, agentConfig, adapter, sandboxProvider, db, runId } = input;
+  const startTime = Date.now();
+  let sandbox = null;
+  let passed = false;
+  let score = 0;
+  let stepsCount = 0;
+  let tokensIn = 0;
+  let tokensOut = 0;
+  let costUsd = 0;
+  let durationMs = 0;
+  let errorMsg;
+  let finalDiff = "";
+  let agentDiff = "";
+  let agentResult;
+  const metrics = {};
+  const emittedSteps = [];
+  if (db) {
+    await createRun(db, {
+      id: runId,
+      testCaseId: testCase.id || testCase.name,
+      agentConfigId: agentConfig.id || agentConfig.name,
+      sandboxProvider: sandboxProvider.name,
+      status: "running",
+      createdAt: Math.floor(startTime / 1e3)
+    });
+  }
+  let baseline;
+  try {
+    baseline = await getOrComputeBaseline({ testCase, sandboxProvider, db });
+  } catch (err) {
+    console.error(`Failed to compute baseline: ${err.message}`);
+  }
+  const toolkits = Array.from(
+    /* @__PURE__ */ new Set([...agentConfig.toolkits ?? [], ...testCase.toolkits ?? []])
+  );
+  const setupSandboxStep = createStep({
+    id: "setupSandbox",
+    inputSchema: z.any(),
+    outputSchema: z.object({}),
+    execute: async () => {
+      sandbox = await sandboxProvider.create({
+        image: testCase.image,
+        gitSnapshot: testCase.fixture,
+        toolkits
+      });
+      return {};
+    }
+  });
+  const solveStep = createStep({
+    id: "solve",
+    inputSchema: z.any(),
+    outputSchema: z.object({}),
+    execute: async () => {
+      if (!sandbox) throw new Error("Sandbox not initialized");
+      const onStepCallback = (stepEvent) => {
+        emittedSteps.push(stepEvent);
+        stepsCount++;
+        tokensIn += stepEvent.tokensIn || 0;
+        tokensOut += stepEvent.tokensOut || 0;
+        costUsd += stepEvent.costUsd || 0;
+        if (db) {
+          addTrace(db, {
+            runId,
+            stepIndex: stepEvent.index,
+            kind: stepEvent.kind,
+            tool: stepEvent.tool,
+            tokensIn: stepEvent.tokensIn,
+            tokensOut: stepEvent.tokensOut,
+            costUsd: stepEvent.costUsd,
+            timestamp: stepEvent.timestamp,
+            content: stepEvent.content
+          }).catch((err) => {
+            console.error(`Failed to persist trace step: ${err.message}`);
+          });
+        }
+      };
+      let effectiveConfig = agentConfig;
+      if (toolkits.length > 0) {
+        try {
+          const skills = discoverSkillsForToolkits(toolkits);
+          const addendum = buildSkillsPromptAddendum(skills);
+          if (addendum) {
+            effectiveConfig = {
+              ...agentConfig,
+              system_prompt: agentConfig.system_prompt ? `${agentConfig.system_prompt}
+
+${addendum}` : addendum
+            };
+          }
+        } catch (e) {
+          console.error(`Failed to build skills prompt addendum: ${e.message}`);
+        }
+      }
+      const result = await adapter.solve({
+        prompt: testCase.prompt,
+        sandbox,
+        config: effectiveConfig,
+        onStep: onStepCallback
+      });
+      try {
+        agentDiff = await sandbox.gitDiff();
+      } catch (e) {
+        console.error(`Failed to capture agent diff: ${e.message}`);
+      }
+      agentResult = { ...result, finalDiff: agentDiff || result.finalDiff };
+      if (testCase.test_patch) {
+        try {
+          const patchResult = await sandbox.applyPatch(testCase.test_patch);
+          metrics.testPatchApply = patchResult;
+        } catch (e) {
+          metrics.testPatchApply = { applied: false, repaired: false, output: e.message };
+        }
+      }
+      return { result };
+    }
+  });
+  const scoreStep = createStep({
+    id: "score",
+    inputSchema: z.any(),
+    outputSchema: z.object({
+      passed: z.boolean(),
+      detail: z.string(),
+      score: z.number()
+    }),
+    execute: async () => {
+      if (!sandbox) throw new Error("Sandbox not initialized");
+      const cmdScorer = new CommandScorer();
+      const cmdResult = await cmdScorer.score({
+        testCase,
+        sandbox
+      });
+      metrics.command = { passed: cmdResult.passed, detail: cmdResult.detail };
+      if (!cmdResult.passed) {
+        return { passed: false, detail: cmdResult.detail, score: 0 };
+      }
+      const assertScorer = new AssertionScorer();
+      const trace = { runId, steps: emittedSteps };
+      const assertResult = await assertScorer.score({
+        testCase,
+        trace
+      });
+      metrics.assertion = { passed: assertResult.passed, detail: assertResult.detail };
+      if (!assertResult.passed) {
+        return { passed: false, detail: assertResult.detail, score: 0 };
+      }
+      let overallPassed = true;
+      let overallDetail = "All tests passed";
+      const regressionScorer = new RegressionScorer();
+      const regressionResult = await regressionScorer.score({
+        testCase,
+        sandbox,
+        baseline: baseline?.statusMap
+      });
+      metrics.regression = {
+        passed: regressionResult.passed,
+        score: regressionResult.score,
+        detail: regressionResult.detail
+      };
+      if (!regressionResult.passed) {
+        overallPassed = false;
+        overallDetail = regressionResult.detail;
+      }
+      if (agentResult) {
+        const diffScorer = new DiffScorer();
+        const diffResult = await diffScorer.score({ testCase, result: agentResult });
+        metrics.diff = { score: diffResult.score, detail: diffResult.detail };
+        const localizationScorer = new LocalizationScorer();
+        const localizationResult = await localizationScorer.score({
+          testCase,
+          result: agentResult
+        });
+        metrics.localization = {
+          score: localizationResult.score,
+          detail: localizationResult.detail
+        };
+      }
+      if (baseline) {
+        metrics.baseline = { cached: baseline.cached, fixtureHash: baseline.fixtureHash };
+      }
+      return {
+        passed: overallPassed,
+        detail: overallDetail,
+        score: overallPassed ? 100 : 0
+      };
+    }
+  });
+  const cleanupStep = createStep({
+    id: "cleanup",
+    inputSchema: z.any(),
+    outputSchema: z.object({}),
+    execute: async () => {
+      if (sandbox) {
+        try {
+          finalDiff = agentDiff || await sandbox.gitDiff();
+          await sandbox.destroy();
+        } catch (e) {
+          console.error(`Failed to clean up sandbox: ${e.message}`);
+        }
+        sandbox = null;
+      }
+      return {};
+    }
+  });
+  const workflow = createWorkflow({
+    id: `run-single-${runId}`,
+    inputSchema: z.any(),
+    outputSchema: z.any()
+  }).then(setupSandboxStep).then(solveStep).then(scoreStep).then(cleanupStep).commit();
+  try {
+    const runState = {};
+    const run = await workflow.createRun();
+    const res = await run.start({
+      inputData: {},
+      initialState: runState
+    });
+    const scoreResults = res.results?.score;
+    passed = scoreResults?.passed ?? false;
+    score = scoreResults?.score ?? 0;
+    errorMsg = scoreResults?.passed ? void 0 : scoreResults?.detail;
+  } catch (err) {
+    errorMsg = err.message || "Unknown execution error";
+    passed = false;
+    if (sandbox) {
+      try {
+        await sandbox.destroy();
+      } catch (e) {
+      }
+    }
+  }
+  durationMs = Date.now() - startTime;
+  if (db) {
+    await updateRun(db, runId, {
+      status: errorMsg ? "failed" : "completed",
+      passed,
+      score,
+      stepsCount,
+      tokensIn,
+      tokensOut,
+      costUsd,
+      durationMs,
+      error: errorMsg,
+      finalDiff,
+      metrics: Object.keys(metrics).length > 0 ? JSON.stringify(metrics) : void 0,
+      completedAt: Math.floor(Date.now() / 1e3)
+    });
+  }
+  return {
+    runId,
+    passed,
+    score,
+    stepsCount,
+    tokensIn,
+    tokensOut,
+    costUsd,
+    durationMs,
+    error: errorMsg,
+    finalDiff,
+    metrics
+  };
+}
+async function runBenchmark(input) {
+  const { testCases, agentConfigs, adapter, sandboxProvider, db, concurrency = 2, onRunUpdate } = input;
+  const generateCombinationsStep = createStep({
+    id: "generateCombinations",
+    inputSchema: z.any(),
+    outputSchema: z.array(z.any()),
+    execute: async ({ getInitData }) => {
+      const initData = getInitData();
+      const combinations = [];
+      for (const tc of initData.testCases) {
+        for (const config of initData.agentConfigs) {
+          combinations.push({
+            testCase: tc,
+            agentConfig: config
+          });
+        }
+      }
+      return combinations;
+    }
+  });
+  const executeSingleRunStep = createStep({
+    id: "executeSingleRun",
+    inputSchema: z.any(),
+    outputSchema: z.any(),
+    execute: async ({ inputData, requestContext }) => {
+      const { testCase, agentConfig } = inputData;
+      const ctx = requestContext?.context || requestContext;
+      const getVal = (key) => {
+        if (ctx instanceof Map) return ctx.get(key);
+        if (ctx && typeof ctx === "object" && key in ctx) return ctx[key];
+        if (typeof ctx?.get === "function") return ctx.get(key);
+        return void 0;
+      };
+      const adapter2 = getVal("adapter");
+      const sandboxProvider2 = getVal("sandboxProvider");
+      const db2 = getVal("db");
+      const onRunUpdate2 = getVal("onRunUpdate");
+      const runId = randomUUID();
+      if (onRunUpdate2) {
+        onRunUpdate2({
+          runId,
+          testCaseId: testCase.id || testCase.name,
+          agentConfigId: agentConfig.id || agentConfig.name,
+          status: "running",
+          passed: false,
+          stepsCount: 0,
+          tokensIn: 0,
+          tokensOut: 0,
+          costUsd: 0,
+          durationMs: 0
+        });
+      }
+      try {
+        const res2 = await runSingle({
+          testCase,
+          agentConfig,
+          adapter: adapter2,
+          sandboxProvider: sandboxProvider2,
+          db: db2,
+          runId
+        });
+        if (onRunUpdate2) {
+          onRunUpdate2({
+            ...res2,
+            testCaseId: testCase.id || testCase.name,
+            agentConfigId: agentConfig.id || agentConfig.name,
+            status: res2.error ? "failed" : "completed"
+          });
+        }
+        return res2;
+      } catch (err) {
+        const failedResult = {
+          runId,
+          passed: false,
+          stepsCount: 0,
+          tokensIn: 0,
+          tokensOut: 0,
+          costUsd: 0,
+          durationMs: 0,
+          error: err.message || "Failed during execution"
+        };
+        if (onRunUpdate2) {
+          onRunUpdate2({
+            ...failedResult,
+            testCaseId: testCase.id || testCase.name,
+            agentConfigId: agentConfig.id || agentConfig.name,
+            status: "failed"
+          });
+        }
+        return failedResult;
+      }
+    }
+  });
+  const workflow = createWorkflow({
+    id: `benchmark-orchestrator-${randomUUID()}`,
+    inputSchema: z.any(),
+    outputSchema: z.any()
+  }).then(generateCombinationsStep).foreach(executeSingleRunStep, { concurrency }).commit();
+  const runState = {};
+  const executionContext = /* @__PURE__ */ new Map([
+    ["adapter", adapter],
+    ["sandboxProvider", sandboxProvider],
+    ["db", db],
+    ["onRunUpdate", onRunUpdate]
+  ]);
+  const run = await workflow.createRun();
+  const res = await run.start({
+    inputData: { testCases, agentConfigs },
+    initialState: runState,
+    requestContext: executionContext
+  });
+  const rawRuns = res.results?.executeSingleRunStepResult || [];
+  return {
+    runs: Array.isArray(rawRuns) ? rawRuns : []
+  };
+}
+
+// src/runner/validate-test-case.ts
+async function validateTestCase(input) {
+  const { testCase, sandboxProvider } = input;
+  const checks = [];
+  checks.push(...checkStaticFields(testCase));
+  if (!testCase.test_command) {
+    checks.push({
+      name: "execution-checks",
+      passed: true,
+      detail: "No test_command configured; skipping pre/post-patch execution checks."
+    });
+    return finalize(checks);
+  }
+  const sandbox = await sandboxProvider.create({ gitSnapshot: testCase.fixture });
+  try {
+    if (testCase.test_patch) {
+      const patchRes = await sandbox.applyPatch(testCase.test_patch);
+      checks.push({
+        name: "test_patch applies cleanly",
+        passed: patchRes.applied,
+        detail: patchRes.applied ? `Applied${patchRes.repaired ? " (required repair/fallback)" : ""}.` : `Failed to apply test_patch:
+${patchRes.output}`
+      });
+      if (!patchRes.applied) return finalize(checks);
+    }
+    const preRun = await sandbox.exec(testCase.test_command);
+    const preStatus = new TapTestResultParser().parse(`${preRun.stdout}
+${preRun.stderr}`);
+    checks.push(
+      checkStatusExpectation(
+        "pre-patch: fail_to_pass tests are currently failing",
+        testCase.fail_to_pass ?? [],
+        preStatus,
+        "FAIL"
+      )
+    );
+    checks.push(
+      checkStatusExpectation(
+        "pre-patch: pass_to_pass tests are currently passing",
+        testCase.pass_to_pass ?? [],
+        preStatus,
+        "PASS"
+      )
+    );
+    if (testCase.solution) {
+      const solutionRes = await sandbox.applyPatch(testCase.solution);
+      checks.push({
+        name: "solution patch applies cleanly",
+        passed: solutionRes.applied,
+        detail: solutionRes.applied ? `Applied${solutionRes.repaired ? " (required repair/fallback)" : ""}.` : `Failed to apply solution patch:
+${solutionRes.output}`
+      });
+      if (solutionRes.applied) {
+        const postRun = await sandbox.exec(testCase.test_command);
+        const postStatus = new TapTestResultParser().parse(`${postRun.stdout}
+${postRun.stderr}`);
+        checks.push(
+          checkStatusExpectation(
+            "post-patch: fail_to_pass tests now pass",
+            testCase.fail_to_pass ?? [],
+            postStatus,
+            "PASS"
+          )
+        );
+        checks.push(
+          checkStatusExpectation(
+            "post-patch: pass_to_pass tests still pass",
+            testCase.pass_to_pass ?? [],
+            postStatus,
+            "PASS"
+          )
+        );
+      }
+    }
+  } finally {
+    await sandbox.destroy();
+  }
+  return finalize(checks);
+}
+function checkStaticFields(testCase) {
+  const checks = [];
+  checks.push({
+    name: "has name and prompt",
+    passed: Boolean(testCase.name && testCase.prompt),
+    detail: testCase.name && testCase.prompt ? "ok" : "Test case is missing `name` and/or `prompt`."
+  });
+  const failToPass = testCase.fail_to_pass ?? [];
+  const passToPass = testCase.pass_to_pass ?? [];
+  if (failToPass.length > 0 || passToPass.length > 0) {
+    checks.push({
+      name: "test_command configured for fail_to_pass/pass_to_pass",
+      passed: Boolean(testCase.test_command),
+      detail: testCase.test_command ? "ok" : "fail_to_pass/pass_to_pass are set but no test_command was provided."
+    });
+    const overlap = failToPass.filter((n) => passToPass.includes(n));
+    checks.push({
+      name: "fail_to_pass and pass_to_pass do not overlap",
+      passed: overlap.length === 0,
+      detail: overlap.length === 0 ? "ok" : `Tests listed in both sets: ${overlap.join(", ")}`
+    });
+  }
+  if (testCase.forbid_modified?.length) {
+    const invalid = testCase.forbid_modified.filter((p) => !isLikelyValidGlob(p));
+    checks.push({
+      name: "forbid_modified patterns look valid",
+      passed: invalid.length === 0,
+      detail: invalid.length === 0 ? "ok" : `Suspicious glob pattern(s): ${invalid.join(", ")}`
+    });
+  }
+  if (testCase.expected_files?.length) {
+    const invalid = testCase.expected_files.filter((p) => !isLikelyValidGlob(p));
+    checks.push({
+      name: "expected_files patterns look valid",
+      passed: invalid.length === 0,
+      detail: invalid.length === 0 ? "ok" : `Suspicious glob pattern(s): ${invalid.join(", ")}`
+    });
+  }
+  if (testCase.created_at) {
+    const date = new Date(testCase.created_at);
+    checks.push({
+      name: "created_at is a parseable date (contamination/date-cutoff check)",
+      passed: !Number.isNaN(date.getTime()),
+      detail: !Number.isNaN(date.getTime()) ? `ok (${date.toISOString()})` : `Could not parse created_at: "${testCase.created_at}"`
+    });
+  }
+  return checks;
+}
+function isLikelyValidGlob(pattern) {
+  if (!pattern || pattern.trim().length === 0) return false;
+  try {
+    matchGlob(pattern, "x");
+    return true;
+  } catch {
+    return false;
+  }
+}
+function checkStatusExpectation(name, testNames, statusMap, expected) {
+  if (testNames.length === 0) {
+    return { name, passed: true, detail: "No tests configured for this check." };
+  }
+  const problems = [];
+  for (const testName of testNames) {
+    const actual = statusMap[testName];
+    if (actual === void 0) {
+      problems.push(`"${testName}" not found in test output`);
+    } else if (actual !== expected) {
+      problems.push(`"${testName}" is ${actual}, expected ${expected}`);
+    }
+  }
+  return {
+    name,
+    passed: problems.length === 0,
+    detail: problems.length === 0 ? `ok (${testNames.length} test(s))` : problems.join("; ")
+  };
+}
+function finalize(checks) {
+  return { ok: checks.every((c) => c.passed), checks };
+}
+
+export { AgentConfigSchema, AssertionScorer, CommandScorer, DiffScorer, LocalizationScorer, McpServerConfigSchema, RegressionScorer, RunSchema, SkillFrontmatterSchema, StepEventSchema, SuccessCriterionSchema, TapTestResultParser, TestCaseSchema, TraceSchema, buildSkillsPromptAddendum, discoverSkills, discoverSkillsForToolkits, getOrComputeBaseline, hashFixture, matchAnyGlob, matchGlob, parseDiffStats, parseSkillMarkdown, runBenchmark, runSingle, validateTestCase };