agentgrader 1.1.0 → 1.2.0

package/dist/index.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import 'dotenv/config';
 import { cac } from 'cac';
+import { initDb, saveTestCase, saveAgentConfig, getRun, getTraces, getRunsByMatrixId } from '@agentgrader/store';
 import { randomUUID } from 'crypto';
 import { resolve, dirname, isAbsolute, basename } from 'path';
 import { render, Box, Text } from 'ink';
-import { initDb, saveTestCase, saveAgentConfig, getRun, getTraces, getRunsByMatrixId } from '@agentgrader/store';
 import { runSingle, runBenchmark, validateTestCase, TestCaseSchema, AgentConfigSchema } from '@agentgrader/core';
 import { DockerSandboxProvider } from '@agentgrader/sandbox-docker';
 import { AiSdkAgentAdapter } from '@agentgrader/agent-openrouter';
@@ -16,6 +16,155 @@ import { parse, stringify } from 'yaml';
 import { z, ZodError } from 'zod';
 import { execFileSync } from 'child_process';
 
+var CONTENT_PREVIEW_MAX = 200;
+var ANSI = {
+  reset: "\x1B[0m",
+  gray: "\x1B[90m",
+  yellow: "\x1B[33m",
+  cyan: "\x1B[36m",
+  blue: "\x1B[34m"};
+function paint(text, code) {
+  if (!process.stdout.isTTY) return text;
+  return `${code}${text}${ANSI.reset}`;
+}
+function truncateContent(content, full) {
+  if (full || content.length <= CONTENT_PREVIEW_MAX) return content;
+  return `${content.slice(0, CONTENT_PREVIEW_MAX)}...`;
+}
+function normalizeContent(content) {
+  return (content ?? "").trim();
+}
+function formatStepSummary(step, full) {
+  if (!step) return "(no step)";
+  const label = step.tool ? `${step.kind}:${step.tool}` : step.kind;
+  if (!step.content) return label;
+  const preview = truncateContent(step.content.replace(/\n/g, " "), full);
+  return `${label} ${preview}`;
+}
+function stepsByIndex(traces) {
+  const map = /* @__PURE__ */ new Map();
+  for (const step of traces) {
+    map.set(step.stepIndex, step);
+  }
+  return map;
+}
+function stepsDiverge(a, b) {
+  if (!a || !b) return true;
+  if (a.kind !== b.kind) return true;
+  if ((a.tool ?? "") !== (b.tool ?? "")) return true;
+  if (normalizeContent(a.content) !== normalizeContent(b.content)) return true;
+  return false;
+}
+function formatRunStatus(run) {
+  const passed = run.passed === true ? " (passed)" : run.passed === false ? " (failed)" : "";
+  return `${run.status}${passed}`;
+}
+function printRunHeader(label, run) {
+  const tag = label === "A" ? paint(`Run A (${run.id})`, ANSI.cyan) : paint(`Run B (${run.id})`, ANSI.blue);
+  console.log(tag);
+  console.log(`  test case:    ${run.testCaseId}`);
+  console.log(`  agent config: ${run.agentConfigId}`);
+  console.log(`  status:       ${formatRunStatus(run)}`);
+  console.log(`  steps:        ${run.stepsCount}`);
+  console.log(`  cost:         $${run.costUsd.toFixed(4)}`);
+  console.log(`  duration:     ${run.durationMs}ms`);
+  if (run.error) console.log(`  error:        ${run.error}`);
+}
+async function compareCommand(runIdA, runIdB, opts) {
+  const db = initDb();
+  const [runA, runB, tracesA, tracesB] = await Promise.all([
+    getRun(db, runIdA),
+    getRun(db, runIdB),
+    getTraces(db, runIdA),
+    getTraces(db, runIdB)
+  ]);
+  if (!runA) {
+    console.error(`Run not found: ${runIdA}`);
+    process.exit(1);
+  }
+  if (!runB) {
+    console.error(`Run not found: ${runIdB}`);
+    process.exit(1);
+  }
+  console.log("");
+  printRunHeader("A", runA);
+  console.log("");
+  printRunHeader("B", runB);
+  console.log("");
+  if (runA.testCaseId !== runB.testCaseId) {
+    console.log(
+      paint(
+        "\u26A0\uFE0F  Comparing runs of different test cases - step alignment may not be meaningful.",
+        ANSI.yellow
+      )
+    );
+    console.log("");
+  }
+  const mapA = stepsByIndex(tracesA);
+  const mapB = stepsByIndex(tracesB);
+  const maxIndex = Math.max(
+    tracesA.length > 0 ? Math.max(...tracesA.map((s) => s.stepIndex)) : -1,
+    tracesB.length > 0 ? Math.max(...tracesB.map((s) => s.stepIndex)) : -1,
+    -1
+  );
+  if (maxIndex < 0) {
+    console.log("No steps recorded for either run.");
+    return;
+  }
+  const divergentIndices = /* @__PURE__ */ new Set();
+  for (let i = 0; i <= maxIndex; i++) {
+    if (stepsDiverge(mapA.get(i), mapB.get(i))) {
+      divergentIndices.add(i);
+    }
+  }
+  const visibleIndices = /* @__PURE__ */ new Set();
+  if (opts.onlyDiff) {
+    for (const idx of divergentIndices) {
+      visibleIndices.add(idx);
+      if (idx > 0) visibleIndices.add(idx - 1);
+      if (idx < maxIndex) visibleIndices.add(idx + 1);
+    }
+  } else {
+    for (let i = 0; i <= maxIndex; i++) visibleIndices.add(i);
+  }
+  const sortedVisible = [...visibleIndices].sort((a, b) => a - b);
+  if (sortedVisible.length === 0) {
+    console.log("No divergent steps (nothing to show with --only-diff).");
+  } else {
+    console.log("Step comparison:");
+    for (const i of sortedVisible) {
+      const stepA = mapA.get(i);
+      const stepB = mapB.get(i);
+      const divergent = stepsDiverge(stepA, stepB);
+      if (divergent) {
+        console.log(paint(`[step ${i}] DIVERGENT`, ANSI.yellow));
+        console.log(`  A: ${formatStepSummary(stepA, opts.full ?? false)}`);
+        console.log(`  B: ${formatStepSummary(stepB, opts.full ?? false)}`);
+      } else {
+        const line = formatStepSummary(stepA ?? stepB, opts.full ?? false);
+        console.log(paint(`[step ${i}] (same)`, ANSI.gray));
+        console.log(`  ${line}`);
+      }
+    }
+  }
+  const totalSteps = maxIndex + 1;
+  const diffCount = divergentIndices.size;
+  let firstDivergence = null;
+  for (let i = 0; i <= maxIndex; i++) {
+    if (divergentIndices.has(i)) {
+      firstDivergence = i;
+      break;
+    }
+  }
+  console.log("");
+  console.log(`${diffCount} of ${totalSteps} step(s) differ.`);
+  if (firstDivergence !== null) {
+    console.log(`First divergence at step ${firstDivergence}.`);
+  } else {
+    console.log("No divergence detected.");
+  }
+  console.log("");
+}
 var CONFIG_COL_WIDTH = 24;
 var CONFIG_LABEL_MAX = 20;
 function truncateLabel(name, max = CONFIG_LABEL_MAX) {
@@ -1119,6 +1268,17 @@ cli.command("trace <runId>", "Show the step trace and metrics for a single run")
     process.exit(1);
   }
 });
+cli.command("compare <runIdA> <runIdB>", "Compare the step traces of two runs side by side").option("--full", "Print full step content without truncation").option(
+  "--only-diff",
+  "Show only divergent steps plus one step of context before and after each"
+).example("agr compare <runIdA> <runIdB> --only-diff").action(async (runIdA, runIdB, options) => {
+  try {
+    await compareCommand(runIdA, runIdB, options);
+  } catch (err) {
+    console.error(`Error executing compare: ${err.message}`);
+    process.exit(1);
+  }
+});
 cli.help();
 try {
   cli.parse();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentgrader",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "CLI for the Agentgrader benchmarking framework — run and bench coding agents",
   "license": "MIT",
   "type": "module",