@sean.holung/minicode 0.3.2 → 0.3.3

package/dist/tests/agent.test.js

@@ -29,6 +29,22 @@ class RepeatingModelClient {
         };
     }
 }
+class InfiniteToolModelClient {
+    callCount = 0;
+    getCalls() {
+        return this.callCount;
+    }
+    async chat(params) {
+        void params;
+        this.callCount += 1;
+        return {
+            text: `step ${this.callCount}`,
+            toolCalls: [{ id: `tool-${this.callCount}`, name: "echo_tool", input: { value: String(this.callCount) } }],
+            stopReason: "tool_use",
+            usage: { inputTokens: 1, outputTokens: 1 },
+        };
+    }
+}
 function createEchoTool() {
     return {
         name: "echo_tool",
@@ -81,6 +97,21 @@ test("agent stops on repeated identical tool calls", async () => {
     const { text } = await agent.runTurn("Do something");
     assert.match(text, /repeated identical tool calls/);
 });
+test("agent tells the user how to continue when the turn call limit is reached", async () => {
+    const config = createTestAgentConfig("/tmp");
+    config.maxSteps = 2;
+    const modelClient = new InfiniteToolModelClient();
+    const agent = new CodingAgent({
+        config,
+        modelClient,
+        toolRegistry: new ToolRegistry([createEchoTool()]),
+    });
+    const { text } = await agent.runTurn("Keep working");
+    assert.match(text, /turn call limit/);
+    assert.match(text, /Type "continue"/);
+    assert.match(text, /Settings/);
+    assert.equal(modelClient.getCalls(), 2);
+});
 test("agent omits code map when projectIndex is not provided", async () => {
     let capturedSystem = "";
     const spyClient = {

package/dist/tests/analysis-helpers.test.js

@@ -0,0 +1,89 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { buildFindingGraphContext, buildFindingMetricChips, countFindingsByType, filterFindings, findingTypeLabel, } from "../src/web/analysis-helpers.js";
+test("buildFindingGraphContext limits cycle highlights to internal cycle edges", () => {
+    const finding = {
+        id: "cycle:alpha->beta",
+        type: "cycle",
+        severity: "warning",
+        title: "Cycle across 2 symbols",
+        summary: "alpha and beta are mutually dependent.",
+        symbols: ["alpha", "beta"],
+        files: ["src/a.ts", "src/b.ts"],
+        metrics: { cycleSize: 2, edgeCount: 2, fileCount: 2 },
+        rationale: ["Strongly connected component detected."],
+    };
+    const context = buildFindingGraphContext(finding, [
+        { source: "alpha", target: "beta", kind: "calls" },
+        { source: "beta", target: "alpha", kind: "calls" },
+        { source: "alpha", target: "gamma", kind: "calls" },
+    ]);
+    assert.deepEqual(context.nodes, ["alpha", "beta"]);
+    assert.deepEqual(context.edgeIds, [
+        "alpha->beta:calls",
+        "beta->alpha:calls",
+    ]);
+});
+test("buildFindingGraphContext includes incident edges for hotspot-style findings", () => {
+    const finding = {
+        id: "hotspot:service",
+        type: "hotspot",
+        severity: "info",
+        title: "service is a structural hotspot",
+        summary: "service has total degree 4.",
+        symbols: ["service"],
+        files: ["src/service.ts"],
+        metrics: { totalDegree: 4, fanIn: 1, fanOut: 3, threshold: 4 },
+        rationale: ["Total degree exceeds hotspot threshold."],
+    };
+    const context = buildFindingGraphContext(finding, [
+        { source: "entry", target: "service", kind: "calls" },
+        { source: "service", target: "repo", kind: "calls" },
+        { source: "service", target: "util", kind: "calls" },
+        { source: "other", target: "elsewhere", kind: "calls" },
+    ]);
+    assert.deepEqual(context.nodes, ["service"]);
+    assert.deepEqual(context.edgeIds, [
+        "entry->service:calls",
+        "service->repo:calls",
+        "service->util:calls",
+    ]);
+});
+test("analysis helpers summarize types and metrics for rendering", () => {
+    const findings = [
+        {
+            id: "fanin:util",
+            type: "fanInOutlier",
+            severity: "info",
+            title: "util has high fan-in",
+            summary: "util is widely referenced.",
+            symbols: ["util"],
+            files: ["src/util.ts"],
+            metrics: { fanIn: 5, threshold: 3 },
+            rationale: [],
+        },
+        {
+            id: "file:service",
+            type: "fileCoupling",
+            severity: "warning",
+            title: "service.ts is highly coupled",
+            summary: "service.ts has high afferent/efferent coupling.",
+            symbols: ["service"],
+            files: ["src/service.ts"],
+            metrics: { totalCoupling: 4, instability: 0.75 },
+            rationale: [],
+        },
+    ];
+    assert.deepEqual(countFindingsByType(findings), {
+        cycle: 0,
+        fanInOutlier: 1,
+        fanOutOutlier: 0,
+        hotspot: 0,
+        fileCoupling: 1,
+    });
+    assert.deepEqual(buildFindingMetricChips(findings[0]), ["fan-in 5", "threshold 3"]);
+    assert.deepEqual(buildFindingMetricChips(findings[1]), ["coupling 4", "instability 0.75"]);
+    assert.equal(findingTypeLabel("fileCoupling"), "File coupling");
+    assert.deepEqual(filterFindings(findings, "all").map((finding) => finding.id), ["fanin:util", "file:service"]);
+    assert.deepEqual(filterFindings(findings, "fileCoupling").map((finding) => finding.id), ["file:service"]);
+});

package/dist/tests/analysis-ui.test.js

@@ -0,0 +1,29 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { test } from "node:test";
+const distWeb = join(import.meta.dirname, "..", "dist", "src", "web");
+test("built HTML contains analysis entry point and drawer shell", () => {
+    const html = readFileSync(join(distWeb, "index.html"), "utf8");
+    assert.ok(html.includes('id="graph-analyze"'), "HTML should contain the Analyze toolbar button");
+    assert.ok(html.includes('id="analysis-panel"'), "HTML should contain the analysis panel");
+    assert.ok(html.includes("These signals come from the dependency graph itself, not from agent judgment."), "HTML should explain the deterministic scope of the analysis");
+});
+test("built CSS contains analysis drawer and finding card styles", () => {
+    const css = readFileSync(join(distWeb, "style.css"), "utf8");
+    assert.ok(css.includes("#analysis-panel"), "CSS should contain analysis panel styles");
+    assert.ok(css.includes(".analysis-finding"), "CSS should contain finding card styles");
+    assert.ok(css.includes(".analysis-summary-card"), "CSS should contain summary card styles");
+    assert.ok(css.includes(".analysis-summary-card.active"), "CSS should style the active analysis filter");
+    assert.ok(css.includes(".analysis-explanation"), "CSS should contain AI explanation styles");
+});
+test("built JS contains structural analysis loading and highlighting logic", () => {
+    const js = readFileSync(join(distWeb, "app.js"), "utf8");
+    assert.ok(js.includes("/api/analysis"), "JS should fetch the analysis API");
+    assert.ok(js.includes("/api/analysis/explain"), "JS should call the analysis explanation API");
+    assert.ok(js.includes("analysis-selected"), "JS should apply selected analysis highlight classes");
+    assert.ok(js.includes("graph-derived structural signals"), "JS should surface deterministic analysis messaging");
+    assert.ok(js.includes("AI interpretation"), "JS should label advisory AI interpretation distinctly");
+    assert.ok(js.includes("activeAnalysisFilter"), "JS should track the active analysis filter");
+    assert.ok(js.includes("data-filter"), "JS should wire summary cards into finding filters");
+});

package/dist/tests/benchmark-harness.test.js

@@ -0,0 +1,527 @@
+import assert from "node:assert/strict";
+import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+import path from "node:path";
+import { tmpdir } from "node:os";
+import { test } from "node:test";
+import { loadBenchmarkTasks, loadBenchmarkTask } from "../src/benchmark/task-loader.js";
+import { evaluate } from "../src/benchmark/evaluator.js";
+import { runBenchmarkTask } from "../src/benchmark/runner.js";
+import { buildReport, formatReport, compareReports, } from "../src/benchmark/reporter.js";
+import { createTestAgentConfig } from "./test-utils.js";
+// ─── Helpers ───────────────────────────────────────────────────
+async function createTempTaskDir() {
+    const tmpDir = await mkdtemp(path.join(tmpdir(), "bench-test-"));
+    await mkdir(path.join(tmpDir, "navigation", "find-foo"), { recursive: true });
+    await mkdir(path.join(tmpDir, "editing", "fix-bar"), { recursive: true });
+    await writeFile(path.join(tmpDir, "navigation", "find-foo", "task.json"), JSON.stringify({
+        title: "Find foo",
+        prompt: "Find where foo is defined",
+        rubric: {
+            expectedOutputPatterns: ["foo"],
+            maxToolCalls: 5,
+        },
+    }));
+    await writeFile(path.join(tmpDir, "editing", "fix-bar", "task.json"), JSON.stringify({
+        title: "Fix bar",
+        prompt: "Fix the bar function",
+        rubric: {
+            expectedOutputPatterns: ["bar"],
+            expectedFilesRead: ["bar.ts"],
+            forbiddenPatterns: ["error"],
+            maxToolCalls: 10,
+            maxTotalTokens: 5000,
+        },
+    }));
+    return tmpDir;
+}
+function makeTrace(overrides = {}) {
+    return {
+        taskId: "navigation/find-foo",
+        model: "test-model",
+        variant: "baseline",
+        commitSha: "abc123",
+        response: "Found foo in src/foo.ts at line 10.",
+        toolCalls: [
+            { name: "search", input: { query: "foo" }, output: "src/foo.ts:10", durationMs: 50 },
+            { name: "read_file", input: { path: "src/foo.ts" }, output: "function foo() {}", durationMs: 30 },
+        ],
+        filesRead: ["src/foo.ts"],
+        symbolsQueried: [],
+        usage: { inputTokens: 500, outputTokens: 100, totalTokens: 600 },
+        durationMs: 200,
+        startedAt: new Date().toISOString(),
+        ...overrides,
+    };
+}
+class MockModelClient {
+    response;
+    constructor(response) {
+        this.response = response;
+    }
+    async chat(params) {
+        void params;
+        return {
+            text: this.response,
+            toolCalls: [],
+            stopReason: "end_turn",
+            usage: { inputTokens: 100, outputTokens: 50 },
+        };
+    }
+}
+class ToolCallingMockClient {
+    callCount = 0;
+    async chat(params) {
+        void params;
+        this.callCount += 1;
+        if (this.callCount === 1) {
+            return {
+                text: "Let me search for that.",
+                toolCalls: [{ id: "t1", name: "echo_tool", input: { value: "hello" } }],
+                stopReason: "tool_use",
+                usage: { inputTokens: 100, outputTokens: 50 },
+            };
+        }
+        return {
+            text: "Found: echo:hello",
+            toolCalls: [],
+            stopReason: "end_turn",
+            usage: { inputTokens: 150, outputTokens: 60 },
+        };
+    }
+}
+// ─── Task Loader Tests ─────────────────────────────────────────
+test("loadBenchmarkTasks loads tasks from directory structure", async () => {
+    const tmpDir = await createTempTaskDir();
+    try {
+        const tasks = await loadBenchmarkTasks(tmpDir);
+        assert.equal(tasks.length, 2);
+        const first = tasks[0];
+        const second = tasks[1];
+        assert.equal(first.id, "editing/fix-bar");
+        assert.equal(second.id, "navigation/find-foo");
+        assert.equal(first.category, "editing");
+        assert.equal(second.category, "navigation");
+        assert.equal(second.prompt, "Find where foo is defined");
+    }
+    finally {
+        await rm(tmpDir, { recursive: true });
+    }
+});
+test("loadBenchmarkTasks returns empty array for empty directory", async () => {
+    const tmpDir = await mkdtemp(path.join(tmpdir(), "bench-empty-"));
+    try {
+        const tasks = await loadBenchmarkTasks(tmpDir);
+        assert.equal(tasks.length, 0);
+    }
+    finally {
+        await rm(tmpDir, { recursive: true });
+    }
+});
+test("loadBenchmarkTasks ignores invalid category directories", async () => {
+    const tmpDir = await mkdtemp(path.join(tmpdir(), "bench-invalid-"));
+    await mkdir(path.join(tmpDir, "invalid-category", "task1"), { recursive: true });
+    await writeFile(path.join(tmpDir, "invalid-category", "task1", "task.json"), JSON.stringify({ title: "X", prompt: "X", rubric: {} }));
+    try {
+        const tasks = await loadBenchmarkTasks(tmpDir);
+        assert.equal(tasks.length, 0);
+    }
+    finally {
+        await rm(tmpDir, { recursive: true });
+    }
+});
+test("loadBenchmarkTask loads a single task by id", async () => {
+    const tmpDir = await createTempTaskDir();
+    try {
+        const task = await loadBenchmarkTask(tmpDir, "navigation/find-foo");
+        assert.ok(task);
+        assert.equal(task.title, "Find foo");
+        assert.equal(task.category, "navigation");
+    }
+    finally {
+        await rm(tmpDir, { recursive: true });
+    }
+});
+test("loadBenchmarkTask returns undefined for missing task", async () => {
+    const tmpDir = await createTempTaskDir();
+    try {
+        const task = await loadBenchmarkTask(tmpDir, "navigation/nonexistent");
+        assert.equal(task, undefined);
+    }
+    finally {
+        await rm(tmpDir, { recursive: true });
+    }
+});
+// ─── Evaluator Tests ───────────────────────────────────────────
+test("evaluate passes when all expected patterns match", () => {
+    const rubric = {
+        expectedOutputPatterns: ["foo", "line \\d+"],
+    };
+    const trace = makeTrace({ response: "Found foo at line 10" });
+    const result = evaluate("test/task", rubric, trace);
+    assert.equal(result.passed, true);
+    assert.equal(result.checks.length, 2);
+    assert.ok(result.checks.every((c) => c.passed));
+});
+test("evaluate fails when expected pattern is missing", () => {
+    const rubric = {
+        expectedOutputPatterns: ["bar"],
+    };
+    const trace = makeTrace({ response: "Found foo at line 10" });
+    const result = evaluate("test/task", rubric, trace);
+    assert.equal(result.passed, false);
+    assert.equal(result.checks[0].passed, false);
+    assert.ok(result.checks[0].detail?.includes("Pattern not found"));
+});
+test("evaluate checks expected files read", () => {
+    const rubric = {
+        expectedFilesRead: ["src/foo.ts", "src/bar.ts"],
+    };
+    const trace = makeTrace({ filesRead: ["src/foo.ts"] });
+    const result = evaluate("test/task", rubric, trace);
+    assert.equal(result.passed, false);
+    assert.equal(result.checks[0].passed, true);
+    assert.equal(result.checks[1].passed, false);
+});
+test("evaluate checks expected symbols queried", () => {
+    const rubric = {
+        expectedSymbols: ["buildProjectIndex"],
+    };
+    const trace = makeTrace({ symbolsQueried: ["buildProjectIndex"] });
+    const result = evaluate("test/task", rubric, trace);
+    assert.equal(result.passed, true);
+});
+test("evaluate checks forbidden patterns", () => {
+    const rubric = {
+        forbiddenPatterns: ["error"],
+    };
+    const traceOk = makeTrace({ response: "All good" });
+    const traceBad = makeTrace({ response: "There was an error" });
+    assert.equal(evaluate("t", rubric, traceOk).passed, true);
+    assert.equal(evaluate("t", rubric, traceBad).passed, false);
+});
+test("evaluate computes efficiency metrics", () => {
+    const rubric = {
+        maxToolCalls: 3,
+        maxTotalTokens: 1000,
+    };
+    const trace = makeTrace({
+        toolCalls: [
+            { name: "a", input: {}, output: "", durationMs: 10 },
+            { name: "b", input: {}, output: "", durationMs: 10 },
+        ],
+        usage: { inputTokens: 400, outputTokens: 200, totalTokens: 600 },
+    });
+    const result = evaluate("t", rubric, trace);
+    assert.equal(result.efficiency.toolCallCount, 2);
+    assert.equal(result.efficiency.totalTokens, 600);
+    assert.equal(result.efficiency.withinBudget, true);
+});
+test("evaluate marks over-budget when tool calls exceed limit", () => {
+    const rubric = { maxToolCalls: 1 };
+    const trace = makeTrace({
+        toolCalls: [
+            { name: "a", input: {}, output: "", durationMs: 10 },
+            { name: "b", input: {}, output: "", durationMs: 10 },
+        ],
+    });
+    const result = evaluate("t", rubric, trace);
+    assert.equal(result.efficiency.withinBudget, false);
+});
+test("evaluate marks over-budget when tokens exceed limit", () => {
+    const rubric = { maxTotalTokens: 100 };
+    const trace = makeTrace({
+        usage: { inputTokens: 500, outputTokens: 100, totalTokens: 600 },
+    });
+    const result = evaluate("t", rubric, trace);
+    assert.equal(result.efficiency.withinBudget, false);
+});
+test("evaluate passes with empty rubric", () => {
+    const rubric = {};
+    const trace = makeTrace();
+    const result = evaluate("t", rubric, trace);
+    assert.equal(result.passed, true);
+    assert.equal(result.checks.length, 0);
+    assert.equal(result.efficiency.withinBudget, true);
+});
+// ─── Runner Tests ──────────────────────────────────────────────
+test("runBenchmarkTask captures trace with tool calls", async () => {
+    const task = {
+        id: "navigation/test-task",
+        title: "Test task",
+        category: "navigation",
+        prompt: "Find the thing",
+        rubric: {},
+    };
+    const echoTool = {
+        name: "echo_tool",
+        description: "Echoes a value",
+        inputSchema: {
+            type: "object",
+            properties: { value: { type: "string" } },
+            required: ["value"],
+        },
+        execute: async (input) => `echo:${String(input.value)}`,
+    };
+    const config = createTestAgentConfig(process.cwd());
+    const trace = await runBenchmarkTask(task, {
+        modelClient: new ToolCallingMockClient(),
+        config,
+        tools: [echoTool],
+        variant: "test",
+    });
+    assert.equal(trace.taskId, "navigation/test-task");
+    assert.equal(trace.variant, "test");
+    assert.equal(trace.toolCalls.length, 1);
+    assert.equal(trace.toolCalls[0].name, "echo_tool");
+    assert.equal(trace.toolCalls[0].output, "echo:hello");
+    assert.ok(trace.durationMs >= 0);
+    assert.ok(trace.usage.totalTokens > 0);
+});
+test("runBenchmarkTask captures trace without tool calls", async () => {
+    const task = {
+        id: "planning/simple",
+        title: "Simple task",
+        category: "planning",
+        prompt: "Explain something",
+        rubric: {},
+    };
+    const config = createTestAgentConfig(process.cwd());
+    const trace = await runBenchmarkTask(task, {
+        modelClient: new MockModelClient("Here is my explanation about foo."),
+        config,
+        tools: [],
+        variant: "baseline",
+    });
+    assert.equal(trace.response, "Here is my explanation about foo.");
+    assert.equal(trace.toolCalls.length, 0);
+    assert.equal(trace.variant, "baseline");
+});
+test("runBenchmarkTask calls onTaskComplete callback", async () => {
+    const task = {
+        id: "nav/cb-test",
+        title: "Callback test",
+        category: "navigation",
+        prompt: "Do something",
+        rubric: {},
+    };
+    let callbackTaskId;
+    const config = createTestAgentConfig(process.cwd());
+    await runBenchmarkTask(task, {
+        modelClient: new MockModelClient("done"),
+        config,
+        tools: [],
+        variant: "v1",
+        onTaskComplete: (taskId) => {
+            callbackTaskId = taskId;
+        },
+    });
+    assert.equal(callbackTaskId, "nav/cb-test");
+});
+test("runBenchmarkTask tracks files read from read_file tool", async () => {
+    const task = {
+        id: "nav/file-track",
+        title: "File tracking test",
+        category: "navigation",
+        prompt: "Read a file",
+        rubric: {},
+    };
+    let callCount = 0;
+    const mockClient = {
+        async chat(params) {
+            void params;
+            callCount += 1;
+            if (callCount === 1) {
+                return {
+                    text: "Reading file",
+                    toolCalls: [{ id: "t1", name: "read_file", input: { path: "src/main.ts" } }],
+                    stopReason: "tool_use",
+                    usage: { inputTokens: 100, outputTokens: 50 },
+                };
+            }
+            return {
+                text: "Found it",
+                toolCalls: [],
+                stopReason: "end_turn",
+                usage: { inputTokens: 100, outputTokens: 50 },
+            };
+        },
+    };
+    const readFileTool = {
+        name: "read_file",
+        description: "Read a file",
+        inputSchema: {
+            type: "object",
+            properties: { path: { type: "string" } },
+            required: ["path"],
+        },
+        execute: async () => "file contents here",
+    };
+    const config = createTestAgentConfig(process.cwd());
+    const trace = await runBenchmarkTask(task, {
+        modelClient: mockClient,
+        config,
+        tools: [readFileTool],
+        variant: "v1",
+    });
+    assert.ok(trace.filesRead.includes("src/main.ts"));
+});
+test("runBenchmarkTask tracks symbols from structural tools", async () => {
+    const task = {
+        id: "nav/sym-track",
+        title: "Symbol tracking test",
+        category: "navigation",
+        prompt: "Find references",
+        rubric: {},
+    };
+    let callCount = 0;
+    const mockClient = {
+        async chat(params) {
+            void params;
+            callCount += 1;
+            if (callCount === 1) {
+                return {
+                    text: "Looking up symbol",
+                    toolCalls: [{ id: "t1", name: "read_symbol", input: { symbol: "CodingAgent" } }],
+                    stopReason: "tool_use",
+                    usage: { inputTokens: 100, outputTokens: 50 },
+                };
+            }
+            return {
+                text: "Found CodingAgent",
+                toolCalls: [],
+                stopReason: "end_turn",
+                usage: { inputTokens: 100, outputTokens: 50 },
+            };
+        },
+    };
+    const readSymbolTool = {
+        name: "read_symbol",
+        description: "Read a symbol",
+        inputSchema: {
+            type: "object",
+            properties: { symbol: { type: "string" } },
+            required: ["symbol"],
+        },
+        execute: async () => "class CodingAgent { ... }",
+    };
+    const config = createTestAgentConfig(process.cwd());
+    const trace = await runBenchmarkTask(task, {
+        modelClient: mockClient,
+        config,
+        tools: [readSymbolTool],
+        variant: "v1",
+    });
+    assert.ok(trace.symbolsQueried.includes("CodingAgent"));
+});
+// ─── Reporter Tests ────────────────────────────────────────────
+test("buildReport generates correct summary", () => {
+    const tasks = [
+        {
+            id: "navigation/find-foo",
+            title: "Find foo",
+            category: "navigation",
+            prompt: "Find foo",
+            rubric: { expectedOutputPatterns: ["foo"] },
+        },
+        {
+            id: "editing/fix-bar",
+            title: "Fix bar",
+            category: "editing",
+            prompt: "Fix bar",
+            rubric: { expectedOutputPatterns: ["bar"] },
+        },
+    ];
+    const traces = [
+        makeTrace({ taskId: "navigation/find-foo", response: "Found foo" }),
+        makeTrace({ taskId: "editing/fix-bar", response: "No match here" }),
+    ];
+    const report = buildReport(tasks, traces, "baseline", "test-model");
+    assert.equal(report.summary.totalTasks, 2);
+    assert.equal(report.summary.passed, 1);
+    assert.equal(report.summary.failed, 1);
+    assert.equal(report.summary.passRate, 0.5);
+    const navStats = report.summary.byCategory["navigation"];
+    assert.ok(navStats);
+    assert.equal(navStats.passed, 1);
+    const editStats = report.summary.byCategory["editing"];
+    assert.ok(editStats);
+    assert.equal(editStats.passed, 0);
+});
+test("buildReport throws for unknown task id in trace", () => {
+    assert.throws(() => buildReport([], [makeTrace({ taskId: "unknown/task" })], "v1", "m"), /No task definition found/);
+});
+test("formatReport produces readable output", () => {
+    const tasks = [
+        {
+            id: "navigation/find-foo",
+            title: "Find foo",
+            category: "navigation",
+            prompt: "Find foo",
+            rubric: { expectedOutputPatterns: ["foo"] },
+        },
+    ];
+    const traces = [makeTrace({ taskId: "navigation/find-foo", response: "Found foo" })];
+    const report = buildReport(tasks, traces, "baseline", "test-model");
+    const output = formatReport(report);
+    assert.ok(output.includes("Benchmark Report: baseline"));
+    assert.ok(output.includes("Model: test-model"));
+    assert.ok(output.includes("1/1 passed"));
+    assert.ok(output.includes("[PASS] navigation/find-foo"));
+});
+test("compareReports shows improvements and regressions", () => {
+    const tasks = [
+        {
+            id: "navigation/find-foo",
+            title: "Find foo",
+            category: "navigation",
+            prompt: "Find foo",
+            rubric: { expectedOutputPatterns: ["foo"] },
+        },
+    ];
+    const baseTraces = [makeTrace({ taskId: "navigation/find-foo", response: "no match" })];
+    const candTraces = [makeTrace({ taskId: "navigation/find-foo", response: "Found foo" })];
+    const baseline = buildReport(tasks, baseTraces, "baseline", "test-model");
+    const candidate = buildReport(tasks, candTraces, "v2", "test-model");
+    const comparison = compareReports(baseline, candidate);
+    assert.ok(comparison.includes("baseline"));
+    assert.ok(comparison.includes("v2"));
+    assert.ok(comparison.includes("FIXED"));
+});
+test("compareReports detects regressions", () => {
+    const tasks = [
+        {
+            id: "navigation/find-foo",
+            title: "Find foo",
+            category: "navigation",
+            prompt: "Find foo",
+            rubric: { expectedOutputPatterns: ["foo"] },
+        },
+    ];
+    const baseTraces = [makeTrace({ taskId: "navigation/find-foo", response: "Found foo here" })];
+    const candTraces = [makeTrace({ taskId: "navigation/find-foo", response: "no match" })];
+    const baseline = buildReport(tasks, baseTraces, "baseline", "test-model");
+    const candidate = buildReport(tasks, candTraces, "v2", "test-model");
+    const comparison = compareReports(baseline, candidate);
+    assert.ok(comparison.includes("REGRESSED"));
+});
+// ─── Integration: Load real tasks from benchmarks/ ─────────────
+test("loads real benchmark tasks from benchmarks/tasks/", async () => {
+    const tasksDir = path.resolve(import.meta.dirname, "..", "benchmarks", "tasks");
+    const tasks = await loadBenchmarkTasks(tasksDir);
+    assert.ok(tasks.length >= 20, `Expected at least 20 tasks, got ${tasks.length}`);
+    // Verify each task has required fields
+    for (const task of tasks) {
+        assert.ok(task.id, "task must have an id");
+        assert.ok(task.title, "task must have a title");
+        assert.ok(task.prompt, "task must have a prompt");
+        assert.ok(task.category, "task must have a category");
+        assert.ok(task.rubric, "task must have a rubric");
+    }
+    // Check categories are covered
+    const categories = new Set(tasks.map((t) => t.category));
+    assert.ok(categories.has("navigation"));
+    assert.ok(categories.has("editing"));
+    assert.ok(categories.has("refactors"));
+    assert.ok(categories.has("debugging"));
+    assert.ok(categories.has("planning"));
+});