cclaw-cli 0.26.0 → 0.28.0

package/dist/cli.d.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import type { FlowTrack, HarnessId, InitProfile } from "./types.js";
-import type { EvalTier } from "./eval/types.js";
+import type { EvalMode } from "./eval/types.js";
 type CommandName = "init" | "sync" | "doctor" | "upgrade" | "uninstall" | "archive" | "eval";
 interface ParsedArgs {
     command?: CommandName;
@@ -18,7 +18,7 @@ interface ParsedArgs {
     archiveSkipRetro?: boolean;
     archiveSkipRetroReason?: string;
     evalStage?: string;
-    evalTier?: EvalTier;
+    evalMode?: EvalMode;
     evalSchemaOnly?: boolean;
     evalRules?: boolean;
     evalJudge?: boolean;
@@ -26,6 +26,14 @@ interface ParsedArgs {
     evalNoWrite?: boolean;
     evalUpdateBaseline?: boolean;
     evalConfirm?: boolean;
+    evalQuiet?: boolean;
+    evalMaxCostUsd?: number;
+    /** Optional subcommand after `eval`. */
+    evalSubcommand?: "diff" | "runs";
+    /** Positional arguments for eval subcommands (e.g. `diff <old> <new>`). */
+    evalArgs?: string[];
+    evalBackground?: boolean;
+    evalCompareModel?: string;
     showHelp?: boolean;
     showVersion?: boolean;
 }

package/dist/cli.js

@@ -1,5 +1,7 @@
 #!/usr/bin/env node
-import { readFileSync, realpathSync } from "node:fs";
+import { createReadStream, readFileSync, realpathSync } from "node:fs";
+import { spawn } from "node:child_process";
+import fs from "node:fs/promises";
 import process from "node:process";
 import path from "node:path";
 import { createInterface } from "node:readline/promises";
@@ -14,9 +16,13 @@ import { createDefaultConfig, createProfileConfig } from "./config.js";
 import { detectHarnesses } from "./init-detect.js";
 import { HARNESS_ADAPTERS } from "./harness-adapters.js";
 import { runEval } from "./eval/runner.js";
+import { createStderrProgressLogger } from "./eval/progress.js";
 import { writeBaselinesFromReport } from "./eval/baseline.js";
 import { writeJsonReport, writeMarkdownReport } from "./eval/report.js";
-import { EVAL_TIERS } from "./eval/types.js";
+import { formatDiffMarkdown, runEvalDiff } from "./eval/diff.js";
+import { ensureRunDir, generateRunId, isRunAlive, listRuns, readRunStatus, resolveRunId, runLogPath, writeRunStatus } from "./eval/runs.js";
+import { EVAL_MODES } from "./eval/types.js";
+import { parseModeInput } from "./eval/mode.js";
 import { FLOW_STAGES } from "./types.js";
 const INSTALLER_COMMANDS = [
     "init",
@@ -55,16 +61,41 @@ Commands:
                     --skip-retro       Bypass mandatory retro gate (requires --retro-reason).
                     --retro-reason=<t> Reason for bypassing retro gate.
   eval       Run cclaw evals against .cclaw/evals/corpus (Phase 7: structural verifier + baselines).
-             Flags: --stage=<id>         Limit to one flow stage (${FLOW_STAGES.join("|")}).
-                    --tier=<A|B|C>       Fidelity tier (A=single-shot, B=tools, C=workflow).
+             Flags: --stage=<id>         Limit to one flow stage (${FLOW_STAGES.join("|")}) for fixture/agent modes.
+                    --mode=<${EVAL_MODES.join("|")}>
+                                         Evaluation mode:
+                                           fixture  = verify existing artifacts with structural/rule/judge verifiers.
+                                           agent    = LLM drafts one stage's artifact in a sandbox with tools.
+                                           workflow = LLM runs the full multi-stage flow (brainstorm→plan).
+                                         Legacy --tier=A|B|C still works (deprecated).
                     --schema-only        Run only structural verifiers (default).
                     --rules              Also run rule-based verifiers (keywords, regex, counts, uniqueness, traceability).
-                    --judge              Run the LLM judge (median-of-N) against each case's rubric. Requires CCLAW_EVAL_API_KEY; Tier A runs the single-shot agent, Tier B runs the sandbox tool-using agent (read_file/write_file/glob/grep).
+                    --judge              Run the LLM judge (median-of-N) against each case's rubric. Requires CCLAW_EVAL_API_KEY; fixture mode judges an existing artifact, agent/workflow modes draft first and then judge.
                     --dry-run            Validate config + corpus, print summary, do not execute.
                     --json               Emit machine-readable JSON on stdout.
                     --no-write           Skip writing the report to .cclaw/evals/reports/.
                     --update-baseline    Overwrite baselines from the current run (requires --confirm).
                     --confirm            Acknowledge --update-baseline (prevents accidental resets).
+                    --quiet              Silence the stderr progress logger (default: emit one
+                                         line per case / stage to stderr so long runs are visible).
+                    --max-cost-usd=<n>   Abort the run if committed USD spend crosses <n>
+                                         (independent from the daily cap). Also readable from
+                                         CCLAW_EVAL_MAX_COST_USD.
+                    --compare-model=<id> Run the same corpus twice — once with the configured model
+                                         and once with <id> — then diff the summaries. Exit code 1
+                                         when the override model regressed.
+                    --background         Spawn the run as a detached child process, write the
+                                         combined output to .cclaw/evals/runs/<id>/run.log, and
+                                         return immediately. Attach later with
+                                         \`cclaw eval runs tail <id|latest>\`.
+
+             Subcommands:
+                    diff <old> <new>     Compare two reports under .cclaw/evals/reports/.
+                                         Each argument is a cclawVersion (e.g. 0.26.0), a filename,
+                                         or the literal "latest". Exit code 1 when the diff shows a
+                                         regression. Accepts --json to emit machine-readable output.
+                    runs [action] [id]   Inspect background runs under .cclaw/evals/runs/.
+                                         Actions: list (default) | status <id|latest> | tail <id|latest>.
   upgrade    Refresh generated files in .cclaw without modifying user artifacts.
   uninstall  Remove .cclaw runtime and the generated harness shim files.
 
@@ -78,8 +109,10 @@ Examples:
   cclaw archive --name=payments-revamp
   cclaw eval --dry-run
   cclaw eval --stage=brainstorm --schema-only
-  cclaw eval --judge --tier=A --stage=brainstorm
-  cclaw eval --judge --tier=B --stage=spec
+  cclaw eval --judge --mode=fixture --stage=brainstorm
+  cclaw eval --judge --mode=agent --stage=spec
+  cclaw eval --mode=workflow --judge
+  cclaw eval diff 0.26.0 latest
 
 Docs:   https://github.com/zuevrs/cclaw
 Issues: https://github.com/zuevrs/cclaw/issues
@@ -135,12 +168,17 @@ function parseProfile(raw) {
     }
     return trimmed;
 }
-function parseEvalTier(raw) {
-    const trimmed = raw.trim().toUpperCase();
-    if (!EVAL_TIERS.includes(trimmed)) {
-        throw new Error(`Unknown eval tier: ${raw}. Supported: ${EVAL_TIERS.join(", ")}`);
-    }
-    return trimmed;
+function parseLegacyTier(raw) {
+    return parseModeInput(raw.toUpperCase(), {
+        source: "cli",
+        raw: `--tier=${raw}`
+    });
+}
+function parseEvalMode(raw) {
+    return parseModeInput(raw, {
+        source: "cli",
+        raw: `--mode=${raw}`
+    });
 }
 function parseEvalStage(raw) {
     const trimmed = raw.trim();
@@ -363,6 +401,18 @@ function printDoctorText(ctx, checks, options) {
         ctx.stdout.write("Doctor status: HEALTHY (no failing error checks)\n");
     }
 }
+function resolveMaxCostOption(fromCli, env) {
+    if (fromCli !== undefined)
+        return { maxCostUsd: fromCli };
+    const raw = env.CCLAW_EVAL_MAX_COST_USD;
+    if (raw === undefined || raw.trim() === "")
+        return {};
+    const value = Number(raw);
+    if (!Number.isFinite(value) || value <= 0) {
+        throw new Error(`CCLAW_EVAL_MAX_COST_USD must be a positive number, got: ${raw}`);
+    }
+    return { maxCostUsd: value };
+}
 function parseArgs(argv) {
     const parsed = {};
     const helpFlag = argv.find((arg) => arg === "--help" || arg === "-h");
@@ -373,10 +423,45 @@ function parseArgs(argv) {
     if (versionFlag) {
         parsed.showVersion = true;
     }
-    const [commandRaw, ...flags] = argv.filter((arg) => arg !== "--help" && arg !== "-h" && arg !== "--version" && arg !== "-v");
+    const filteredArgv = argv.filter((arg) => arg !== "--help" && arg !== "-h" && arg !== "--version" && arg !== "-v");
+    const [commandRaw, ...rest] = filteredArgv;
     parsed.command = INSTALLER_COMMANDS.includes(commandRaw)
         ? commandRaw
         : undefined;
+    // For `eval`, the next non-flag argument is an optional subcommand. Any
+    // subsequent non-flag tokens are captured as evalArgs (consumed by the
+    // subcommand handler). This preserves backwards compat: callers that run
+    // `cclaw eval --dry-run` see no subcommand and no positional args.
+    let flags = rest;
+    if (parsed.command === "eval") {
+        const evalArgs = [];
+        const remainder = [];
+        let sawSubcommand = false;
+        for (const token of rest) {
+            if (token.startsWith("--")) {
+                remainder.push(token);
+                continue;
+            }
+            if (!sawSubcommand) {
+                if (token === "diff") {
+                    parsed.evalSubcommand = "diff";
+                    sawSubcommand = true;
+                }
+                else if (token === "runs") {
+                    parsed.evalSubcommand = "runs";
+                    sawSubcommand = true;
+                }
+                else {
+                    evalArgs.push(token);
+                }
+                continue;
+            }
+            evalArgs.push(token);
+        }
+        if (evalArgs.length > 0)
+            parsed.evalArgs = evalArgs;
+        flags = remainder;
+    }
     for (const flag of flags) {
         if (flag.startsWith("--harnesses=")) {
             parsed.harnesses = parseHarnesses(flag.replace("--harnesses=", ""));
@@ -438,8 +523,12 @@ function parseArgs(argv) {
             parsed.evalStage = parseEvalStage(flag.replace("--stage=", ""));
             continue;
         }
+        if (flag.startsWith("--mode=")) {
+            parsed.evalMode = parseEvalMode(flag.replace("--mode=", ""));
+            continue;
+        }
         if (flag.startsWith("--tier=")) {
-            parsed.evalTier = parseEvalTier(flag.replace("--tier=", ""));
+            parsed.evalMode = parseLegacyTier(flag.replace("--tier=", ""));
             continue;
         }
         if (flag === "--schema-only") {
@@ -466,14 +555,245 @@ function parseArgs(argv) {
             parsed.evalConfirm = true;
             continue;
         }
+        if (flag === "--background") {
+            parsed.evalBackground = true;
+            continue;
+        }
+        if (flag.startsWith("--compare-model=")) {
+            const value = flag.replace("--compare-model=", "").trim();
+            if (value.length === 0) {
+                throw new Error(`--compare-model requires a non-empty model id (e.g. --compare-model=gpt-4o-mini).`);
+            }
+            parsed.evalCompareModel = value;
+            continue;
+        }
+        if (flag.startsWith("--max-cost-usd=")) {
+            const raw = flag.replace("--max-cost-usd=", "").trim();
+            const value = Number(raw);
+            if (!Number.isFinite(value) || value <= 0) {
+                throw new Error(`--max-cost-usd requires a positive number, got: ${raw}`);
+            }
+            parsed.evalMaxCostUsd = value;
+            continue;
+        }
     }
     // `--json` is shared between doctor and eval. Disambiguate by command.
     if (parsed.command === "eval" && parsed.doctorJson === true) {
         parsed.evalJson = true;
         parsed.doctorJson = undefined;
     }
+    // `--quiet` on `eval` silences the stderr progress logger. On doctor it
+    // continues to mean "print only failing checks" — the flag slot is the
+    // same, the semantics depend on which command owns the invocation.
+    if (parsed.command === "eval" && parsed.doctorQuiet === true) {
+        parsed.evalQuiet = true;
+        parsed.doctorQuiet = undefined;
+    }
     return parsed;
 }
+/**
+ * Spawn `cclaw eval` (without `--background`) in a detached child process
+ * and return immediately. The child's stdout+stderr are piped to
+ * `.cclaw/evals/runs/<id>/run.log` so the user can attach later with
+ * `cclaw eval runs tail`. We do NOT wait for the child — the whole point
+ * is to free the terminal while a multi-minute workflow-mode run
+ * proceeds in the background.
+ */
+async function spawnBackgroundEval(parsed, ctx) {
+    const id = generateRunId();
+    await ensureRunDir(ctx.cwd, id);
+    const logPath = runLogPath(ctx.cwd, id);
+    const childArgv = process.argv.slice(2).filter((a) => a !== "--background");
+    const cliEntry = process.argv[1];
+    if (!cliEntry) {
+        error(ctx, "Could not resolve cclaw entrypoint for --background.");
+        return 1;
+    }
+    const logHandle = await fs.open(logPath, "a");
+    try {
+        const child = spawn(process.execPath, [cliEntry, ...childArgv], {
+            cwd: ctx.cwd,
+            detached: true,
+            stdio: ["ignore", logHandle.fd, logHandle.fd],
+            env: process.env
+        });
+        const pid = child.pid ?? -1;
+        await writeRunStatus(ctx.cwd, {
+            id,
+            startedAt: new Date().toISOString(),
+            pid,
+            argv: childArgv,
+            cwd: ctx.cwd,
+            state: "running"
+        });
+        child.unref();
+        const finalize = async (code) => {
+            const current = await readRunStatus(ctx.cwd, id);
+            if (!current)
+                return;
+            const exitCode = typeof code === "number" ? code : -1;
+            await writeRunStatus(ctx.cwd, {
+                ...current,
+                endedAt: new Date().toISOString(),
+                exitCode,
+                state: exitCode === 0 ? "succeeded" : "failed"
+            });
+        };
+        child.on("exit", (code) => {
+            void finalize(code);
+        });
+        child.on("error", (err) => {
+            void writeRunStatus(ctx.cwd, {
+                id,
+                startedAt: new Date().toISOString(),
+                pid,
+                argv: childArgv,
+                cwd: ctx.cwd,
+                endedAt: new Date().toISOString(),
+                exitCode: -1,
+                state: "failed"
+            });
+            error(ctx, `Background eval failed to start: ${err.message}`);
+        });
+        ctx.stdout.write(`cclaw eval: background run id=${id} pid=${pid}\n` +
+            `  log:    ${logPath}\n` +
+            `  tail:   cclaw eval runs tail ${id}\n` +
+            `  status: cclaw eval runs status ${id}\n`);
+        return 0;
+    }
+    finally {
+        await logHandle.close();
+    }
+}
+function formatRunRow(status) {
+    const ended = status.endedAt ? ` ended=${status.endedAt}` : "";
+    const exitCode = status.exitCode !== undefined ? ` exit=${status.exitCode}` : "";
+    const alive = status.state === "running" ? (isRunAlive(status) ? "" : " (stale)") : "";
+    return `${status.id}  state=${status.state}${alive}  pid=${status.pid}  started=${status.startedAt}${ended}${exitCode}`;
+}
+async function runEvalRunsSubcommand(parsed, ctx) {
+    const args = parsed.evalArgs ?? [];
+    const action = args[0] ?? "list";
+    if (action === "list") {
+        const runs = await listRuns(ctx.cwd);
+        if (runs.length === 0) {
+            ctx.stdout.write("No eval runs recorded under .cclaw/evals/runs/.\n");
+            return 0;
+        }
+        if (parsed.evalJson === true) {
+            ctx.stdout.write(`${JSON.stringify(runs, null, 2)}\n`);
+            return 0;
+        }
+        for (const run of runs)
+            ctx.stdout.write(`${formatRunRow(run)}\n`);
+        return 0;
+    }
+    if (action === "status") {
+        const id = await resolveRunId(ctx.cwd, args[1]);
+        if (!id) {
+            error(ctx, `No such run: ${args[1] ?? "(none recorded)"}`);
+            return 1;
+        }
+        const status = await readRunStatus(ctx.cwd, id);
+        if (!status) {
+            error(ctx, `Run ${id} has no status file.`);
+            return 1;
+        }
+        if (parsed.evalJson === true) {
+            ctx.stdout.write(`${JSON.stringify(status, null, 2)}\n`);
+        }
+        else {
+            ctx.stdout.write(`${formatRunRow(status)}\n`);
+            ctx.stdout.write(`log: ${runLogPath(ctx.cwd, id)}\n`);
+        }
+        return status.state === "failed" ? 1 : 0;
+    }
+    if (action === "tail") {
+        const id = await resolveRunId(ctx.cwd, args[1]);
+        if (!id) {
+            error(ctx, `No such run: ${args[1] ?? "(none recorded)"}`);
+            return 1;
+        }
+        const logFile = runLogPath(ctx.cwd, id);
+        const stream = createReadStream(logFile, { encoding: "utf8" });
+        await new Promise((resolve, reject) => {
+            stream.on("data", (chunk) => ctx.stdout.write(chunk));
+            stream.on("end", () => resolve());
+            stream.on("error", reject);
+        });
+        return 0;
+    }
+    error(ctx, `Unknown \`cclaw eval runs\` action: ${action}. Use list | status | tail.`);
+    return 1;
+}
+/**
+ * Run the same corpus twice — once against the configured model, once
+ * against `--compare-model=<id>` — and print a summary comparing the
+ * two. Both reports are written to `.cclaw/evals/reports/` (unless
+ * `--no-write` is set) and a unified diff is emitted to stdout. Exit
+ * code is 1 when the override model regressed against the baseline
+ * model, 0 otherwise.
+ */
+async function runCompareModel(parsed, ctx, progress) {
+    const baselineOpts = {
+        projectRoot: ctx.cwd,
+        stage: parsed.evalStage,
+        mode: parsed.evalMode,
+        schemaOnly: parsed.evalSchemaOnly === true,
+        rules: parsed.evalRules === true,
+        judge: parsed.evalJudge === true,
+        ...(progress ? { progress } : {}),
+        ...resolveMaxCostOption(parsed.evalMaxCostUsd, process.env)
+    };
+    ctx.stderr.write(`[cclaw eval] compare: running baseline model...\n`);
+    const baseline = await runEval(baselineOpts);
+    if ("kind" in baseline) {
+        error(ctx, "--compare-model is incompatible with --dry-run.");
+        return 1;
+    }
+    ctx.stderr.write(`[cclaw eval] compare: running ${parsed.evalCompareModel} ...\n`);
+    const candidate = await runEval({
+        ...baselineOpts,
+        modelOverride: parsed.evalCompareModel
+    });
+    if ("kind" in candidate) {
+        error(ctx, "--compare-model received an unexpected dry-run response.");
+        return 1;
+    }
+    if (parsed.evalNoWrite !== true) {
+        await writeJsonReport(ctx.cwd, baseline);
+        await writeMarkdownReport(ctx.cwd, baseline);
+        await writeJsonReport(ctx.cwd, candidate);
+        await writeMarkdownReport(ctx.cwd, candidate);
+    }
+    const passDelta = candidate.summary.passed - baseline.summary.passed;
+    const failDelta = candidate.summary.failed - baseline.summary.failed;
+    const costDelta = candidate.summary.totalCostUsd - baseline.summary.totalCostUsd;
+    if (parsed.evalJson === true) {
+        ctx.stdout.write(`${JSON.stringify({
+            baseline: {
+                model: baseline.model,
+                summary: baseline.summary
+            },
+            candidate: {
+                model: candidate.model,
+                summary: candidate.summary
+            },
+            delta: { passed: passDelta, failed: failDelta, costUsd: costDelta }
+        }, null, 2)}\n`);
+    }
+    else {
+        ctx.stdout.write(`cclaw eval compare-model:\n` +
+            `  baseline   ${baseline.model}: pass=${baseline.summary.passed}/${baseline.summary.totalCases} ` +
+            `fail=${baseline.summary.failed} cost=$${baseline.summary.totalCostUsd.toFixed(4)}\n` +
+            `  candidate  ${candidate.model}: pass=${candidate.summary.passed}/${candidate.summary.totalCases} ` +
+            `fail=${candidate.summary.failed} cost=$${candidate.summary.totalCostUsd.toFixed(4)}\n` +
+            `  delta: passed=${passDelta >= 0 ? "+" : ""}${passDelta} ` +
+            `failed=${failDelta >= 0 ? "+" : ""}${failDelta} ` +
+            `cost=${costDelta >= 0 ? "+" : ""}$${costDelta.toFixed(4)}\n`);
+    }
+    return failDelta > 0 ? 1 : 0;
+}
 async function runCommand(parsed, ctx) {
     if (parsed.showHelp) {
         ctx.stdout.write(usage());
@@ -567,15 +887,59 @@ async function runCommand(parsed, ctx) {
         info(ctx, "Upgraded .cclaw runtime and regenerated generated files");
         return 0;
     }
+    if (command === "eval" && parsed.evalSubcommand === "runs") {
+        return runEvalRunsSubcommand(parsed, ctx);
+    }
+    if (command === "eval" && parsed.evalBackground === true) {
+        return spawnBackgroundEval(parsed, ctx);
+    }
+    if (command === "eval" && parsed.evalSubcommand === "diff") {
+        const args = parsed.evalArgs ?? [];
+        if (args.length !== 2) {
+            error(ctx, `\`cclaw eval diff\` requires two arguments: <old> <new>. ` +
+                `Example: cclaw eval diff 0.26.0 latest`);
+            return 1;
+        }
+        const [oldSel, newSel] = args;
+        try {
+            const diff = await runEvalDiff({
+                projectRoot: ctx.cwd,
+                old: oldSel,
+                new: newSel
+            });
+            if (parsed.evalJson === true) {
+                ctx.stdout.write(`${JSON.stringify(diff, null, 2)}\n`);
+            }
+            else {
+                ctx.stdout.write(formatDiffMarkdown(diff));
+            }
+            return diff.regressed ? 1 : 0;
+        }
+        catch (err) {
+            error(ctx, err instanceof Error ? err.message : String(err));
+            return 1;
+        }
+    }
     if (command === "eval") {
+        const wantProgress = parsed.evalQuiet !== true &&
+            parsed.dryRun !== true &&
+            parsed.evalJson !== true;
+        const progress = wantProgress
+            ? createStderrProgressLogger({ writer: (s) => ctx.stderr.write(s) })
+            : undefined;
+        if (parsed.evalCompareModel !== undefined) {
+            return runCompareModel(parsed, ctx, progress);
+        }
         const result = await runEval({
             projectRoot: ctx.cwd,
             stage: parsed.evalStage,
-            tier: parsed.evalTier,
+            mode: parsed.evalMode,
             schemaOnly: parsed.evalSchemaOnly === true,
             rules: parsed.evalRules === true,
             judge: parsed.evalJudge === true,
-            dryRun: parsed.dryRun === true
+            dryRun: parsed.dryRun === true,
+            ...(progress ? { progress } : {}),
+            ...resolveMaxCostOption(parsed.evalMaxCostUsd, process.env)
         });
         if ("kind" in result) {
             if (parsed.evalJson === true) {
@@ -588,11 +952,17 @@ async function runCommand(parsed, ctx) {
             ctx.stdout.write(`  model: ${result.config.model}\n`);
             ctx.stdout.write(`  source: ${result.config.source}\n`);
             ctx.stdout.write(`  apiKey: ${result.config.apiKey ? "set" : "unset"}\n`);
-            ctx.stdout.write(`  tier: ${result.plannedTier}\n`);
+            ctx.stdout.write(`  mode: ${result.plannedMode}\n`);
             ctx.stdout.write(`  corpus: ${result.corpus.total} case(s)\n`);
             for (const [stage, count] of Object.entries(result.corpus.byStage)) {
                 ctx.stdout.write(`    - ${stage}: ${count}\n`);
             }
+            if (result.workflowCorpus.total > 0 || result.plannedMode === "workflow") {
+                ctx.stdout.write(`  workflow corpus: ${result.workflowCorpus.total} case(s)\n`);
+                for (const wf of result.workflowCorpus.cases) {
+                    ctx.stdout.write(`    - ${wf.id}: ${wf.stages.join(" → ")}\n`);
+                }
+            }
             ctx.stdout.write(`  verifiers available:\n`);
             for (const [key, value] of Object.entries(result.verifiersAvailable)) {
                 ctx.stdout.write(`    - ${key}: ${value ? "yes" : "no"}\n`);

package/dist/content/eval-scaffold.d.ts

@@ -4,8 +4,8 @@
  * scaffold is intentionally minimal: a usable default config plus short
  * READMEs that point at `docs/evals.md` for authoring guidance.
  */
-export declare const EVAL_CONFIG_YAML = "# cclaw eval config\n# See docs/evals.md for the full schema and rollout plan.\n#\n# All values can be overridden at runtime with CCLAW_EVAL_* environment\n# variables (env wins). Secrets like CCLAW_EVAL_API_KEY never live here.\nprovider: zai\nbaseUrl: https://api.z.ai/api/coding/paas/v4\nmodel: glm-5.1\n\n# Default fidelity tier when --tier is not supplied.\n#   A = single-shot API call (cheap)\n#   B = SDK with tool use     (realistic)\n#   C = multi-stage workflow  (end-to-end)\ndefaultTier: A\n\n# Per-call timeout and retry budget.\ntimeoutMs: 120000\nmaxRetries: 2\n\n# Optional hard-stop on estimated USD spend per day. Leave unset for no cap.\n# dailyUsdCap: 5\n\n# Regression thresholds used by CI.\nregression:\n  # Fail when overall score drops by more than this fraction (e.g. -0.15 = 15%).\n  failIfDeltaBelow: -0.15\n  # Fail when any single critical rubric drops below this absolute score.\n  failIfCriticalBelow: 3.0\n";
-export declare const EVAL_CORPUS_README = "# Eval Corpus\n\nSeed cases live in `./<stage>/<id>.yaml`, one file per case.\nSee `docs/evals.md` for the schema.\n\nMinimal shape:\n\n```yaml\nid: brainstorm-01\nstage: brainstorm\ninput_prompt: |\n  One short paragraph describing the user's task.\ncontext_files: []\nexpected:\n  # verifier-specific hints; optional\n```\n\nStart with 3 structural cases per stage (24 total), then expand to 5 per\nstage (40 total) once rule verifiers land. Tier B/C runs may add\n`context_files` pulled from real projects to exercise the sandbox.\n";
+export declare const EVAL_CONFIG_YAML = "# cclaw eval config\n# See docs/evals.md for the full schema and rollout plan.\n#\n# All values can be overridden at runtime with CCLAW_EVAL_* environment\n# variables (env wins). Secrets like CCLAW_EVAL_API_KEY never live here.\nprovider: zai\nbaseUrl: https://api.z.ai/api/coding/paas/v4\nmodel: glm-5.1\n\n# Default evaluation mode when --mode is not supplied.\n#   fixture  = verify existing artifacts (cheap, LLM-free unless --judge is set)\n#   agent    = LLM drafts one stage's artifact in a sandbox with tools\n#   workflow = LLM runs the full multi-stage flow (brainstorm \u2192 plan)\n# (Legacy alias --tier=A|B|C still works; A\u2192fixture, B\u2192agent, C\u2192workflow.)\ndefaultMode: fixture\n\n# Per-call timeout and retry budget.\ntimeoutMs: 120000\nmaxRetries: 2\n\n# Optional hard-stop on estimated USD spend per day. Leave unset for no cap.\n# dailyUsdCap: 5\n\n# Regression thresholds used by CI.\nregression:\n  # Fail when overall score drops by more than this fraction (e.g. -0.15 = 15%).\n  failIfDeltaBelow: -0.15\n  # Fail when any single critical rubric drops below this absolute score.\n  failIfCriticalBelow: 3.0\n";
+export declare const EVAL_CORPUS_README = "# Eval Corpus\n\nSeed cases live in `./<stage>/<id>.yaml`, one file per case.\nSee `docs/evals.md` for the schema.\n\nMinimal shape:\n\n```yaml\nid: brainstorm-01\nstage: brainstorm\ninput_prompt: |\n  One short paragraph describing the user's task.\ncontext_files: []\nexpected:\n  # verifier-specific hints; optional\n```\n\nStart with 3 structural cases per stage (24 total), then expand to 5 per\nstage (40 total) once rule verifiers land. Agent/workflow runs may add\n`context_files` pulled from real projects to exercise the sandbox.\n";
 export declare const EVAL_RUBRICS_README = "# Eval Rubrics\n\nLLM-judge rubrics. Each rubric is a short list of checks scored on a\n`1\u20135` scale with a rationale. The runner picks `<stage>.yaml` when\n`cclaw eval --judge` is invoked; every stage ships a starter rubric\nbelow \u2014 edit the checks to match what your team cares about, and add\n`critical: true` to the checks that should hard-fail nightly CI on\nregression.\n\n```yaml\nstage: brainstorm\nchecks:\n  - id: distinctness\n    prompt: \"Are the proposed directions genuinely distinct (not rephrasings)?\"\n    scale: \"1-5 where 5=fully distinct approaches\"\n    weight: 1.0\n    critical: false\n```\n\nSee `docs/evals.md` for the full schema.\n";
 export declare const EVAL_RUBRIC_FILES: ReadonlyArray<{
     stage: string;

package/dist/content/eval-scaffold.js

@@ -13,11 +13,12 @@ provider: zai
 baseUrl: https://api.z.ai/api/coding/paas/v4
 model: glm-5.1
 
-# Default fidelity tier when --tier is not supplied.
-#   A = single-shot API call (cheap)
-#   B = SDK with tool use     (realistic)
-#   C = multi-stage workflow  (end-to-end)
-defaultTier: A
+# Default evaluation mode when --mode is not supplied.
+#   fixture  = verify existing artifacts (cheap, LLM-free unless --judge is set)
+#   agent    = LLM drafts one stage's artifact in a sandbox with tools
+#   workflow = LLM runs the full multi-stage flow (brainstorm → plan)
+# (Legacy alias --tier=A|B|C still works; A→fixture, B→agent, C→workflow.)
+defaultMode: fixture
 
 # Per-call timeout and retry budget.
 timeoutMs: 120000
@@ -51,7 +52,7 @@ expected:
 \`\`\`
 
 Start with 3 structural cases per stage (24 total), then expand to 5 per
-stage (40 total) once rule verifiers land. Tier B/C runs may add
+stage (40 total) once rule verifiers land. Agent/workflow runs may add
 \`context_files\` pulled from real projects to exercise the sandbox.
 `;
 export const EVAL_RUBRICS_README = `# Eval Rubrics

package/dist/eval/agents/single-shot.d.ts

@@ -23,5 +23,5 @@ export interface SingleShotOutput {
     userPrompt: string;
 }
 export declare function loadStageSkill(projectRoot: string, stage: FlowStage): Promise<string>;
-/** Run the Tier A single-shot AUT and return the produced artifact. */
+/** Run the single-shot AUT (fixture mode + --judge) and return the produced artifact. */
 export declare function runSingleShot(input: SingleShotInput): Promise<SingleShotOutput>;

package/dist/eval/agents/single-shot.js

@@ -1,5 +1,5 @@
 /**
- * Tier A single-shot agent.
+ * Single-shot agent used by fixture mode when `--judge` is set.
  *
  * Simplest realistic AUT: one LLM call with the stage's SKILL.md as the
  * system prompt and the case's `inputPrompt` as the user message. Output
@@ -9,7 +9,7 @@
  * Design notes:
  *
  * - No tools. No multi-turn. No reads of the project beyond the one
- *   SKILL.md. Tier B/C layer complexity on top in later steps.
+ *   SKILL.md. agent/workflow modes layer complexity on top.
  * - Errors are propagated as-is (`EvalLlmError` subclasses) so the
  *   runner can surface them as verifier failures without swallowing the
  *   cause.
@@ -27,7 +27,7 @@ export async function loadStageSkill(projectRoot, stage) {
     const file = path.join(projectRoot, RUNTIME_ROOT, "skills", folder, "SKILL.md");
     if (!(await exists(file))) {
         throw new Error(`Stage skill not found: ${path.relative(projectRoot, file)}. ` +
-            `Run \`cclaw init\` (or \`cclaw sync\`) before \`cclaw eval --tier=A --judge\`.`);
+            `Run \`cclaw init\` (or \`cclaw sync\`) before \`cclaw eval --mode=fixture --judge\`.`);
     }
     return fs.readFile(file, "utf8");
 }
@@ -50,7 +50,7 @@ function buildUserPrompt(caseEntry) {
         `Do not wrap in code fences, do not add commentary before or after.`);
     return lines.join("\n");
 }
-/** Run the Tier A single-shot AUT and return the produced artifact. */
+/** Run the single-shot AUT (fixture mode + --judge) and return the produced artifact. */
 export async function runSingleShot(input) {
     const { caseEntry, config, projectRoot, client } = input;
     const started = Date.now();

package/dist/eval/agents/with-tools.d.ts

@@ -1,5 +1,5 @@
 import type { ChatUsage, EvalLlmClient } from "../llm-client.js";
-import { createSandbox } from "../sandbox.js";
+import { createSandbox, type Sandbox } from "../sandbox.js";
 import type { SandboxTool } from "../tools/index.js";
 import type { EvalCase, ResolvedEvalConfig, ToolUseSummary } from "../types.js";
 export declare class MaxTurnsExceededError extends Error {
@@ -16,6 +16,19 @@ export interface WithToolsInput {
     loadSkill?: (stage: EvalCase["stage"]) => Promise<string>;
     /** Override for the sandbox factory (test hook). */
     createSandboxFn?: typeof createSandbox;
+    /**
+     * Reuse an externally-managed sandbox instead of creating + disposing a
+     * per-call one. Workflow mode uses this so every stage shares the same
+     * sandbox and earlier artifacts remain visible. When set, the caller is
+     * responsible for `dispose()`.
+     */
+    externalSandbox?: Sandbox;
+    /**
+     * Optional override of the default user prompt prefix. Workflow mode uses
+     * this to tell the model which stage it is on and where the prior
+     * artifacts are located.
+     */
+    promptPreamble?: string;
 }
 export interface WithToolsOutput {
     artifact: string;