cclaw-cli 0.26.0 → 0.27.0

package/dist/cli.d.ts

@@ -26,6 +26,10 @@ interface ParsedArgs {
     evalNoWrite?: boolean;
     evalUpdateBaseline?: boolean;
     evalConfirm?: boolean;
+    /** Optional subcommand after `eval`. Currently only `diff` is supported. */
+    evalSubcommand?: "diff";
+    /** Positional arguments for eval subcommands (e.g. `diff <old> <new>`). */
+    evalArgs?: string[];
     showHelp?: boolean;
     showVersion?: boolean;
 }

package/dist/cli.js

@@ -16,6 +16,7 @@ import { HARNESS_ADAPTERS } from "./harness-adapters.js";
 import { runEval } from "./eval/runner.js";
 import { writeBaselinesFromReport } from "./eval/baseline.js";
 import { writeJsonReport, writeMarkdownReport } from "./eval/report.js";
+import { formatDiffMarkdown, runEvalDiff } from "./eval/diff.js";
 import { EVAL_TIERS } from "./eval/types.js";
 import { FLOW_STAGES } from "./types.js";
 const INSTALLER_COMMANDS = [
@@ -55,16 +56,22 @@ Commands:
                     --skip-retro       Bypass mandatory retro gate (requires --retro-reason).
                     --retro-reason=<t> Reason for bypassing retro gate.
   eval       Run cclaw evals against .cclaw/evals/corpus (Phase 7: structural verifier + baselines).
-             Flags: --stage=<id>         Limit to one flow stage (${FLOW_STAGES.join("|")}).
-                    --tier=<A|B|C>       Fidelity tier (A=single-shot, B=tools, C=workflow).
+             Flags: --stage=<id>         Limit to one flow stage (${FLOW_STAGES.join("|")}) for Tier A/B.
+                    --tier=<A|B|C>       Fidelity tier (A=single-shot, B=tools, C=multi-stage workflow).
                     --schema-only        Run only structural verifiers (default).
                     --rules              Also run rule-based verifiers (keywords, regex, counts, uniqueness, traceability).
-                    --judge              Run the LLM judge (median-of-N) against each case's rubric. Requires CCLAW_EVAL_API_KEY; Tier A runs the single-shot agent, Tier B runs the sandbox tool-using agent (read_file/write_file/glob/grep).
+                    --judge              Run the LLM judge (median-of-N) against each case's rubric. Requires CCLAW_EVAL_API_KEY; Tier A runs the single-shot agent, Tier B/C the sandbox tool-using agent (read_file/write_file/glob/grep).
                     --dry-run            Validate config + corpus, print summary, do not execute.
                     --json               Emit machine-readable JSON on stdout.
                     --no-write           Skip writing the report to .cclaw/evals/reports/.
                     --update-baseline    Overwrite baselines from the current run (requires --confirm).
                     --confirm            Acknowledge --update-baseline (prevents accidental resets).
+
+             Subcommands:
+                    diff <old> <new>     Compare two reports under .cclaw/evals/reports/.
+                                         Each argument is a cclawVersion (e.g. 0.26.0), a filename,
+                                         or the literal "latest". Exit code 1 when the diff shows a
+                                         regression. Accepts --json to emit machine-readable output.
   upgrade    Refresh generated files in .cclaw without modifying user artifacts.
   uninstall  Remove .cclaw runtime and the generated harness shim files.
 
@@ -80,6 +87,8 @@ Examples:
   cclaw eval --stage=brainstorm --schema-only
   cclaw eval --judge --tier=A --stage=brainstorm
   cclaw eval --judge --tier=B --stage=spec
+  cclaw eval --tier=C --judge
+  cclaw eval diff 0.26.0 latest
 
 Docs:   https://github.com/zuevrs/cclaw
 Issues: https://github.com/zuevrs/cclaw/issues
@@ -373,10 +382,42 @@ function parseArgs(argv) {
     if (versionFlag) {
         parsed.showVersion = true;
     }
-    const [commandRaw, ...flags] = argv.filter((arg) => arg !== "--help" && arg !== "-h" && arg !== "--version" && arg !== "-v");
+    const filteredArgv = argv.filter((arg) => arg !== "--help" && arg !== "-h" && arg !== "--version" && arg !== "-v");
+    const [commandRaw, ...rest] = filteredArgv;
     parsed.command = INSTALLER_COMMANDS.includes(commandRaw)
         ? commandRaw
         : undefined;
+    // For `eval`, the next non-flag argument is an optional subcommand. Any
+    // subsequent non-flag tokens are captured as evalArgs (consumed by the
+    // subcommand handler). This preserves backwards compat: callers that run
+    // `cclaw eval --dry-run` see no subcommand and no positional args.
+    let flags = rest;
+    if (parsed.command === "eval") {
+        const evalArgs = [];
+        const remainder = [];
+        let sawSubcommand = false;
+        for (const token of rest) {
+            if (token.startsWith("--")) {
+                remainder.push(token);
+                continue;
+            }
+            if (!sawSubcommand) {
+                if (token === "diff") {
+                    parsed.evalSubcommand = "diff";
+                    sawSubcommand = true;
+                }
+                else {
+                    // Treat unknown positional as an eval arg for forward compat.
+                    evalArgs.push(token);
+                }
+                continue;
+            }
+            evalArgs.push(token);
+        }
+        if (evalArgs.length > 0)
+            parsed.evalArgs = evalArgs;
+        flags = remainder;
+    }
     for (const flag of flags) {
         if (flag.startsWith("--harnesses=")) {
             parsed.harnesses = parseHarnesses(flag.replace("--harnesses=", ""));
@@ -567,6 +608,33 @@ async function runCommand(parsed, ctx) {
         info(ctx, "Upgraded .cclaw runtime and regenerated generated files");
         return 0;
     }
+    if (command === "eval" && parsed.evalSubcommand === "diff") {
+        const args = parsed.evalArgs ?? [];
+        if (args.length !== 2) {
+            error(ctx, `\`cclaw eval diff\` requires two arguments: <old> <new>. ` +
+                `Example: cclaw eval diff 0.26.0 latest`);
+            return 1;
+        }
+        const [oldSel, newSel] = args;
+        try {
+            const diff = await runEvalDiff({
+                projectRoot: ctx.cwd,
+                old: oldSel,
+                new: newSel
+            });
+            if (parsed.evalJson === true) {
+                ctx.stdout.write(`${JSON.stringify(diff, null, 2)}\n`);
+            }
+            else {
+                ctx.stdout.write(formatDiffMarkdown(diff));
+            }
+            return diff.regressed ? 1 : 0;
+        }
+        catch (err) {
+            error(ctx, err instanceof Error ? err.message : String(err));
+            return 1;
+        }
+    }
     if (command === "eval") {
         const result = await runEval({
             projectRoot: ctx.cwd,
@@ -593,6 +661,12 @@ async function runCommand(parsed, ctx) {
             for (const [stage, count] of Object.entries(result.corpus.byStage)) {
                 ctx.stdout.write(`    - ${stage}: ${count}\n`);
             }
+            if (result.workflowCorpus.total > 0 || result.plannedTier === "C") {
+                ctx.stdout.write(`  workflow corpus: ${result.workflowCorpus.total} case(s)\n`);
+                for (const wf of result.workflowCorpus.cases) {
+                    ctx.stdout.write(`    - ${wf.id}: ${wf.stages.join(" → ")}\n`);
+                }
+            }
             ctx.stdout.write(`  verifiers available:\n`);
             for (const [key, value] of Object.entries(result.verifiersAvailable)) {
                 ctx.stdout.write(`    - ${key}: ${value ? "yes" : "no"}\n`);

package/dist/eval/agents/with-tools.d.ts

@@ -1,5 +1,5 @@
 import type { ChatUsage, EvalLlmClient } from "../llm-client.js";
-import { createSandbox } from "../sandbox.js";
+import { createSandbox, type Sandbox } from "../sandbox.js";
 import type { SandboxTool } from "../tools/index.js";
 import type { EvalCase, ResolvedEvalConfig, ToolUseSummary } from "../types.js";
 export declare class MaxTurnsExceededError extends Error {
@@ -16,6 +16,19 @@ export interface WithToolsInput {
     loadSkill?: (stage: EvalCase["stage"]) => Promise<string>;
     /** Override for the sandbox factory (test hook). */
     createSandboxFn?: typeof createSandbox;
+    /**
+     * Reuse an externally-managed sandbox instead of creating + disposing a
+     * per-call one. Tier C workflow orchestration uses this so every stage
+     * shares the same sandbox and earlier artifacts remain visible. When
+     * set, the caller is responsible for `dispose()`.
+     */
+    externalSandbox?: Sandbox;
+    /**
+     * Optional override of the default user prompt prefix. Tier C uses this
+     * to tell the model which stage it is on and where the prior artifacts
+     * are located.
+     */
+    promptPreamble?: string;
 }
 export interface WithToolsOutput {
     artifact: string;

package/dist/eval/agents/with-tools.js

@@ -62,10 +62,12 @@ export async function runWithTools(input) {
     const toolMap = toolsByName(tools);
     const toolsBody = toolsForRequest(tools);
     const sandboxFactory = input.createSandboxFn ?? createSandbox;
-    const sandbox = await sandboxFactory({
-        projectRoot,
-        ...(caseEntry.contextFiles ? { contextFiles: caseEntry.contextFiles } : {})
-    });
+    const externalSandbox = input.externalSandbox;
+    const sandbox = externalSandbox ??
+        (await sandboxFactory({
+            projectRoot,
+            ...(caseEntry.contextFiles ? { contextFiles: caseEntry.contextFiles } : {})
+        }));
     const toolUse = {
         turns: 0,
         calls: 0,
@@ -76,7 +78,7 @@ export async function runWithTools(input) {
     const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
     let lastModel = config.model;
     let totalAttempts = 0;
-    const userPrompt = buildUserPrompt(caseEntry, sandbox, tools);
+    const userPrompt = buildUserPrompt(caseEntry, sandbox, tools, input.promptPreamble);
     const messages = [
         { role: "system", content: systemPrompt },
         { role: "user", content: userPrompt }
@@ -150,7 +152,8 @@ export async function runWithTools(input) {
         throw new MaxTurnsExceededError(maxTurns);
     }
     finally {
-        await sandbox.dispose();
+        if (!externalSandbox)
+            await sandbox.dispose();
     }
 }
 function finalize(artifact, usage, model, attempts, started, toolUse, systemPrompt, userPrompt, config) {
@@ -196,16 +199,18 @@ function clampPositive(value, fallback) {
         return fallback;
     return Math.floor(value);
 }
-function buildUserPrompt(caseEntry, sandbox, tools) {
+function buildUserPrompt(caseEntry, sandbox, tools, preamble) {
     const toolList = tools.map((t) => `- ${t.descriptor.name}: ${t.descriptor.description}`);
     const files = caseEntry.contextFiles ?? [];
     const contextLines = files.length > 0
         ? files.map((f) => `- ${f}`).join("\n")
         : "(no files seeded)";
-    const lines = [
-        `Stage: ${caseEntry.stage}`,
-        `Case id: ${caseEntry.id}`,
-        ``,
+    const lines = [];
+    if (preamble && preamble.trim().length > 0) {
+        lines.push(preamble.trim(), ``);
+    }
+    lines.push(`Stage: ${caseEntry.stage}`, `Case id: ${caseEntry.id}`, ``);
+    const rest = [
         `Sandbox root: ${sandbox.root}`,
         `You may call the following tools to read or modify files inside the sandbox.`,
         `All paths are relative to the sandbox root.`,
@@ -225,6 +230,7 @@ function buildUserPrompt(caseEntry, sandbox, tools) {
         `You may optionally write the artifact to \`artifact.md\` in the sandbox; ` +
             `if you do, the last written \`artifact.md\` is preferred over the chat reply.`
     ];
+    lines.push(...rest);
     return lines.join("\n");
 }
 async function resolveArtifact(sandbox, fallback) {

package/dist/eval/agents/workflow.d.ts

@@ -0,0 +1,24 @@
+import type { EvalLlmClient } from "../llm-client.js";
+import { createSandbox } from "../sandbox.js";
+import type { SandboxTool } from "../tools/index.js";
+import type { ResolvedEvalConfig, WorkflowCase, WorkflowStageName, WorkflowStageResult } from "../types.js";
+export interface WorkflowInput {
+    workflow: WorkflowCase;
+    config: Pick<ResolvedEvalConfig, "model" | "agentTemperature" | "timeoutMs" | "tokenPricing" | "toolMaxTurns" | "toolMaxArgumentsBytes" | "toolMaxResultBytes" | "workflowMaxTotalTurns">;
+    projectRoot: string;
+    client: EvalLlmClient;
+    tools?: SandboxTool[];
+    /** Override for the SKILL.md loader (test hook). */
+    loadSkill?: (stage: WorkflowStageName) => Promise<string>;
+    /** Override for the sandbox factory (test hook). */
+    createSandboxFn?: typeof createSandbox;
+}
+export interface WorkflowOutput {
+    caseId: string;
+    stages: WorkflowStageResult[];
+    /** Map from stage name to produced artifact (also persisted in sandbox). */
+    artifacts: Map<WorkflowStageName, string>;
+    totalUsageUsd: number;
+    totalDurationMs: number;
+}
+export declare function runWorkflow(input: WorkflowInput): Promise<WorkflowOutput>;

package/dist/eval/agents/workflow.js

@@ -0,0 +1,133 @@
+/**
+ * Tier C workflow agent.
+ *
+ * Runs the Tier B with-tools loop once per stage in a workflow case,
+ * sharing a single sandbox across stages so every new stage can read
+ * the earlier artifacts the model produced. The shape of the run is:
+ *
+ *   1. Create one sandbox seeded with `contextFiles`.
+ *   2. For each stage in `workflow.stages`:
+ *      a. Delete any leftover `artifact.md` so the resolver doesn't
+ *         accidentally pick the previous stage's output.
+ *      b. Invoke `runWithTools({ externalSandbox: sandbox, promptPreamble })`.
+ *         The preamble tells the model which stage it is on and lists the
+ *         `stages/*.md` files available for reading.
+ *      c. Persist the returned artifact to `stages/<stage>.md` inside the
+ *         sandbox (deterministic, regardless of whether the model wrote
+ *         `artifact.md` itself).
+ *      d. Record `WorkflowStageResult` with usage, duration, and tool use.
+ *   3. Dispose the sandbox in a `finally` so temp directories never leak.
+ *
+ * Errors bubble up from `runWithTools`:
+ *   - `MaxTurnsExceededError` stops the workflow at the current stage.
+ *   - `DailyCostCapExceededError` (surfaced by the cost-guard wrapper in
+ *     the runner) aborts immediately.
+ *   - Generic `EvalLlmError` subclasses propagate as-is so the runner can
+ *     record a workflow-level verifier failure.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { createSandbox } from "../sandbox.js";
+import { loadStageSkill } from "./single-shot.js";
+import { runWithTools } from "./with-tools.js";
+const STAGES_SUBDIR = "stages";
+const ARTIFACT_CANDIDATES = ["artifact.md", "artifact.txt", "ARTIFACT.md"];
+export async function runWorkflow(input) {
+    const { workflow, config, projectRoot, client } = input;
+    const sandboxFactory = input.createSandboxFn ?? createSandbox;
+    const sandbox = await sandboxFactory({
+        projectRoot,
+        ...(workflow.contextFiles ? { contextFiles: workflow.contextFiles } : {})
+    });
+    const stageResults = [];
+    const artifacts = new Map();
+    let totalUsageUsd = 0;
+    let totalDurationMs = 0;
+    try {
+        await fs.mkdir(await sandbox.resolve(STAGES_SUBDIR, { allowMissing: true }), { recursive: true });
+        for (const step of workflow.stages) {
+            await clearArtifactFile(sandbox);
+            const priorStages = stageResults.map((r) => r.stage);
+            const preamble = buildStagePreamble(workflow, step.name, priorStages);
+            const caseEntry = {
+                id: `${workflow.id}/${step.name}`,
+                stage: step.name,
+                inputPrompt: step.inputPrompt,
+                ...(workflow.contextFiles ? { contextFiles: workflow.contextFiles } : {})
+            };
+            const result = await runWithTools({
+                caseEntry,
+                config,
+                projectRoot,
+                client,
+                ...(input.tools ? { tools: input.tools } : {}),
+                ...(input.loadSkill
+                    ? { loadSkill: input.loadSkill }
+                    : {
+                        loadSkill: (stage) => loadStageSkill(projectRoot, stage)
+                    }),
+                externalSandbox: sandbox,
+                promptPreamble: preamble
+            });
+            await persistStageArtifact(sandbox, step.name, result.artifact);
+            artifacts.set(step.name, result.artifact);
+            const stageResult = {
+                stage: step.name,
+                artifact: result.artifact,
+                durationMs: result.durationMs,
+                usageUsd: result.usageUsd,
+                toolUse: result.toolUse,
+                attempts: result.attempts,
+                model: result.model,
+                promptTokens: result.usage.promptTokens,
+                completionTokens: result.usage.completionTokens
+            };
+            stageResults.push(stageResult);
+            totalUsageUsd += result.usageUsd;
+            totalDurationMs += result.durationMs;
+        }
+        return {
+            caseId: workflow.id,
+            stages: stageResults,
+            artifacts,
+            totalUsageUsd: Number(totalUsageUsd.toFixed(6)),
+            totalDurationMs
+        };
+    }
+    finally {
+        await sandbox.dispose();
+    }
+}
+async function clearArtifactFile(sandbox) {
+    for (const candidate of ARTIFACT_CANDIDATES) {
+        try {
+            const abs = await sandbox.resolve(candidate);
+            await fs.rm(abs, { force: true });
+        }
+        catch {
+            // candidate did not exist — resolve threw SandboxEscapeError for
+            // missing realpath; safe to ignore.
+        }
+    }
+}
+async function persistStageArtifact(sandbox, stage, artifact) {
+    const rel = `${STAGES_SUBDIR}/${stage}.md`;
+    const abs = await sandbox.resolve(rel, { allowMissing: true });
+    await fs.mkdir(path.dirname(abs), { recursive: true });
+    await fs.writeFile(abs, artifact.endsWith("\n") ? artifact : `${artifact}\n`, "utf8");
+}
+function buildStagePreamble(workflow, current, priorStages) {
+    const lines = [];
+    lines.push(`You are running stage "${current}" of the Tier C workflow "${workflow.id}".`);
+    if (workflow.description) {
+        lines.push(`Case description: ${workflow.description}`);
+    }
+    if (priorStages.length === 0) {
+        lines.push(`This is the first stage. Any context_files have been seeded into the sandbox root.`);
+    }
+    else {
+        lines.push(`Earlier stage artifacts are available via read_file:`, ...priorStages.map((name) => `  - ${STAGES_SUBDIR}/${name}.md`), `Read the prior artifacts before drafting your output so decisions and ` +
+            `ids carry through.`);
+    }
+    return lines.join("\n");
+}

package/dist/eval/config-loader.js

@@ -35,7 +35,8 @@ const NUMERIC_ENVS = new Set([
     "CCLAW_EVAL_AGENT_TEMPERATURE",
     "CCLAW_EVAL_TOOL_MAX_TURNS",
     "CCLAW_EVAL_TOOL_MAX_ARG_BYTES",
-    "CCLAW_EVAL_TOOL_MAX_RESULT_BYTES"
+    "CCLAW_EVAL_TOOL_MAX_RESULT_BYTES",
+    "CCLAW_EVAL_WORKFLOW_MAX_TOTAL_TURNS"
 ]);
 function evalConfigError(configFilePath, reason) {
     return new Error(`Invalid cclaw eval config at ${configFilePath}: ${reason}\n` +
@@ -166,6 +167,7 @@ function validateFileConfig(raw, configFilePath) {
     assignPositiveInt("toolMaxTurns", raw.toolMaxTurns, "toolMaxTurns");
     assignPositiveInt("toolMaxArgumentsBytes", raw.toolMaxArgumentsBytes, "toolMaxArgumentsBytes");
     assignPositiveInt("toolMaxResultBytes", raw.toolMaxResultBytes, "toolMaxResultBytes");
+    assignPositiveInt("workflowMaxTotalTurns", raw.workflowMaxTotalTurns, "workflowMaxTotalTurns");
     if (raw.regression !== undefined) {
         if (!isRecord(raw.regression)) {
             throw evalConfigError(configFilePath, `"regression" must be a mapping`);
@@ -203,7 +205,8 @@ function validateFileConfig(raw, configFilePath) {
         "tokenPricing",
         "toolMaxTurns",
         "toolMaxArgumentsBytes",
-        "toolMaxResultBytes"
+        "toolMaxResultBytes",
+        "workflowMaxTotalTurns"
     ]);
     const unknown = Object.keys(raw).filter((key) => !knownKeys.has(key));
     if (unknown.length > 0) {
@@ -326,6 +329,7 @@ function applyEnvOverrides(base, env) {
         void label;
     };
     readPositiveInt("CCLAW_EVAL_TOOL_MAX_TURNS", "toolMaxTurns", "toolMaxTurns");
+    readPositiveInt("CCLAW_EVAL_WORKFLOW_MAX_TOTAL_TURNS", "workflowMaxTotalTurns", "workflowMaxTotalTurns");
     readPositiveInt("CCLAW_EVAL_TOOL_MAX_ARG_BYTES", "toolMaxArgumentsBytes", "toolMaxArgumentsBytes");
     readPositiveInt("CCLAW_EVAL_TOOL_MAX_RESULT_BYTES", "toolMaxResultBytes", "toolMaxResultBytes");
     const apiKey = read("CCLAW_EVAL_API_KEY");

package/dist/eval/diff.d.ts

@@ -0,0 +1,64 @@
+import type { EvalReport } from "./types.js";
+export interface EvalDiffInput {
+    projectRoot: string;
+    /** Version string, filename, or "latest". */
+    old: string;
+    /** Version string, filename, or "latest". */
+    new: string;
+}
+export interface EvalDiffCaseEntry {
+    caseId: string;
+    stage: string;
+    /** Pass/fail transition: `same`, `regressed`, `recovered`, `added`, `removed`. */
+    transition: "same" | "regressed" | "recovered" | "added" | "removed";
+    previousPassed?: boolean;
+    currentPassed?: boolean;
+    durationDeltaMs?: number;
+    costDeltaUsd?: number;
+    verifierDeltas: EvalDiffVerifierEntry[];
+    stageDeltas?: EvalDiffStageEntry[];
+}
+export interface EvalDiffVerifierEntry {
+    verifierId: string;
+    kind: string;
+    transition: "same" | "regressed" | "recovered" | "added" | "removed" | "score-drop";
+    previousScore?: number;
+    currentScore?: number;
+    previousOk?: boolean;
+    currentOk?: boolean;
+}
+export interface EvalDiffStageEntry {
+    stage: string;
+    durationDeltaMs: number;
+    costDeltaUsd: number;
+    turnsDelta: number;
+    callsDelta: number;
+}
+export interface EvalDiffReport {
+    old: EvalDiffReportMeta;
+    new: EvalDiffReportMeta;
+    summaryDelta: {
+        totalCasesDelta: number;
+        passedDelta: number;
+        failedDelta: number;
+        skippedDelta: number;
+        totalCostUsdDelta: number;
+        totalDurationMsDelta: number;
+    };
+    cases: EvalDiffCaseEntry[];
+    /** True when any case regressed or any verifier dropped. */
+    regressed: boolean;
+}
+export interface EvalDiffReportMeta {
+    runId: string;
+    cclawVersion: string;
+    generatedAt: string;
+    tier: string;
+    model: string;
+    sourcePath: string;
+}
+export declare function resolveReportPath(projectRoot: string, selector: string): Promise<string>;
+export declare function diffReports(previous: EvalReport, current: EvalReport, prevPath: string, currPath: string): EvalDiffReport;
+export declare function runEvalDiff(input: EvalDiffInput): Promise<EvalDiffReport>;
+/** Render the diff as a terse human-readable Markdown block. */
+export declare function formatDiffMarkdown(diff: EvalDiffReport): string;